From 06df0f0fc428678284ab1a40d307c1f90dbe80dc Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Wed, 31 May 2023 08:27:00 +0000 Subject: [PATCH 001/103] Update .gitreview for stable/2023.1 Change-Id: Icacd1624f705154bf8fe8f73e83de46b1e845179 --- .gitreview | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitreview b/.gitreview index b9adc8dd9..b0241214f 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/kayobe-config.git +defaultbranch=stable/2023.1 From b6a94dfe4607216a4c4357b9f7319516a817580f Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Wed, 31 May 2023 08:27:01 +0000 Subject: [PATCH 002/103] Update TOX_CONSTRAINTS_FILE for stable/2023.1 Update the URL to the upper-constraints file to point to the redirect rule on releases.openstack.org so that anyone working on this branch will switch to the correct upper-constraints list automatically when the requirements repository branches. Until the requirements repository has as stable/2023.1 branch, tests will continue to use the upper-constraints list on master. Change-Id: I33b0d87a32d5853f529c304546fd01f03edfbe1e --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 47adec2e2..223ef4ebc 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ skipsdist = True [testenv] install_command = pip install {opts} {packages} deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/test-requirements.txt [testenv:pep8] From 8b7900c185a773bc410a153816a76c9b1c30f27d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 1 Jun 2023 09:15:12 +0100 Subject: [PATCH 003/103] 2023.1 - Sync defaults with Kayobe skydive != skyline Change-Id: If57a9d39a77ca02a1cd7f59ae5691fbf39b78964 (cherry picked from commit 3adf46cfd1137aae68c729c33b53b8c691f03ca3) --- etc/kayobe/kolla.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 2d9a1e696..81e48c55b 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -432,7 +432,7 @@ #kolla_enable_redis: #kolla_enable_sahara: #kolla_enable_senlin: -#kolla_enable_skydive: +#kolla_enable_skyline: #kolla_enable_solum: #kolla_enable_swift: #kolla_enable_swift_recon: From bf09c41ddcec60d3ab4404ea8860d99e4380d948 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 20 Sep 2023 09:14:17 +0100 Subject: [PATCH 004/103] kayobe-env: Unstick KOLLA_SOURCE_PATH and KOLLA_VENV_PATH The kayobe-env script does not update the KOLLA_SOURCE_PATH and KOLLA_VENV_PATH variables if they are already set. This can lead to dangerous and difficult to diagnose issues where Kayobe uses a different version of Kolla Ansible than expected. This change updates these variables each time the kayobe-env script is sourced. Change-Id: I3b4b0b611750b9c7846ff5f74554aee2f14939e4 Closes-Bug: #2036711 (cherry picked from commit 651b8be1a0a2dd38ab46253a0c4a9c7d617cf7bc) --- kayobe-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kayobe-env b/kayobe-env index 5137927e5..28b1cccdb 100644 --- a/kayobe-env +++ b/kayobe-env @@ -30,8 +30,8 @@ export KOLLA_CONFIG_PATH=$KAYOBE_CONFIG_ROOT/etc/kolla # kayobe/ # kolla-ansible/ base_path=$(realpath $KAYOBE_CONFIG_ROOT/../../) -export KOLLA_SOURCE_PATH=${KOLLA_SOURCE_PATH:-${base_path}/src/kolla-ansible} -export KOLLA_VENV_PATH=${KOLLA_VENV_PATH:-${base_path}/venvs/kolla-ansible} +export KOLLA_SOURCE_PATH=${base_path}/src/kolla-ansible +export KOLLA_VENV_PATH=${base_path}/venvs/kolla-ansible function check_and_export_env { # Look for existing Kayobe environments From 63f3f6a09d5398cb1a293346b5a93550d90a9f67 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 4 Dec 2023 17:47:46 +0100 Subject: [PATCH 005/103] Synchronise with stable/2023.1 Synchronise with kayobe @ 96e09e21a4e0820feb10861257453132f7d8c8ba. Change-Id: I9561aac93097159977a93f836f857da5d7d5096d --- etc/kayobe/bifrost.yml | 7 +++++++ etc/kayobe/compute.yml | 8 ++++---- etc/kayobe/docker.yml | 5 +++++ etc/kayobe/infra-vms.yml | 8 ++++---- etc/kayobe/kolla.yml | 13 ++++++------- etc/kayobe/monitoring.yml | 6 +++--- etc/kayobe/seed-hypervisor.yml | 4 ++-- etc/kayobe/seed.yml | 12 ++++++++---- etc/kayobe/storage.yml | 8 ++++---- 9 files changed, 43 insertions(+), 28 deletions(-) diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index a9eba19dd..53921a1b4 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -71,6 +71,10 @@ # Default is none. #kolla_bifrost_deploy_image_rootfs: +# Custom cloud-init user-data passed to deploy of the deployment image. +# Default is an empty string. +#kolla_bifrost_deploy_image_user_data_content: + ############################################################################### # Ironic configuration. @@ -116,6 +120,9 @@ # Ironic inspector deployment ramdisk location. #kolla_bifrost_inspector_deploy_ramdisk: +# Ironic inspector legacy deployment kernel location. +#kolla_bifrost_inspector_legacy_deploy_kernel: + # Timeout of hardware inspection on overcloud nodes, in seconds. Default is # {{ inspector_inspection_timeout }}. #kolla_bifrost_inspection_timeout: diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index 03c41e3db..15c4cbcc6 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -63,15 +63,15 @@ ############################################################################### # Compute node LVM configuration. -# List of compute volume groups. See mrlesmithjr.manage-lvm role for +# List of compute volume groups. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_groups: -# Default list of compute volume groups. See mrlesmithjr.manage-lvm role for +# Default list of compute volume groups. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_groups_default: -# Additional list of compute volume groups. See mrlesmithjr.manage-lvm role +# Additional list of compute volume groups. See mrlesmithjr.manage_lvm role # for format. #compute_lvm_groups_extra: @@ -82,7 +82,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #compute_lvm_group_data_enabled: -# Compute LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Compute LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_group_data: diff --git a/etc/kayobe/docker.yml b/etc/kayobe/docker.yml index 4ec00a1f4..ba3f22c9a 100644 --- a/etc/kayobe/docker.yml +++ b/etc/kayobe/docker.yml @@ -24,6 +24,11 @@ # URL of docker registry #docker_registry: +# Whether docker should be configured to use an insecure registry. +# Default is false, unless docker_registry_enabled is true and +# docker_registry_enable_tls is false. +#docker_registry_insecure: + # CA of docker registry #docker_registry_ca: diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index c8050d050..3e1db5d5e 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -92,15 +92,15 @@ ############################################################################### # Infrastructure VM node LVM configuration. -# List of infrastructure vm volume groups. See mrlesmithjr.manage-lvm role for +# List of infrastructure vm volume groups. See mrlesmithjr.manage_lvm role for # format. #infra_vm_lvm_groups: -# Default list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# Default list of infrastructure vm volume groups. See mrlesmithjr.manage_lvm # role for format. #infra_vm_lvm_groups_default: -# Additional list of infrastructure vm volume groups. See mrlesmithjr.manage-lvm +# Additional list of infrastructure vm volume groups. See mrlesmithjr.manage_lvm # role for format. #infra_vm_lvm_groups_extra: @@ -111,7 +111,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #infra_vm_lvm_group_data_enabled: -# Infrastructure VM LVM volume group for data. See mrlesmithjr.manage-lvm role +# Infrastructure VM LVM volume group for data. See mrlesmithjr.manage_lvm role # for format. #infra_vm_lvm_group_data: diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 81e48c55b..2ab22bb8f 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -85,11 +85,6 @@ # Docker namespace to use for Kolla images. Default is 'kolla'. #kolla_docker_namespace: -# Whether docker should be configured to use an insecure registry for Kolla -# images. Default is false, unless docker_registry_enabled is true and -# docker_registry_enable_tls is false. -#kolla_docker_registry_insecure: - # Username to use to access a docker registry. Default is not set, in which # case the registry will be used without authentication. #kolla_docker_registry_username: @@ -454,6 +449,10 @@ # Kolla passwords file. #kolla_ansible_default_custom_passwords: +# Dictionary containing extra custom passwords to add or override in the Kolla +# passwords file. +#kolla_ansible_extra_custom_passwords: + # Dictionary containing custom passwords to add or override in the Kolla # passwords file. #kolla_ansible_custom_passwords: @@ -493,7 +492,7 @@ # Path to a CA certificate file to use for the OS_CACERT environment variable # in public-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_external_fqdn_cacert: +#kolla_public_openrc_cacert: # Internal API certificate bundle. # @@ -506,7 +505,7 @@ # Path to a CA certificate file to use for the OS_CACERT environment variable # in admin-openrc.sh file when TLS is enabled, instead of Kolla-Ansible's # default. -#kolla_internal_fqdn_cacert: +#kolla_admin_openrc_cacert: ############################################################################### # Proxy configuration diff --git a/etc/kayobe/monitoring.yml b/etc/kayobe/monitoring.yml index f332ab938..5468936d3 100644 --- a/etc/kayobe/monitoring.yml +++ b/etc/kayobe/monitoring.yml @@ -63,15 +63,15 @@ ############################################################################### # Monitoring node LVM configuration. -# List of monitoring node volume groups. See mrlesmithjr.manage-lvm role for +# List of monitoring node volume groups. See mrlesmithjr.manage_lvm role for # format. #monitoring_lvm_groups: -# Default list of monitoring node volume groups. See mrlesmithjr.manage-lvm +# Default list of monitoring node volume groups. See mrlesmithjr.manage_lvm # role for format. #monitoring_lvm_groups_default: -# Additional list of monitoring node volume groups. See mrlesmithjr.manage-lvm +# Additional list of monitoring node volume groups. See mrlesmithjr.manage_lvm # role for format. #monitoring_lvm_groups_extra: diff --git a/etc/kayobe/seed-hypervisor.yml b/etc/kayobe/seed-hypervisor.yml index ac72fcd3d..dd8fbca23 100644 --- a/etc/kayobe/seed-hypervisor.yml +++ b/etc/kayobe/seed-hypervisor.yml @@ -36,7 +36,7 @@ ############################################################################### # Seed hypervisor node LVM configuration. -# List of seed hypervisor volume groups. See mrlesmithjr.manage-lvm role for +# List of seed hypervisor volume groups. See mrlesmithjr.manage_lvm role for # format. Set to "{{ seed_hypervisor_lvm_groups_with_data }}" to create a # volume group for libvirt storage. #seed_hypervisor_lvm_groups: @@ -45,7 +45,7 @@ # default. #seed_hypervisor_lvm_groups_with_data: -# Seed LVM volume group for data. See mrlesmithjr.manage-lvm role for format. +# Seed LVM volume group for data. See mrlesmithjr.manage_lvm role for format. #seed_hypervisor_lvm_group_data: # List of disks for use by seed hypervisor LVM data volume group. Default to an diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index ade99307d..bc86fa627 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -36,14 +36,14 @@ ############################################################################### # Seed node LVM configuration. -# List of seed volume groups. See mrlesmithjr.manage-lvm role for format. +# List of seed volume groups. See mrlesmithjr.manage_lvm role for format. #seed_lvm_groups: -# Default list of seed volume groups. See mrlesmithjr.manage-lvm role for +# Default list of seed volume groups. See mrlesmithjr.manage_lvm role for # format. #seed_lvm_groups_default: -# Additional list of seed volume groups. See mrlesmithjr.manage-lvm role for +# Additional list of seed volume groups. See mrlesmithjr.manage_lvm role for # format. #seed_lvm_groups_extra: @@ -54,7 +54,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #seed_lvm_group_data_enabled: -# Seed LVM volume group for data. See mrlesmithjr.manage-lvm role for format. +# Seed LVM volume group for data. See mrlesmithjr.manage_lvm role for format. #seed_lvm_group_data: # List of disks for use by seed LVM data volume group. Default to an invalid @@ -106,6 +106,10 @@ # #seed_containers: +# Whether to attempt a basic authentication login to a registry when +# deploying seed containers +#seed_deploy_containers_registry_attempt_login: + ############################################################################### # Seed node firewalld configuration. diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index 535666c95..e9e52dfe6 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -68,15 +68,15 @@ ############################################################################### # Storage node LVM configuration. -# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# List of storage volume groups. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_groups: -# Default list of storage volume groups. See mrlesmithjr.manage-lvm role for +# Default list of storage volume groups. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_groups_default: -# Additional list of storage volume groups. See mrlesmithjr.manage-lvm role +# Additional list of storage volume groups. See mrlesmithjr.manage_lvm role # for format. #storage_lvm_groups_extra: @@ -87,7 +87,7 @@ # 'docker_storage_driver' is set to 'devicemapper', or false otherwise. #storage_lvm_group_data_enabled: -# Storage LVM volume group for data. See mrlesmithjr.manage-lvm role for +# Storage LVM volume group for data. See mrlesmithjr.manage_lvm role for # format. #storage_lvm_group_data: From 205d070a13ade0c259f2f3c0e13d1d9593a1bbae Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 7 Jun 2024 08:50:20 +0000 Subject: [PATCH 006/103] Update .gitreview for stable/2024.1 Change-Id: I6a1a16e473c805a1993c3c5c8eff7484181278f9 --- .gitreview | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitreview b/.gitreview index b9adc8dd9..088e01d84 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/kayobe-config.git +defaultbranch=stable/2024.1 From 982bfae457ac0ab188a215e6d1f0183e9f02d565 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 7 Jun 2024 08:50:22 +0000 Subject: [PATCH 007/103] Update TOX_CONSTRAINTS_FILE for stable/2024.1 Update the URL to the upper-constraints file to point to the redirect rule on releases.openstack.org so that anyone working on this branch will switch to the correct upper-constraints list automatically when the requirements repository branches. Until the requirements repository has as stable/2024.1 branch, tests will continue to use the upper-constraints list on master. Change-Id: Ie1f7284da9b7d4a7ecf226c02e13ca45f87451ea --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 47adec2e2..3b00c00af 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ skipsdist = True [testenv] install_command = pip install {opts} {packages} deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2024.1} -r{toxinidir}/test-requirements.txt [testenv:pep8] From 3c070c87024ce048479693023051c2236f387de9 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 24 Jun 2024 10:08:43 +0100 Subject: [PATCH 008/103] Adding inventory hosts --- etc/kayobe/inventory/hosts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 etc/kayobe/inventory/hosts diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts new file mode 100644 index 000000000..70b21fe5c --- /dev/null +++ b/etc/kayobe/inventory/hosts @@ -0,0 +1,31 @@ +# This host acts as the configuration management Ansible control host. This must be +# localhost. +localhost ansible_connection=local + +[seed] +srcseed + +[controllers] +srcseed + +[compute] +c-1 +c-2 +c-3 +c-4 +c-5 +c-6 +c-7 +c-14 +# c-8 +# c-9 +# c-10 +# c-11 +# c-12 +# c-13 + +[storage] +ds1 +ds2 +mds1 +mds2 From 8679ea96087d15601fa0c0b8ebaa2546055d0243 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 10:17:35 +0100 Subject: [PATCH 009/103] Add network interfaces --- etc/kayobe/inventory/group_vars/compute/network-interfaces | 3 +++ etc/kayobe/inventory/group_vars/controllers/network-interfaces | 3 +++ etc/kayobe/inventory/group_vars/seed/network-interfaces | 3 +++ 3 files changed, 9 insertions(+) diff --git a/etc/kayobe/inventory/group_vars/compute/network-interfaces b/etc/kayobe/inventory/group_vars/compute/network-interfaces index 0592581a4..704d41237 100644 --- a/etc/kayobe/inventory/group_vars/compute/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute/network-interfaces @@ -15,6 +15,9 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. +prod_interface: ens9f0np0 +data_interface: ens3f0np0 + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 1c6f81f0e..0e483ff70 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -15,6 +15,9 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. +prod_interface: eno12399 +data_interface: ens2f0np0 + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/seed/network-interfaces b/etc/kayobe/inventory/group_vars/seed/network-interfaces index 542cf751a..ea5da98b4 100644 --- a/etc/kayobe/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/inventory/group_vars/seed/network-interfaces @@ -15,6 +15,9 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. +prod_interface: eno12399 +data_interface: ens2f0np0 + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 83564eda5ca5fe77751b6924ca7163272eb4c108 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 10:41:03 +0100 Subject: [PATCH 010/103] Import proxy support from stackhpc-kayobe-config https://github.com/stackhpc/stackhpc-kayobe-config/commit/bb97a8ac9b4ed8a296707d5fd3085a4e025b20ee --- etc/kayobe/containers/squid.conf | 79 +++++++++++++++++++ etc/kayobe/containers/squid_proxy/pre.yml | 14 ++++ .../inventory/group_vars/overcloud/proxy | 38 +++++++++ .../inventory/group_vars/overcloud/time | 7 ++ 4 files changed, 138 insertions(+) create mode 100644 etc/kayobe/containers/squid.conf create mode 100644 etc/kayobe/containers/squid_proxy/pre.yml create mode 100644 etc/kayobe/inventory/group_vars/overcloud/proxy create mode 100644 etc/kayobe/inventory/group_vars/overcloud/time diff --git a/etc/kayobe/containers/squid.conf b/etc/kayobe/containers/squid.conf new file mode 100644 index 000000000..c7aadf879 --- /dev/null +++ b/etc/kayobe/containers/squid.conf @@ -0,0 +1,79 @@ +# +# Recommended minimum configuration: +# + +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 10.0.0.0/8 # RFC1918 possible internal network +acl localnet src 172.16.0.0/12 # RFC1918 possible internal network +acl localnet src 192.168.0.0/16 # RFC1918 possible internal network +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines + +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT + +# +# Recommended minimum Access Permission configuration: +# +# Deny requests to certain unsafe ports +http_access deny !Safe_ports + +# Deny CONNECT to other than secure SSL ports +http_access deny CONNECT !SSL_ports + +# Only allow cachemgr access from localhost +http_access allow localhost manager +http_access deny manager + +# We strongly recommend the following be uncommented to protect innocent +# web applications running on the proxy server who think the only +# one who can access services on "localhost" is a local user +#http_access deny to_localhost + +# +# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS +# + +# Example rule allowing access from your local networks. +# Adapt localnet in the ACL section to list your (internal) IP networks +# from where browsing should be allowed +http_access allow localnet +http_access allow localhost + +# And finally deny all other access to this proxy +http_access deny all + +# Squid normally listens to port 3128 +http_port 3128 + +# Uncomment and adjust the following to add a disk cache directory. +cache_dir ufs /var/spool/squid 4096 16 256 +cache_mem 768 MB +maximum_object_size_in_memory 64 MB + +# Set a sensible cap on the number of open file descriptors. +# Rocky Linux 8.6 sets this to ~2^30 which results in alloc failure on start. +max_filedescriptors 10000 + +# Leave coredumps in the first cache dir +coredump_dir /var/spool/squid + +# +# Add any of your own refresh_pattern entries above these. +# +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern . 0 20% 4320 \ No newline at end of file diff --git a/etc/kayobe/containers/squid_proxy/pre.yml b/etc/kayobe/containers/squid_proxy/pre.yml new file mode 100644 index 000000000..5fa390290 --- /dev/null +++ b/etc/kayobe/containers/squid_proxy/pre.yml @@ -0,0 +1,14 @@ +--- +- name: Ensure /srv/docker/squid directory exists + file: + path: /srv/docker/squid + state: directory + mode: 0755 + become: true + +- name: Copy modified squid.conf + copy: + src: "{{ kayobe_config_path }}/containers/squid_proxy/squid.conf" + dest: /srv/docker/squid/ + mode: 0644 + become: true \ No newline at end of file diff --git a/etc/kayobe/inventory/group_vars/overcloud/proxy b/etc/kayobe/inventory/group_vars/overcloud/proxy new file mode 100644 index 000000000..ed3ca7c4c --- /dev/null +++ b/etc/kayobe/inventory/group_vars/overcloud/proxy @@ -0,0 +1,38 @@ +--- +# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port). By +# default no proxy is used. +http_proxy: "http://{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}:3128" + +# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port). By +# default no proxy is used. +https_proxy: "{{ http_proxy }}" + +# List of domains, hostnames, IP addresses and networks for which no proxy is +# used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~ +# docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or +# ["127.0.0.1", "localhost"] otherwise. This is configured only if either +# http_proxy or https_proxy is set. +no_proxy: + - "127.0.0.1" + - "localhost" + - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" + - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[groups.seed.0] }}" + - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[inventory_hostname] }}" + - "{{ kolla_external_fqdn }}" + - "{{ kolla_internal_fqdn }}" + +# PyPI proxy URL (format: http(s)://[user:password@]proxy_name:port) +pip_proxy: "{{ https_proxy }}" + +# dnf_config: +# proxy: http://proxy.example.com +dnf_config: + proxy: "{{ http_proxy }}" + +# Use a proxy for external Docker image pulls +docker_http_proxy: "http://:3128" +docker_https_proxy: "http://:3128" +docker_no_proxy: + - "127.0.0.1" + - "localhost" + - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" \ No newline at end of file diff --git a/etc/kayobe/inventory/group_vars/overcloud/time b/etc/kayobe/inventory/group_vars/overcloud/time new file mode 100644 index 000000000..42ff4208e --- /dev/null +++ b/etc/kayobe/inventory/group_vars/overcloud/time @@ -0,0 +1,7 @@ +--- +# NTP services for overcloud hosts +# During early initialisation we use the seed Node +# Following deployment we include the OpenStack VIP + +chrony_ntp_servers: + - server: "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" \ No newline at end of file From dbb2a00f23f3b1d97ff6a5b5b85022260c9d4cae Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 10:46:35 +0100 Subject: [PATCH 011/103] Config seed proxy container --- etc/kayobe/seed.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index bc86fa627..299c05fab 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -105,6 +105,16 @@ # post: "{{ kayobe_env_config_path }}/containers/squid/post.yml" # #seed_containers: +seed_containers: + squid: + image: stackhpc/squid + pre: "{{ kayobe_config_path }}/containers/squid_proxy/pre.yml" + tag: "4.15" + network_mode: host + volumes: + - squid_spool:/var/spool/squid + - /srv/docker/squid/squid.conf:/etc/squid/squid.conf + - /var/log/squid:/var/log/squid # Whether to attempt a basic authentication login to a registry when # deploying seed containers From a2f6546bcf36a12cfb5c3c821df072196ecd74c6 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 10:46:54 +0100 Subject: [PATCH 012/103] Add storage node interface --- etc/kayobe/inventory/group_vars/storage/network-interfaces | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/kayobe/inventory/group_vars/storage/network-interfaces b/etc/kayobe/inventory/group_vars/storage/network-interfaces index 45efd636c..fe3abff34 100644 --- a/etc/kayobe/inventory/group_vars/storage/network-interfaces +++ b/etc/kayobe/inventory/group_vars/storage/network-interfaces @@ -15,6 +15,10 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. +prod_interface: eno8303 +# TODO - check this! +data_interface: ens2f0np0 + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 8280349e173f118f70465db59de706ffd479cb09 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 11:01:47 +0100 Subject: [PATCH 013/103] First attempt at network allocations --- .../group_vars/seed/network-interfaces | 1 + etc/kayobe/network-allocation.yml | 41 +++++++++++++++++++ etc/kayobe/networks.yml | 16 +++++++- 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/network-allocation.yml diff --git a/etc/kayobe/inventory/group_vars/seed/network-interfaces b/etc/kayobe/inventory/group_vars/seed/network-interfaces index ea5da98b4..04ccd742c 100644 --- a/etc/kayobe/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/inventory/group_vars/seed/network-interfaces @@ -17,6 +17,7 @@ prod_interface: eno12399 data_interface: ens2f0np0 +ipmi_interface: enp177s0 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml new file mode 100644 index 000000000..c01a23da5 --- /dev/null +++ b/etc/kayobe/network-allocation.yml @@ -0,0 +1,41 @@ +--- +#prod_cidr: 192.168.10.0/24 +#data_cidr: 192.168.100.0/24 +#ipmi_cidr: 192.168.1.0/24 + +ipmi_ips: + srcseed: 192.168.1.201 + +prod_ips: + srcseed: 192.168.10.201 + c-1: 192.168.10.1 + c-2: 192.168.10.2 + c-3: 192.168.10.3 + c-4: 192.168.10.4 + c-5: 192.168.10.5 + c-6: 192.168.10.6 + c-7: 192.168.10.7 + c-8: 192.168.10.8 + c-9: 192.168.10.9 + c-10: 192.168.10.10 + c-11: 192.168.10.11 + c-12: 192.168.10.12 + c-13: 192.168.10.13 + c-14: 192.168.10.14 + +data_ips: + srcseed: 192.168.100.201 + c-1: 192.168.100.1 + c-2: 192.168.100.2 + c-3: 192.168.100.3 + c-4: 192.168.100.4 + c-5: 192.168.100.5 + c-6: 192.168.100.6 + c-7: 192.168.100.7 + c-8: 192.168.100.8 + c-9: 192.168.100.9 + c-10: 192.168.100.10 + c-11: 192.168.100.11 + c-12: 192.168.100.12 + c-13: 192.168.100.13 + c-14: 192.168.100.14 diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index e58ab44cf..2880e276b 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -5,11 +5,12 @@ # Network role to network mappings. # Name of the network used for admin access to the overcloud -#admin_oc_net_name: +admin_oc_net_name: prod # Name of the network used by the seed to manage the bare metal overcloud # hosts via their out-of-band management controllers. #oob_oc_net_name: +oob_oc_net_name: ipmi # Name of the network used by the seed to provision the bare metal overcloud # hosts. @@ -25,6 +26,7 @@ # Name of the network used to expose the internal OpenStack API endpoints. #internal_net_name: +internal_net_name: data # List of names of networks used to provide external network access via # Neutron. @@ -32,18 +34,24 @@ # If external_net_name is defined, external_net_names will default to a list # containing one item, external_net_name. #external_net_names: +external_net_names: +- prod # Name of the network used to expose the public OpenStack API endpoints. #public_net_name: +public_net_name: prod # Name of the network used by Neutron to carry tenant overlay network traffic. #tunnel_net_name: +tunnel_net_name: data # Name of the network used to carry storage data traffic. #storage_net_name: +storage_net_name: data # Name of the network used to carry storage management traffic. #storage_mgmt_net_name: +storage_mgmt_net_name: data # Name of the network used to carry swift storage data traffic. #swift_storage_net_name: @@ -79,6 +87,12 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. +prod_cidr: 192.168.10.0/24 + +data_cidr: 192.168.100.0/24 + +ipmi_cidr: 192.168.1.0/24 + ############################################################################### # Network virtual patch link configuration. From 0367d6480da3c495b16de4f8374d1087d5f8fcd6 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 11:16:15 +0100 Subject: [PATCH 014/103] Set bootstrap user account --- etc/kayobe/compute.yml | 1 + etc/kayobe/controllers.yml | 1 + etc/kayobe/globals.yml | 4 ++++ etc/kayobe/kolla/globals.yml | 2 +- etc/kayobe/seed.yml | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index 15c4cbcc6..d251291ce 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -5,6 +5,7 @@ # User with which to access the computes via SSH during bootstrap, in order # to setup the Kayobe user account. Default is {{ os_distribution }}. #compute_bootstrap_user: +compute_bootstrap_user: ileigh ############################################################################### # Compute network interface configuration. diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index 983251c6c..720ff52b1 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -5,6 +5,7 @@ # User with which to access the controllers via SSH during bootstrap, in order # to setup the Kayobe user account. Default is {{ os_distribution }}. #controller_bootstrap_user: +controller_bootstrap_user: ileigh ############################################################################### # Controller network interface configuration. diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index 50af1cb14..5680720e1 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -47,11 +47,13 @@ # OS distribution name. Valid options are "centos", "rocky", "ubuntu". Default # is "rocky". #os_distribution: +os_distribution: "rocky" # OS release. Valid options are "9-stream" when os_distribution is "centos", or # "9" when os_distribution is "rocky", or "jammy" when os_distribution is # "ubuntu". #os_release: +os_release: 9 ############################################################################### # Ansible configuration. @@ -64,6 +66,8 @@ # to not specify a gather subset. #kayobe_ansible_setup_gather_subset: +selinux_do_reboot: false + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 794c5fee5..733467697 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,2 +1,2 @@ --- -# Add extra Kolla global configuration here. +# Add extra Kolla global configuration here. \ No newline at end of file diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 299c05fab..cbb0a2528 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -5,6 +5,7 @@ # User with which to access the seed via SSH during bootstrap, in order to # setup the Kayobe user account. Default is {{ os_distribution }}. #seed_bootstrap_user: +seed_bootstrap_user: ileigh ############################################################################### # Seed network interface configuration. From 35f0a0ab8d750c8b82fd67e400ee800648dc0881 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 11:41:37 +0100 Subject: [PATCH 015/103] Add seed node as prod gateway --- etc/kayobe/networks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 2880e276b..574d613f0 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -88,6 +88,7 @@ storage_mgmt_net_name: data # network configuration reference in the Kayobe documentation. prod_cidr: 192.168.10.0/24 +prod_gateway: 192.168.10.201 data_cidr: 192.168.100.0/24 From e108d56a4cc528252b7fcb8219598d0354926d21 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 11:42:41 +0100 Subject: [PATCH 016/103] Remove extra proxy settings for now --- .../inventory/group_vars/overcloud/proxy | 38 ------------------- .../inventory/group_vars/overcloud/time | 7 ---- 2 files changed, 45 deletions(-) delete mode 100644 etc/kayobe/inventory/group_vars/overcloud/proxy delete mode 100644 etc/kayobe/inventory/group_vars/overcloud/time diff --git a/etc/kayobe/inventory/group_vars/overcloud/proxy b/etc/kayobe/inventory/group_vars/overcloud/proxy deleted file mode 100644 index ed3ca7c4c..000000000 --- a/etc/kayobe/inventory/group_vars/overcloud/proxy +++ /dev/null @@ -1,38 +0,0 @@ ---- -# HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port). By -# default no proxy is used. -http_proxy: "http://{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}:3128" - -# HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port). By -# default no proxy is used. -https_proxy: "{{ http_proxy }}" - -# List of domains, hostnames, IP addresses and networks for which no proxy is -# used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~ -# docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or -# ["127.0.0.1", "localhost"] otherwise. This is configured only if either -# http_proxy or https_proxy is set. -no_proxy: - - "127.0.0.1" - - "localhost" - - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" - - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[groups.seed.0] }}" - - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[inventory_hostname] }}" - - "{{ kolla_external_fqdn }}" - - "{{ kolla_internal_fqdn }}" - -# PyPI proxy URL (format: http(s)://[user:password@]proxy_name:port) -pip_proxy: "{{ https_proxy }}" - -# dnf_config: -# proxy: http://proxy.example.com -dnf_config: - proxy: "{{ http_proxy }}" - -# Use a proxy for external Docker image pulls -docker_http_proxy: "http://:3128" -docker_https_proxy: "http://:3128" -docker_no_proxy: - - "127.0.0.1" - - "localhost" - - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" \ No newline at end of file diff --git a/etc/kayobe/inventory/group_vars/overcloud/time b/etc/kayobe/inventory/group_vars/overcloud/time deleted file mode 100644 index 42ff4208e..000000000 --- a/etc/kayobe/inventory/group_vars/overcloud/time +++ /dev/null @@ -1,7 +0,0 @@ ---- -# NTP services for overcloud hosts -# During early initialisation we use the seed Node -# Following deployment we include the OpenStack VIP - -chrony_ntp_servers: - - server: "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" \ No newline at end of file From c1bec5907d9c2bbefd2d61146fea11b96e22ac66 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 24 Jun 2024 11:43:43 +0100 Subject: [PATCH 017/103] Drop squid config for now --- etc/kayobe/containers/squid.conf | 79 ----------------------- etc/kayobe/containers/squid_proxy/pre.yml | 14 ---- etc/kayobe/seed.yml | 10 --- 3 files changed, 103 deletions(-) delete mode 100644 etc/kayobe/containers/squid.conf delete mode 100644 etc/kayobe/containers/squid_proxy/pre.yml diff --git a/etc/kayobe/containers/squid.conf b/etc/kayobe/containers/squid.conf deleted file mode 100644 index c7aadf879..000000000 --- a/etc/kayobe/containers/squid.conf +++ /dev/null @@ -1,79 +0,0 @@ -# -# Recommended minimum configuration: -# - -# Example rule allowing access from your local networks. -# Adapt to list your (internal) IP networks from where browsing -# should be allowed -acl localnet src 10.0.0.0/8 # RFC1918 possible internal network -acl localnet src 172.16.0.0/12 # RFC1918 possible internal network -acl localnet src 192.168.0.0/16 # RFC1918 possible internal network -acl localnet src fc00::/7 # RFC 4193 local private network range -acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines - -acl SSL_ports port 443 -acl Safe_ports port 80 # http -acl Safe_ports port 21 # ftp -acl Safe_ports port 443 # https -acl Safe_ports port 70 # gopher -acl Safe_ports port 210 # wais -acl Safe_ports port 1025-65535 # unregistered ports -acl Safe_ports port 280 # http-mgmt -acl Safe_ports port 488 # gss-http -acl Safe_ports port 591 # filemaker -acl Safe_ports port 777 # multiling http -acl CONNECT method CONNECT - -# -# Recommended minimum Access Permission configuration: -# -# Deny requests to certain unsafe ports -http_access deny !Safe_ports - -# Deny CONNECT to other than secure SSL ports -http_access deny CONNECT !SSL_ports - -# Only allow cachemgr access from localhost -http_access allow localhost manager -http_access deny manager - -# We strongly recommend the following be uncommented to protect innocent -# web applications running on the proxy server who think the only -# one who can access services on "localhost" is a local user -#http_access deny to_localhost - -# -# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS -# - -# Example rule allowing access from your local networks. -# Adapt localnet in the ACL section to list your (internal) IP networks -# from where browsing should be allowed -http_access allow localnet -http_access allow localhost - -# And finally deny all other access to this proxy -http_access deny all - -# Squid normally listens to port 3128 -http_port 3128 - -# Uncomment and adjust the following to add a disk cache directory. -cache_dir ufs /var/spool/squid 4096 16 256 -cache_mem 768 MB -maximum_object_size_in_memory 64 MB - -# Set a sensible cap on the number of open file descriptors. -# Rocky Linux 8.6 sets this to ~2^30 which results in alloc failure on start. -max_filedescriptors 10000 - -# Leave coredumps in the first cache dir -coredump_dir /var/spool/squid - -# -# Add any of your own refresh_pattern entries above these. -# -refresh_pattern ^ftp: 1440 20% 10080 -refresh_pattern ^gopher: 1440 0% 1440 -refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 -refresh_pattern . 0 20% 4320 \ No newline at end of file diff --git a/etc/kayobe/containers/squid_proxy/pre.yml b/etc/kayobe/containers/squid_proxy/pre.yml deleted file mode 100644 index 5fa390290..000000000 --- a/etc/kayobe/containers/squid_proxy/pre.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Ensure /srv/docker/squid directory exists - file: - path: /srv/docker/squid - state: directory - mode: 0755 - become: true - -- name: Copy modified squid.conf - copy: - src: "{{ kayobe_config_path }}/containers/squid_proxy/squid.conf" - dest: /srv/docker/squid/ - mode: 0644 - become: true \ No newline at end of file diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index cbb0a2528..eb5a8d6c7 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -106,16 +106,6 @@ seed_bootstrap_user: ileigh # post: "{{ kayobe_env_config_path }}/containers/squid/post.yml" # #seed_containers: -seed_containers: - squid: - image: stackhpc/squid - pre: "{{ kayobe_config_path }}/containers/squid_proxy/pre.yml" - tag: "4.15" - network_mode: host - volumes: - - squid_spool:/var/spool/squid - - /srv/docker/squid/squid.conf:/etc/squid/squid.conf - - /var/log/squid:/var/log/squid # Whether to attempt a basic authentication login to a registry when # deploying seed containers From 7ecebee90a819234dd8692517c6b0d61556aaf39 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 24 Jun 2024 13:31:01 +0000 Subject: [PATCH 018/103] Fix-up gateway --- etc/kayobe/dns.yml | 5 ++++- .../group_vars/compute/network-interfaces | 1 + .../group_vars/storage/network-interfaces | 1 + etc/kayobe/inventory/hosts | 22 +++++++++---------- etc/kayobe/networks.yml | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/etc/kayobe/dns.yml b/etc/kayobe/dns.yml index 2b91aa941..ddbd01467 100644 --- a/etc/kayobe/dns.yml +++ b/etc/kayobe/dns.yml @@ -6,7 +6,10 @@ #resolv_is_managed: # List of DNS nameservers. -#resolv_nameservers: +resolv_nameservers: + - 10.255.250.1 + - 10.255.250.2 + # DNS domain suffix. #resolv_domain: diff --git a/etc/kayobe/inventory/group_vars/compute/network-interfaces b/etc/kayobe/inventory/group_vars/compute/network-interfaces index 704d41237..eb5f0014f 100644 --- a/etc/kayobe/inventory/group_vars/compute/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute/network-interfaces @@ -17,6 +17,7 @@ prod_interface: ens9f0np0 data_interface: ens3f0np0 +prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/storage/network-interfaces b/etc/kayobe/inventory/group_vars/storage/network-interfaces index fe3abff34..89f9f419d 100644 --- a/etc/kayobe/inventory/group_vars/storage/network-interfaces +++ b/etc/kayobe/inventory/group_vars/storage/network-interfaces @@ -18,6 +18,7 @@ prod_interface: eno8303 # TODO - check this! data_interface: ens2f0np0 +prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 70b21fe5c..e226befcb 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -10,13 +10,13 @@ srcseed [compute] c-1 -c-2 -c-3 -c-4 -c-5 -c-6 -c-7 -c-14 +#c-2 +#c-3 +#c-4 +#c-5 +#c-6 +#c-7 +#c-14 # c-8 # c-9 # c-10 @@ -25,7 +25,7 @@ c-14 # c-13 [storage] -ds1 -ds2 -mds1 -mds2 +#ds1 +#ds2 +#mds1 +#mds2 diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 574d613f0..aa632724e 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -88,7 +88,7 @@ storage_mgmt_net_name: data # network configuration reference in the Kayobe documentation. prod_cidr: 192.168.10.0/24 -prod_gateway: 192.168.10.201 +#prod_gateway: 192.168.10.201 data_cidr: 192.168.100.0/24 From fc684b0f295e5ff8c07fd7378eb14a667f16fe7d Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 24 Jun 2024 14:19:15 +0000 Subject: [PATCH 019/103] generate passwords --- etc/kayobe/inventory/hosts | 4 +- etc/kayobe/kolla/passwords.yml | 1622 ++++++++++++++++++++++++++++++++ etc/kayobe/networks.yml | 2 + 3 files changed, 1626 insertions(+), 2 deletions(-) create mode 100644 etc/kayobe/kolla/passwords.yml diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index e226befcb..8439f7e87 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -2,8 +2,8 @@ # localhost. localhost ansible_connection=local -[seed] -srcseed +#[seed] +#srcseed [controllers] srcseed diff --git a/etc/kayobe/kolla/passwords.yml b/etc/kayobe/kolla/passwords.yml new file mode 100644 index 000000000..557f36f39 --- /dev/null +++ b/etc/kayobe/kolla/passwords.yml @@ -0,0 +1,1622 @@ +$ANSIBLE_VAULT;1.1;AES256 +62646166396335343566643535626635373665656463373638366164623736386665666666663663 +3265393730656330343265393636643765333530663538350a396463343866623562313834303562 +35653432373066343965303638373931396334626362313739613533326162306434626662336365 +3836643665383563660a376531383935303766613231303666346536663764393132303134656535 +31616164326433666562656166306661656564326265393538353666363737613836383735373938 +35353632363964613330363761316663633532313566643461383038383632663531383837616230 +39393563633036333131643836646561613537336230633866633465646336646337326233356666 +32646334313530326534396235306436653066356239636332313235656339373735383139393238 +35663361323238336465363038653063323465653939383363313130363535383765653930323636 +36343636636231303263656635303133363463643961353331633232393439303366646138366361 +37646532663632303732663731623734333837353962623863336362643637636432386335623834 +66663632363566333537386665326135643932306231323631636166636638396464363661626135 +31343136646563626438353038336132343166336334303134313238636663343163626165333962 +66633062306132386338383034386364336434636565313837626637356131396665343866643332 +39303330353134663739356434363535376163666561366666306236303337333433396531623732 +38616266336134343833333939396233396563643961633933656537626532396631633965396437 +31333935613539646363666239353766633261613537343962326232643832646630633130326461 +62646133666132386266393234613362383132316463376663383462376135363431396465613533 +31356265333063656133393765653832363761323030643733653338396663353434383132306461 +38313763383863383933646666626165393365626536633364313864363031633262393939663331 +64346239666563333533646634663334623930626338356537643934633630353435633734616432 +38633231343337633030383766353366636665643832646135356534376333373662336166376661 +61383032386130373961396437366365616361313931366131653064613133353931623864656533 +64343864313164633838316663663639623439333261366431616534396534313266383566386639 +32643262643437646434353531613639656262643432636561383531386361646264323938346635 +34343534343661366565613361386335323530393233653066646331646238353161626631623262 +31306264613430393830396531353035316164393764393863393935323335396134323138343131 +30636462663430393134383532633737653462643336663735383538663837356364363065396366 +32623866656534616265613463623933613831386137613032663135303036333961396236313538 +35333831396436303131663962643865306235336462653461316635636565313934633061633633 +39343962633132346364343265646435363333326237636235616564353833326333663936346337 +32313332376532666138383834383066643337383264303534323632656530333936373264633730 +66653865383262346239626562323137363731343863653931646431356437616538393635316361 +64653861643130313366396335663835323339363934613438376162313832326664373864333039 +39393930663465303637326438343631663731653433663431396535613136333135633735303935 +35623065376339366361393561323237363937633164376463313664363037633461626434626664 +62653230653062333837353534666535653862366531656231356136663163653766306266636238 +38636134656132313233663038306566393335316536353462616163366535336362353935346633 +65616639626437633536656364613031656632663666336439393062373331326331666438376639 +65343565363666616264383566306432636236306339646231326436336233306335343838626564 +36303262316664613839366434333063393331383262653134303530343732323565346333656238 +64313838343337343730343461383137326561643262373864623436333331636134613865313765 +36623933393764383465326636643237656664666339373039656531343238333737646239303866 +36613330656462313963373865333234643331663635363039633162643561303530376336646634 +63373536303863623639326237393664353362643164303437663738646466643463383031626637 +39643862326535636166353061363833373136343530363666376330616162373062653061616531 +66633430396536666131396536386337303934303461653334343762616132393039303834356234 +63326630626166306463393961613135363738373765343930646330313764613935326635643531 +65633038373630353864306232356332393064366434393761633331353862386539643663353531 +39656566356439393739326165353739386535303862336266633266623335376230326464653732 +31326537353866313233663762616165633730653365373034626463666338336138616466323939 +66346538626335643462643664353632366636613630353161663663643233393537616632393038 +36373431383962613734356435336134656162396465623966363465616563326363653231343034 +61623833343134656238303433626662396562386237303735663239333734383731663234656562 +32363337336634623965653962393032373166313531333336313566663235373763303131356231 +63656137346435353534616532396231356464356163363033356431313639636437303739343365 +37646430663333646436333165323165303566613538386632336431323137663864326263386536 +36336531363965376232373232346137623039386632633932623663626166623238633539333339 +66396539653534363536653439326338313535356165666131393964366165333130356634396565 +62646366326364646238636530653163373237646364626337396533353566663433363437353164 +30663631333362376665373530343634383431356162363132326631656464363230313934373337 +66353639393734623936613134626239303466353766323738383637316335396534626561663861 +34616664333738313130366535336638323631666231363835643161386663663038313237336534 +31396462306466353336626639343661386665653334386439396430623033626562336565636437 +38613964663966343838373238376664333730653638633566346131326430353665613033376133 +36633035643533363761373163326362343639613839323839306239303330623736653837633964 +39643864373262366535396330303166356333646466303030306632386632653638653138653062 +35383033373833663232636536623738623266663464393232666336643934663838616633336461 +36383038333564633532336161313731613333323564323633633766343039656231643138653030 +63366661383261343535636432363339366565376139383435333530613363623539646537313839 +31643636353766333436363636306533303930393937336631313136373431653637616536613364 +36373861356364323437326432363936316435316338346134396630366330666334643361643937 +32323661363962636332363539666438373336663862643534383663386233623165326266613036 +30333366383036343639333538666533373863396535616233323139666261646439353739623739 +36316133323132396465383838613834346230303763386532656563646638363136666331323365 +32613936363662623332326133633664323733666537386162323737333938663361653065653337 +32633362373939396663646666343636363039363830636234646232646534663030383333613663 +64666634346637383831313638313638323537373562316337633936323565663439353363633733 +37376437643465303331353461356333646334646635313764353734636539343434306333373233 +63613832383631373638616663336637386132383135663838353366666237653162333036326337 +37396566373138323266393662373230623136336564656636326233323335303962653034643631 +30633331336339646465353636663930363234336335386666343461643737353639643661386331 +36633262343831356166333232363738633731306465653463613866333632663937376362356137 +35386362373733613365396538636636386565633637326262306265333834376564663836613638 +62623964343363343734306437343433383839313038623731356264646234646537653938623963 +30313438396162643032313633356463376637356364303031346431313366343466336434643166 +63366334386139366637353439623934613932653337393561353132393633653966366134316434 +36616263633366393635643439323562333466373636356661343038376439643535613238656435 +38376533313464356531633336366564613436613434316161663937393737346631343437636433 +65666436653735393963353538646462643638353464396333666238343433393165366135303139 +31646164333539303035656361643966363830336439373437333833633530326461383463343939 +34316337646262353332303166663036653266383333373364663838323337333531393830363232 +31303438366366303033353630643161626134663934333639333262373461663963386562373036 +31636363643963393238346364636262396532353238346534333662646564303131356663333937 +61303039653466346265383338393764343664366265353338343466653538396234663238666531 +31353965333765323964373237646162363639336365333234323538613162326262336237323232 +36393538393939623261643566313337666237333233643161323139383032323264633335336634 +33313136353761383134353065616534643133336166653064343365643464666431636633666337 +33376130323438343864316564653337623835396136616466646536376330373331663039393435 +64366464653932393335303661393235626636643265313139376638643236623730346130643663 +61373735366238666333383865326232643232313233386234396361626565326431623666326537 +63393639333664393530643436356532393435643033616361373536313732393963623337383761 +35386335613036636365393134363764343932376534643563613261333661333536363333623461 +35636162373335306130303437383439396662626330383936363039323330643037336438363765 +38313232323935643338306237623937343539396538363463613939663639393838333332333766 +34653138306530626536323966666336643834333031363133376530353732663836666565643735 +63376435636466343837613061633030336365346339313532333533353764383939316335643861 +37306231393030363961613564613536313639376363363034333230653632346631313739383337 +61333363313931303937643535376266616630333062303738613237343632663831373863306432 +34373037376638663061626365623963373334306538383632623737633265656363666637343031 +33633239353236353437653434333266376164373338323863323832363839376265653162666161 +62303764393462613035663535303739396639303964616436336465373761373939616464333465 +62343634393837613032623263356265313561663432623139373261303537643463636338383732 +38326338356437376633616538633637313161363039623465363035343961633835326137633462 +37323837653731383031396262663139653363396439666662393436363631393765653362623663 +61633863643737383565346139333238386537356238353466333134376362623132343463643439 +63363939643834623534653530323664386231353932333136326433366232633663633338613763 +33393736303230323066393935383964393534353133666166353536653962313231633538373330 +32633534353264386134373837636533333430663135626562323933656331363739613833613861 +61663261656361353039646263326563376534333562633239373537393137363336383839366262 +38303462636632356662643364316163653562363935636337666334353963383463363538313835 +36393062613732353930313961636536633431623031646363373931646639653362323562643331 +34336364636266346334373663383763386430613337353537396165343364646332303237376163 +36643436356165303861643833353636646635393866623531343066313438613439396666363138 +35663565653537343736623633363865333733303231346233333065373464373163306534313765 +32303833356465393965636262633136636461346532393731663031353034643432333866326133 +66393837636166643234333036616239356432343330353933623765663635643036386166303361 +62376631383163306165343533356439663066353162313065326136663639646233653766623830 +36373730393461333732306533633035323564613530333863313338306465616238663235613766 +66613337346364326364626162383435663234313132336462383963373866346361323336613962 +38656261383335396134336335663265356132316237646230613131366232653033393466333862 +30343331316335373661336262636631303165323365303766383836346166323762356133323464 +38303334323363653431336430653439656436616337633963336238346236633134353639613636 +33643731343636613363623137653864303331653263306462623366663437626638376438393563 +37383137636163646638383333643734303534653731323465623564653638353631343634303163 +35303965333866383731353937663833303132343861323734306333326566303930306561386465 +32303535653833653134373462386231396434353433323031333031316335303362616432633865 +33383761633037616666366461333332313765356333613334396435303936303165656530373731 +35323638383339386139306164626634383066343834363430316334373336613830393631353162 +30366535333134663862306661363465333032336565353331363538396561353434393463376263 +32613765643966613730383364613537323834346661643834323338626661663635363337353635 +62616432366438303661383463633365643332323161323939333663316134653662303466313663 +37393164653561643438376463343339363339353932346538623237366533396464346662366533 +37353161373564643166616262313135366261363839303331343538653032326332336332333730 +64373530303332643432393836333365313534383437323461663239613463316565623563313434 +61623332623364663136306138336361646137356464623062626163353662356634336330336236 +62383462613863643433613964353737383665376336333532353830306262356263393432386639 +31393165303264393035393234646266626137326465396230313366343361396264623336633061 +64326237633337613462616430616535633233373161336361666531653466663830303363383765 +31323537383233653437666162376234663363393930626665313361623334326565616266643930 +30373165626566333834343534333539623639333965666236343235643663303637616131396265 +37366435393934666435613935313461376139653733306565323537393037363930623565333834 +32343231616466386366646261383863386237623731396238363838333762616666336564646366 +38386235393634376631663465396330616666623466316637336361613734363064313835643961 +33366433316663623364336562656161656661666134356564616438653839643636323130313436 +38396238333264353965393066316261396261316430366638356462353233343234366433363562 +34633039336439346331656262373534663664313435333938363834343666633531306139666463 +64666533653036316639356264303538623439653439613835333763316433386436623561356632 +36343938373037376633626565356564316235353134316261643031376134363030663233646437 +37623463353135346365386139333562653065343332663136383331313732333963306632663035 +34303830656336323162386163363962313861373437316332663761663764656236363935313436 +33346662643139376562313864383834356663396565363435613061393034303534336239663134 +65333837646263656433373730383961393631313561613965623735636134663538616164663036 +37323130666636356162616631646132363734616263356536633531646665643961663837623166 +35373239336562306264363339333936346134323434343730393365623538383462336537333136 +62343934383234646137356234393736643939623832366230663166316235333134303139633733 +34396264663937633632343133353337373765373838623535663531656331343566653931343463 +37636563303766653366323964306134356236623534333238323735386530333461343733363634 +38323665306230616562623566643334396635316136643135346666303265386233376463343661 +36396231396661326361336436343966353566336635333262363335626533656565316132643932 +37316431363464376539356162373031373863383939313461613765326661343234663463666366 +64666330373763326132633530353839303262313933623364393632336136636137336335303562 +32356430383961396135383465393935333336666131323233376563303237346630363535623530 +39653937613064313762306562613164623163643232376331393537646138333337626661663963 +38323131656335353765323638326131663133623539636638353033323134623736343864613730 +65616361663035633734353466616162623132643136303766336239666533313261643165636536 +38336161626139326230623432663865326532333734363435623332383139363235393937303731 +62363533353030353165666465373930383132363036626433623434663539653366343661666336 +34333730616664363535383738373632346534323833326366626432626132643333383335346536 +35353631306436633435633361643834363330353237343761336439383965653265656635396131 +35336233613465373136646362663561343764343264346362653231666366363637646133666238 +61323837303234323134613530386463666139303735306339333030333562376334333264626237 +63373234646234666662313061613634636463373137626261643531373863646633313733316231 +63393931653665386636656162326239363330333733626466613533383365303062663436343638 +65333632343735353730373536613939306461663765383632316431373562313430346139366234 +36313739393837626338643633323536366630323432646463646233393334616266316561386433 +62623736303933313166343965643232666461363237396131633866393332373861396537636336 +39316232336364373830616539343335656166326531613230326334643731326639333965393261 +30306231623562356536616334373732393632333833343730383837343133356631636337373836 +35393031303336316465323530646137363463366435623961366635316433313138383930386362 +36613662636362343532376534666132393839653839353064343361356264333662393039646661 +64326631343366613433326338353137626332356330363836343333633530396238353630373864 +36636666363566663662373336636638353563373565313034376437356434323136393332383364 +37326637333630383661616231343139393161313339316363616363316138366631326463346139 +33623435333230383464316262656537623264303737646531613464613337316566326166333838 +34383566393961346463363463613730356538343534636438666530633536313435656234663432 +61613761306262633933623231343735633135336432383666623330333037323836313362646535 +37646537366539616436666266376561393436343261633266666166393761623831626436343935 +37363138353831663066653164613162613064363663383732633836623738643930313663653131 +36366634633334393261623735323634386166643238376530626431646466326339353936326533 +34386330643335623338393164613962656636353635386439313432663732346461656332623236 +33653437346463663937616437626465623537313835323930626132376535623630636337376364 +33643330643866653337663731613964306664383161376665323835623334316433613566633432 +66313062356437323539656433386535313065313230666331303938653662393665333137333135 +32366462646334643666666434343962623661313337626538663262326138633537633562386238 +63333631363936336466663630643964316438623939636530303136393066356631326664393265 +65616562343139336234656336663436336162373734663736373231373732323664643634323265 +61663834386361656164356336353263336131373934633939393037366632336365326536343331 +66346137363762316239666539646564316237613735646461356661623632653136313064373331 +37616534346439323936386136323439653633343633376166623466633737386564363432396265 +65383332323035346564643866316264373036633635366361336630386637386437666162323537 +34323933643065393934613136353035666132643638303061313939346438386233663939373763 +33333035313361646165663339393966646634666262626638396338396236383961333563623732 +37383238633032626563383037643637303634643937303231633938306462646639613061653334 +32356136653163353531343366613062383837356432656637323734636534653939656437356563 +36343263343163396336646165663230643630313635353837666662383363373834656365643232 +38653938313763613762623734653963623234393838333264363032663038636264393065633261 +66313032383163643762623261663433313134616230306331363932336232383038633062363537 +61303934326230656436623730356632356663396231366464633262383434646331616632376438 +35316339386364353933653163313932376634333665326263326439373361343731306136383762 +64396334386238653964646138396234313361636231356462366537336237363064613464316562 +64643439313339393039393165363436616238303730633530313932643934316161613531623562 +37303061656336396439653366623136633964313932303139616130393033363633343863666231 +32346564633131643364363865366532666465353837376533633638633666353364346137353736 +61393938303437626234643865316637326664336633633731646237316362333865366235373561 +63636332326435356533653236633138336539393434366566613538373133393933306337313437 +34613034376434663932383930396666323866326637353166396236396531623661666632313232 +62343066353366636231623338663230633966646437343830643833636532306339333635336433 +34303166636337346238346263383034326361316564326237666636316132303564653336333664 +35343464326566386464353032326535666265643939313236653136646330393932366161383635 +39303863663132313633336337623661303532323534336565343436316465656231313531366630 +35616362633166383835363665646536316337393766343364373138376664613635393032633737 +36353130383033653936633862393061373830636262366434393837653836653232643862313665 +38333062626335643737313934373231323962626265663633303432376632363664613564363066 +64663532326530333463656163663232313135653538303031373530343965363136393438653337 +61613461626464313336353531373963633630663638346131323639653830386635613665333535 +61666533353462326264303538376335363032616661636435643237376662326330623035353834 +38313966653366336163663663336364366565343036656437303837373764613663376131646361 +61316663306437346665313036343164383130623331396264346236643135343936313738623837 +64656432613538653163336463613135366464353664653133303063363831636334336637656264 +64333932303135623935386461356662393462666235383466373635333865613135656333616632 +37353135643139303136316538356164616337393263363439623762336162313166346266393233 +62643535643866613264316632316264623732666662363439363839376661353232316235323666 +33323732343861333837316261666639613732396639383463386666333834353862346561323536 +36643030653236383061336532333835656636386133346436356331373832376666373235333639 +66616232313233386638366664313734616137326639663061323762633231336538303836643561 +65633430336232396432323566346463333063616630626236633531633032316666383737323433 +61646366343363623366646665636361363162656634313430313538616338626536613431393038 +32616434353466643736636465303865626235633966653739646139653433646533303066346438 +36663335383466306334663832633230323337373631386134366664666438386566366561616165 +33646636353236336663623261613863383835326432303261393563313739613232353539636332 +32613236316263643836323534653132656132646639663634653033313365643063373537306466 +64346165373333343334316230356136346130376534393337663738363838393437333461633930 +33306139353561623661373833343737313237393462343337383639393738636535643563626339 +30666335306332646564643464656364616664316464653564643834323066643736346631373939 +64333963313864613334643538346238396634383333303531636161306131363735373563323663 +38376630363234646235313263396164316330623536323861323363353336316462376664626465 +66303864353035663566663361383132663034623133383639333961376263643663366635363239 +34666333666333303365383830393166373163356530376461353131363033363039383337313737 +37366236636532623837643065373233313237393966313764386161663235653337383331373262 +37656330616262313062303939656465323165616235623132346138366635333135663761323630 +32303763376166643037616533636266393064373561653335363831393835386464643033613037 +36333832303132373631323231393761306434336230303437363332333862373339643964386330 +39613062383765366531613062363836303538303565393936393465343335643862343233633436 +61613330363438633632353265613833356430623064323363373239663737303733633034656332 +66346432326534356335633236393066373335303763306265356330366238633237646532633066 +30393738643262396563656565303334383561356631346333366632633165316365633336623630 +66343862353561383631643864623862386139623364353034333735623066616437636536643365 +33663362336563373630636130386438613836343136656664393462383730373363666264653535 +34393733623135313037353830633561646139653363333839343164326632303465346165393163 +63663539323465316164373034663135313139306538633866346238666436393065393030616237 +39383132366132346533333139393732616439373136363966393139396431313334336266343861 +62343336653339323963393838383838383337366636303935383435663438616366643632643037 +34623563643930353031653930393439303533623932346333353364353935643435336535363363 +61616464313836303732633334643536366133373930306235323935623136303033316434336139 +35643639353364626632623633626263333431316562666336343131626134343962636236396336 +39306130303233313133613264323836636363613132626536653938386162643266373962313539 +33323463333239643563323539376338373836366536663639666233636438323439313766316432 +39356438383136336562386365336265306336633833303465336666663564373538373631363437 +64393336633535636438313833396537333365306131346138633964653066323061363861336338 +33386433393464393036346266376631653863646135303063333135663933666436336133373737 +37613663633134303439623338653333613366366538343430643461303034636637376565396366 +35313830663464313235356635363261653233363561323033346363393163656139323630383232 +36386236373538613133316366326164396538303734636462653338376362343666656363613836 +61373934343461316531613834313530623564376561313236323765363735633033623162323436 +32613765646337323738613561333231313862383439373262653230663038636666623962383533 +62623431353265346434613235343639613936373433633837336435343863326634636433356639 +37616635303761306264633032663538333738616138303266613133313565663435343936356132 +38383936623761356338323430336439323363646235626137363666356334613665383134353134 +35303435346462656436616436313331343763643362333732396331663536656265643665366166 +39643165333062336637386363363065636261373739643962373134343965336665316666333661 +34356439336162653366303035613235303530623564353266373439663338386361373539393232 +33363637663363306433663634653832346562323031623732633864636334393933363439373565 +34306538353164633763366264333563316461383237636161333836623066363238373466303838 +66343232666565656537653034363936623265626566613264363466323630663237613737313132 +65613039653938323363643438386238656364636136393066653830346133613865353366353462 +65346230663161656431313531333131613830316335306263396233313733316333376330386436 +30356138313133333734656633376264383437613439346636656435623235356130313532336163 +62663433313133366539306438646531613962353330633436643732356666363239653632656130 +39666431383033303530386236386531613032636566626661306263366361383563643462303231 +33356434393263393434663064643561393163623666306363396337343138373462383536663535 +32613461336537616361383832366163663233323737376232653233306432376563623632343137 +35306163616165383764386533626261306463373432306533393737366131613736376666613037 +35633231393137316339346437366262613239633039646139653435383961633366623636383539 +64613837343133373231323862353435336663366538646539366166653033636234303138313765 +32343063613532663631613062636634363538623437363265333631356132633833343433646365 +38653561366132306639353666663934396564626265313930646163336630653631653335363137 +37643863373962323638643635346438383539323762346563383439396437633630613938666262 +39393830613564646238343261343265323730656430323666623161306261363463356330646335 +31346432663266383163326234313565343434636363636366646136396437303933633533353762 +37653263653465383438653763363732366665643138396635623866663766353636613363346135 +63656432323138633762373330373739653739373836353531393930633433366535653537333539 +34663037396561393733376262653366626561366164633463633464336534336139666435616131 +62656264383731326632613239656661623633336136393465326338313265323039643765336232 +65653131633665653435633164306561316130643437396538356533636463363263663163356162 +30663531616534636637303738633562373064663262383239623666656266346537616330643635 +30666231376134343066656336323238373939303766383438393065613039323930353133366637 +65386531323166333961613139313431316535653964373339653834336537633366323565653939 +36316335646339623436636430373565346666356666666133626632623164336466363031396634 +39393363353664653166373962646132323839343061373031373336356131623133653837383139 +66616263663236653432663837636137303963643637313436323466313735386236656430626535 +34636634303639656661343262613931636637336435356539346539313165323933643233306132 +34636361333035623238366635613965633134393031323963656637303231663738616133613761 +64303063613062393439323638363138383562636131393239633866323266303535633232353834 +39633064623064336234313866616431326131653538313531326565373737643261346335393831 +66623261643531623330336165373865316232656533356462393831636334646535653737373238 +65323737663864356437656161303661623431326263656137366335393566613938316165383932 +66393233313962326263633463383833363438373762316439663130363064346563363936363131 +35636437386666333062663731623662633635303833386535336561393833613465313738393134 +61323536333364646437633730343136373830656532613037666434623265373061346634323632 +30613361306538353766373139333263626565333231353365646134326234613733363165363636 +36306132613962333362663932643865386534346139386265666262383838346434623765353765 +61336333643832333434393165653935623931613862326330336339613964303333333732393933 +65363562316266383663393437623363663038386430366537353432643733643932353739616530 +38343735646666613637666365663537393731363162316163636330646331363264333235313432 +36316335366439373336396163363534376465393539326466373933356231383264386638356130 +38636461646663663861393939353336386663316163633738623933323863393534313232623033 +38303535326438656334646338623966306536616664353231363330653265366165303536363839 +38343961653661373834376134656238333564656238653964366663306364643438323336663338 +32666661653161356138346138623866353363373430616531623537643232376338356334373862 +62356139663732643336326262326137346564333734366536623439383364623534313865353333 +37613163356139393334333066633037636439633061623538386531613863656563356566366634 +65653261303437626262643531353435346131623935363434356331383063663239353962303262 +38636563633662333338633466303033363130393333303730323231646134303035653832383330 +33316631656561333235653931643136336530366535653466333766353761393265643438633332 +36353536336363303864663632333538363634623066383938316636626133626238616661366631 +63323063373534363832303338343631366239313536303565386564396164366331363533343064 +33393032333562656266656530373663613036393833303833343761613064353331643765333665 +61303934316535303162613563343165366239666664656132366238306162633564623134316235 +35396235336332323234653930323762366664343135623661623836646535343834323636393634 +62663562313533393062663166643338646461336432363333373864346533343337383361346636 +62366334383065303863323062613735333130643737306137656636343633363639636638343630 +38303438616439393136303465306161366432643036653132383661343236393431643863353436 +66646564616430383038373031633336396238623962353435633331303463383766393531326165 +30343936336162633130633162373063303134636363316438383565373733323461613932643163 +61363130623033383731373834383837373861376633386565323161303463663932303134366339 +33396335636530653965346361613232306430376335306533663164393664333633393965373335 +66663732613562386438326335666637323133363362346336343935323264346432323032373038 +34653132323363346664393265316663396135366638663735646364323032343235643039353935 +35653831393166363233633161663163343839386434383638343039613437643037356666626138 +36626231393333313763386430326234393131383438613061643538393732623936323735613536 +66313335653861303936366337353537623263306436386663383765323435336434373665346563 +35633663333031303939303361656563373930666635653433353634303835633666656638376335 +37303131646239613561373530383662633434353434343865666335633533383662336261663461 +64656464366262366637366161316438653238643432626661646231636238343765343030383566 +63323336626633363965653331333834376533633636653331313737616539356661353638656333 +35653765323464326339663563333036393162353436636238616332383539373035623962616662 +64303166643332323934326562653662356362643664313739336264653562663733386666313038 +35626364643163613963656630363433393439666366316239626566336136353861396130386362 +36633831663336666333623539653531366338373562646561373863323730373966393166393362 +34636337643939363731653764333730346233393437633336366665626264393936666431663034 +31623536323161393132613461313031396561376633333164363065393066353034333634663338 +31396134633939643835346434646530356537613064383565383765633461373238306361633833 +63613732353961366139376466343961316638343439613532343665306165626437663138333233 +31303865343838336238363233636337396338343238336365646466333966616336666138366432 +35663630306530623233313865376665626137356566353630623262366565323639313862326631 +32306434623464633833363137616332623435303634376363613935623734323831316164346531 +34623764336237313230613330393435333636336237656338346164313164653161623336356534 +37333662356366323531303030663432363061613461376333616366653263333265643932393531 +62333238303833393131363863313039356235363332656631636361306230373265363339343236 +64653064383535393663633864663265626663653835366439636263353731343965393636323762 +38306436663663646666656439353433336533306165383830383864333832663561316563626366 +62636531323164323863616632663732313730316566356331666335373965353132313364343831 +34366261396537303866363961643330636461383963363164313834386336333531376161383661 +63636363323231376165623466663738616335643966396233636264373234653032663933643066 +31333437656434303366356261356432326534336563333831633736316435326362613230396237 +32303234326233326461303332323635616332333737633464396438323633336532623530323034 +32636639386434356262366564396138636635653862663934366339366635623536643533326561 +33356365636461363233376466316431313231643039643334343663613661626336666261353931 +62313230386165373439353431616530373436643664393535663433326565613066643863656232 +37653861623035646539636439363162383834626637373530376563313233636662633664626531 +31363164646163343266393362626464643734353834626331376363313261346136383062643834 +37326631386564383036316535343132616632336335633461633863373836633161616233613634 +39353561376430666461333166623635333061636261343938353231343339316430386434663264 +32613262303961666263326363386136313134386337333434653830333637613539616663623736 +37653838316436646464306263616564363665313337383665393163336261663163626135373936 +62646234303466316234333330666233393465376164623033323437363431663139333661656235 +62306466383666343833366530333866636362616132363964383232323332356536373335386533 +38656130616632386338613965363030356138303234376433303661353466663335323464623363 +65373031343531346632623338303634666432343435323532386633366461333963323839666136 +39663333643139306461656563356232393033643165346635323336323331393763636530393738 +30366666326465313037393064366663333739663332356234633962643062353164643237383266 +38663561326638633265326630303939393036316237643164336337643364643036343238663838 +63383736663061363735323932623833346462313364616332373130353465306437653037303137 +31373135663261656630313564346634633766643833623438623234346561343834313563323938 +39373131653236396235323866636666366438386261613164633035333630656462323161643637 +38356263623131666331643634346463316165323331383963326632613261336139396236646332 +36613265303864303661323736313062356165616634643733336463636462643437623334313365 +32666134653362666436633138396664396564386633653932363366636564636338323565656161 +34303030376135363262656562383465333036306335393033383561333037363436643434666633 +64346564613862393637343433643933343437303430633236393237383633336462333838656361 +38333661323639316433383032336331373866666465613165373566666332376637653964306232 +33383263663931303236326139633931336538383232653664353861666535613339306235346266 +31363439383263666432663235643236633834373765306635633935363966333462376563303138 +63353035643664656131636339353066616433643031393865383130663363316436326334316364 +61373238646562633766666264323562383238646537613166616639646433326532323237383532 +36303330333131363433383865346237306633313632393365616564663636313138383165663264 +65363761633065373537356635646435363236623932613563656264343731616465323431346633 +38616536636661396138653565663335376131323630353063633739356462323438326462313639 +34356365613535333033306364643265613131626461353064613266343233653865383636323237 +32393763333636646465343137356166336136303363353031323361333539646266306261636562 +32386334326638646433326239613735393638613037663262343863393262343963363834303339 +37363130306338626639393464623831653332623934343363356130313866633738326432363061 +35393237333263643562326365363331623036386132393237623365366164356230666631343663 +33303236343038663439313335623364383763623666623234393136626238316131646535356465 +62653333393339383766613564373636646464316438313038386434376137336533303937313337 +34646363376634643731636534396432626236633734626131633533336134633536356264346631 +32653363616330353832666239626665316465643238373031373035333965663437343565326536 +62333837393238363964393935313461336165303765623338303532313363303632363632653931 +31333634363765623662373862316233376437663166636331343963303262616665386365633131 +65666637636336313634303061343732343238616236383935633338666137353664323064376634 +33666561313932333864653261653863353130626337363736383662383564343539656265323064 +34356535656166356266393064383762303165663365393231306564396661626236633736653664 +32306666653631666632363136353862353132386634336361346361396337626333363733313237 +64343130633635393534393365383833313863633831326539363433643431313432663635366230 +33613364326434303966666565623037383032626535663631313639646539323765613938386465 +36656339316666633230303539363235613335383532656236323261656464356639396162316161 +65616135316137323237313932303263343531656563643038643830616135373864303038343663 +37646438363834316335623334346433663536313961336633356137336130373738376439373237 +62663734663439613434316532393135653066303130666432316232326330656465303863313165 +36316133633731636135313133383463626132633561643932613238663239636532383761663330 +30626665393065306432396531626430336438356630353163393366366133653533626234646337 +37303732346130343832623630366138316665666539666336363934393537363636393638336233 +39656163346333333335376364343662613263613532633038346466626637366438353938363933 +38326537666364623765373537373265623630616561316230653061313438396561336434373135 +32393566643764643836393662323063353938373634653233333963383139383132356135346264 +38383235623437626364376336306361326133386230616531313266346530626535306237346338 +32336334353561333530643663623164636363353365306264633039653663356130646265636137 +30623437666630393962343664356639326466346636616337623432653937333466346137303632 +64663562633131313634663737393535356134636264636436666664326638313137303733336365 +66383535633533373263613362383061396135356163623761393436653463363763666236313864 +33653334636231326137363031323762343933366161623236383962633932343564613163303638 +30623237643464363930386130616135636665656662303039356165356533646462613764393632 +34333633663662346163653938396333356537643836663036643439633732306639343336353939 +33346334363236383065643431646565613364353835306439343639306262646361373866333035 +64313466646635306261343532646236363637666232316237366237623132663635636165613133 +31343063396361316434386135666137326639313561623337393462346663396238656535393638 +35336331316131313937323061653464343031643865323338383165313062363234636233656536 +64393862613466396136313237393037313165643666356531373436363831373862353937623362 +33663634313839323466663761383365303136663661613331326135623266353461636635306164 +35383337373739633764373565333363643431623336633431313363636633376261616564383036 +34323665333466346138613138343063363433393133623664623034666333343466386234656165 +65373133666335376434646462643662653735316137306465343032363632326266323366373432 +61343665356435376637653730393466633833633431333363326431656534356139333965363539 +64336633653864353162346434353335653865633433643331383530316266343663366164643665 +33616661646564326161633830356535353530393338613331383565613630343463643564653437 +64656661353334663138383137633862643133303535323232386661396561303762653331633466 +36633937636636613461343937326132363264323938353736393463313164656133396365353035 +36366336383232656237353330396535313537373634643938613165653162313233376566383336 +36343866633138393163663837323132343634363564663734303461643562313461363137623934 +64323034333165313838666131373861373131326661363964343233366461366465333738323336 +31633234396637343865373139333537623336316465393931313830353237383338356665393337 +31316166376434643865633839333363646136313230313763316363343634386362656266363831 +39343362636135626565633432383332643834346336386266356535643739336130636665623635 +63636563306536316439666361623439363932363432383432336264313236656464326234326239 +38396631663136363462313533626163393334336537313133646666633633316161313962613138 +64336336656631383134383237333735313864373665666132373339353865396332643335626134 +37616439366132363933363333333838393030613463653739356165323530316339383439366161 +38356435626465653362636665306463346232343632366461326165303434383830656365373266 +38613437646232386336373936383533336232393639303437626430326366356135666231326334 +39366236356239353236646364393463336261613661356365393430386232653536623639383166 +39663866356166343830666132393764643462353635396235336430333661653964396637623265 +38346236306139323432383562646537333034303232313866376466623432323630623135336135 +36393938373535366530373335653464633464353033356330653034393065626237333239343032 +32663636653735346330353563383830623835373763396665363661353134623162323563613366 +62633630636464323564303633643262353532626161666636346334353434323430366163393432 +36623838663635303234663137633566316261333432316264353539613838663563303164623035 +64626366643364666332646436376331633839383635376131363461393861323532646638343363 +66653935333965386439626132633739313336353665326130653665386231313837386265613837 +63633465656266333431323566656431616362396337343063616135633965343537613334373432 +35623532633461383634316236323436376264636162323664373434613064363963303762616130 +63333062383936653632363031393533346334613965393236626437313333626661623633313664 +64366265633262653931626161383931343632613463613635396538303661626263666662303037 +38393638633466636239646539366563613961363939346231333030653138346263313630386633 +32333739613561356131636166623365663934616361373331333962396333363764313434643531 +64393166376532643533333633633034313131343537653333393165616565316664646230336532 +65343135346538306631386566346162306639323965383931333364623037346434613532353163 +63653231366135323232396339613661643337373838623438343066326464643865626565643031 +65363235656638363566636336356265633561643934346130653066343230343261353331376431 +35393538363065306635306637636165343932373638386235343962326663393730356432396335 +35643134323133353065393531306633643162613731353265626636313862643765646664306634 +38633330343431633832363765346239353639643336643237663061333765356665646436663135 +35356366653430633335306361356266373131376466373366626436653038666666323436316661 +38346432353662316166623761343164663466666462363434316461306665353466313537316166 +63613562343963396463653639643865356630656333616538666230366235363963643633363032 +66646665373365323964653366376530316262396536636635343535313034666262343031663734 +30313831626635646461396264313966636661393963386338633939313736633964656239333033 +65323863353639313935666538636563363037646562623162343731613034386561396664343835 +36386164356465363061376131646636373032303538643865356433363562383863376331336135 +62393131663631303331663131316637336332343362376333643638333536643433633731313062 +30353331663336336436613732663362303531323139653231323361323663373762336133333638 +62616165306631373636336661326136356564343531383565323963393135356139306337383461 +30313938353037333737383064653764336138396235353738323031333435636630313436653536 +36373462623134346136363031643832356264346135323438333330353033376231326465363836 +38376661646265336632333566353439393335343337323965353933303636303166663231633066 +61353631663233373064383432663539323734343461396431336666386565333431653461376637 +39613030626363613461373761383439343565653631636234356531653865346263666262656661 +63636664333830663362363530356233323838343832303335643939363964323562353231353938 +37663138346239633566656139663262383939306663653830663466376432333965656364323666 +34656162666138383965613061636333336565636338633535323436656230326531666466383864 +62393662623439363561663434313663623139356464663161393162663765306137653863616639 +65376330303261393334303731623435303266303461343436643432633066613262306238663039 +61636236646638323939356461383338656666333938376230396134656635663163646265333862 +65623033666534343163393866336465343435663362646135313963313731353939653463396638 +66353039363230343466653364653931393433386462656236616364303936646437393232666262 +32613030333036633531363938336337396436613735383265343131343530373932626661353933 +38306632373339306230316363626232313538663265303766316430613737386561396339663261 +36323863366136363366333830333736613962343761393163623765663031623162393836396333 +39336635613939653762353330333330353661333662386265356338656433646261383063333234 +37366338336435633762653133666331366566323532333765323562393764303132373836313263 +36343435383438643536313161343431666163316265343835346466376534396433393737663866 +37393234383166393965326233376166303735396664356232316265633836653931333261343666 +36306565663664383534653837353165383934323035303261613262666334376336663462316434 +39623032396535643333356333633162613137303838383630666234636661386630343430306534 +64643566366666343230353436373939396166363035356330643333326163383165623664326130 +62303237323063383063393865306138393630363030643464633932633531346431343966643964 +65663838343862363635333161643236326332353465323239303730366663656265613261663061 +66616632326664343533383537363461303461346233633234373866326333326533653963363438 +61616364633630666563323436353766366635363532656537306463613764356436303666373134 +37663433656539356464623638623565653731303962353933613762303136623836653865626336 +38666430393137383461376233313562646264623635326137363930336539396134626332353030 +66336239623063623239316432353333626265633631643530653638333165346661636364663762 +39376563306436646566653835626662616639303636643334336631343135626139373363613866 +64666239326332626631623436326438396539316165383033383962363561303939326538386234 +65326531373533373133343566633638623231356630623633316132643631373839663135326265 +33346663613765323464323963373038643437663433333763623165363165623937316464623831 +32663739656434653961313462623866373439363261343465633164313932626332346366656336 +32663833316564616266656165396330323666393834393734323432336166336234363965623836 +63613138626333373561393038656161313361633531663035313934636237613036643737306232 +38333035303461393636323937663334626331623239383363373432653065333239353865326239 +61663430393338633464653739633133336334323063363263373533646564373933653932346235 +63323737336366316666366464623431656238303838373062383366356166376139396137356235 +37396530373338343464666136393665323934323566383865356637353030663432623365353165 +34643434313732636137366563373865616334643139373735396161333731343234366565643464 +36343337666535333733623032613032353837643831316462356665326331343561633164616566 +61346433626362616434366266663434623264353834383333313235633039646164376631383762 +30363238333166623631363061373634653165326163336165353037656333316231333361626431 +61633964633431383261623664316636396338633639316365633565383839356430356262646636 +30633235353837623064376136666339366465393066336137333435376138653737393333393465 +61653664653234636537623064373239313064323461633336363836326233353933623463386235 +39313763623737383934303236306464613662396234636537323661326264383666376533326432 +30343630373666306164333035326233646163363065366233383466646630366138393336373033 +65303836363566363330366432316139343865653032323831336261343731393834653462653664 +31373863363264633031373939373235333661376331383362656262396161333133313338396564 +38323532646234393236643431343430373837393538623331636365626332616465346434613034 +36343765353730373735343964383463366232623735633232616538376433376631333962323136 +34623032363865333664346630313337333432653831393763653661363166313661626465346364 +31363035396463393464623634303864323331306631373537656263336138303065616265626630 +37313465313831313766616330303766613063633162333662316665666335343536313038663738 +36333039666636616130643631313235653033336635363238333265353638363636323664333930 +31633662646635313364313539613134336437663065363365306431613132656339636564353039 +61323365666561663364393435393163343163653964623335656338333631393635306264396435 +37306431336135623063366662326335653934383430303532303537646162663263373530356238 +37306530343161633462313534303334616364363934386531363133383338623461376437363535 +39303839363464666535393636303965663562346330366335363734333732636361316236343731 +66393338613831396538346435326535633530303930613964636232626539633631303965366166 +36356162613134326262383136336265343931396631373230336335393132663066623562343234 +36663931373933383066643732653461353566653362386261383031336136313838366563646566 +38613332393063643538383430633061303634623633376564343061323264313662643331396664 +64396438323637613039376463313236313938366162666266353735653533626337626437363666 +30386135663733396634346363303762363936643632386363623363326330663837656565353664 +63653633646333626166643636656238363130393761616262643462363065383734363034323763 +65393264313632646639393739653161646633303432343033316532396263336631323031363832 +31336534633533626538646365613535346435613636376165656234353761643661363138363133 +38343036653463393064666661623534393962323665636563636238323362343033343035313130 +61656266396437373766373461323036343761653239626537636665666531346536363838343131 +38613265316530383338373965633561353764346362666530363230663430383033393036666564 +33616632336434656538356334393437383166656537333962646262636161306265383662313962 +35623431356463313661623337346335626234633833303035626363623131663834663036383836 +34646161623532353637303638373030623263643438633961343866376137346331346364323032 +32386331333965336536323931373761306238663233343266656266316562313131663936333265 +30616339633739366232613636393739663661343838373862656463663665323439396464343237 +34643735663030643864373631623665343039663064396631653232633033363461306265336134 +35616365636336396236636131373064613232366337373862663839316461663266343433383466 +65303361366333336135316331323738633861653935366434396166643234373738626465366164 +61316465656135616634383466396566313932666564363330653238663634333566636131636439 +62643337376564303138656634353132653064313439346136663061353061616438643561323936 +64653033386639393039383335343236383039383036376336393966656234393033303537386430 +36353864656361653364613263643265663536363961643738633161613438333062633838623662 +33326236316134643933303537356263396632323565626637313738396162303064376239306461 +36663231303137373466323865363933303264303661333865613734333865383632636436666539 +62663364393866356561613534643637656435366162663461636362383332633063623431356666 +63633130336566323563323764333562373566383861333865323933326336336131373763613836 +32313564346164323433353937623366376332343630353766656339623564613332626361306130 +32313932326438316430373138613037643134623063346565303339663538316661393362356464 +31333831663665396230643163663636383530346533383933313432316561323234663862646265 +34303035336266373263363135393564636536333834303464363338336632636438383539316161 +33306636306630643832303838643835333034376363613930323866613433343661386438313538 +63373431303638393863333230643462343966373033613133636336616336333935623239326338 +37306262646165363265316363383837653837643435616237383162663462393161383862306331 +39653936653964303636633137613062326462376339333834623435313531313763393134366435 +63616137346364333931333731316131366563373165343839323336623630376235326131326236 +33316634623037343866643336316135356231636434393363636336393938343831393931613932 +35303363343062613636346263303438393062663564623131356461323865633462323736313832 +62313063386361323238376363646334396266653932663935343438303436633730646563326265 +65356330373236313061323535313332373235626134346663633662613134636431383265353665 +64373136306564653832363430333632333666333462396136643839643563333835373963353030 +31623930653965613762306137313335366533363564383963303832303339316532616161633433 +66613230623731333538643061333762373135303834663263346464663466353736313834666631 +62653633646264303534663535306336643061613238363935336661303463656136346133383866 +32323935386232386539616539373433366235343032393365323862353265626562316636303432 +35363530666162303264323565353837313032333739383938626233373637336538356238366235 +37626563393961316432663838656336656566636561616663613262663362663932643938646663 +61333331636665383632653135343131613161393266366632636461663539643663303264303936 +36396637316536623136333639356364636337653963643564323865363462393038643339386638 +66326131393963303230616339393833643766373135303732303562306534613263386433636636 +62373830306335383162383534353163383663653836653539356638323261613433383935663837 +35653765363766303730646237623936353734313934376463336134366262646363616538356333 +38656663623033656439646538643838383637366431333930616639666266323861333366396163 +61393166356634633635323030313238653164643735613533373630396264376461623836323561 +35656230646634313235343932616332653063623136303739343233616462346630663936386236 +36323263653464393737366162363933386163386163363166633630353139653132666663623166 +31643830353132363361646161303339343939376633356435393362346532633938643138346530 +61623339663162383932653131616234356264316461653930613264373562393237323164653264 +62333233666365313136366439336331663133356334326432343337363064353235616535336634 +37663963323430363631376233643433613965616633623335653163303663613936363836626338 +32373433656465646163396332313263633032653236346435646531643439653036663831383761 +38323635636161343666356566656237626439356132376138633337376337303462386165353361 +37373632343537303363356564326334376533633631653832616330323838616564663939623939 +37306633643231623632616566343461313930303064633733613234343832306331353539636666 +35343838393866666664656339366532636666356436326535383538346432303531623537353138 +32373831373136353562616436663061396262326463323366393135326132623338383337643462 +33626330626166323962646630363561343662353264343965356237626337353862303730393630 +31353564373562386239313731306165653937386663623566363831363564393935393138313636 +30656137656236336464323863323138343661636265653032633437633635356663633037353764 +33336664336439613435303466363766653531663333616364613632303934363136383336353739 +32316135643962346431613837366463396230313465613363646264303936393161326536326463 +36353135386534376161336530353766373265336132653236653839373236383838623564336239 +66303533383932633065373565393137313666373039323564376664653338333464633162333061 +66613361643462663564656663333266386535303764333162663837346535326335393962636330 +65626363306333333863366663353063393039623335316261636535663363626134613237326332 +34363638356634643833346332616132383331643938663633356135643037303532626532653736 +33353965393633323763323962666265306539663834346261643432313932623936623135623263 +62383731333938613962353565356134336566306562303639323434613534316534363663623231 +33643163623932313161366433316435613033366434626562326234646164326339346461343662 +37316163323731646634636634356464643935646232393761316466336438613035363435633130 +31313962303830646561643234303963613262393232643031346130386266393137383534366135 +38353133323035366239656536616634346430636230653638363938366338343531366363663564 +39313633353838616439363734323133663666633637323964306137366138643665623064316436 +37396363316239303761333230623461363261623339656364316263653430616463343033376162 +65623761333335396332313662343639616533343862353639626135396134353765316666376532 +62366330386266386664653133623534633837326233646662333066613838393036303230396465 +63396463653535373136643864623965366263376265306263356265353036636533346261636265 +66343031346162353733613933393164393261613763383165653835333762383130313938303064 +64313330653566333632313138636238393865333563366139313832323035613336313761303539 +36376530343963343961633031303536376664396239373832636335636136636330646334393466 +37336462663438366536346537306230373535636335316330376163363136333564313635373231 +65623232393334393634666236646630666439343630626334306334383935626335343137616161 +36303762653336623936366163383866656364343234626237336230303036333632363836346165 +35636361366639336265303536373736653663373337343432303030373162303433646161613838 +39646435646233626532353233636437313130656330663938383334653038663065306136353637 +63356138353839653136663336303432623663313635653633303532616166333334303937616462 +61623537663634323134306332353530626138303833626435383334393130663438303338333335 +35633162333234653935643130363233633564386563636332666463353330613835613331653864 +64333266323539346536636439633232393132333439386535363965366565343330626337626661 +63383864353636336439396636346632623464643732303336306338333864323935663563303564 +62363436613833343837386434373332656136626266353239626433623831303561653532313839 +34633961613434343234346335643831653564366330393735636136616231333037663065616265 +39383630643636313065636339313466323633643337353862396462303935326531306435636532 +31633531383534393239316231636632306530333733613936383461623038616231633337363838 +62383836396434656335393239373062353737623230623830346664396237656135303838326263 +38666331343239656666343239393666326661663535336338373536353236633234626239333533 +31326130663932383662396564383130613865333763653035343831316539666465636437373066 +32623239346565346239363837383231396133643236653735326537353038333031336162356437 +61336461616438343833383932313934623035343666613035333338323265373837303865366261 +64366338653131653562626532393266626166363361356533353030336461643432663262306466 +39303366636136313163363366313634613863386435303662323265626631316338336530323138 +37383033616538383539643436346134313466396534653132633737383862366664313533306632 +39636338303632353361646633333034326132653933376333376439386362316139653339373738 +65633834353034323839326339363930626332396366623762346163343264353164623738383234 +32376133393766356434303938663536323734643964623338306634626665623233306334306333 +33653534393465313261323761633531336131356461346231356639323864303163316532626636 +66316661626261323534326163653762616633343936396630393438373532633864303861613439 +33656330663433343334616635373464643363643331613537656233613134353336303137343939 +65346436343965636232383362366231356661303164363962613763356661663630333838643833 +33383030323638663663363937323264663533656263363465316439663136383765363039636538 +39393936353933383635636531653230316234626331376364643138663830656330343530346435 +32613962306663303062383837653830313063333966646233323538653363303730353466373165 +35303439366134376639666636393933613935663965653961303265393461376436376134646262 +61633466393535353363336263666137313832353461653562633664636232373837326531646464 +32353432356231373362333935616661383435623339663965613030636366353337303864353665 +32393961363364326233353632613865383730326236373239356338306235333635653136343537 +31363163353464653936306232643235316235383836306536303961383238623165323163343836 +38343731333834636535386166313634616262366265306239636435653364656534643436343437 +39616532656636666364363738613532633566623230323663326534646366326130313534656466 +61343131643164353336323737303436366335346638326633366466613935323262323836646137 +62316230333235313530353563373165343366373138346538303166396364653137343033623066 +36303861326363653662666561376433393033663836656535343165636635663763373462316166 +65363762663930373035626639316331366435636136616638643963626535343761313537393135 +34376136346537323566343064613961393439386536373166383833353365343562353662306665 +38663538313463666663323531346332643536663132643736623462346366306538373436343436 +38333265336438643436623630313935376239323735303161666538366437623830666536386330 +66346331393639636166323135356234303965316332616137663533373365333932386431373836 +34333133643532303932363139343764363861316639643136343461366164663035353838323663 +62626265313632373631616363323266376463633539383038343961633738366637646233633366 +39373838636562313237386430363737613032373864633364373065323763323439623265626238 +32336236386138323531653061323633356130643335306266316362323062646162396531353262 +37353934623134613838323361373465316339626239333433363563613536323933323533623235 +66343039636436653537373337326132626661336238626566613062313031633266613136666161 +36376236636330316231623732396361333938656139626562343836353263623338386133633361 +33366262336363383032623564393262323932373235623330386135383033316133383239383164 +31313432656636386562616332656433383164363539636232373863643233393031376436373832 +35346235303034366638663962333961643064303564393361623636363364326366666633323830 +61376538626439363034396436356165396239663163306338353931323264303462386264376636 +38383363626632313734316138346238333930353966376336323764616237303062366530353362 +30656262313165366232616339383038666636663464393837626263626161626337376534306639 +35393263653239313932343366316665653736303533353538363736373533633861363031313366 +65363639363639393538616234343738376136376365336164353237346263366131373233626561 +38623461303335633762646439653536646234383565663264633562653564343066366535613837 +37323765373563646361396663353836383334656666383963373533306665623334346136313563 +31366630343963376437353434653334343630393137613061313163396636386533666234666134 +34343333356262643964643039646264616265623639376466623664663566663862376166663332 +38653635303130656637303436643637343461383231306532376363366466633734346436363062 +35626165646131353565316166643666346139366265353936633662396165393466323664616539 +30613630373030353239666233363031336666623466613130623937643062633561333935643934 +37373964633933393764613765666532396336666432653837396334353337653439313663393733 +66356465316330353438356632396662663537306362343631373831613535313336366665303334 +33343466623134646665303030373439313730646539373166353831376166643963393533326630 +65366138656536303135323038666236346134393133616230343464393635316262356634333262 +65666436353834333233333730333535373938396135333534376463333537343031323439336537 +34306538343536643937336434313032626531636366366162616434393933643835383632323463 +33373264643435333032663836633566386436363432666366663264613933613964386434636263 +31323438636532393861313330343535376433373263323637373738646233643862333265363434 +34626133393335656161326264356261373733613638373963333564313138386563643963643631 +31303131613363393662393231393732643831363266656433373464643633626566646532643966 +30616639313536656334623664386336376464343033383938363539326533376161336331306432 +61336431623432393865653462393164383935656364383139623538643035366564363537613062 +62623237343537366132333763383966336166333230396537313663633062616631326633336263 +65373936633639393139383332303761363266363832646630616331376166623232373331333862 +64393762316337396538316239656139333563613836343232636335343730613333636136303565 +33326235336262326161666430613634633135373637393565653532663164323936633632313237 +35633838303562303039666130386131376537653862366533653736303931646135396565616364 +64643539616133373834666231636332633636326161623732376537633838646532636538643439 +32303534633262626561643362613730646663326262336334323339643838376231633163613462 +33613036343031633830613562333637333934613934343262353164313337303839313633353065 +35336231386464623038303562643739313763313132356561376433663763353030373830303531 +35303464623361383637626236633132666330623065626131376561613663343634323831383635 +32666630636562323835393337333662616637306631303235336266343362386631663863333033 +64376131666265393231376266623837343038633462353734376431333139333262326637393130 +38656131653639383735303333346530353865626231343739666435633537343434323830623663 +66646133393834363739633937366130643366656162616565626435323061346234633066343865 +63396537643435383634376134633030333561643837393232623563636265393166303230303564 +64653637336635303964316434393436316536356664333236346132376433616462366162653064 +61396433353362623631643033346234306466653637303062663431303034623831353861323434 +62346636323735633639653465343764326461336263373131643439613536356230646634666434 +63366264653437393963303864353865376561396666663431613535366237393161633533313437 +38396635633863346337643837353936386564656339313265663466666137616334306363313739 +32616339396530353064353934643465663035613661646138396136663337363631383535303932 +38313162313930663931373866346363383863663633386535393365343537303264316362393039 +30363732643839303730303133303334643362353265623838383964636663386330376237633364 +66666262373138333863666664616339613665663161306263636266646132326665393337646361 +63643963313965306366616362376234626464643565326464366666666436383662646533636237 +33333866323236326636313936393164313963643034336238383239343131643162313162343164 +61613537356465376135333162623233383536383936623663376230643365653761303636636231 +36386464663832343436653464303738643633396432653066336131666639376539306432386263 +61333935376561613932366163316134373636353461353563663261346564633639356235623665 +64636236333637333033643431333138353436396230316664313231356639313535326332623131 +64343866643761613736303031303762336161333766613465393835636639653666333164333435 +34643039366632386463386663343531306632323638386430396437323738363134376631356463 +37353039346638626433623632623338306264643163323233353838623238393465396464646239 +35636165613032633637316130663539626635623539653539636265376130306363333734383566 +37643439373635393031326565653765386635313731643863616334626539396333646462343632 +33343134653530343134353630633961346139633436373238653633333437386336313363343034 +37393632316637646535316266326532333064343739356366326631313664323537373264326639 +32663633396130333965336333383130386266623431343062376639346532656262656634303835 +39656230353636646161653661373832623534663630353436383739383039323539393435616430 +62363464616663383565346431613033323361626530353261646636616239653866663961656532 +33616663336236376335643566626433396635623435633732346661333262366536393362396330 +66616537353130663663306434373638626439306633313133643537303232376633363061306531 +65393366643937356563623733393238393133633933383563653763353034396435396336316166 +64663637376633663433363031396335383838643232343133306535633631653831393035333765 +63393162343530316565376538313939356165616130626136376464663562336665373832336531 +33356265353532316262303239336163363165336339333638636163316230633932656133626364 +66333563616430646166626361653934613039633931633537653962613164376161393533323535 +32306634383166646437623662616634343835346639313333313030303236626330663031356431 +30383733376235636463616263633461373462313437616235313365376530653264343964323930 +36343633353964326635323635633666323662343731616665613738616666386337336662353236 +32326534383031393137646636643438353662643063336266313266363534616362303238373439 +38616431643330373563646465623030386366373537653736313130633263633239316234636464 +39646266363530323039663534373338323139646361326337343033383337623439353161393739 +38323631663938303437633764386266643066323764633261323930393937613735663135346163 +30636238616462623232373765653033383966343862366139343134303834643531333934343865 +35333936303530636265626664363530353134613632666634373864383632333632633030343235 +65623762356331666563616465363661303661333061323232616138656262663739633762356430 +37366266326364396536613536663935363834356131383838663234373138356339643930373739 +37386134663633383231643166393464326132626238633235383039383461373963613836393361 +61653366356237363563396166366463373136313739313931336131383261373861363830316332 +35326638623334663065383132636133363430343331633632316435333466633730366639646530 +31333039633430656562326534663333346630643639346664666661353431343261363033373366 +37643166653339393235366262346163663937376131623531323461306536666362376539346536 +61326436383363336330636565316536303530383061646134666234386633346634343662623936 +37613861623235353238373134356639663739623938396435373532393562313033316665333439 +63353332303562396666346265366561643234633937376562613736343039376130346332383635 +39323164306436306363613839346262306631313931336162623063353436373036656638373362 +33396262636537336538306130343862363531656161316636613432386635616263383561316263 +30643733666230313163353965343664626636373836366534333963303662656231633863386466 +66393137663461333762373433623135623735656131353138306562633065326534323565383030 +35346235393831346162633332323461636137616463666132306637303361313863613061306533 +35386665383739356630343865333561336366663735633632383932623635663738613032343938 +34303664666337393833623636346333353062303832386138303463333464353362656562616630 +31666165393931316234633162353235613963363539333336663636623439616563373637643333 +64306430373134313262383762316430663130663962393330316235326436633532633238633839 +34623763623231356433626263313030313062323935326430653265383136303234613661343237 +63313566303065373632316461326331663336656532303366623065313561646532393162383238 +31653139353232633238393835373163306564343466343161623363303230646237663336343762 +33363735656135333265366332666536396361383839333135643130376337653061643032613134 +66623932313763656336653432326662636663343062373337303066303163666239626161343735 +64326333616163393334353038306233306666313738396130666233656637316161306266316139 +30353565656431363136653238386638633139333565653635616266363165363139343334623361 +35306139303237336635353031393238333366326631303763306531376637373666393765373931 +36303835323834313730663665373463316664303863323562363234323065383036373061616330 +37316130313633666563366439333532643330666133656366353339353565663065353137373835 +62353836333366376633373761386166636331363236353031613335643130396634303337366534 +34653332626636363631323531313461396538343731316265323137346336353737316234353735 +35333839336433366561376130346335343835306139363831376537663634306361656136396165 +61656266333536353335613461353936616339343134626334303366333735613332636230633465 +61356333363661386564653164613663633962623630343734323335336636323939366534363066 +38353836333933386332396232343161343933653132386335343432396564613838643165363165 +34396136343933653930613635646334643331656330386562333366653162336135303365643539 +30396366656365633531613038666561663531353838313338663630356138383536633261663839 +66376264393466623032666633626331356166623863353432333464653563663736323665356565 +32663766303931656635326463343762316338393437323762393630356530646339353233613839 +36613836376361333438643536613138376435353331393565646537393534383734666166343035 +38366562363732386435643061663639643961346262373834363035306261396132656365383762 +35316564613435613630623161363738383436623962386236613637373136613534626631656162 +34656432346637663965663563393838656638333635323731306661366565336666313935353233 +39613161626234306231356532373335386261373733653731363230303937343539636262633637 +35653364393039366238373735333861313134353064303264663537623131343935666566656138 +37643563363164333537646465616332383933626162356639633064373834643631316566643861 +35646665396238376132353239333030306636643636646266386166343435373135373666323766 +35343433363035386630633664356562623632373132366634646361396538353737363932663730 +65343737303532363437653965616166656137366564343537366663343832313032306532636164 +63376666386533623438626232386362396632623163386463353937336438383864613732626439 +33633265666436323863323632333039333165323563653662356431616239626261326165366262 +62363638393731393936336463333033366263636531316234636463313433306538633033633633 +34616462346635333838626231366332393535656365663665666636613838363662613934313762 +35613665343761333463336266633066333333613835306264376363653939303666333462643433 +30623064646439383062646266326564323036623237363635326538313065653235373139633464 +39386362653361616434373539613332356232303562363434656364626533653462613462356637 +39383030356236326562343239363638333837656333303761636632643738373935353034396464 +63646536366139376237663335623762313439366463616235333566616662656339643661313463 +66343737336138333562303666653436623432306330613636353064376433333839643564636134 +38386163616566383637623237373666623466393861643330386133613737393732356564326435 +31336235623736633431613739386133633366643336646230653636383333316661616462303033 +35646131303431356363373430623039366331303230366565646364656466353962666661373235 +34323730633339393531656138363130336432666262646139366664333263323561306163326136 +39653131393331366530613838663937353735373737666630616535353366353532646339363239 +63626630306339346439643934353962663264333465386639393734303265616263646237326466 +37303663376261616632393064353266663432633639653262363435663462323262613739656664 +61616661346464623434386637656363633161383338336638306635323739353135626365653836 +30383838373637366633303132623730303461666463383662343630363437386631313634373661 +34356138613432363339636334393661313063373533323631663839636538306337386538376665 +38373739643439633530656666303463316239626130323932333165303730323363326630343433 +36336362313831373562646331666335306361353831653835306335356430666263633037643261 +33306365356366373930613330303565646461346434613838636536316265313831666265336532 +61633465653863336437313162376362656338636131613133313639353361313133323963316462 +61343638343638663538653465643231343064316162646565663764373731353034353737623539 +65326665316563383562363661396236643439346538633938663865353663623637663266323062 +38353662663033366532393133313130363239383237633139313439376232313330613966633437 +64373335333737353266333466313436653231373431306437616135366465383434323535663762 +33636366623734386165363430363134666238316138636461343164613466653837336232343733 +39373465616633663665363865373466633731656663643134343534643332386462363935353034 +63663732376662643263393864393133666365343432376530636666636138313434393230333130 +64623564326335643264316432663963376337386563346331366366353732633432643832656339 +31633132306161306130383930393032643431613630373137613165316236353763373339346536 +39313861623037303164343064303538663738653763643838313430366662316233323238323833 +38663131393565616536613939316166386661373237656135336261313832643935636535663562 +39323033326432386666353762333531636331613065383666333331623065613237313961666661 +34313738303635363265623465353665663263616336623831316161353232666465313866373531 +61306337353032336362646163623538626336333866356461613335363165383366616564373132 +35356462376237363538313365383436383662386461663539316234303761663536653261663961 +63343839623564616331323234653932626232643966396637663166663031633965643264343162 +65636533376236643130363036613535333738376362333435336261363037643039386364313363 +61343835313635396639396334316635303166313837666339376331646635633030343463323762 +36383836326239386539653634626261393161303736626131333762346332336330366336333039 +32643831633361623731306566616131623439343630396661346331343933656538653734343831 +34623166313632353864656231353263306339613863646236396235333130636538386565303063 +32666132626535653134666565383163636533633635613562326434636637656630666361653534 +62316162303764303733326134613536333231333737653861343937356238323865333462646637 +36393561363130623534396362613231376530323936636335323035666461643462643630626439 +35326339303931363235333039383634663362333234313438633838353865396330343738653239 +65393439633130303130323665663663393266346663363762366338616236313961393836303935 +35346533373763613139656661356638353765656139366263313631303061366331373530393537 +34303266353931326538353734333234323563356166323965623265323866656266393733356461 +39393161323561313465343833356631353662376632393864396565386636303334343032383266 +61663932616339663265323064663436323663303631383438656635626565646663396637376631 +39306136626338626332363038636462633831646632653038346262643266373963326261383763 +38316431303634623337353835396330656164643939653132376166366134643561353263336463 +33373931616538363665633362353762353963353438633139613934646265646664356135633530 +65373632313939666662396439353137376238373466323832376638363865663336326565333831 +37306630363236643431383532363666623334366430623364313837616636626230616530353038 +38656135326130616166656338383136313362626166386465663932343738306162666630656331 +39653139313534333535356637653835363432633864386564666537346139656564393762656230 +34396135343337626363306665326338353732396233363437663966373139366330323735656334 +30393962636631663432653961303632343635633564346638616332363132653765613833353531 +66336361316234323864333533316234303030613535346464663132383563653363643831626235 +30653535653439306632623839326366316138343138346236643261316534326334373763303433 +39393461393734386237303464306331306262303732373666386336633635353063323239366537 +33363161663337613866666136643533643933373038613636306437313961633665333730653239 +36373361626361313435313662343338303463363935316461313063353261323135633266396466 +38646665643064653731666437613164663031343863646463303630363535313461616363316362 +63303539373537346435306333326666656231303837376263353863343962343536633935613964 +35653230616564623330366664373536303338336666313561313365343134663437373462633039 +33373039306536656233376364656534643863313733313638353433303565653234643633616439 +35313838383737363237623033323132643536366663383761323738383563623264303763313230 +62316361306136353333653062653865613836633161626163616264613638633433303961623730 +32306232303131353562373865303334323936643065386133663636383865396435393237663537 +38643735346235323561356163656361666434333435346438306362343433303563323730663061 +33396166326463386538613937393766666133653036373839306661373165636633353133376131 +36643137663531646433616132663337613134306466303866326531663864386463396130376638 +33616661313032663536383231393861373934323130376164383337366461663538336563393361 +65333531343761633563306266366235386262306337663335323235383961393732636338666465 +66643461323039396636653232373231663539333831656530386433623934383264616264353535 +39363039346633666436663265616530383539643961336130313839633966326362326466653233 +65313766646636616338386265643437653262623361346534363739666661336532316535643331 +32333561366363343163313732373437333130396166346135343230663230393561316363643735 +62326364623033636436613830653039336630353030313162376631633634353134626265633134 +63313433313139616430623765616534353661396136383936626630313031613664386633613063 +65633431646462643438663035646130613266383634376537356635383663656362663364326336 +32633863653432613565346532316436323665363932636335303039303936306532623964613664 +62363334346365626433633735333435326661366232356538393339373261363364363366333137 +64373133343537393863653365373863346661333265353865343265346261633863336461393131 +30396633633962333966633832613265646262613431623431643938316432303764376266633062 +35623661336636326334316661306132393961376262303363656464616661326537636663313162 +39306165363838363736336230333634353938656430386234303864376262383236616238383962 +33663533393939653736363031666439323834383361633637653534383036353034353635326161 +66386561343731346532333735623932333334373435323664646235653332646431356466373432 +31343534633236656238653366633636313837383234623532333534623761663835363135373530 +37386537366438333630313433303164643266353139353030643963653533636533393561633331 +37646561626131366566326636303030373034623230623161316337306532353236393938346464 +61396231303631326665376462383461326533623362376437396462396139333266316238363732 +37643532353962383861383933303633323264353662633065343730376231643634313165366539 +33313539353538303730656239643331643333356662343530323739333961306266613462623537 +31623366353336626362623436636565333837336235356562333636323831306630646534316135 +33663165373661303063383839356461346633346337636535373461643162643961616539343234 +32633663303066323435343832663662353737633931313031336631653834616233386438383461 +35356161346439653665343432373166383361396536366239316131663364386261383730363439 +61323838326265663466346133626139623165356461643036366663643031343863306364326563 +65396663643831386634653531666133353965613839346437376537613834656230646165666634 +37616264363436383331343430653136353532626637643432623631613734653462393564663565 +63626633646166363138613062313865616362323436663534343332663932396632633734643966 +61333132613730623336303361626236383035613939386661613966393061363264333337346161 +61626665316237313764633665396535376538313362653365373731333138613161336561316661 +30363536653231646138653034323665363465663764353162613065333131623762623137303565 +34383663616263383635663562633261373966656530306362376162343561616531386333386231 +35623062363361373238383637656166396133663565653461363634326138313961353331383661 +35366661663539646637373335326237383064303533316534343866313335333664373336393434 +65356135396664333633353035653433393938386531376635626339316137333562353038613634 +35363733303032326164386231626435646239366464386262303466383666623465303538363335 +33373434623235373762303661306632363262326264323166643831356661353334663264316437 +30303535386136393761623537633833653939376164383839303462656139656539663630383739 +39376234353836396162396337613339303930666637633463333039353331386235336338326562 +63356366353365623335363764363634666431323534353035653966333534333031316631326531 +64363761653333383430653433396538393065643965316235326330616335336539393165303134 +34323139346339376232666537653461656130386431623065316232373737386337306636306638 +33386636333964366434646130636466633837313030616133336630333438643435333566633639 +32353863363161316364383366636164396262393065633331333238646633333066643033303437 +63303062396134636464333937653135393266313663653538303261333664656536313535613133 +30356564653838646634646134346366653965323433356266656438656662376561326332366161 +35393739373339623963623532653734613136666164376633626431616539613732313932396664 +34373636333539356132346537366263656633383461333437393165393561333734663430363366 +34656432303330346236326132353635356463303233613531343261393832393866653332393139 +66373732356666386337346132613336303237393530616235373835303534616639663334373261 +64663762613536636364666564306236633030336539653430616435636366616332626631333364 +35333932333238313233313439303737343066616633363466383862353037633837316232303763 +65376131636639653834643365316534326336303336373862626365396662616239346264623964 +31376565373335343736353864303866613139383837366235616232323531383862646166386339 +65653961343437336138653161316262333433306364616161623431643531346634646139353834 +30633933613030636462366334653062653262663934356236303438353062633139333166633664 +33303033383965316165333635343536376364386135663435383136633438633035633966353639 +38343737633237356331343031653564616265353639666334353061383437633962386430623334 +35373239633036626566316561323861326566666464383930373464343063313965663230376138 +30656435643732623761373461343666346437383166363265353339356438323864356532616161 +31366430316363343461346135383332373535353032623638316535656239353864313739663934 +63646634396637303034623466336237653131323235326535316334653262343165333766656165 +39656437313835326533623835383533366366346639323132333330366231626364656137396535 +34333930626430396539346230383232383832396138313239326163613038303735643135663434 +62383532323863323938323038616434653235393931653032653330653138356436313134663038 +33633961373138663735323438636636376565353536303432623430633330333763666431366536 +38356139356338323535313732623961366631346461313234323265663737643435363731333136 +35333461666437613834376230643234666334346338383036636430343839383434343863636662 +30383561386633313837343863386133656337353931376132646364613263656461646262653862 +34643264653138663433303930376564386665633139393264653963343033343431353834336164 +62373935353266313262383661633165643033636439316538666232613838393637343739306135 +38316661633438326263313436313233323834626138663466373536633538626166653331643236 +63326331633064383533663264396630663565653430653539666635306235393564373131353364 +39636638343739616663303237626361323364306562386239306131393533326332613963353230 +64636332616263623239373335626465393030653232653736376661373862376535356337656666 +33346335616331666361393861373163333461393933386230626632393038376333353036353964 +38373464636665306131363362663765626133383732613361303661343566366333623436316563 +34306662376336613630383665626138623765316539613632336532343462343866663239636436 +36666630336139623637643936343037613034333462303036323933306366663664623139653133 +39666635663233626330303631333236333131356536623431333636336630626133383763326164 +62336130333033303865623730386134653530626533656637613635613163386135653830343135 +31356366353464343238333738316538363135303763356438306330623633643862616139646136 +33313866643830323232366331616166653138363266616633383435366662623163373537613038 +65646662346563346635326236646231346634353232313636366266383439383532376262623366 +65356234663631643731623064316561343864316531636132393036623630633862353338343565 +30623163613162303861363338626166376339643735643032633935343561616330393164656432 +62306461366666623734626663666264663734356131393535376537636433386233323866393534 +35643939333635646163306262636635323539346462323136646535333635666164313662623933 +36623164653631383465613439626562623236353366663063346263613934336436326630336135 +61333264653430353835376637363638393531313037353363316633346463303966356335623337 +31323765663239653031666138636633613739303466363631323965306361353631653938393565 +38316535663833656239373838666137366139326461613164666237393638313737323039376531 +34376131326463346237336635333533373034393039336463376638336363646437616535353731 +38356430376334643337653865636164343534306537643731326337643462633838363039303765 +66363865393634653731613238643835353731376631643333363664343634313533373633666630 +33623262663234626231383534393231643735386362646639396562653837306362333139616238 +30323133383565626462386334653131396437363465346132383238333939336561653330646332 +30663734643662646635313236393534343936636465623438346235636562373762663330393938 +62326633376131396562653239313831373239386438313930366331643539653731376230386435 +38623433646136363237346263353963633831623461383737326563316464626339666365306230 +62616539633835366431626530333031313337373338363938313130396664323131633533353762 +36343264643632633536366532333132636631363265386466623033303063623563393034666130 +34393833663066623433643632623637333632653362336338383739663365636565643439633232 +32333133663933303931343633363031613532386339323537306265643565393634303064383532 +35653430363931326333383761316238396261626534353161636538616438653135386661303930 +31326561656333666533386364346364376464303735303530383666356361376438356563646364 +35363130386432303432376537383665643239353762623936363036343930636164303136333563 +37666237616232383937666262363561373666656632323664656563333838343164636532646561 +61656366343064393461323530333364666232613566316430633363666263613736646237343536 +32333430643263363732333965636539663236656638323936616234383731346363623830623932 +61636635313966356561363335636262653062393634633965666331623863313963656531623039 +37323664626638323836303962386630666638646663396466656161323439383335376333623832 +63396235663630323936303932616163323830353166393533656161363965313035396164636538 +38666530336165636461363839383766313963306134653537356131656239323239376333613935 +35326163373765323237373136323338663632383465303261613538626135383435666661666666 +38396338343031373135373439666363663333623637376138396431643438333837613862313637 +65363865383332613366663264663934346135646237653238323339663432626564336364336564 +63616434393064346366633063626339366465613565333739353030343561336336346437316361 +63366366333230636663316435313737353764303832663532313235653765653664363431386231 +66336137663464343833393735323532346164336564353765303364353835383362646462393336 +63363038336439353439363338313661643234646639303866656137643663356535343164666563 +32633639623732393430363436396235366563363737363865383465663261656461316330643833 +30343631623861396261306631303266623335333136303131626639613933643038653862346538 +39376638323364396439333935303139373838626264336232393035343061626135613465616363 +64633133636362633561383834626630376331323163313134356137666439653036336638613739 +64336537393134323363326431653639373832613939333766363535656131626466663763326338 +33376565326133626531613337393165623737323362343263653135656233346630396564326331 +35316165346631373438396535303732373035326166306366616165333062613532326137303732 +38396262303933316534313932643539303535376565353662363565356237336336666338373237 +32346561613163663564383338646331373439626262383233643266383964353532373264633134 +61396636396138346465326561386330633035633063376130626365623630633036363537333432 +32636237346536396631613937613166363936613963383936303865626639353633636131663439 +36396566653530653965363231383337343638346239396335643534383866656332636634653735 +34343736636238386630623665313762643861393438363733353231336539613037346363393337 +64333836663731376430333237626335323533363037343065393130333730643131306632653563 +64643833306536343038613833356535323862646433396438626535306335333363356232353064 +34393035663037633636386630396662636264326138383363623337326136393730303335303664 +39653334383964363539373433393666646663633336353562366338636564303033366662386234 +39616661326538323734366637646536663235396162396136653165383161636230666537626166 +66396236343137396665313037386239643361623134366334643163663963666466636236633935 +37333432343162353031313264636131613939633665316434303435373265643230353463313130 +61363265356432326163373436386664336135623966653161313038633533323332666437653764 +65613735613838653264346362373062383339323862666630356164353635376336623133353966 +36653264393838326433343934646364343362643531366561356539633133656561613930626239 +65623033326236636536383634396163626635633236386134396238643939313166616530393663 +61396332393138333165653331666465323266636364323162653464336330333035366435653563 +37653161373239346432366265346466386463633532633562323936313434653833376537303935 +34376264303563646362666331613338336130346434636432366538663663613331623662346137 +31396139393664303164313461376366313237633566616637633862306235343961383561303432 +36396234313334396231373264666564623362656263613836363336636233373361646236326431 +30626566623830313130303862313331356264666164323461393539343039383132396661376531 +64653034616231636134633634353065343339346663663539343331356339333433376436616665 +61613636653034336331336138363039616161646438316532653166613136636139663138663139 +36356631303961656137663662376662643734623735393032363739353535303865303630663837 +35626232383134316261643330373437666531366238643033353035323233386534393236316366 +64353030626135396133323461646135353737313634393037633236323533396431343466313634 +30323262656430663762663963313762353635633361653830336531306566643438613566363261 +61636362383462626235663931333662333031653337343061643264636366356661383034393739 +38643133636139343031303137316336356232653564653661623933623635306265363833656235 +65323464386232623737383232353331613266353235653364373832643562393331356132636335 +65306638656438323863326365383831343266323562346262343266346537396233626235303165 +33323162643832333062353465376463633965613333323031363033326663333133663963333335 +62653938373232643638636436336138613939383163336133653836633835376333346463343034 +34303461303863376234646234373462326430333361303436303565313164663633366536363333 +65336139643363386537396230633530303137346565386633326564383764343563646231653035 +61633533396533616333623931373330643637336538636339363337346535303239643164343364 +32316534343162663837383765336234653362336534336532353634306263376162613536383430 +36646532303465346634616364303433326638616463643735636435643266383833666235626337 +37346166626338343238653433353466366634626637633039313263313334336433656632626539 +37623530623138343263646335653863373731633861323266613138373461616336313962663261 +66656137346434613630616335386631306263613637663737343138643936336564393934623038 +61643531636266396330323961613930646163303263303936393865303764303263373830366336 +66326536653863663164636563336530613565323839373831666561383430373737376565666338 +65306332313165313364343262366431376139626130343132393964636236313034383634613733 +64383335626534313861623738396639663061396161613439363432623864323834643336623966 +63353333376665626233643138366339366332326562643464363238646630613764666436313537 +39653331343133666166343139643634666262353932323131643262346365613931326234613864 +37643862316266383464663938636433663266373437393732663935656461653461666261386434 +33633432356433353865303138643232346339386430346265633364636235623334336436386232 +30646665316364646663353834626363303134623863653834616237616331626436306439346432 +31653636633061333964353733353634636231626638616164343162653132613338343766333233 +38626235336232656164656264386138356536636364336164656136386137366265396564626263 +39373533396435633139626135623362383162356235383764643630306661323262653237356361 +32396563323964666632303565346666356566373261373532316535303133346537373037643139 +65333731653939323664316434383532633065373661646364343161343762663235323364643835 +32623734623661616466383034613063333235363765656661323961666463323866383966393238 +32666437373061386639346538656165643963343231343462393561356635653764646333646466 +65343962653237383337666265336531373833386133373835643538656366643963643862336165 +36643338313735393632623464376532336538333336313239333339373535306365336463646134 +64303835333136653933643834306333366431633362366362666363643536643638376434363864 +30306630653262303566396231623737613665333239353261333632323864613662393935653365 +33626237323862633062326365663062313761363137613235376131306561366436326166353966 +33346661386566366630313464656361616638376230313630313266353736303363363031353333 +39643431633638643762613465396366393336313162333032323738306238356264346230656366 +33653334383733353962663033663965346636616636333633653430656631366537316339383265 +30383161306464646131376238643664633465633831653336343164666264326166363661303831 +65376564396564303562646234316630383834356665386664306535666262323938616432316266 +34366130623137316366386133613035323232336137363835646531663065636638303439373538 +39616631666435383932333634646231373462663730363536326337613734643931333332663661 +62326161373763346633386233383562356630303237623935323834363932376431613966303834 +30623130633234313764636432613930363438393965396537633736643039636164353434376632 +31393533646334333832356433393164376134303566386635616237376236373631393233323330 +32313836663038663462306431646433613239323633623165313838616439663362356436633934 +65613433663065643263363564653131613033376331313762356332396466356337636362316137 +33303234623530643733643162636133336534396139393635633130303437373163623930333461 +36623932663932366664663531633634376135663165656238353030393739333035363362373935 +31363835666131323132663066636436636638616637353734343231333637306665363366373734 +32663064313833376234623765663630313265363638613731393138346438383666306237363937 +32316235383338366566646666623537303764306235306266623433353565366266633962613562 +36326234353533383665376331656136343064656164663431616531666438306532663461313031 +39333331386330363662396261613365646436356536356366326262376462313531623237356165 +35333937396630643830346335636530353637363835636535643230616162643962303538633934 +66356534666562393430613932666338636533376263376238303438653830653264313665633231 +66346565663032336664363565306633663536323135616530333536336336386534383233616232 +33653036663539336564623735313561336338643937396562346162346533323033333164396432 +35306635353337643230663532333130326466643661333233336436346335383537306463316466 +37356565636430326237323161373033623733376532633064646165336538636466323961386163 +32353432386532306434343735343361353066326331636334343835316164646138313632643935 +38376533326134303165396138633133626133303635366561353464363333343064366561663431 +64643934313036343630373638353739633266343638316263653337326431333264336362316330 +63653662613430336430386130383338343237383765653936386661316361623834313731343266 +36666130386462396635363039376436336639343237323165616434356436333530653131636263 +38356234326135366239303632306333323630333561333138306537663662623332333365323933 +62393162343630373030386264636162366634313936343538646334633466386537323564633934 +63313239343165623462646538656364386162666239363162393162646463616431356137323339 +63636664343331613533336531623437353461363833393363396631366434346237386439376131 +37363565326634326336343265613337366338626536653261663761666336343863363236333962 +63636434383538316636333762396265653937313632326135303335323366633330353464323866 +34653934333237373736363666326565323066386636373230353361343835313362313635323435 +38303036396266656438343031393066353361383731313132623963366564373838326265633438 +39636262306130643863386236383461323932663534323533326162646439633762323737646563 +32373535626561616632386435633661343666663665333664376132343166356566666532643833 +31323438623531633762376235656464613463623865623538313031306430326235343865326139 +66363737613833616464643862373138393964386665646139393165643963376137663362633431 +34313738356334333230626166313032363838653631343531626364373637613165666338383038 +35656235646431343766303833643062663338373733666436323936303462333137333863373935 +37363263336632393135636132363966666265313665646335663261616633656439303537353932 +38613861326633356430636338343937323630646438356362303066666366376432653431343466 +64306264333930663930366530396565343531616666646361383734313562343063656234326335 +63343665393661356432383837303261353065636462666230353632643664356432333461356634 +39323630353263616565316132383536656531313536633732633966643864393534323138353630 +35333738313061623763306165326365653330653661383463623762366534636161363432623833 +36303839313362613137613036636663343361646131626230353038336139303433313766333263 +64393966653165333562303961326232633262346465353065663861343262326236393635646361 +30646164316233366238323365383463383462303837383033326464306335623761353136326530 +64636562636334333535303839623865636366326139613262333866626566356530303635366335 +38383564346266666365393563306331323330356431646465313361383532323964616664373961 +34303033346431383634366638333361323035643366373661316536306434653434323265323738 +38643665376137396130643862303137346138326561663265626639333234373130636635333232 +62636236356236363361633435353262653134643039646563386330323163633166623739666331 +36623835373134303536653561343762613136633634396363373161353839366630643361343238 +33663130663336386264616533626530313537346566653135663636356236346434663464623333 +37363137333032353666323763653066666439653662376334613534393766613638346435363337 +61363866303466626164646238323334636335303066356433303030316231643139653064656137 +63666263366232373638646335663863343461373834633265353364393731643239326463323830 +32643935336137653763343934643365333236623534333731343261636138303833653261333231 +39333236616361363435323866363630396162386264333234646631653136323534623633386466 +35343634336564623762326439393038346232383963366234346161393934306637323962616435 +30363366383562313963626530626430383766666630646631376135333839653161326534373164 +36616330343932653066396531616563306638313565316462333530393965336633643735376239 +62326565613034396337346637386564613034386633346364303639333537313161346261353930 +63343238386335383561663763306436393839393130353335323133666430383763653062316235 +64353464643964326561386636623932303061646333636263386137393436636239313336386661 +32313864656634623234656632613635656666373834373833376563663034393638626463356531 +35653436313166333861656535633366633733613736633432653464653764306435393336353733 +38663561346630363238646331393961396665393630333363323237616238636466633861643630 +30323339333131383537396635383765383534333539336234313061623731383065653437613032 +37343763373164313530666566386633653138383934616235626337633036316461353366393634 +36326161626264333035653632626463393262613031316634353237666131626335303366623434 +38623963623530656332306633386235663764356563666362656162323239393262393165653334 +37386261356536616361613862663963636236633331643263306138316337643234306533663330 +61373735333138323532343333323130386364343165663131326130393932396236356534383165 +39313361653837643432663235633961613437643530316363643865373233666137373637366538 +32316431633932326435323136656261353532326466393234623035326636306564306536393364 +32316463623566643166393634303933323135313535356335353265376661316537313434313763 +63326536636363346335373062366363333432383036393534323030373739623633613033633665 +65303962366237663130393037323165653436653364376665613933373565616464393839306332 +62313332393562386636353164306364656430363432386237396533373566636664616637653531 +37386438616331653139346235613663636436383037386666363831363738376436653333303033 +63663939323463636564626237376163353366663065366366643437666135303363623634396465 +35653735313239663035643737326531613035643136626435333662376561323934646232336436 +32346338313231386138633261313666303866656239386336363836333931646463636461623865 +35653139633862363636333466303561643035393235626133336131396232623630363336376632 +63653437316662656664356261343132386531373137666465663365326635343765333362616535 +36636432336431346365336532333437633366643034303739333830313934306534363138623433 +34386562346161346531386166363862326433386665396634313333393530363366623637366637 +39376364316164356332353335303061363332633735343433653262373563336137336138643935 +35353964356363346635656262633563346335666533366166366135353735323831393931373132 +34623065313361613837336536613863623430613130633437323062633262663865623065306136 +34333763623937316236613736353037666364653537653238343039393331363232366665393832 +39656663643964613536376365396334626430636535646139333461623737363137376231376431 +63346166626562366463383134326532383730383535633833373736643138336231653865636233 +62323434666332346433653538363938666233383439616165653765663564346463653639333739 +34383066396236616630323136383166616433353033646434636265306539643230376335343531 +61366662616666373137663763656463326433353764346561636631356565343266643236623763 +66663063373832636265313962366261343066366436386432313139303866663235656131643830 +38333430613835306630633139393534373164303661633362383336643065376464653936316630 +34623333666430333939393734613538633965373033343235373339316234643663373430333035 +33383936353531303735616335313164636362373434616636363161666233356336653363386664 +63323932656231306262376464373936633137613337663438346439356631366132323334313532 +64653132623562633735643663643837656266613663396361393066353562636461303836636533 +34316134306564336162323936343638633165653066393065653238626537346134306132626337 +65336365373062393634663135343133623963613139616632656633373834353933346562326539 +36643862343862346232316363346530393637616434336562653561376333623332663931303061 +33313961626465353564313435303165363335643563353738646264326563306232656463623833 +36656332313739613137343536313130353432343365363439393430346265623039373331643562 +31303331373762663030623463613166306136343065383563646335373730343839613363343135 +34383332363130383936616461613162333030626634393432323461633833313236346233373439 +62393964383932666565333031326362333734356135393238366634313937613262376462386135 +32373064623566316430653666626639303139363439613966313563313530373233386561353937 +33306331646263313334626361373236643337323464663165376239633335343365326239313831 +36313534643532653531613136373331376335363037666630616132623662613835626632636661 +64613331656531346264366462656538663266326135366133356536313762353261316137666331 +65613338626363646135383333366564393636373730336564376334373464323832356565366133 +64633238323666363864383935366263656434356166363562616336643466643235626335653261 +38653263643737633763376635626266353930663838393062323166396434323965633633316533 +61316531383133373663383133643435656332366661313165643762613766333134666237393132 +65633862326266383333613231363061386561663364366564653666323066643237343537373335 +31366433636364656638613365633662626434363661383038333364616565366531626562306133 +34353938636238383264363635643565316665666338326462303636346232613066306131613766 +61383435346131633139376133626531666666323239316138613365656539623639316663316539 +36393630616539623765306464326233313761366533653864316637643766373666326638636134 +35666537666564656131633563646264336332636562356431633835636662366331626365306436 +61363937313335633935626465396638653234656366663630333230333139663234613965613534 +36336237653931636437633631393465373630353634393337316433303533643739303636643464 +62656433333533663334393537343435326637363639643433366633376365363238303238396337 +33343736303935336464316637643637313261333733663764323361326563336562663539616635 +31663935346163343134613162663031346138333635396134653266343539663331343766306334 +34303366316537666338393062653639363763616334363833393134356535303661663663646330 +66633832323032383566343464643636303334363064636436336134393561363836313434636261 +31626432333130636161313831373833326232323733353936333762633362663239626232303730 +63323334323935336161363432323331653839326666626137316433616465326261353731373130 +36663631623436356165373134663466303039346232343530343565326338346462623063643464 +39343333376566663239663064373036343232376634373166643637353035393865636431366363 +37336263653534326465356338633937316337623232653931336264373134383365613565383130 +36353532666564653063646437393266306233363039616237333730633737303734323232303636 +62303634303363316531643365343966333730353631613366376636656666353964643331393831 +63383033663335653732393863633064666630373533333133383335353030663365613639353135 +30663766643135396165383234396132656336363766643665336331666233353135626332653735 +34323130393336616461643538366562323366383064323834636431623530313938303435363335 +66316136386332623234313162313838356238646138616465613236353466616566613134346637 +30643632666339663162383264363561336237656661306331656566393638343434666461636139 +63656266323731383237343732633234623164316639396633373261303764316138623733313065 +34653937393630623364346236346664303864666665376235363163306534313038343062316133 +64666462373265393133333531373132336165653963343139616664376438656361326632353566 +34306334363736666466626136663964636430613734616232383331383235343365663964646438 +33656261616464323564376137373332663435356337653237636133356432663464613132393038 +65373738323061633866343334343366646561396264373164383465343862363163333761313662 +66666461383263376237303461653937356233353236633665636233613732613739356239666335 +31373731666131363166383364373862313666316561303036666265663365626335346166346333 +30376233373138383564366632316137373838356435616338326264653564346633343561363830 +32363235313038643564613939613065383463666566383632356334303835306162643663383736 +30366162353739363432373038626134303266626663393366663833383131656139613738653165 +34373262303233393439373639396566626630363564663765393032383961333030643862363931 +37663863333166356236346463613166613934346133653735613635373734383864366536313330 +32383239666364316635366433653162656132616462396663393263373064383534666363393864 +66663563663566633934313533333766666464383735653139646436616132613634333638383730 +66356466386662393236353931636262633937303537633231636661326332353764356563633833 +62326231353832306466623532613038646464643563366636653964653630333066626431643661 +33646637623661373665643963663835666661326562363336353964643537396663353239666334 +37323031356132663761333830666537313337396263383730363237656232383638643764656633 +32363332386565356631363632313064306431343163323333313262313630383932666632306362 +61393137336464366635666665316636383165313761343339373237323339316565306636626238 +66396537653235353638633330663332396362376561626431323831363337313338313038353933 +66373963333266633532393837353563393761306531366632613161626662306462313631633236 +65386663353263313438333065633466636464616364363464313139353038316139663365636636 +66353130383638306638396135663939643733643665396439333834303339303032393038633338 +63646664353831353666666261343562376462353063626233393466613366386336326132383238 +32336233333063353862333261633730633261613538353439373730363966316532666366666466 +30353032346536393836393661373630653738363462346230306466613962323866306666626132 +63313661363965666335336537326339393062353765386534393338396362326232326363663630 +65616636323931313332333334336233623934386361366162393964353632643232613264623335 +30613266306237333065656239643363643166383839626266336366613562303666343936323063 +34643866353166303434396665613634653731393835373836633237386637653733373133653265 +63376532356461643033313930376536333930353633353433656364663537303261373532303634 +36626132393761626433316333313566396362343939623261653439346533303038366363633138 +32353438333065656166366265393630363534346237326663356132333764333464333133386361 +32306531393430366131616233656630666136373165363337323937653364303864303530636166 +37313537623065636461373638313635636466346365343738363039326566363964633437376662 +62363531346537303562363038303934666331666530326665396632343536393932363032653735 +33656430316531323931636534356334663362316561396630633737333962393066663336353162 +63643537663036306462623763343236333666613333383361623664356563663334666333393232 +32626465303962373530656530303139616538316665386431353338386634333361626166373462 +65363634613163613364666366666631313536303164613566376632343838383866663034346438 +33393136663834356162386432306665373230326666303064323033326235366131383430373764 +31383430383530343838646539326532343366386361656233316131326361666337313731616439 +66373732333531333337373332613735376637303462376662373837303261383462373135386361 +63623731383663346133643939643738333439633765616634653161626339316233646136653532 +30653438636331333839306631346466346230346266643761303465666337343939393966393665 +64656633346431393430663236636433666538346364633339623861633461356535376537316661 +37303435663638323463373637346436323561353939373162386232366330643665306232663237 +64383532666362623537333865386231343561623335353863336666303139643763396130623163 +34613033633338633061613266386165346336323533303463336465633861613136356631633961 +62616131626334633939663965316631386330393534343061333337323739313230353639393464 +31363430643936343138343765363738663937376162323334643966653566323731353261653334 +65393237646633663161313462626366313830393238333566656266353866636162376536366334 +61393962633633663736343764643765633861643434316632616638393561613237666635613766 +30353836353966373836323262323734353438326430623839356536663265623235613032353264 +65666564636235346163616433323465323066643162633033623266626630343834313035393634 +37613765646333396235306565343164373435623432656430383662346636663561643833306332 +61666239363037323037363436306338623637393031363639646262646335653265363035393937 +62663865323236613632663931616564396130303062363738643164333533333164383537653937 +34613963363362336563636163396339656335366466353466363963323536633136333439653936 +65383566366237326137616336366166633839376361303336306536653166393939633636663366 +66306432613034323264363631623063373863653632336331636464303433646535623839626365 +62383330313061383439366461313836646264343436633562623066343865303664336664393264 +33383162366330373031353533353866303936393865303535656232393566376538303565333835 +33363032386331303237366431343235393161623034386163626164623836393765346366333065 +36303831363961356530356136306266396362346337663434363964303936613239316365306162 +66393836333464346338383432336138616330666362326562613239383165393530633734636637 +66373339336133323561306633653233303864333763633137366335356534373334613461356337 +62343835643763636264373639646633373164636562343033346137303234616133313161316533 +35613863633430363936653962303033363836323761373838366665306136353737643938663061 +34356363376538356238306333623265313836666538383062346432316163633733353139646537 +35313866333436623434643830323732623165303862316466373231373564353833316534623234 +31366137366461336163333837353261303131623333643739626634373832376634346262633633 +37653833633733373638616133313933616334646434363834356338303166356562383134376134 +66303366363562646630333062343138353137313734643463383837646562653036643635373136 +31326237646131313633666432346461636137396162663265376166353239306563643865363334 +65343261613565613638326637303432643136373630383761653934636532373661373661383432 +36656131326466326665613364633537633238646332643261316165346534303835653363643533 +65336532306366646438646337303030316466396534656531353230653230333039643935613735 +39663536336163363431666436396364666537653037653134326330366434633239663862366165 +37643864383937343437373964383562633930636230373039363731326536356531633036393137 +39333036353765323638663263316530376231653232356336396534366538643536323734393235 +32353038343163303561343038383532653564393531386264313834336439336134636261386631 +66643533363561326464383565373938646339646534396561333732356136363235376533383736 +39613633333032633539386465386533653932353738313237373766313264353064633837646565 +62396164363234646630326165663666313261316165633631386138643533303037663062326238 +31613337626562643166636239633932663833373065636439373836656663616537326336663131 +39616566393766363764666134353337333938343234333866353566656237326435336464323439 +65646663626238323861353034333564393233323038333563613462336232356133346539336164 +64343662313561653964323566303862353861386663613830366366303463616232376332306565 +30656436653035633239386534393937383635326235633833386264333463653438663563333566 +66303862373536666135326434316331663534356133663533343166363336323034623139303931 +39353830386632396262373537396566656237326161323166626463386337303432306261383830 +38373435663130306565316639613232356261393937626631353336373265326163326163346535 +35613737623565303237356136653338643662663635373263666232326539343264653839363336 +64633737613435623066303664326536356364663231373362376538626262643731313739663639 +65343237353834353131366463333532653739313636666664343239383966623031353033326561 +31316530623830663830666561633832626635643565613966663261646466313131306239313838 +64646538353538646634316231656239633637653330356337376161386635663038383465633765 +38646331623536313861383465386261643333383333356361316130383733336131356238383939 +38343739306337373837363661346430623965646536363361623636313630393537323566626134 +31333737613163366239393065346131653434346135656330346533636137636232383031346330 +39343664323935356238663865393666346462616639613061326464396164336634636463633436 +65636666316365346435336365333661343961386331656264323938666237393830366330633639 +33363732326533313239326130656331656461346661656336356366336434646163376266633563 +34333131643839613662303862316433623934396363616533323261383238643332633036616164 +63346232373233373034636639363139663331323233613936393032623935633230396362663235 +36623864303332623863613036396461376262303165353564613135373735356532616165343032 +38616331303239306239393162333130343164333433613938653663323835353135613433376134 +35313732333438373734363362643031316339613962346466363838343739396230653733376334 +30386139313061323230373464353364313237663061656363333862653661666563653631616262 +31366164363830393662656564633665616331316162303266643163313937343061336135663266 +39626232656362623831303131386662333862396334393162633138376636643632303236666262 +62306436316662303436313962666465353861613031373862393532393336353664613438613938 +33323832326561663861396530316639643038316365313639656236353766623933386335373662 +36663164336531356433383937646166396661313865626263613462643930623735393566343130 +39653765316130626137343564383736366261366566643738313439383138336131373634363362 +38383365646435653331393939616361346364376465656332383536633136343765336330363064 +62353432353430393236346139326135323939616135363763373035313439316563656561623831 +38356232623534303264383239323865353731333464663434643233346339343566363937336636 +30656263663163386462336263663733636264633666386362353735646433363134346436303333 +61303739666431373464306539383765333836303564353736363762383830363161393130366636 +32633865393431383934623765653934643466346436656633386465303134633564336264656430 +30353833376336363835303863333137343235633762393166353566393437343835623537333935 +66653865333334326237656263326139333465613239366434346530306463313061663337326435 +39643065356261643731366138633433626464663965643062636233313338376632306636383231 +39623233663537636137633738626138343466363164656239323432366161323962313764656439 +35373237303931336635386130343962336633323836356439626538333963353937353632633366 +38306236623266343038643439383532653839363161393063393039356565663964636463343238 +66376233326562653934393964353637373537663366613464633038613265393832313832363436 +39396364336164663734316536356234623262653639626530303164303662653531323631653866 +62303863353037326134663565363462363664376362396338326237643633366164303835616165 +30616331376563643134626563663863633561336133313931663761643739366161376637333235 +39303266363537623763653930376164333861356537663264646361613066346534363863643738 +65646331626162313030623032353136633931363666653965623531323437383363363433666265 +63343563323632366539643234366466303137313837376136353434326566666231666633303134 +36373562376639626532313664333631346535386435636233336439666462666265396535363235 +35316339353266623962396262333464613430316362363432626135323763666364383235653934 +33356430366337643964626464353632356133373433653034663664636231616664383832343263 +38386635356335383831653265656162643463373831343531613830393636363066353262636630 +33366532646539373466306461393433393430376464353536323236316362346163306265303764 +38633330666637346133643632316336393435303135313962306237376666346335316135306363 +64636163616466333561666661313834336232663730636636346238623033333235666666633261 +31653431373833383631373364363361326239383535383465366666373862346664386534323331 +65303531663965363133323739306531306563626538623361646663626637326562333130306163 +61353332666236666231656331346235323039646632383830313932616363326339346139373031 +64613563393133376332356561396536613465663832353364396165623438333561633664393938 +61313230666237313765636131306261336238616134356336613434323536633139623735663836 +65623238346362613530333237323135356432376630333436326564616364396235336564633639 +37666161623961376166643431306634316262353630303138306531323432333030343439363438 +32316235323831366632326163386237383163663062383933343763643139613064646330653736 +37343236366234323766393364363333643864313166386165393436613064396638613235643935 +39646536353934636335383731633838626232363464373936373233646230613662373539306131 +63366233363539316134633965333035353031663064393435396334356338376264306533616164 +66393564613231643734363132613964663433353831376536353237333135626164663466656338 +38373835343232646230303430326533633733643231316130326634663239343366396138376534 +30643138353666366166343234323834343631623037313630666262636162316161376235353461 +65383935383230303265656563383035363837656134613165626434623566336531303632386635 +65383139633732623739373436326461393764376461343663613966323363623434663434393664 +38303230396336336636313439316431313835616534656363316230383634616132646465353630 +63663632363039633039636462303537646633616465646132336639353938326539626436343336 +39363235323134633235636339616130333061303464663436613262613032363138336261633938 +64393130313863373166316538353962323035383833353761333966663661306237346135353564 +62313136656263343763306564303533636464613564343934303036663236393463356533626332 +38663664356239656161363464303139376131353162653030396162306335353632333732303636 +66343732316665663964303938303036333063666331383337616264306561636563373961313739 +39356662313330333134646662373334393734363737323964316639353539323933323265663837 +39633234313230383465653235643134636563343032336539613765303239636534383363666166 +66623430346634663665356365346638376661626465353530373735333762656366643733383665 +37646139653963633732306333663363623066323135363135303265376263316163396635313362 +36656133373761643765663139613332376563343666646531303536363865643334336538636333 +63373730343662386235313533653462643164303237623839343037373332363865323165366638 +33336434663639373562613564393838393437363762613430356162653139646433303262633163 +62623965356163643739623466383737306638643533353262656361373339316666396262636462 +36663632613737623636343938633138653766616134636465366263383133343434366639373262 +65373436653230326332336561373962386266323335633864363161376238333062326232613162 +39393131303865623530666637333131386164656131383039663561613931323862376565663936 +64636665373732646339393362666137316139336332316134623061383039613331626630626331 +38623830383331323562393063333561323766633566643064323565356230343439396333316333 +62303238363762323836336533643862376661356235613138633162363364656461653263363031 +37643162343031323662373564373535623234356430393164353638653735323038613130393636 +36346233653639363535303538333732303637643363633037653735306331373639343862323831 +66303833356239666132326132323232626635353431643563383562633463633830643761633366 +33663861333433623064346630363338366438636162663230383561323030666333316230626232 +39656538613034323936306630653266653066333733326137383734666433363730393866343066 +61303236356263323766333665653533373465383630333133653934643861376438316436343766 +33633935353937396633333938326266353766626539326161343434616433356164306335306335 +64376334613866326639653835353061383062303332626230306430353437616635366133326665 +33616664303134346537656562336566666631663835333566376338643632393231646362303039 +62613039303361616339303565313539663630613635303133393632386134636535346665616262 +33373339303532616466613465336636326639643933363030613334623162623661616537336662 +39383862313936626264636630653038646339323635633936633961623863613464623864373638 +35663733326339323032636339666662633064643637336132363266616631366437323439373130 +65383336613162383730373462356433383234633539663332613865653730343430616637303864 +33313536636336636231643532643963313666353836363935373234306134326431393134383438 +32653435383038353532396336343563613666666530663862306265623138333838383539653662 +66343639393734393965326163343733663934386535373463386533316638353563653464663563 +37636630666665393161663930626539393963366339343335313932363961386162346632343432 +63323637396665643532636437626538316533343434326363346231623538393966646334333065 +33353132386532636239643436643766633764316530326561393230313533323064366137316336 +32626334636339373932393065393636306264656461313332613063646239643162663339373035 +32313637386538316665363166633831373332643766646332616434346534656135396237653065 +34356562313839363866643639366435376635336263336265373866313238346130653730313763 +66323865616535393861356666333332393065356431653935646535636431363731643031343562 +63373661396132366566636230623035333031636563373239666633363733386238336661643434 +63313762303537366365626262616537623133373031623036316661626363353639356163333564 +63653636643637303832303838306164343633326534613534623739353038373764326430306332 +35656335373764313464323464323764646439373832643838653464663263656131643937336130 +31666234303839366264333832643334653735366232396232643436356566353432313830666466 +65366638383137666163323737373839666430306465663364313061323561383437626230623433 +63343664363930373331333331656465656433333230333233346433643537626635616533333739 +65386362326533383866636564346337643466363637663133643030333562336230346363303436 +36633234313031303063333438353933663138366666353638653931623062363639313935393534 +62316637646464366665373636396236636466313735643465383666393563646636613665343966 +39623363616336386561353261303339393230323530303164313164363464616166356330323261 +36326566346661313966306661666333643437333639366339356534616166613533366561323265 +63623933623064383234613962353337653838613438653032303863343963303966643732306339 +66343539323238323565303032303534663131333435626664386462303033386535356433326661 +37373265383335383232353534613661393933356366346439613033313563376661653863353733 +33346431353164363834323639363036396464336134616266633739663363346133643365343262 +38366663373139323962633238363866613466306632396666613965383062613861656430653434 +66653931306465343136636136613863653232643136376666616132623837346135303464633833 +63386637353733376336646637336161666234353465663932646332323562343232643861326535 +37303065313534306662356338323935633836653832663233396564326430313662626462373033 +30303833303262323961633633663861616237336332376433326161326134636364623732303265 +62653764633064326335353330306332626365313532346635313033663937636634613637353965 +65326536376362653239383964393866303039343538623737656133343139366536343834646361 +62333330353832356265353034326563383731663038326333303833313761393364316362633639 +38636130303236316537656364326434663636303038326161336238643065323465313937376634 +61613062666339346331616639396564393638623234376565613766346135353061363835656364 +36306466353735646561363436643035316533363365316266323731623765613332613837323436 +30386361363537376165633236623832343536626132623266326265643963373338336235383630 +38306566346163646462383666373436373436323637363466396134626434326433323233323663 +33653135643563303164353066656530646664326433663631336536393962373164626333396636 +36346362346132653861633834613965343762333762336331653739643535383432636332313030 +34376464636532323261383735653630643534316539323662313138363064663866626537663163 +62386233643662663030623264343330623139333337303335393265616662333039663031393039 +37363334623336313663306437623938396264323031326632663663376532333065393431306233 +64663261353732303332316230626233643236353034666537343331646131386437613238643966 +63333132343634613561336364306636316362353134353163663733363539393761643731396164 +63623235343934366630313764353563636563663033303333323434303732666164393066666238 +37313361373733613537613164306539316663343032353135376537306137663634316532323331 +36323739363732616432353833326435333833373538613764343066346136376231383137353836 +32386461633937636131663761366339663238333866613832363937643033643838306332636330 +65303364366136613930353332393362346537626230383031636239313834663631643964306561 +64356333663535636437613637393336613064616236353437656135613562386666656636336430 +37313839393064616331363436633666313930633663303261653865636636613264316332363863 +64616538383332343831343964333833356137366566306435623561613062316132396562333037 +64343963366233316230636435323930653238636165663432323333326536333432356231303234 +35336438393638326239316163383538306265326266333761383037636330363963616435336539 +32643462316435623763363164363264333561613362643330623937396563393837656336393964 +65333065636363616364323832653961636661383233396131626439393939373832303234616137 +35636361353464303736346335613334623936386130383634316538663561303833333966656139 +36373037366633373130353838613764356436336239613835643866633533383164333136303266 +35373830643933303466633431393936616330323164383964613864626134653235653636643164 +64306239373234316431383139386463666132316637633364356230363735343836393532313061 +36393065323236363664653639643436353231643330396333366236663436316361623632626139 +65376538396436343965313533333466663237626639383539323131633737356237313631366533 +65613137356561383734376137346636636339333631316565356233656163356234376166336137 +61656338386536613161373031653732373639613834613638616634336432646432366439356338 +30313532356563356264326430646463663135366637623335626137386230653163643036343435 +62626463333461383664643134313763643664333930336133623531313733373465306132343530 +39616563373737633935363962386233616634366535306261613439636561653036323131333132 +38343633633833393337393832376431353430303930373063333535386139623837383265313264 +38353131653366363964663962623966353933356664323834333164366233653032636634626364 +30666265626130343765333632343939326439393237646330666464373232373066633139353737 +65616363663662346537393632336334653437356232323133386432306536343866653063666461 +32373563313539333763663765336332346437386334646261353261353962386434643531346130 +32353463373436313966653235393831623933343135313031316136626366656263326664386530 +36396530633661346332356364356130636638383363336665363934363430396536653134626233 +66326562656637313261663134613131363363353231306661646666666130623935643536326566 +34656438353136386337623133373030626362316464323765663531626462303061653766636466 +66366238653161396237633261643265643234616633363363303834353633346365383432346564 +30346230373636333664386162326535323633313865663066643065623961356334663632306662 +37663130656163386664373961633630613861343136373939633066343463343663646335656436 +64333539613538336166333862373739656161633063323335393938336633386566366163303938 +65663931306634383237633265376437626636393863383331636361653435653837316332303134 +39613237306234633663366538343135656431623530656661313961616636316330333165623331 +62613538346132303134363762386331356465373466313838373862613931343135643334643063 +38653938376262623364356637386433343530363633303436323263643065393066653631373963 +31643863633432636131373532326138336335346235303139316536666663386635643261376465 +65646234663266323266656634376563333136383663616535356662353961316439326364326564 +39646136356238363364346633373164613437383466663537653836653461646238626632396331 +37346364396262396334363639303963356531396565613438613830306162366439326433313238 +35303337356130656138336530653965353964393337633432333465306530386566336432656166 +38626333616461616337383065643634313635656633386161653861316538386661363937383163 +66323463656561623535616462626239626165316636613639623966633966626462313230353166 +31376330343631386538666561373866653831623939303338356337623430383935346335323836 +36366136633638373461313963313761356230333765396438633634366431623531303536353131 +31366639306634343261333434303264643439363961333566323462333435326237643165336232 +36373935636633343336666362393832326563356666623965623531373233326366356536396237 +65373734623539383939393330653230326235656238346536616335663333396438326231363065 +33643339616137613537643266376533326435303237646564383463623233656432613161313333 +33386639383733353334316662336339323737643364616436323732313134373733666466356137 +37316463656230366637323062356165623732626435376634306361663739303632623633346137 +37323634353338343837303064366364323965663534616466316439353336623238373639616337 +31613364316330363363333761306463303361396136383735626166326161346161323963623062 +33316563623637626364383661636164366631613839386466383032616130623661613764616435 +32356336653565623463363166613237393530636330323236656333323438646439653636353838 +36666561343834353037643731636135353132363430643564373563346531393936373233303336 +37666535323135316133616334363366336162363366383266303737353266613731386663353666 +64386264633563663735653330623766326261336538393034333365396334363135633937616131 +31663565356465303065393636363039393164663535323234616231353531323862633039313865 +39396662353866326335633138306261363263336231643332396130386430653962643134313436 +66366632303561326635636462316263643034333761616561613132626438656366646538633664 +63643038653261643865303531643662306637333062303965363961323761323536326537666265 +32616235656531356463643738306530326664663932366134393362376466383661643430626630 +63643837393461363231633937643866636431636166633237343866653935653263653434616532 +61636631303566396330323537613862333164353234363031616462343566356465373335616461 +62366564623432613466616135633064613665633463353032656337376438303531343862373864 +35396139366362346637626661626563656561656431313737393730636338346137366163383139 +39313937396361366565306135333034633035663234616334333638323237666232333662633335 +62323536643666326633383637323862663566353735386266343335366634663266323439316634 +30326237313565343033393739616533306163363430373566303262393962653431316438633265 +38383433363634316636623932656563356162666364373233323735646366653065313638303636 +64633365373239633030636332323032376339626166653634636330393266613835373237386439 +63326434623863303430386361393263313963373736643538623765333935346432623461646263 +61303665653436383266646263626134363865613665643338386663366139396362343936383566 +62636333663939656636623939396530633161343266376462633234636538383566663663376439 +61396533653662613635646165383561393539306362313835633966613764376663663838323632 +66383436643666383062326138383466656262353531333062323732333365656333333866303436 +36343664336438616431303266373034663238313736316530643361623564363535613130656139 +35363164353832643064356462313761363331396165356134343863383438626333663133616339 +39306534656430353430303966303066623231613833346562376662316435383635616637393636 +65333762336164303133636363613735353262636261616663353837393164646138333264386530 +65653136626166653165656331663335616638353337356539663430386537343734356436323238 +30313966363637376533616232613464333662363839646433616430353034303032396539626663 +65376363303761326235313333303061383965633061656664346531613431363438636563396133 +35396431333033386230636163356261353266353239323663643531353562613466663065323834 +36653638626633666133633830633237333863333162383131626130393036343065646430333236 +61626333613635323233643533363264356438306538626163643264316161336566373439383863 +64376231373163323534623437306336333839386163633739626465373566656537636466396162 +34373032333630663538373964316235666264393061366231626566353438353963313763303030 +36613630316234393130306432316331383365633031326331633664613333613463313065376566 +30356261653263393537306432356335373963366665346265313731363631326533653238653733 +64366539626134313939323034316464626636373663666161386363623939626235356265373462 +63643032653362323939396539636365346335373966623262396264393035343166373737313630 +31656662323733653434633138306339306233653031343137366263363333333761376566663563 +66363734316562613538623932636432343265626663313935313365663236316531356433623438 +66313939323665396564623162313762343934623363373164646630646265616432356337313036 +66303464323563326332333361333035313732623933373838633363656338643139313861386265 +63353331646633316230373635333233326638383737333936316364366634356436393864656236 +36303962613066313738353337383763343732633261653535303632643562323539303632316339 +30666437353438366139366130663366356230626435643566633266613964623730393630343263 +3966 diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index aa632724e..3b0b755db 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -89,8 +89,10 @@ storage_mgmt_net_name: data prod_cidr: 192.168.10.0/24 #prod_gateway: 192.168.10.201 +prod_vip_address: 192.168.10.200 data_cidr: 192.168.100.0/24 +data_vip_address: 192.168.100.200 ipmi_cidr: 192.168.1.0/24 From 9976b00c8179007c0076e7b98f0d09dd2342b0d9 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 1 Jul 2024 09:45:33 +0000 Subject: [PATCH 020/103] uncommented c-2 --- etc/kayobe/inventory/hosts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 8439f7e87..33623eba3 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -10,7 +10,7 @@ srcseed [compute] c-1 -#c-2 +c-2 #c-3 #c-4 #c-5 From e73e019c3287f1ebb135409d48307b320b400e61 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 4 Jul 2024 06:58:00 +0000 Subject: [PATCH 021/103] uncommented c-3 to c-12 --- etc/kayobe/inventory/hosts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 33623eba3..a8c13c2dd 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -11,18 +11,18 @@ srcseed [compute] c-1 c-2 -#c-3 -#c-4 -#c-5 -#c-6 -#c-7 -#c-14 -# c-8 -# c-9 -# c-10 -# c-11 -# c-12 +c-3 +c-4 +c-5 +c-6 +c-7 +c-8 +c-9 +c-10 +c-11 +c-12 # c-13 +# c-14 [storage] #ds1 From 9c734e7d0c0e16262f0e952c4e6b5b6f5ed4d7f4 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 4 Jul 2024 09:21:55 +0000 Subject: [PATCH 022/103] Uncommented ds1, ds2, mds1, mds2 --- etc/kayobe/inventory/hosts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index a8c13c2dd..05f22324f 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -25,7 +25,7 @@ c-12 # c-14 [storage] -#ds1 -#ds2 -#mds1 -#mds2 +ds1 +ds2 +mds1 +mds2 From 4de8ef1f620b01049a80b7ac55e033ac6ee535ab Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 8 Jul 2024 08:16:09 +0000 Subject: [PATCH 023/103] Updated time.yml with JBO NTP servers --- etc/kayobe/time.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/time.yml b/etc/kayobe/time.yml index 8bfbd3391..844251c76 100644 --- a/etc/kayobe/time.yml +++ b/etc/kayobe/time.yml @@ -28,7 +28,13 @@ # - option: maxsources # val: 3 # -#chrony_ntp_servers: +chrony_ntp_servers: + - server: tycho.ast.man.ac.uk + type: server + - server: goedel.ast.man.ac.uk + type: server + - server: escher.ast.man.ac.uk + type: server # Synchronise hardware clock with system time. Default is true. #chrony_rtcsync_enabled: From e2e8e91e44c94253b484ef04a26c364eca4f80b0 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 8 Jul 2024 08:28:39 +0000 Subject: [PATCH 024/103] Revert "uncommented c-3 to c-12" This reverts commit e73e019c3287f1ebb135409d48307b320b400e61. --- etc/kayobe/inventory/hosts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 05f22324f..cb391f4f4 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -11,18 +11,18 @@ srcseed [compute] c-1 c-2 -c-3 -c-4 -c-5 -c-6 -c-7 -c-8 -c-9 -c-10 -c-11 -c-12 +#c-3 +#c-4 +#c-5 +#c-6 +#c-7 +#c-14 +# c-8 +# c-9 +# c-10 +# c-11 +# c-12 # c-13 -# c-14 [storage] ds1 From cf9d1e53cfd479007a85509c5648f765e1c7b1ba Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 8 Jul 2024 08:29:50 +0000 Subject: [PATCH 025/103] Revert "Uncommented ds1, ds2, mds1, mds2" This reverts commit 9c734e7d0c0e16262f0e952c4e6b5b6f5ed4d7f4. --- etc/kayobe/inventory/hosts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index cb391f4f4..33623eba3 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -25,7 +25,7 @@ c-2 # c-13 [storage] -ds1 -ds2 -mds1 -mds2 +#ds1 +#ds2 +#mds1 +#mds2 From 73ac97dd0d5803a3a306401e84873a154210ece7 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 8 Jul 2024 08:30:37 +0000 Subject: [PATCH 026/103] Revert "uncommented c-2" This reverts commit 9976b00c8179007c0076e7b98f0d09dd2342b0d9. --- etc/kayobe/inventory/hosts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 33623eba3..8439f7e87 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -10,7 +10,7 @@ srcseed [compute] c-1 -c-2 +#c-2 #c-3 #c-4 #c-5 From 46cf7479ef0f361bee5f764013f3f3f0be8bbcd3 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 15 Jul 2024 08:56:01 +0000 Subject: [PATCH 027/103] moving control service to C-1. --- .../group_vars/controllers/network-interfaces | 7 +- etc/kayobe/inventory/hosts | 5 +- etc/kayobe/kolla/passwords.yml | 3242 ++++++++--------- etc/kayobe/network-allocation.yml | 59 +- 4 files changed, 1658 insertions(+), 1655 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 0e483ff70..eb5f0014f 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -1,6 +1,6 @@ --- ############################################################################### -# Network interface definitions for the controller group. +# Network interface definitions for the compute group. # NOTE: The content of this section is very deployment-specific, since it # depends on the names and types of networks in the deployment. It should @@ -15,8 +15,9 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. -prod_interface: eno12399 -data_interface: ens2f0np0 +prod_interface: ens9f0np0 +data_interface: ens3f0np0 +prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 8439f7e87..90efa8205 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -6,11 +6,12 @@ localhost ansible_connection=local #srcseed [controllers] -srcseed +#srcseed +c-1 [compute] c-1 -#c-2 +c-2 #c-3 #c-4 #c-5 diff --git a/etc/kayobe/kolla/passwords.yml b/etc/kayobe/kolla/passwords.yml index 557f36f39..17b62042a 100644 --- a/etc/kayobe/kolla/passwords.yml +++ b/etc/kayobe/kolla/passwords.yml @@ -1,1622 +1,1622 @@ $ANSIBLE_VAULT;1.1;AES256 -62646166396335343566643535626635373665656463373638366164623736386665666666663663 -3265393730656330343265393636643765333530663538350a396463343866623562313834303562 -35653432373066343965303638373931396334626362313739613533326162306434626662336365 -3836643665383563660a376531383935303766613231303666346536663764393132303134656535 -31616164326433666562656166306661656564326265393538353666363737613836383735373938 -35353632363964613330363761316663633532313566643461383038383632663531383837616230 -39393563633036333131643836646561613537336230633866633465646336646337326233356666 -32646334313530326534396235306436653066356239636332313235656339373735383139393238 -35663361323238336465363038653063323465653939383363313130363535383765653930323636 -36343636636231303263656635303133363463643961353331633232393439303366646138366361 -37646532663632303732663731623734333837353962623863336362643637636432386335623834 -66663632363566333537386665326135643932306231323631636166636638396464363661626135 -31343136646563626438353038336132343166336334303134313238636663343163626165333962 -66633062306132386338383034386364336434636565313837626637356131396665343866643332 -39303330353134663739356434363535376163666561366666306236303337333433396531623732 -38616266336134343833333939396233396563643961633933656537626532396631633965396437 -31333935613539646363666239353766633261613537343962326232643832646630633130326461 -62646133666132386266393234613362383132316463376663383462376135363431396465613533 -31356265333063656133393765653832363761323030643733653338396663353434383132306461 -38313763383863383933646666626165393365626536633364313864363031633262393939663331 -64346239666563333533646634663334623930626338356537643934633630353435633734616432 -38633231343337633030383766353366636665643832646135356534376333373662336166376661 -61383032386130373961396437366365616361313931366131653064613133353931623864656533 -64343864313164633838316663663639623439333261366431616534396534313266383566386639 -32643262643437646434353531613639656262643432636561383531386361646264323938346635 -34343534343661366565613361386335323530393233653066646331646238353161626631623262 -31306264613430393830396531353035316164393764393863393935323335396134323138343131 -30636462663430393134383532633737653462643336663735383538663837356364363065396366 -32623866656534616265613463623933613831386137613032663135303036333961396236313538 -35333831396436303131663962643865306235336462653461316635636565313934633061633633 -39343962633132346364343265646435363333326237636235616564353833326333663936346337 -32313332376532666138383834383066643337383264303534323632656530333936373264633730 -66653865383262346239626562323137363731343863653931646431356437616538393635316361 -64653861643130313366396335663835323339363934613438376162313832326664373864333039 -39393930663465303637326438343631663731653433663431396535613136333135633735303935 -35623065376339366361393561323237363937633164376463313664363037633461626434626664 -62653230653062333837353534666535653862366531656231356136663163653766306266636238 -38636134656132313233663038306566393335316536353462616163366535336362353935346633 -65616639626437633536656364613031656632663666336439393062373331326331666438376639 -65343565363666616264383566306432636236306339646231326436336233306335343838626564 -36303262316664613839366434333063393331383262653134303530343732323565346333656238 -64313838343337343730343461383137326561643262373864623436333331636134613865313765 -36623933393764383465326636643237656664666339373039656531343238333737646239303866 -36613330656462313963373865333234643331663635363039633162643561303530376336646634 -63373536303863623639326237393664353362643164303437663738646466643463383031626637 -39643862326535636166353061363833373136343530363666376330616162373062653061616531 -66633430396536666131396536386337303934303461653334343762616132393039303834356234 -63326630626166306463393961613135363738373765343930646330313764613935326635643531 -65633038373630353864306232356332393064366434393761633331353862386539643663353531 -39656566356439393739326165353739386535303862336266633266623335376230326464653732 -31326537353866313233663762616165633730653365373034626463666338336138616466323939 -66346538626335643462643664353632366636613630353161663663643233393537616632393038 -36373431383962613734356435336134656162396465623966363465616563326363653231343034 -61623833343134656238303433626662396562386237303735663239333734383731663234656562 -32363337336634623965653962393032373166313531333336313566663235373763303131356231 -63656137346435353534616532396231356464356163363033356431313639636437303739343365 -37646430663333646436333165323165303566613538386632336431323137663864326263386536 -36336531363965376232373232346137623039386632633932623663626166623238633539333339 -66396539653534363536653439326338313535356165666131393964366165333130356634396565 -62646366326364646238636530653163373237646364626337396533353566663433363437353164 -30663631333362376665373530343634383431356162363132326631656464363230313934373337 -66353639393734623936613134626239303466353766323738383637316335396534626561663861 -34616664333738313130366535336638323631666231363835643161386663663038313237336534 -31396462306466353336626639343661386665653334386439396430623033626562336565636437 -38613964663966343838373238376664333730653638633566346131326430353665613033376133 -36633035643533363761373163326362343639613839323839306239303330623736653837633964 -39643864373262366535396330303166356333646466303030306632386632653638653138653062 -35383033373833663232636536623738623266663464393232666336643934663838616633336461 -36383038333564633532336161313731613333323564323633633766343039656231643138653030 -63366661383261343535636432363339366565376139383435333530613363623539646537313839 -31643636353766333436363636306533303930393937336631313136373431653637616536613364 -36373861356364323437326432363936316435316338346134396630366330666334643361643937 -32323661363962636332363539666438373336663862643534383663386233623165326266613036 -30333366383036343639333538666533373863396535616233323139666261646439353739623739 -36316133323132396465383838613834346230303763386532656563646638363136666331323365 -32613936363662623332326133633664323733666537386162323737333938663361653065653337 -32633362373939396663646666343636363039363830636234646232646534663030383333613663 -64666634346637383831313638313638323537373562316337633936323565663439353363633733 -37376437643465303331353461356333646334646635313764353734636539343434306333373233 -63613832383631373638616663336637386132383135663838353366666237653162333036326337 -37396566373138323266393662373230623136336564656636326233323335303962653034643631 -30633331336339646465353636663930363234336335386666343461643737353639643661386331 -36633262343831356166333232363738633731306465653463613866333632663937376362356137 -35386362373733613365396538636636386565633637326262306265333834376564663836613638 -62623964343363343734306437343433383839313038623731356264646234646537653938623963 -30313438396162643032313633356463376637356364303031346431313366343466336434643166 -63366334386139366637353439623934613932653337393561353132393633653966366134316434 -36616263633366393635643439323562333466373636356661343038376439643535613238656435 -38376533313464356531633336366564613436613434316161663937393737346631343437636433 -65666436653735393963353538646462643638353464396333666238343433393165366135303139 -31646164333539303035656361643966363830336439373437333833633530326461383463343939 -34316337646262353332303166663036653266383333373364663838323337333531393830363232 -31303438366366303033353630643161626134663934333639333262373461663963386562373036 -31636363643963393238346364636262396532353238346534333662646564303131356663333937 -61303039653466346265383338393764343664366265353338343466653538396234663238666531 -31353965333765323964373237646162363639336365333234323538613162326262336237323232 -36393538393939623261643566313337666237333233643161323139383032323264633335336634 -33313136353761383134353065616534643133336166653064343365643464666431636633666337 -33376130323438343864316564653337623835396136616466646536376330373331663039393435 -64366464653932393335303661393235626636643265313139376638643236623730346130643663 -61373735366238666333383865326232643232313233386234396361626565326431623666326537 -63393639333664393530643436356532393435643033616361373536313732393963623337383761 -35386335613036636365393134363764343932376534643563613261333661333536363333623461 -35636162373335306130303437383439396662626330383936363039323330643037336438363765 -38313232323935643338306237623937343539396538363463613939663639393838333332333766 -34653138306530626536323966666336643834333031363133376530353732663836666565643735 -63376435636466343837613061633030336365346339313532333533353764383939316335643861 -37306231393030363961613564613536313639376363363034333230653632346631313739383337 -61333363313931303937643535376266616630333062303738613237343632663831373863306432 -34373037376638663061626365623963373334306538383632623737633265656363666637343031 -33633239353236353437653434333266376164373338323863323832363839376265653162666161 -62303764393462613035663535303739396639303964616436336465373761373939616464333465 -62343634393837613032623263356265313561663432623139373261303537643463636338383732 -38326338356437376633616538633637313161363039623465363035343961633835326137633462 -37323837653731383031396262663139653363396439666662393436363631393765653362623663 -61633863643737383565346139333238386537356238353466333134376362623132343463643439 -63363939643834623534653530323664386231353932333136326433366232633663633338613763 -33393736303230323066393935383964393534353133666166353536653962313231633538373330 -32633534353264386134373837636533333430663135626562323933656331363739613833613861 -61663261656361353039646263326563376534333562633239373537393137363336383839366262 -38303462636632356662643364316163653562363935636337666334353963383463363538313835 -36393062613732353930313961636536633431623031646363373931646639653362323562643331 -34336364636266346334373663383763386430613337353537396165343364646332303237376163 -36643436356165303861643833353636646635393866623531343066313438613439396666363138 -35663565653537343736623633363865333733303231346233333065373464373163306534313765 -32303833356465393965636262633136636461346532393731663031353034643432333866326133 -66393837636166643234333036616239356432343330353933623765663635643036386166303361 -62376631383163306165343533356439663066353162313065326136663639646233653766623830 -36373730393461333732306533633035323564613530333863313338306465616238663235613766 -66613337346364326364626162383435663234313132336462383963373866346361323336613962 -38656261383335396134336335663265356132316237646230613131366232653033393466333862 -30343331316335373661336262636631303165323365303766383836346166323762356133323464 -38303334323363653431336430653439656436616337633963336238346236633134353639613636 -33643731343636613363623137653864303331653263306462623366663437626638376438393563 -37383137636163646638383333643734303534653731323465623564653638353631343634303163 -35303965333866383731353937663833303132343861323734306333326566303930306561386465 -32303535653833653134373462386231396434353433323031333031316335303362616432633865 -33383761633037616666366461333332313765356333613334396435303936303165656530373731 -35323638383339386139306164626634383066343834363430316334373336613830393631353162 -30366535333134663862306661363465333032336565353331363538396561353434393463376263 -32613765643966613730383364613537323834346661643834323338626661663635363337353635 -62616432366438303661383463633365643332323161323939333663316134653662303466313663 -37393164653561643438376463343339363339353932346538623237366533396464346662366533 -37353161373564643166616262313135366261363839303331343538653032326332336332333730 -64373530303332643432393836333365313534383437323461663239613463316565623563313434 -61623332623364663136306138336361646137356464623062626163353662356634336330336236 -62383462613863643433613964353737383665376336333532353830306262356263393432386639 -31393165303264393035393234646266626137326465396230313366343361396264623336633061 -64326237633337613462616430616535633233373161336361666531653466663830303363383765 -31323537383233653437666162376234663363393930626665313361623334326565616266643930 -30373165626566333834343534333539623639333965666236343235643663303637616131396265 -37366435393934666435613935313461376139653733306565323537393037363930623565333834 -32343231616466386366646261383863386237623731396238363838333762616666336564646366 -38386235393634376631663465396330616666623466316637336361613734363064313835643961 -33366433316663623364336562656161656661666134356564616438653839643636323130313436 -38396238333264353965393066316261396261316430366638356462353233343234366433363562 -34633039336439346331656262373534663664313435333938363834343666633531306139666463 -64666533653036316639356264303538623439653439613835333763316433386436623561356632 -36343938373037376633626565356564316235353134316261643031376134363030663233646437 -37623463353135346365386139333562653065343332663136383331313732333963306632663035 -34303830656336323162386163363962313861373437316332663761663764656236363935313436 -33346662643139376562313864383834356663396565363435613061393034303534336239663134 -65333837646263656433373730383961393631313561613965623735636134663538616164663036 -37323130666636356162616631646132363734616263356536633531646665643961663837623166 -35373239336562306264363339333936346134323434343730393365623538383462336537333136 -62343934383234646137356234393736643939623832366230663166316235333134303139633733 -34396264663937633632343133353337373765373838623535663531656331343566653931343463 -37636563303766653366323964306134356236623534333238323735386530333461343733363634 -38323665306230616562623566643334396635316136643135346666303265386233376463343661 -36396231396661326361336436343966353566336635333262363335626533656565316132643932 -37316431363464376539356162373031373863383939313461613765326661343234663463666366 -64666330373763326132633530353839303262313933623364393632336136636137336335303562 -32356430383961396135383465393935333336666131323233376563303237346630363535623530 -39653937613064313762306562613164623163643232376331393537646138333337626661663963 -38323131656335353765323638326131663133623539636638353033323134623736343864613730 -65616361663035633734353466616162623132643136303766336239666533313261643165636536 -38336161626139326230623432663865326532333734363435623332383139363235393937303731 -62363533353030353165666465373930383132363036626433623434663539653366343661666336 -34333730616664363535383738373632346534323833326366626432626132643333383335346536 -35353631306436633435633361643834363330353237343761336439383965653265656635396131 -35336233613465373136646362663561343764343264346362653231666366363637646133666238 -61323837303234323134613530386463666139303735306339333030333562376334333264626237 -63373234646234666662313061613634636463373137626261643531373863646633313733316231 -63393931653665386636656162326239363330333733626466613533383365303062663436343638 -65333632343735353730373536613939306461663765383632316431373562313430346139366234 -36313739393837626338643633323536366630323432646463646233393334616266316561386433 -62623736303933313166343965643232666461363237396131633866393332373861396537636336 -39316232336364373830616539343335656166326531613230326334643731326639333965393261 -30306231623562356536616334373732393632333833343730383837343133356631636337373836 -35393031303336316465323530646137363463366435623961366635316433313138383930386362 -36613662636362343532376534666132393839653839353064343361356264333662393039646661 -64326631343366613433326338353137626332356330363836343333633530396238353630373864 -36636666363566663662373336636638353563373565313034376437356434323136393332383364 -37326637333630383661616231343139393161313339316363616363316138366631326463346139 -33623435333230383464316262656537623264303737646531613464613337316566326166333838 -34383566393961346463363463613730356538343534636438666530633536313435656234663432 -61613761306262633933623231343735633135336432383666623330333037323836313362646535 -37646537366539616436666266376561393436343261633266666166393761623831626436343935 -37363138353831663066653164613162613064363663383732633836623738643930313663653131 -36366634633334393261623735323634386166643238376530626431646466326339353936326533 -34386330643335623338393164613962656636353635386439313432663732346461656332623236 -33653437346463663937616437626465623537313835323930626132376535623630636337376364 -33643330643866653337663731613964306664383161376665323835623334316433613566633432 -66313062356437323539656433386535313065313230666331303938653662393665333137333135 -32366462646334643666666434343962623661313337626538663262326138633537633562386238 -63333631363936336466663630643964316438623939636530303136393066356631326664393265 -65616562343139336234656336663436336162373734663736373231373732323664643634323265 -61663834386361656164356336353263336131373934633939393037366632336365326536343331 -66346137363762316239666539646564316237613735646461356661623632653136313064373331 -37616534346439323936386136323439653633343633376166623466633737386564363432396265 -65383332323035346564643866316264373036633635366361336630386637386437666162323537 -34323933643065393934613136353035666132643638303061313939346438386233663939373763 -33333035313361646165663339393966646634666262626638396338396236383961333563623732 -37383238633032626563383037643637303634643937303231633938306462646639613061653334 -32356136653163353531343366613062383837356432656637323734636534653939656437356563 -36343263343163396336646165663230643630313635353837666662383363373834656365643232 -38653938313763613762623734653963623234393838333264363032663038636264393065633261 -66313032383163643762623261663433313134616230306331363932336232383038633062363537 -61303934326230656436623730356632356663396231366464633262383434646331616632376438 -35316339386364353933653163313932376634333665326263326439373361343731306136383762 -64396334386238653964646138396234313361636231356462366537336237363064613464316562 -64643439313339393039393165363436616238303730633530313932643934316161613531623562 -37303061656336396439653366623136633964313932303139616130393033363633343863666231 -32346564633131643364363865366532666465353837376533633638633666353364346137353736 -61393938303437626234643865316637326664336633633731646237316362333865366235373561 -63636332326435356533653236633138336539393434366566613538373133393933306337313437 -34613034376434663932383930396666323866326637353166396236396531623661666632313232 -62343066353366636231623338663230633966646437343830643833636532306339333635336433 -34303166636337346238346263383034326361316564326237666636316132303564653336333664 -35343464326566386464353032326535666265643939313236653136646330393932366161383635 -39303863663132313633336337623661303532323534336565343436316465656231313531366630 -35616362633166383835363665646536316337393766343364373138376664613635393032633737 -36353130383033653936633862393061373830636262366434393837653836653232643862313665 -38333062626335643737313934373231323962626265663633303432376632363664613564363066 -64663532326530333463656163663232313135653538303031373530343965363136393438653337 -61613461626464313336353531373963633630663638346131323639653830386635613665333535 -61666533353462326264303538376335363032616661636435643237376662326330623035353834 -38313966653366336163663663336364366565343036656437303837373764613663376131646361 -61316663306437346665313036343164383130623331396264346236643135343936313738623837 -64656432613538653163336463613135366464353664653133303063363831636334336637656264 -64333932303135623935386461356662393462666235383466373635333865613135656333616632 -37353135643139303136316538356164616337393263363439623762336162313166346266393233 -62643535643866613264316632316264623732666662363439363839376661353232316235323666 -33323732343861333837316261666639613732396639383463386666333834353862346561323536 -36643030653236383061336532333835656636386133346436356331373832376666373235333639 -66616232313233386638366664313734616137326639663061323762633231336538303836643561 -65633430336232396432323566346463333063616630626236633531633032316666383737323433 -61646366343363623366646665636361363162656634313430313538616338626536613431393038 -32616434353466643736636465303865626235633966653739646139653433646533303066346438 -36663335383466306334663832633230323337373631386134366664666438386566366561616165 -33646636353236336663623261613863383835326432303261393563313739613232353539636332 -32613236316263643836323534653132656132646639663634653033313365643063373537306466 -64346165373333343334316230356136346130376534393337663738363838393437333461633930 -33306139353561623661373833343737313237393462343337383639393738636535643563626339 -30666335306332646564643464656364616664316464653564643834323066643736346631373939 -64333963313864613334643538346238396634383333303531636161306131363735373563323663 -38376630363234646235313263396164316330623536323861323363353336316462376664626465 -66303864353035663566663361383132663034623133383639333961376263643663366635363239 -34666333666333303365383830393166373163356530376461353131363033363039383337313737 -37366236636532623837643065373233313237393966313764386161663235653337383331373262 -37656330616262313062303939656465323165616235623132346138366635333135663761323630 -32303763376166643037616533636266393064373561653335363831393835386464643033613037 -36333832303132373631323231393761306434336230303437363332333862373339643964386330 -39613062383765366531613062363836303538303565393936393465343335643862343233633436 -61613330363438633632353265613833356430623064323363373239663737303733633034656332 -66346432326534356335633236393066373335303763306265356330366238633237646532633066 -30393738643262396563656565303334383561356631346333366632633165316365633336623630 -66343862353561383631643864623862386139623364353034333735623066616437636536643365 -33663362336563373630636130386438613836343136656664393462383730373363666264653535 -34393733623135313037353830633561646139653363333839343164326632303465346165393163 -63663539323465316164373034663135313139306538633866346238666436393065393030616237 -39383132366132346533333139393732616439373136363966393139396431313334336266343861 -62343336653339323963393838383838383337366636303935383435663438616366643632643037 -34623563643930353031653930393439303533623932346333353364353935643435336535363363 -61616464313836303732633334643536366133373930306235323935623136303033316434336139 -35643639353364626632623633626263333431316562666336343131626134343962636236396336 -39306130303233313133613264323836636363613132626536653938386162643266373962313539 -33323463333239643563323539376338373836366536663639666233636438323439313766316432 -39356438383136336562386365336265306336633833303465336666663564373538373631363437 -64393336633535636438313833396537333365306131346138633964653066323061363861336338 -33386433393464393036346266376631653863646135303063333135663933666436336133373737 -37613663633134303439623338653333613366366538343430643461303034636637376565396366 -35313830663464313235356635363261653233363561323033346363393163656139323630383232 -36386236373538613133316366326164396538303734636462653338376362343666656363613836 -61373934343461316531613834313530623564376561313236323765363735633033623162323436 -32613765646337323738613561333231313862383439373262653230663038636666623962383533 -62623431353265346434613235343639613936373433633837336435343863326634636433356639 -37616635303761306264633032663538333738616138303266613133313565663435343936356132 -38383936623761356338323430336439323363646235626137363666356334613665383134353134 -35303435346462656436616436313331343763643362333732396331663536656265643665366166 -39643165333062336637386363363065636261373739643962373134343965336665316666333661 -34356439336162653366303035613235303530623564353266373439663338386361373539393232 -33363637663363306433663634653832346562323031623732633864636334393933363439373565 -34306538353164633763366264333563316461383237636161333836623066363238373466303838 -66343232666565656537653034363936623265626566613264363466323630663237613737313132 -65613039653938323363643438386238656364636136393066653830346133613865353366353462 -65346230663161656431313531333131613830316335306263396233313733316333376330386436 -30356138313133333734656633376264383437613439346636656435623235356130313532336163 -62663433313133366539306438646531613962353330633436643732356666363239653632656130 -39666431383033303530386236386531613032636566626661306263366361383563643462303231 -33356434393263393434663064643561393163623666306363396337343138373462383536663535 -32613461336537616361383832366163663233323737376232653233306432376563623632343137 -35306163616165383764386533626261306463373432306533393737366131613736376666613037 -35633231393137316339346437366262613239633039646139653435383961633366623636383539 -64613837343133373231323862353435336663366538646539366166653033636234303138313765 -32343063613532663631613062636634363538623437363265333631356132633833343433646365 -38653561366132306639353666663934396564626265313930646163336630653631653335363137 -37643863373962323638643635346438383539323762346563383439396437633630613938666262 -39393830613564646238343261343265323730656430323666623161306261363463356330646335 -31346432663266383163326234313565343434636363636366646136396437303933633533353762 -37653263653465383438653763363732366665643138396635623866663766353636613363346135 -63656432323138633762373330373739653739373836353531393930633433366535653537333539 -34663037396561393733376262653366626561366164633463633464336534336139666435616131 -62656264383731326632613239656661623633336136393465326338313265323039643765336232 -65653131633665653435633164306561316130643437396538356533636463363263663163356162 -30663531616534636637303738633562373064663262383239623666656266346537616330643635 -30666231376134343066656336323238373939303766383438393065613039323930353133366637 -65386531323166333961613139313431316535653964373339653834336537633366323565653939 -36316335646339623436636430373565346666356666666133626632623164336466363031396634 -39393363353664653166373962646132323839343061373031373336356131623133653837383139 -66616263663236653432663837636137303963643637313436323466313735386236656430626535 -34636634303639656661343262613931636637336435356539346539313165323933643233306132 -34636361333035623238366635613965633134393031323963656637303231663738616133613761 -64303063613062393439323638363138383562636131393239633866323266303535633232353834 -39633064623064336234313866616431326131653538313531326565373737643261346335393831 -66623261643531623330336165373865316232656533356462393831636334646535653737373238 -65323737663864356437656161303661623431326263656137366335393566613938316165383932 -66393233313962326263633463383833363438373762316439663130363064346563363936363131 -35636437386666333062663731623662633635303833386535336561393833613465313738393134 -61323536333364646437633730343136373830656532613037666434623265373061346634323632 -30613361306538353766373139333263626565333231353365646134326234613733363165363636 -36306132613962333362663932643865386534346139386265666262383838346434623765353765 -61336333643832333434393165653935623931613862326330336339613964303333333732393933 -65363562316266383663393437623363663038386430366537353432643733643932353739616530 -38343735646666613637666365663537393731363162316163636330646331363264333235313432 -36316335366439373336396163363534376465393539326466373933356231383264386638356130 -38636461646663663861393939353336386663316163633738623933323863393534313232623033 -38303535326438656334646338623966306536616664353231363330653265366165303536363839 -38343961653661373834376134656238333564656238653964366663306364643438323336663338 -32666661653161356138346138623866353363373430616531623537643232376338356334373862 -62356139663732643336326262326137346564333734366536623439383364623534313865353333 -37613163356139393334333066633037636439633061623538386531613863656563356566366634 -65653261303437626262643531353435346131623935363434356331383063663239353962303262 -38636563633662333338633466303033363130393333303730323231646134303035653832383330 -33316631656561333235653931643136336530366535653466333766353761393265643438633332 -36353536336363303864663632333538363634623066383938316636626133626238616661366631 -63323063373534363832303338343631366239313536303565386564396164366331363533343064 -33393032333562656266656530373663613036393833303833343761613064353331643765333665 -61303934316535303162613563343165366239666664656132366238306162633564623134316235 -35396235336332323234653930323762366664343135623661623836646535343834323636393634 -62663562313533393062663166643338646461336432363333373864346533343337383361346636 -62366334383065303863323062613735333130643737306137656636343633363639636638343630 -38303438616439393136303465306161366432643036653132383661343236393431643863353436 -66646564616430383038373031633336396238623962353435633331303463383766393531326165 -30343936336162633130633162373063303134636363316438383565373733323461613932643163 -61363130623033383731373834383837373861376633386565323161303463663932303134366339 -33396335636530653965346361613232306430376335306533663164393664333633393965373335 -66663732613562386438326335666637323133363362346336343935323264346432323032373038 -34653132323363346664393265316663396135366638663735646364323032343235643039353935 -35653831393166363233633161663163343839386434383638343039613437643037356666626138 -36626231393333313763386430326234393131383438613061643538393732623936323735613536 -66313335653861303936366337353537623263306436386663383765323435336434373665346563 -35633663333031303939303361656563373930666635653433353634303835633666656638376335 -37303131646239613561373530383662633434353434343865666335633533383662336261663461 -64656464366262366637366161316438653238643432626661646231636238343765343030383566 -63323336626633363965653331333834376533633636653331313737616539356661353638656333 -35653765323464326339663563333036393162353436636238616332383539373035623962616662 -64303166643332323934326562653662356362643664313739336264653562663733386666313038 -35626364643163613963656630363433393439666366316239626566336136353861396130386362 -36633831663336666333623539653531366338373562646561373863323730373966393166393362 -34636337643939363731653764333730346233393437633336366665626264393936666431663034 -31623536323161393132613461313031396561376633333164363065393066353034333634663338 -31396134633939643835346434646530356537613064383565383765633461373238306361633833 -63613732353961366139376466343961316638343439613532343665306165626437663138333233 -31303865343838336238363233636337396338343238336365646466333966616336666138366432 -35663630306530623233313865376665626137356566353630623262366565323639313862326631 -32306434623464633833363137616332623435303634376363613935623734323831316164346531 -34623764336237313230613330393435333636336237656338346164313164653161623336356534 -37333662356366323531303030663432363061613461376333616366653263333265643932393531 -62333238303833393131363863313039356235363332656631636361306230373265363339343236 -64653064383535393663633864663265626663653835366439636263353731343965393636323762 -38306436663663646666656439353433336533306165383830383864333832663561316563626366 -62636531323164323863616632663732313730316566356331666335373965353132313364343831 -34366261396537303866363961643330636461383963363164313834386336333531376161383661 -63636363323231376165623466663738616335643966396233636264373234653032663933643066 -31333437656434303366356261356432326534336563333831633736316435326362613230396237 -32303234326233326461303332323635616332333737633464396438323633336532623530323034 -32636639386434356262366564396138636635653862663934366339366635623536643533326561 -33356365636461363233376466316431313231643039643334343663613661626336666261353931 -62313230386165373439353431616530373436643664393535663433326565613066643863656232 -37653861623035646539636439363162383834626637373530376563313233636662633664626531 -31363164646163343266393362626464643734353834626331376363313261346136383062643834 -37326631386564383036316535343132616632336335633461633863373836633161616233613634 -39353561376430666461333166623635333061636261343938353231343339316430386434663264 -32613262303961666263326363386136313134386337333434653830333637613539616663623736 -37653838316436646464306263616564363665313337383665393163336261663163626135373936 -62646234303466316234333330666233393465376164623033323437363431663139333661656235 -62306466383666343833366530333866636362616132363964383232323332356536373335386533 -38656130616632386338613965363030356138303234376433303661353466663335323464623363 -65373031343531346632623338303634666432343435323532386633366461333963323839666136 -39663333643139306461656563356232393033643165346635323336323331393763636530393738 -30366666326465313037393064366663333739663332356234633962643062353164643237383266 -38663561326638633265326630303939393036316237643164336337643364643036343238663838 -63383736663061363735323932623833346462313364616332373130353465306437653037303137 -31373135663261656630313564346634633766643833623438623234346561343834313563323938 -39373131653236396235323866636666366438386261613164633035333630656462323161643637 -38356263623131666331643634346463316165323331383963326632613261336139396236646332 -36613265303864303661323736313062356165616634643733336463636462643437623334313365 -32666134653362666436633138396664396564386633653932363366636564636338323565656161 -34303030376135363262656562383465333036306335393033383561333037363436643434666633 -64346564613862393637343433643933343437303430633236393237383633336462333838656361 -38333661323639316433383032336331373866666465613165373566666332376637653964306232 -33383263663931303236326139633931336538383232653664353861666535613339306235346266 -31363439383263666432663235643236633834373765306635633935363966333462376563303138 -63353035643664656131636339353066616433643031393865383130663363316436326334316364 -61373238646562633766666264323562383238646537613166616639646433326532323237383532 -36303330333131363433383865346237306633313632393365616564663636313138383165663264 -65363761633065373537356635646435363236623932613563656264343731616465323431346633 -38616536636661396138653565663335376131323630353063633739356462323438326462313639 -34356365613535333033306364643265613131626461353064613266343233653865383636323237 -32393763333636646465343137356166336136303363353031323361333539646266306261636562 -32386334326638646433326239613735393638613037663262343863393262343963363834303339 -37363130306338626639393464623831653332623934343363356130313866633738326432363061 -35393237333263643562326365363331623036386132393237623365366164356230666631343663 -33303236343038663439313335623364383763623666623234393136626238316131646535356465 -62653333393339383766613564373636646464316438313038386434376137336533303937313337 -34646363376634643731636534396432626236633734626131633533336134633536356264346631 -32653363616330353832666239626665316465643238373031373035333965663437343565326536 -62333837393238363964393935313461336165303765623338303532313363303632363632653931 -31333634363765623662373862316233376437663166636331343963303262616665386365633131 -65666637636336313634303061343732343238616236383935633338666137353664323064376634 -33666561313932333864653261653863353130626337363736383662383564343539656265323064 -34356535656166356266393064383762303165663365393231306564396661626236633736653664 -32306666653631666632363136353862353132386634336361346361396337626333363733313237 -64343130633635393534393365383833313863633831326539363433643431313432663635366230 -33613364326434303966666565623037383032626535663631313639646539323765613938386465 -36656339316666633230303539363235613335383532656236323261656464356639396162316161 -65616135316137323237313932303263343531656563643038643830616135373864303038343663 -37646438363834316335623334346433663536313961336633356137336130373738376439373237 -62663734663439613434316532393135653066303130666432316232326330656465303863313165 -36316133633731636135313133383463626132633561643932613238663239636532383761663330 -30626665393065306432396531626430336438356630353163393366366133653533626234646337 -37303732346130343832623630366138316665666539666336363934393537363636393638336233 -39656163346333333335376364343662613263613532633038346466626637366438353938363933 -38326537666364623765373537373265623630616561316230653061313438396561336434373135 -32393566643764643836393662323063353938373634653233333963383139383132356135346264 -38383235623437626364376336306361326133386230616531313266346530626535306237346338 -32336334353561333530643663623164636363353365306264633039653663356130646265636137 -30623437666630393962343664356639326466346636616337623432653937333466346137303632 -64663562633131313634663737393535356134636264636436666664326638313137303733336365 -66383535633533373263613362383061396135356163623761393436653463363763666236313864 -33653334636231326137363031323762343933366161623236383962633932343564613163303638 -30623237643464363930386130616135636665656662303039356165356533646462613764393632 -34333633663662346163653938396333356537643836663036643439633732306639343336353939 -33346334363236383065643431646565613364353835306439343639306262646361373866333035 -64313466646635306261343532646236363637666232316237366237623132663635636165613133 -31343063396361316434386135666137326639313561623337393462346663396238656535393638 -35336331316131313937323061653464343031643865323338383165313062363234636233656536 -64393862613466396136313237393037313165643666356531373436363831373862353937623362 -33663634313839323466663761383365303136663661613331326135623266353461636635306164 -35383337373739633764373565333363643431623336633431313363636633376261616564383036 -34323665333466346138613138343063363433393133623664623034666333343466386234656165 -65373133666335376434646462643662653735316137306465343032363632326266323366373432 -61343665356435376637653730393466633833633431333363326431656534356139333965363539 -64336633653864353162346434353335653865633433643331383530316266343663366164643665 -33616661646564326161633830356535353530393338613331383565613630343463643564653437 -64656661353334663138383137633862643133303535323232386661396561303762653331633466 -36633937636636613461343937326132363264323938353736393463313164656133396365353035 -36366336383232656237353330396535313537373634643938613165653162313233376566383336 -36343866633138393163663837323132343634363564663734303461643562313461363137623934 -64323034333165313838666131373861373131326661363964343233366461366465333738323336 -31633234396637343865373139333537623336316465393931313830353237383338356665393337 -31316166376434643865633839333363646136313230313763316363343634386362656266363831 -39343362636135626565633432383332643834346336386266356535643739336130636665623635 -63636563306536316439666361623439363932363432383432336264313236656464326234326239 -38396631663136363462313533626163393334336537313133646666633633316161313962613138 -64336336656631383134383237333735313864373665666132373339353865396332643335626134 -37616439366132363933363333333838393030613463653739356165323530316339383439366161 -38356435626465653362636665306463346232343632366461326165303434383830656365373266 -38613437646232386336373936383533336232393639303437626430326366356135666231326334 -39366236356239353236646364393463336261613661356365393430386232653536623639383166 -39663866356166343830666132393764643462353635396235336430333661653964396637623265 -38346236306139323432383562646537333034303232313866376466623432323630623135336135 -36393938373535366530373335653464633464353033356330653034393065626237333239343032 -32663636653735346330353563383830623835373763396665363661353134623162323563613366 -62633630636464323564303633643262353532626161666636346334353434323430366163393432 -36623838663635303234663137633566316261333432316264353539613838663563303164623035 -64626366643364666332646436376331633839383635376131363461393861323532646638343363 -66653935333965386439626132633739313336353665326130653665386231313837386265613837 -63633465656266333431323566656431616362396337343063616135633965343537613334373432 -35623532633461383634316236323436376264636162323664373434613064363963303762616130 -63333062383936653632363031393533346334613965393236626437313333626661623633313664 -64366265633262653931626161383931343632613463613635396538303661626263666662303037 -38393638633466636239646539366563613961363939346231333030653138346263313630386633 -32333739613561356131636166623365663934616361373331333962396333363764313434643531 -64393166376532643533333633633034313131343537653333393165616565316664646230336532 -65343135346538306631386566346162306639323965383931333364623037346434613532353163 -63653231366135323232396339613661643337373838623438343066326464643865626565643031 -65363235656638363566636336356265633561643934346130653066343230343261353331376431 -35393538363065306635306637636165343932373638386235343962326663393730356432396335 -35643134323133353065393531306633643162613731353265626636313862643765646664306634 -38633330343431633832363765346239353639643336643237663061333765356665646436663135 -35356366653430633335306361356266373131376466373366626436653038666666323436316661 -38346432353662316166623761343164663466666462363434316461306665353466313537316166 -63613562343963396463653639643865356630656333616538666230366235363963643633363032 -66646665373365323964653366376530316262396536636635343535313034666262343031663734 -30313831626635646461396264313966636661393963386338633939313736633964656239333033 -65323863353639313935666538636563363037646562623162343731613034386561396664343835 -36386164356465363061376131646636373032303538643865356433363562383863376331336135 -62393131663631303331663131316637336332343362376333643638333536643433633731313062 -30353331663336336436613732663362303531323139653231323361323663373762336133333638 -62616165306631373636336661326136356564343531383565323963393135356139306337383461 -30313938353037333737383064653764336138396235353738323031333435636630313436653536 -36373462623134346136363031643832356264346135323438333330353033376231326465363836 -38376661646265336632333566353439393335343337323965353933303636303166663231633066 -61353631663233373064383432663539323734343461396431336666386565333431653461376637 -39613030626363613461373761383439343565653631636234356531653865346263666262656661 -63636664333830663362363530356233323838343832303335643939363964323562353231353938 -37663138346239633566656139663262383939306663653830663466376432333965656364323666 -34656162666138383965613061636333336565636338633535323436656230326531666466383864 -62393662623439363561663434313663623139356464663161393162663765306137653863616639 -65376330303261393334303731623435303266303461343436643432633066613262306238663039 -61636236646638323939356461383338656666333938376230396134656635663163646265333862 -65623033666534343163393866336465343435663362646135313963313731353939653463396638 -66353039363230343466653364653931393433386462656236616364303936646437393232666262 -32613030333036633531363938336337396436613735383265343131343530373932626661353933 -38306632373339306230316363626232313538663265303766316430613737386561396339663261 -36323863366136363366333830333736613962343761393163623765663031623162393836396333 -39336635613939653762353330333330353661333662386265356338656433646261383063333234 -37366338336435633762653133666331366566323532333765323562393764303132373836313263 -36343435383438643536313161343431666163316265343835346466376534396433393737663866 -37393234383166393965326233376166303735396664356232316265633836653931333261343666 -36306565663664383534653837353165383934323035303261613262666334376336663462316434 -39623032396535643333356333633162613137303838383630666234636661386630343430306534 -64643566366666343230353436373939396166363035356330643333326163383165623664326130 -62303237323063383063393865306138393630363030643464633932633531346431343966643964 -65663838343862363635333161643236326332353465323239303730366663656265613261663061 -66616632326664343533383537363461303461346233633234373866326333326533653963363438 -61616364633630666563323436353766366635363532656537306463613764356436303666373134 -37663433656539356464623638623565653731303962353933613762303136623836653865626336 -38666430393137383461376233313562646264623635326137363930336539396134626332353030 -66336239623063623239316432353333626265633631643530653638333165346661636364663762 -39376563306436646566653835626662616639303636643334336631343135626139373363613866 -64666239326332626631623436326438396539316165383033383962363561303939326538386234 -65326531373533373133343566633638623231356630623633316132643631373839663135326265 -33346663613765323464323963373038643437663433333763623165363165623937316464623831 -32663739656434653961313462623866373439363261343465633164313932626332346366656336 -32663833316564616266656165396330323666393834393734323432336166336234363965623836 -63613138626333373561393038656161313361633531663035313934636237613036643737306232 -38333035303461393636323937663334626331623239383363373432653065333239353865326239 -61663430393338633464653739633133336334323063363263373533646564373933653932346235 -63323737336366316666366464623431656238303838373062383366356166376139396137356235 -37396530373338343464666136393665323934323566383865356637353030663432623365353165 -34643434313732636137366563373865616334643139373735396161333731343234366565643464 -36343337666535333733623032613032353837643831316462356665326331343561633164616566 -61346433626362616434366266663434623264353834383333313235633039646164376631383762 -30363238333166623631363061373634653165326163336165353037656333316231333361626431 -61633964633431383261623664316636396338633639316365633565383839356430356262646636 -30633235353837623064376136666339366465393066336137333435376138653737393333393465 -61653664653234636537623064373239313064323461633336363836326233353933623463386235 -39313763623737383934303236306464613662396234636537323661326264383666376533326432 -30343630373666306164333035326233646163363065366233383466646630366138393336373033 -65303836363566363330366432316139343865653032323831336261343731393834653462653664 -31373863363264633031373939373235333661376331383362656262396161333133313338396564 -38323532646234393236643431343430373837393538623331636365626332616465346434613034 -36343765353730373735343964383463366232623735633232616538376433376631333962323136 -34623032363865333664346630313337333432653831393763653661363166313661626465346364 -31363035396463393464623634303864323331306631373537656263336138303065616265626630 -37313465313831313766616330303766613063633162333662316665666335343536313038663738 -36333039666636616130643631313235653033336635363238333265353638363636323664333930 -31633662646635313364313539613134336437663065363365306431613132656339636564353039 -61323365666561663364393435393163343163653964623335656338333631393635306264396435 -37306431336135623063366662326335653934383430303532303537646162663263373530356238 -37306530343161633462313534303334616364363934386531363133383338623461376437363535 -39303839363464666535393636303965663562346330366335363734333732636361316236343731 -66393338613831396538346435326535633530303930613964636232626539633631303965366166 -36356162613134326262383136336265343931396631373230336335393132663066623562343234 -36663931373933383066643732653461353566653362386261383031336136313838366563646566 -38613332393063643538383430633061303634623633376564343061323264313662643331396664 -64396438323637613039376463313236313938366162666266353735653533626337626437363666 -30386135663733396634346363303762363936643632386363623363326330663837656565353664 -63653633646333626166643636656238363130393761616262643462363065383734363034323763 -65393264313632646639393739653161646633303432343033316532396263336631323031363832 -31336534633533626538646365613535346435613636376165656234353761643661363138363133 -38343036653463393064666661623534393962323665636563636238323362343033343035313130 -61656266396437373766373461323036343761653239626537636665666531346536363838343131 -38613265316530383338373965633561353764346362666530363230663430383033393036666564 -33616632336434656538356334393437383166656537333962646262636161306265383662313962 -35623431356463313661623337346335626234633833303035626363623131663834663036383836 -34646161623532353637303638373030623263643438633961343866376137346331346364323032 -32386331333965336536323931373761306238663233343266656266316562313131663936333265 -30616339633739366232613636393739663661343838373862656463663665323439396464343237 -34643735663030643864373631623665343039663064396631653232633033363461306265336134 -35616365636336396236636131373064613232366337373862663839316461663266343433383466 -65303361366333336135316331323738633861653935366434396166643234373738626465366164 -61316465656135616634383466396566313932666564363330653238663634333566636131636439 -62643337376564303138656634353132653064313439346136663061353061616438643561323936 -64653033386639393039383335343236383039383036376336393966656234393033303537386430 -36353864656361653364613263643265663536363961643738633161613438333062633838623662 -33326236316134643933303537356263396632323565626637313738396162303064376239306461 -36663231303137373466323865363933303264303661333865613734333865383632636436666539 -62663364393866356561613534643637656435366162663461636362383332633063623431356666 -63633130336566323563323764333562373566383861333865323933326336336131373763613836 -32313564346164323433353937623366376332343630353766656339623564613332626361306130 -32313932326438316430373138613037643134623063346565303339663538316661393362356464 -31333831663665396230643163663636383530346533383933313432316561323234663862646265 -34303035336266373263363135393564636536333834303464363338336632636438383539316161 -33306636306630643832303838643835333034376363613930323866613433343661386438313538 -63373431303638393863333230643462343966373033613133636336616336333935623239326338 -37306262646165363265316363383837653837643435616237383162663462393161383862306331 -39653936653964303636633137613062326462376339333834623435313531313763393134366435 -63616137346364333931333731316131366563373165343839323336623630376235326131326236 -33316634623037343866643336316135356231636434393363636336393938343831393931613932 -35303363343062613636346263303438393062663564623131356461323865633462323736313832 -62313063386361323238376363646334396266653932663935343438303436633730646563326265 -65356330373236313061323535313332373235626134346663633662613134636431383265353665 -64373136306564653832363430333632333666333462396136643839643563333835373963353030 -31623930653965613762306137313335366533363564383963303832303339316532616161633433 -66613230623731333538643061333762373135303834663263346464663466353736313834666631 -62653633646264303534663535306336643061613238363935336661303463656136346133383866 -32323935386232386539616539373433366235343032393365323862353265626562316636303432 -35363530666162303264323565353837313032333739383938626233373637336538356238366235 -37626563393961316432663838656336656566636561616663613262663362663932643938646663 -61333331636665383632653135343131613161393266366632636461663539643663303264303936 -36396637316536623136333639356364636337653963643564323865363462393038643339386638 -66326131393963303230616339393833643766373135303732303562306534613263386433636636 -62373830306335383162383534353163383663653836653539356638323261613433383935663837 -35653765363766303730646237623936353734313934376463336134366262646363616538356333 -38656663623033656439646538643838383637366431333930616639666266323861333366396163 -61393166356634633635323030313238653164643735613533373630396264376461623836323561 -35656230646634313235343932616332653063623136303739343233616462346630663936386236 -36323263653464393737366162363933386163386163363166633630353139653132666663623166 -31643830353132363361646161303339343939376633356435393362346532633938643138346530 -61623339663162383932653131616234356264316461653930613264373562393237323164653264 -62333233666365313136366439336331663133356334326432343337363064353235616535336634 -37663963323430363631376233643433613965616633623335653163303663613936363836626338 -32373433656465646163396332313263633032653236346435646531643439653036663831383761 -38323635636161343666356566656237626439356132376138633337376337303462386165353361 -37373632343537303363356564326334376533633631653832616330323838616564663939623939 -37306633643231623632616566343461313930303064633733613234343832306331353539636666 -35343838393866666664656339366532636666356436326535383538346432303531623537353138 -32373831373136353562616436663061396262326463323366393135326132623338383337643462 -33626330626166323962646630363561343662353264343965356237626337353862303730393630 -31353564373562386239313731306165653937386663623566363831363564393935393138313636 -30656137656236336464323863323138343661636265653032633437633635356663633037353764 -33336664336439613435303466363766653531663333616364613632303934363136383336353739 -32316135643962346431613837366463396230313465613363646264303936393161326536326463 -36353135386534376161336530353766373265336132653236653839373236383838623564336239 -66303533383932633065373565393137313666373039323564376664653338333464633162333061 -66613361643462663564656663333266386535303764333162663837346535326335393962636330 -65626363306333333863366663353063393039623335316261636535663363626134613237326332 -34363638356634643833346332616132383331643938663633356135643037303532626532653736 -33353965393633323763323962666265306539663834346261643432313932623936623135623263 -62383731333938613962353565356134336566306562303639323434613534316534363663623231 -33643163623932313161366433316435613033366434626562326234646164326339346461343662 -37316163323731646634636634356464643935646232393761316466336438613035363435633130 -31313962303830646561643234303963613262393232643031346130386266393137383534366135 -38353133323035366239656536616634346430636230653638363938366338343531366363663564 -39313633353838616439363734323133663666633637323964306137366138643665623064316436 -37396363316239303761333230623461363261623339656364316263653430616463343033376162 -65623761333335396332313662343639616533343862353639626135396134353765316666376532 -62366330386266386664653133623534633837326233646662333066613838393036303230396465 -63396463653535373136643864623965366263376265306263356265353036636533346261636265 -66343031346162353733613933393164393261613763383165653835333762383130313938303064 -64313330653566333632313138636238393865333563366139313832323035613336313761303539 -36376530343963343961633031303536376664396239373832636335636136636330646334393466 -37336462663438366536346537306230373535636335316330376163363136333564313635373231 -65623232393334393634666236646630666439343630626334306334383935626335343137616161 -36303762653336623936366163383866656364343234626237336230303036333632363836346165 -35636361366639336265303536373736653663373337343432303030373162303433646161613838 -39646435646233626532353233636437313130656330663938383334653038663065306136353637 -63356138353839653136663336303432623663313635653633303532616166333334303937616462 -61623537663634323134306332353530626138303833626435383334393130663438303338333335 -35633162333234653935643130363233633564386563636332666463353330613835613331653864 -64333266323539346536636439633232393132333439386535363965366565343330626337626661 -63383864353636336439396636346632623464643732303336306338333864323935663563303564 -62363436613833343837386434373332656136626266353239626433623831303561653532313839 -34633961613434343234346335643831653564366330393735636136616231333037663065616265 -39383630643636313065636339313466323633643337353862396462303935326531306435636532 -31633531383534393239316231636632306530333733613936383461623038616231633337363838 -62383836396434656335393239373062353737623230623830346664396237656135303838326263 -38666331343239656666343239393666326661663535336338373536353236633234626239333533 -31326130663932383662396564383130613865333763653035343831316539666465636437373066 -32623239346565346239363837383231396133643236653735326537353038333031336162356437 -61336461616438343833383932313934623035343666613035333338323265373837303865366261 -64366338653131653562626532393266626166363361356533353030336461643432663262306466 -39303366636136313163363366313634613863386435303662323265626631316338336530323138 -37383033616538383539643436346134313466396534653132633737383862366664313533306632 -39636338303632353361646633333034326132653933376333376439386362316139653339373738 -65633834353034323839326339363930626332396366623762346163343264353164623738383234 -32376133393766356434303938663536323734643964623338306634626665623233306334306333 -33653534393465313261323761633531336131356461346231356639323864303163316532626636 -66316661626261323534326163653762616633343936396630393438373532633864303861613439 -33656330663433343334616635373464643363643331613537656233613134353336303137343939 -65346436343965636232383362366231356661303164363962613763356661663630333838643833 -33383030323638663663363937323264663533656263363465316439663136383765363039636538 -39393936353933383635636531653230316234626331376364643138663830656330343530346435 -32613962306663303062383837653830313063333966646233323538653363303730353466373165 -35303439366134376639666636393933613935663965653961303265393461376436376134646262 -61633466393535353363336263666137313832353461653562633664636232373837326531646464 -32353432356231373362333935616661383435623339663965613030636366353337303864353665 -32393961363364326233353632613865383730326236373239356338306235333635653136343537 -31363163353464653936306232643235316235383836306536303961383238623165323163343836 -38343731333834636535386166313634616262366265306239636435653364656534643436343437 -39616532656636666364363738613532633566623230323663326534646366326130313534656466 -61343131643164353336323737303436366335346638326633366466613935323262323836646137 -62316230333235313530353563373165343366373138346538303166396364653137343033623066 -36303861326363653662666561376433393033663836656535343165636635663763373462316166 -65363762663930373035626639316331366435636136616638643963626535343761313537393135 -34376136346537323566343064613961393439386536373166383833353365343562353662306665 -38663538313463666663323531346332643536663132643736623462346366306538373436343436 -38333265336438643436623630313935376239323735303161666538366437623830666536386330 -66346331393639636166323135356234303965316332616137663533373365333932386431373836 -34333133643532303932363139343764363861316639643136343461366164663035353838323663 -62626265313632373631616363323266376463633539383038343961633738366637646233633366 -39373838636562313237386430363737613032373864633364373065323763323439623265626238 -32336236386138323531653061323633356130643335306266316362323062646162396531353262 -37353934623134613838323361373465316339626239333433363563613536323933323533623235 -66343039636436653537373337326132626661336238626566613062313031633266613136666161 -36376236636330316231623732396361333938656139626562343836353263623338386133633361 -33366262336363383032623564393262323932373235623330386135383033316133383239383164 -31313432656636386562616332656433383164363539636232373863643233393031376436373832 -35346235303034366638663962333961643064303564393361623636363364326366666633323830 -61376538626439363034396436356165396239663163306338353931323264303462386264376636 -38383363626632313734316138346238333930353966376336323764616237303062366530353362 -30656262313165366232616339383038666636663464393837626263626161626337376534306639 -35393263653239313932343366316665653736303533353538363736373533633861363031313366 -65363639363639393538616234343738376136376365336164353237346263366131373233626561 -38623461303335633762646439653536646234383565663264633562653564343066366535613837 -37323765373563646361396663353836383334656666383963373533306665623334346136313563 -31366630343963376437353434653334343630393137613061313163396636386533666234666134 -34343333356262643964643039646264616265623639376466623664663566663862376166663332 -38653635303130656637303436643637343461383231306532376363366466633734346436363062 -35626165646131353565316166643666346139366265353936633662396165393466323664616539 -30613630373030353239666233363031336666623466613130623937643062633561333935643934 -37373964633933393764613765666532396336666432653837396334353337653439313663393733 -66356465316330353438356632396662663537306362343631373831613535313336366665303334 -33343466623134646665303030373439313730646539373166353831376166643963393533326630 -65366138656536303135323038666236346134393133616230343464393635316262356634333262 -65666436353834333233333730333535373938396135333534376463333537343031323439336537 -34306538343536643937336434313032626531636366366162616434393933643835383632323463 -33373264643435333032663836633566386436363432666366663264613933613964386434636263 -31323438636532393861313330343535376433373263323637373738646233643862333265363434 -34626133393335656161326264356261373733613638373963333564313138386563643963643631 -31303131613363393662393231393732643831363266656433373464643633626566646532643966 -30616639313536656334623664386336376464343033383938363539326533376161336331306432 -61336431623432393865653462393164383935656364383139623538643035366564363537613062 -62623237343537366132333763383966336166333230396537313663633062616631326633336263 -65373936633639393139383332303761363266363832646630616331376166623232373331333862 -64393762316337396538316239656139333563613836343232636335343730613333636136303565 -33326235336262326161666430613634633135373637393565653532663164323936633632313237 -35633838303562303039666130386131376537653862366533653736303931646135396565616364 -64643539616133373834666231636332633636326161623732376537633838646532636538643439 -32303534633262626561643362613730646663326262336334323339643838376231633163613462 -33613036343031633830613562333637333934613934343262353164313337303839313633353065 -35336231386464623038303562643739313763313132356561376433663763353030373830303531 -35303464623361383637626236633132666330623065626131376561613663343634323831383635 -32666630636562323835393337333662616637306631303235336266343362386631663863333033 -64376131666265393231376266623837343038633462353734376431333139333262326637393130 -38656131653639383735303333346530353865626231343739666435633537343434323830623663 -66646133393834363739633937366130643366656162616565626435323061346234633066343865 -63396537643435383634376134633030333561643837393232623563636265393166303230303564 -64653637336635303964316434393436316536356664333236346132376433616462366162653064 -61396433353362623631643033346234306466653637303062663431303034623831353861323434 -62346636323735633639653465343764326461336263373131643439613536356230646634666434 -63366264653437393963303864353865376561396666663431613535366237393161633533313437 -38396635633863346337643837353936386564656339313265663466666137616334306363313739 -32616339396530353064353934643465663035613661646138396136663337363631383535303932 -38313162313930663931373866346363383863663633386535393365343537303264316362393039 -30363732643839303730303133303334643362353265623838383964636663386330376237633364 -66666262373138333863666664616339613665663161306263636266646132326665393337646361 -63643963313965306366616362376234626464643565326464366666666436383662646533636237 -33333866323236326636313936393164313963643034336238383239343131643162313162343164 -61613537356465376135333162623233383536383936623663376230643365653761303636636231 -36386464663832343436653464303738643633396432653066336131666639376539306432386263 -61333935376561613932366163316134373636353461353563663261346564633639356235623665 -64636236333637333033643431333138353436396230316664313231356639313535326332623131 -64343866643761613736303031303762336161333766613465393835636639653666333164333435 -34643039366632386463386663343531306632323638386430396437323738363134376631356463 -37353039346638626433623632623338306264643163323233353838623238393465396464646239 -35636165613032633637316130663539626635623539653539636265376130306363333734383566 -37643439373635393031326565653765386635313731643863616334626539396333646462343632 -33343134653530343134353630633961346139633436373238653633333437386336313363343034 -37393632316637646535316266326532333064343739356366326631313664323537373264326639 -32663633396130333965336333383130386266623431343062376639346532656262656634303835 -39656230353636646161653661373832623534663630353436383739383039323539393435616430 -62363464616663383565346431613033323361626530353261646636616239653866663961656532 -33616663336236376335643566626433396635623435633732346661333262366536393362396330 -66616537353130663663306434373638626439306633313133643537303232376633363061306531 -65393366643937356563623733393238393133633933383563653763353034396435396336316166 -64663637376633663433363031396335383838643232343133306535633631653831393035333765 -63393162343530316565376538313939356165616130626136376464663562336665373832336531 -33356265353532316262303239336163363165336339333638636163316230633932656133626364 -66333563616430646166626361653934613039633931633537653962613164376161393533323535 -32306634383166646437623662616634343835346639313333313030303236626330663031356431 -30383733376235636463616263633461373462313437616235313365376530653264343964323930 -36343633353964326635323635633666323662343731616665613738616666386337336662353236 -32326534383031393137646636643438353662643063336266313266363534616362303238373439 -38616431643330373563646465623030386366373537653736313130633263633239316234636464 -39646266363530323039663534373338323139646361326337343033383337623439353161393739 -38323631663938303437633764386266643066323764633261323930393937613735663135346163 -30636238616462623232373765653033383966343862366139343134303834643531333934343865 -35333936303530636265626664363530353134613632666634373864383632333632633030343235 -65623762356331666563616465363661303661333061323232616138656262663739633762356430 -37366266326364396536613536663935363834356131383838663234373138356339643930373739 -37386134663633383231643166393464326132626238633235383039383461373963613836393361 -61653366356237363563396166366463373136313739313931336131383261373861363830316332 -35326638623334663065383132636133363430343331633632316435333466633730366639646530 -31333039633430656562326534663333346630643639346664666661353431343261363033373366 -37643166653339393235366262346163663937376131623531323461306536666362376539346536 -61326436383363336330636565316536303530383061646134666234386633346634343662623936 -37613861623235353238373134356639663739623938396435373532393562313033316665333439 -63353332303562396666346265366561643234633937376562613736343039376130346332383635 -39323164306436306363613839346262306631313931336162623063353436373036656638373362 -33396262636537336538306130343862363531656161316636613432386635616263383561316263 -30643733666230313163353965343664626636373836366534333963303662656231633863386466 -66393137663461333762373433623135623735656131353138306562633065326534323565383030 -35346235393831346162633332323461636137616463666132306637303361313863613061306533 -35386665383739356630343865333561336366663735633632383932623635663738613032343938 -34303664666337393833623636346333353062303832386138303463333464353362656562616630 -31666165393931316234633162353235613963363539333336663636623439616563373637643333 -64306430373134313262383762316430663130663962393330316235326436633532633238633839 -34623763623231356433626263313030313062323935326430653265383136303234613661343237 -63313566303065373632316461326331663336656532303366623065313561646532393162383238 -31653139353232633238393835373163306564343466343161623363303230646237663336343762 -33363735656135333265366332666536396361383839333135643130376337653061643032613134 -66623932313763656336653432326662636663343062373337303066303163666239626161343735 -64326333616163393334353038306233306666313738396130666233656637316161306266316139 -30353565656431363136653238386638633139333565653635616266363165363139343334623361 -35306139303237336635353031393238333366326631303763306531376637373666393765373931 -36303835323834313730663665373463316664303863323562363234323065383036373061616330 -37316130313633666563366439333532643330666133656366353339353565663065353137373835 -62353836333366376633373761386166636331363236353031613335643130396634303337366534 -34653332626636363631323531313461396538343731316265323137346336353737316234353735 -35333839336433366561376130346335343835306139363831376537663634306361656136396165 -61656266333536353335613461353936616339343134626334303366333735613332636230633465 -61356333363661386564653164613663633962623630343734323335336636323939366534363066 -38353836333933386332396232343161343933653132386335343432396564613838643165363165 -34396136343933653930613635646334643331656330386562333366653162336135303365643539 -30396366656365633531613038666561663531353838313338663630356138383536633261663839 -66376264393466623032666633626331356166623863353432333464653563663736323665356565 -32663766303931656635326463343762316338393437323762393630356530646339353233613839 -36613836376361333438643536613138376435353331393565646537393534383734666166343035 -38366562363732386435643061663639643961346262373834363035306261396132656365383762 -35316564613435613630623161363738383436623962386236613637373136613534626631656162 -34656432346637663965663563393838656638333635323731306661366565336666313935353233 -39613161626234306231356532373335386261373733653731363230303937343539636262633637 -35653364393039366238373735333861313134353064303264663537623131343935666566656138 -37643563363164333537646465616332383933626162356639633064373834643631316566643861 -35646665396238376132353239333030306636643636646266386166343435373135373666323766 -35343433363035386630633664356562623632373132366634646361396538353737363932663730 -65343737303532363437653965616166656137366564343537366663343832313032306532636164 -63376666386533623438626232386362396632623163386463353937336438383864613732626439 -33633265666436323863323632333039333165323563653662356431616239626261326165366262 -62363638393731393936336463333033366263636531316234636463313433306538633033633633 -34616462346635333838626231366332393535656365663665666636613838363662613934313762 -35613665343761333463336266633066333333613835306264376363653939303666333462643433 -30623064646439383062646266326564323036623237363635326538313065653235373139633464 -39386362653361616434373539613332356232303562363434656364626533653462613462356637 -39383030356236326562343239363638333837656333303761636632643738373935353034396464 -63646536366139376237663335623762313439366463616235333566616662656339643661313463 -66343737336138333562303666653436623432306330613636353064376433333839643564636134 -38386163616566383637623237373666623466393861643330386133613737393732356564326435 -31336235623736633431613739386133633366643336646230653636383333316661616462303033 -35646131303431356363373430623039366331303230366565646364656466353962666661373235 -34323730633339393531656138363130336432666262646139366664333263323561306163326136 -39653131393331366530613838663937353735373737666630616535353366353532646339363239 -63626630306339346439643934353962663264333465386639393734303265616263646237326466 -37303663376261616632393064353266663432633639653262363435663462323262613739656664 -61616661346464623434386637656363633161383338336638306635323739353135626365653836 -30383838373637366633303132623730303461666463383662343630363437386631313634373661 -34356138613432363339636334393661313063373533323631663839636538306337386538376665 -38373739643439633530656666303463316239626130323932333165303730323363326630343433 -36336362313831373562646331666335306361353831653835306335356430666263633037643261 -33306365356366373930613330303565646461346434613838636536316265313831666265336532 -61633465653863336437313162376362656338636131613133313639353361313133323963316462 -61343638343638663538653465643231343064316162646565663764373731353034353737623539 -65326665316563383562363661396236643439346538633938663865353663623637663266323062 -38353662663033366532393133313130363239383237633139313439376232313330613966633437 -64373335333737353266333466313436653231373431306437616135366465383434323535663762 -33636366623734386165363430363134666238316138636461343164613466653837336232343733 -39373465616633663665363865373466633731656663643134343534643332386462363935353034 -63663732376662643263393864393133666365343432376530636666636138313434393230333130 -64623564326335643264316432663963376337386563346331366366353732633432643832656339 -31633132306161306130383930393032643431613630373137613165316236353763373339346536 -39313861623037303164343064303538663738653763643838313430366662316233323238323833 -38663131393565616536613939316166386661373237656135336261313832643935636535663562 -39323033326432386666353762333531636331613065383666333331623065613237313961666661 -34313738303635363265623465353665663263616336623831316161353232666465313866373531 -61306337353032336362646163623538626336333866356461613335363165383366616564373132 -35356462376237363538313365383436383662386461663539316234303761663536653261663961 -63343839623564616331323234653932626232643966396637663166663031633965643264343162 -65636533376236643130363036613535333738376362333435336261363037643039386364313363 -61343835313635396639396334316635303166313837666339376331646635633030343463323762 -36383836326239386539653634626261393161303736626131333762346332336330366336333039 -32643831633361623731306566616131623439343630396661346331343933656538653734343831 -34623166313632353864656231353263306339613863646236396235333130636538386565303063 -32666132626535653134666565383163636533633635613562326434636637656630666361653534 -62316162303764303733326134613536333231333737653861343937356238323865333462646637 -36393561363130623534396362613231376530323936636335323035666461643462643630626439 -35326339303931363235333039383634663362333234313438633838353865396330343738653239 -65393439633130303130323665663663393266346663363762366338616236313961393836303935 -35346533373763613139656661356638353765656139366263313631303061366331373530393537 -34303266353931326538353734333234323563356166323965623265323866656266393733356461 -39393161323561313465343833356631353662376632393864396565386636303334343032383266 -61663932616339663265323064663436323663303631383438656635626565646663396637376631 -39306136626338626332363038636462633831646632653038346262643266373963326261383763 -38316431303634623337353835396330656164643939653132376166366134643561353263336463 -33373931616538363665633362353762353963353438633139613934646265646664356135633530 -65373632313939666662396439353137376238373466323832376638363865663336326565333831 -37306630363236643431383532363666623334366430623364313837616636626230616530353038 -38656135326130616166656338383136313362626166386465663932343738306162666630656331 -39653139313534333535356637653835363432633864386564666537346139656564393762656230 -34396135343337626363306665326338353732396233363437663966373139366330323735656334 -30393962636631663432653961303632343635633564346638616332363132653765613833353531 -66336361316234323864333533316234303030613535346464663132383563653363643831626235 -30653535653439306632623839326366316138343138346236643261316534326334373763303433 -39393461393734386237303464306331306262303732373666386336633635353063323239366537 -33363161663337613866666136643533643933373038613636306437313961633665333730653239 -36373361626361313435313662343338303463363935316461313063353261323135633266396466 -38646665643064653731666437613164663031343863646463303630363535313461616363316362 -63303539373537346435306333326666656231303837376263353863343962343536633935613964 -35653230616564623330366664373536303338336666313561313365343134663437373462633039 -33373039306536656233376364656534643863313733313638353433303565653234643633616439 -35313838383737363237623033323132643536366663383761323738383563623264303763313230 -62316361306136353333653062653865613836633161626163616264613638633433303961623730 -32306232303131353562373865303334323936643065386133663636383865396435393237663537 -38643735346235323561356163656361666434333435346438306362343433303563323730663061 -33396166326463386538613937393766666133653036373839306661373165636633353133376131 -36643137663531646433616132663337613134306466303866326531663864386463396130376638 -33616661313032663536383231393861373934323130376164383337366461663538336563393361 -65333531343761633563306266366235386262306337663335323235383961393732636338666465 -66643461323039396636653232373231663539333831656530386433623934383264616264353535 -39363039346633666436663265616530383539643961336130313839633966326362326466653233 -65313766646636616338386265643437653262623361346534363739666661336532316535643331 -32333561366363343163313732373437333130396166346135343230663230393561316363643735 -62326364623033636436613830653039336630353030313162376631633634353134626265633134 -63313433313139616430623765616534353661396136383936626630313031613664386633613063 -65633431646462643438663035646130613266383634376537356635383663656362663364326336 -32633863653432613565346532316436323665363932636335303039303936306532623964613664 -62363334346365626433633735333435326661366232356538393339373261363364363366333137 -64373133343537393863653365373863346661333265353865343265346261633863336461393131 -30396633633962333966633832613265646262613431623431643938316432303764376266633062 -35623661336636326334316661306132393961376262303363656464616661326537636663313162 -39306165363838363736336230333634353938656430386234303864376262383236616238383962 -33663533393939653736363031666439323834383361633637653534383036353034353635326161 -66386561343731346532333735623932333334373435323664646235653332646431356466373432 -31343534633236656238653366633636313837383234623532333534623761663835363135373530 -37386537366438333630313433303164643266353139353030643963653533636533393561633331 -37646561626131366566326636303030373034623230623161316337306532353236393938346464 -61396231303631326665376462383461326533623362376437396462396139333266316238363732 -37643532353962383861383933303633323264353662633065343730376231643634313165366539 -33313539353538303730656239643331643333356662343530323739333961306266613462623537 -31623366353336626362623436636565333837336235356562333636323831306630646534316135 -33663165373661303063383839356461346633346337636535373461643162643961616539343234 -32633663303066323435343832663662353737633931313031336631653834616233386438383461 -35356161346439653665343432373166383361396536366239316131663364386261383730363439 -61323838326265663466346133626139623165356461643036366663643031343863306364326563 -65396663643831386634653531666133353965613839346437376537613834656230646165666634 -37616264363436383331343430653136353532626637643432623631613734653462393564663565 -63626633646166363138613062313865616362323436663534343332663932396632633734643966 -61333132613730623336303361626236383035613939386661613966393061363264333337346161 -61626665316237313764633665396535376538313362653365373731333138613161336561316661 -30363536653231646138653034323665363465663764353162613065333131623762623137303565 -34383663616263383635663562633261373966656530306362376162343561616531386333386231 -35623062363361373238383637656166396133663565653461363634326138313961353331383661 -35366661663539646637373335326237383064303533316534343866313335333664373336393434 -65356135396664333633353035653433393938386531376635626339316137333562353038613634 -35363733303032326164386231626435646239366464386262303466383666623465303538363335 -33373434623235373762303661306632363262326264323166643831356661353334663264316437 -30303535386136393761623537633833653939376164383839303462656139656539663630383739 -39376234353836396162396337613339303930666637633463333039353331386235336338326562 -63356366353365623335363764363634666431323534353035653966333534333031316631326531 -64363761653333383430653433396538393065643965316235326330616335336539393165303134 -34323139346339376232666537653461656130386431623065316232373737386337306636306638 -33386636333964366434646130636466633837313030616133336630333438643435333566633639 -32353863363161316364383366636164396262393065633331333238646633333066643033303437 -63303062396134636464333937653135393266313663653538303261333664656536313535613133 -30356564653838646634646134346366653965323433356266656438656662376561326332366161 -35393739373339623963623532653734613136666164376633626431616539613732313932396664 -34373636333539356132346537366263656633383461333437393165393561333734663430363366 -34656432303330346236326132353635356463303233613531343261393832393866653332393139 -66373732356666386337346132613336303237393530616235373835303534616639663334373261 -64663762613536636364666564306236633030336539653430616435636366616332626631333364 -35333932333238313233313439303737343066616633363466383862353037633837316232303763 -65376131636639653834643365316534326336303336373862626365396662616239346264623964 -31376565373335343736353864303866613139383837366235616232323531383862646166386339 -65653961343437336138653161316262333433306364616161623431643531346634646139353834 -30633933613030636462366334653062653262663934356236303438353062633139333166633664 -33303033383965316165333635343536376364386135663435383136633438633035633966353639 -38343737633237356331343031653564616265353639666334353061383437633962386430623334 -35373239633036626566316561323861326566666464383930373464343063313965663230376138 -30656435643732623761373461343666346437383166363265353339356438323864356532616161 -31366430316363343461346135383332373535353032623638316535656239353864313739663934 -63646634396637303034623466336237653131323235326535316334653262343165333766656165 -39656437313835326533623835383533366366346639323132333330366231626364656137396535 -34333930626430396539346230383232383832396138313239326163613038303735643135663434 -62383532323863323938323038616434653235393931653032653330653138356436313134663038 -33633961373138663735323438636636376565353536303432623430633330333763666431366536 -38356139356338323535313732623961366631346461313234323265663737643435363731333136 -35333461666437613834376230643234666334346338383036636430343839383434343863636662 -30383561386633313837343863386133656337353931376132646364613263656461646262653862 -34643264653138663433303930376564386665633139393264653963343033343431353834336164 -62373935353266313262383661633165643033636439316538666232613838393637343739306135 -38316661633438326263313436313233323834626138663466373536633538626166653331643236 -63326331633064383533663264396630663565653430653539666635306235393564373131353364 -39636638343739616663303237626361323364306562386239306131393533326332613963353230 -64636332616263623239373335626465393030653232653736376661373862376535356337656666 -33346335616331666361393861373163333461393933386230626632393038376333353036353964 -38373464636665306131363362663765626133383732613361303661343566366333623436316563 -34306662376336613630383665626138623765316539613632336532343462343866663239636436 -36666630336139623637643936343037613034333462303036323933306366663664623139653133 -39666635663233626330303631333236333131356536623431333636336630626133383763326164 -62336130333033303865623730386134653530626533656637613635613163386135653830343135 -31356366353464343238333738316538363135303763356438306330623633643862616139646136 -33313866643830323232366331616166653138363266616633383435366662623163373537613038 -65646662346563346635326236646231346634353232313636366266383439383532376262623366 -65356234663631643731623064316561343864316531636132393036623630633862353338343565 -30623163613162303861363338626166376339643735643032633935343561616330393164656432 -62306461366666623734626663666264663734356131393535376537636433386233323866393534 -35643939333635646163306262636635323539346462323136646535333635666164313662623933 -36623164653631383465613439626562623236353366663063346263613934336436326630336135 -61333264653430353835376637363638393531313037353363316633346463303966356335623337 -31323765663239653031666138636633613739303466363631323965306361353631653938393565 -38316535663833656239373838666137366139326461613164666237393638313737323039376531 -34376131326463346237336635333533373034393039336463376638336363646437616535353731 -38356430376334643337653865636164343534306537643731326337643462633838363039303765 -66363865393634653731613238643835353731376631643333363664343634313533373633666630 -33623262663234626231383534393231643735386362646639396562653837306362333139616238 -30323133383565626462386334653131396437363465346132383238333939336561653330646332 -30663734643662646635313236393534343936636465623438346235636562373762663330393938 -62326633376131396562653239313831373239386438313930366331643539653731376230386435 -38623433646136363237346263353963633831623461383737326563316464626339666365306230 -62616539633835366431626530333031313337373338363938313130396664323131633533353762 -36343264643632633536366532333132636631363265386466623033303063623563393034666130 -34393833663066623433643632623637333632653362336338383739663365636565643439633232 -32333133663933303931343633363031613532386339323537306265643565393634303064383532 -35653430363931326333383761316238396261626534353161636538616438653135386661303930 -31326561656333666533386364346364376464303735303530383666356361376438356563646364 -35363130386432303432376537383665643239353762623936363036343930636164303136333563 -37666237616232383937666262363561373666656632323664656563333838343164636532646561 -61656366343064393461323530333364666232613566316430633363666263613736646237343536 -32333430643263363732333965636539663236656638323936616234383731346363623830623932 -61636635313966356561363335636262653062393634633965666331623863313963656531623039 -37323664626638323836303962386630666638646663396466656161323439383335376333623832 -63396235663630323936303932616163323830353166393533656161363965313035396164636538 -38666530336165636461363839383766313963306134653537356131656239323239376333613935 -35326163373765323237373136323338663632383465303261613538626135383435666661666666 -38396338343031373135373439666363663333623637376138396431643438333837613862313637 -65363865383332613366663264663934346135646237653238323339663432626564336364336564 -63616434393064346366633063626339366465613565333739353030343561336336346437316361 -63366366333230636663316435313737353764303832663532313235653765653664363431386231 -66336137663464343833393735323532346164336564353765303364353835383362646462393336 -63363038336439353439363338313661643234646639303866656137643663356535343164666563 -32633639623732393430363436396235366563363737363865383465663261656461316330643833 -30343631623861396261306631303266623335333136303131626639613933643038653862346538 -39376638323364396439333935303139373838626264336232393035343061626135613465616363 -64633133636362633561383834626630376331323163313134356137666439653036336638613739 -64336537393134323363326431653639373832613939333766363535656131626466663763326338 -33376565326133626531613337393165623737323362343263653135656233346630396564326331 -35316165346631373438396535303732373035326166306366616165333062613532326137303732 -38396262303933316534313932643539303535376565353662363565356237336336666338373237 -32346561613163663564383338646331373439626262383233643266383964353532373264633134 -61396636396138346465326561386330633035633063376130626365623630633036363537333432 -32636237346536396631613937613166363936613963383936303865626639353633636131663439 -36396566653530653965363231383337343638346239396335643534383866656332636634653735 -34343736636238386630623665313762643861393438363733353231336539613037346363393337 -64333836663731376430333237626335323533363037343065393130333730643131306632653563 -64643833306536343038613833356535323862646433396438626535306335333363356232353064 -34393035663037633636386630396662636264326138383363623337326136393730303335303664 -39653334383964363539373433393666646663633336353562366338636564303033366662386234 -39616661326538323734366637646536663235396162396136653165383161636230666537626166 -66396236343137396665313037386239643361623134366334643163663963666466636236633935 -37333432343162353031313264636131613939633665316434303435373265643230353463313130 -61363265356432326163373436386664336135623966653161313038633533323332666437653764 -65613735613838653264346362373062383339323862666630356164353635376336623133353966 -36653264393838326433343934646364343362643531366561356539633133656561613930626239 -65623033326236636536383634396163626635633236386134396238643939313166616530393663 -61396332393138333165653331666465323266636364323162653464336330333035366435653563 -37653161373239346432366265346466386463633532633562323936313434653833376537303935 -34376264303563646362666331613338336130346434636432366538663663613331623662346137 -31396139393664303164313461376366313237633566616637633862306235343961383561303432 -36396234313334396231373264666564623362656263613836363336636233373361646236326431 -30626566623830313130303862313331356264666164323461393539343039383132396661376531 -64653034616231636134633634353065343339346663663539343331356339333433376436616665 -61613636653034336331336138363039616161646438316532653166613136636139663138663139 -36356631303961656137663662376662643734623735393032363739353535303865303630663837 -35626232383134316261643330373437666531366238643033353035323233386534393236316366 -64353030626135396133323461646135353737313634393037633236323533396431343466313634 -30323262656430663762663963313762353635633361653830336531306566643438613566363261 -61636362383462626235663931333662333031653337343061643264636366356661383034393739 -38643133636139343031303137316336356232653564653661623933623635306265363833656235 -65323464386232623737383232353331613266353235653364373832643562393331356132636335 -65306638656438323863326365383831343266323562346262343266346537396233626235303165 -33323162643832333062353465376463633965613333323031363033326663333133663963333335 -62653938373232643638636436336138613939383163336133653836633835376333346463343034 -34303461303863376234646234373462326430333361303436303565313164663633366536363333 -65336139643363386537396230633530303137346565386633326564383764343563646231653035 -61633533396533616333623931373330643637336538636339363337346535303239643164343364 -32316534343162663837383765336234653362336534336532353634306263376162613536383430 -36646532303465346634616364303433326638616463643735636435643266383833666235626337 -37346166626338343238653433353466366634626637633039313263313334336433656632626539 -37623530623138343263646335653863373731633861323266613138373461616336313962663261 -66656137346434613630616335386631306263613637663737343138643936336564393934623038 -61643531636266396330323961613930646163303263303936393865303764303263373830366336 -66326536653863663164636563336530613565323839373831666561383430373737376565666338 -65306332313165313364343262366431376139626130343132393964636236313034383634613733 -64383335626534313861623738396639663061396161613439363432623864323834643336623966 -63353333376665626233643138366339366332326562643464363238646630613764666436313537 -39653331343133666166343139643634666262353932323131643262346365613931326234613864 -37643862316266383464663938636433663266373437393732663935656461653461666261386434 -33633432356433353865303138643232346339386430346265633364636235623334336436386232 -30646665316364646663353834626363303134623863653834616237616331626436306439346432 -31653636633061333964353733353634636231626638616164343162653132613338343766333233 -38626235336232656164656264386138356536636364336164656136386137366265396564626263 -39373533396435633139626135623362383162356235383764643630306661323262653237356361 -32396563323964666632303565346666356566373261373532316535303133346537373037643139 -65333731653939323664316434383532633065373661646364343161343762663235323364643835 -32623734623661616466383034613063333235363765656661323961666463323866383966393238 -32666437373061386639346538656165643963343231343462393561356635653764646333646466 -65343962653237383337666265336531373833386133373835643538656366643963643862336165 -36643338313735393632623464376532336538333336313239333339373535306365336463646134 -64303835333136653933643834306333366431633362366362666363643536643638376434363864 -30306630653262303566396231623737613665333239353261333632323864613662393935653365 -33626237323862633062326365663062313761363137613235376131306561366436326166353966 -33346661386566366630313464656361616638376230313630313266353736303363363031353333 -39643431633638643762613465396366393336313162333032323738306238356264346230656366 -33653334383733353962663033663965346636616636333633653430656631366537316339383265 -30383161306464646131376238643664633465633831653336343164666264326166363661303831 -65376564396564303562646234316630383834356665386664306535666262323938616432316266 -34366130623137316366386133613035323232336137363835646531663065636638303439373538 -39616631666435383932333634646231373462663730363536326337613734643931333332663661 -62326161373763346633386233383562356630303237623935323834363932376431613966303834 -30623130633234313764636432613930363438393965396537633736643039636164353434376632 -31393533646334333832356433393164376134303566386635616237376236373631393233323330 -32313836663038663462306431646433613239323633623165313838616439663362356436633934 -65613433663065643263363564653131613033376331313762356332396466356337636362316137 -33303234623530643733643162636133336534396139393635633130303437373163623930333461 -36623932663932366664663531633634376135663165656238353030393739333035363362373935 -31363835666131323132663066636436636638616637353734343231333637306665363366373734 -32663064313833376234623765663630313265363638613731393138346438383666306237363937 -32316235383338366566646666623537303764306235306266623433353565366266633962613562 -36326234353533383665376331656136343064656164663431616531666438306532663461313031 -39333331386330363662396261613365646436356536356366326262376462313531623237356165 -35333937396630643830346335636530353637363835636535643230616162643962303538633934 -66356534666562393430613932666338636533376263376238303438653830653264313665633231 -66346565663032336664363565306633663536323135616530333536336336386534383233616232 -33653036663539336564623735313561336338643937396562346162346533323033333164396432 -35306635353337643230663532333130326466643661333233336436346335383537306463316466 -37356565636430326237323161373033623733376532633064646165336538636466323961386163 -32353432386532306434343735343361353066326331636334343835316164646138313632643935 -38376533326134303165396138633133626133303635366561353464363333343064366561663431 -64643934313036343630373638353739633266343638316263653337326431333264336362316330 -63653662613430336430386130383338343237383765653936386661316361623834313731343266 -36666130386462396635363039376436336639343237323165616434356436333530653131636263 -38356234326135366239303632306333323630333561333138306537663662623332333365323933 -62393162343630373030386264636162366634313936343538646334633466386537323564633934 -63313239343165623462646538656364386162666239363162393162646463616431356137323339 -63636664343331613533336531623437353461363833393363396631366434346237386439376131 -37363565326634326336343265613337366338626536653261663761666336343863363236333962 -63636434383538316636333762396265653937313632326135303335323366633330353464323866 -34653934333237373736363666326565323066386636373230353361343835313362313635323435 -38303036396266656438343031393066353361383731313132623963366564373838326265633438 -39636262306130643863386236383461323932663534323533326162646439633762323737646563 -32373535626561616632386435633661343666663665333664376132343166356566666532643833 -31323438623531633762376235656464613463623865623538313031306430326235343865326139 -66363737613833616464643862373138393964386665646139393165643963376137663362633431 -34313738356334333230626166313032363838653631343531626364373637613165666338383038 -35656235646431343766303833643062663338373733666436323936303462333137333863373935 -37363263336632393135636132363966666265313665646335663261616633656439303537353932 -38613861326633356430636338343937323630646438356362303066666366376432653431343466 -64306264333930663930366530396565343531616666646361383734313562343063656234326335 -63343665393661356432383837303261353065636462666230353632643664356432333461356634 -39323630353263616565316132383536656531313536633732633966643864393534323138353630 -35333738313061623763306165326365653330653661383463623762366534636161363432623833 -36303839313362613137613036636663343361646131626230353038336139303433313766333263 -64393966653165333562303961326232633262346465353065663861343262326236393635646361 -30646164316233366238323365383463383462303837383033326464306335623761353136326530 -64636562636334333535303839623865636366326139613262333866626566356530303635366335 -38383564346266666365393563306331323330356431646465313361383532323964616664373961 -34303033346431383634366638333361323035643366373661316536306434653434323265323738 -38643665376137396130643862303137346138326561663265626639333234373130636635333232 -62636236356236363361633435353262653134643039646563386330323163633166623739666331 -36623835373134303536653561343762613136633634396363373161353839366630643361343238 -33663130663336386264616533626530313537346566653135663636356236346434663464623333 -37363137333032353666323763653066666439653662376334613534393766613638346435363337 -61363866303466626164646238323334636335303066356433303030316231643139653064656137 -63666263366232373638646335663863343461373834633265353364393731643239326463323830 -32643935336137653763343934643365333236623534333731343261636138303833653261333231 -39333236616361363435323866363630396162386264333234646631653136323534623633386466 -35343634336564623762326439393038346232383963366234346161393934306637323962616435 -30363366383562313963626530626430383766666630646631376135333839653161326534373164 -36616330343932653066396531616563306638313565316462333530393965336633643735376239 -62326565613034396337346637386564613034386633346364303639333537313161346261353930 -63343238386335383561663763306436393839393130353335323133666430383763653062316235 -64353464643964326561386636623932303061646333636263386137393436636239313336386661 -32313864656634623234656632613635656666373834373833376563663034393638626463356531 -35653436313166333861656535633366633733613736633432653464653764306435393336353733 -38663561346630363238646331393961396665393630333363323237616238636466633861643630 -30323339333131383537396635383765383534333539336234313061623731383065653437613032 -37343763373164313530666566386633653138383934616235626337633036316461353366393634 -36326161626264333035653632626463393262613031316634353237666131626335303366623434 -38623963623530656332306633386235663764356563666362656162323239393262393165653334 -37386261356536616361613862663963636236633331643263306138316337643234306533663330 -61373735333138323532343333323130386364343165663131326130393932396236356534383165 -39313361653837643432663235633961613437643530316363643865373233666137373637366538 -32316431633932326435323136656261353532326466393234623035326636306564306536393364 -32316463623566643166393634303933323135313535356335353265376661316537313434313763 -63326536636363346335373062366363333432383036393534323030373739623633613033633665 -65303962366237663130393037323165653436653364376665613933373565616464393839306332 -62313332393562386636353164306364656430363432386237396533373566636664616637653531 -37386438616331653139346235613663636436383037386666363831363738376436653333303033 -63663939323463636564626237376163353366663065366366643437666135303363623634396465 -35653735313239663035643737326531613035643136626435333662376561323934646232336436 -32346338313231386138633261313666303866656239386336363836333931646463636461623865 -35653139633862363636333466303561643035393235626133336131396232623630363336376632 -63653437316662656664356261343132386531373137666465663365326635343765333362616535 -36636432336431346365336532333437633366643034303739333830313934306534363138623433 -34386562346161346531386166363862326433386665396634313333393530363366623637366637 -39376364316164356332353335303061363332633735343433653262373563336137336138643935 -35353964356363346635656262633563346335666533366166366135353735323831393931373132 -34623065313361613837336536613863623430613130633437323062633262663865623065306136 -34333763623937316236613736353037666364653537653238343039393331363232366665393832 -39656663643964613536376365396334626430636535646139333461623737363137376231376431 -63346166626562366463383134326532383730383535633833373736643138336231653865636233 -62323434666332346433653538363938666233383439616165653765663564346463653639333739 -34383066396236616630323136383166616433353033646434636265306539643230376335343531 -61366662616666373137663763656463326433353764346561636631356565343266643236623763 -66663063373832636265313962366261343066366436386432313139303866663235656131643830 -38333430613835306630633139393534373164303661633362383336643065376464653936316630 -34623333666430333939393734613538633965373033343235373339316234643663373430333035 -33383936353531303735616335313164636362373434616636363161666233356336653363386664 -63323932656231306262376464373936633137613337663438346439356631366132323334313532 -64653132623562633735643663643837656266613663396361393066353562636461303836636533 -34316134306564336162323936343638633165653066393065653238626537346134306132626337 -65336365373062393634663135343133623963613139616632656633373834353933346562326539 -36643862343862346232316363346530393637616434336562653561376333623332663931303061 -33313961626465353564313435303165363335643563353738646264326563306232656463623833 -36656332313739613137343536313130353432343365363439393430346265623039373331643562 -31303331373762663030623463613166306136343065383563646335373730343839613363343135 -34383332363130383936616461613162333030626634393432323461633833313236346233373439 -62393964383932666565333031326362333734356135393238366634313937613262376462386135 -32373064623566316430653666626639303139363439613966313563313530373233386561353937 -33306331646263313334626361373236643337323464663165376239633335343365326239313831 -36313534643532653531613136373331376335363037666630616132623662613835626632636661 -64613331656531346264366462656538663266326135366133356536313762353261316137666331 -65613338626363646135383333366564393636373730336564376334373464323832356565366133 -64633238323666363864383935366263656434356166363562616336643466643235626335653261 -38653263643737633763376635626266353930663838393062323166396434323965633633316533 -61316531383133373663383133643435656332366661313165643762613766333134666237393132 -65633862326266383333613231363061386561663364366564653666323066643237343537373335 -31366433636364656638613365633662626434363661383038333364616565366531626562306133 -34353938636238383264363635643565316665666338326462303636346232613066306131613766 -61383435346131633139376133626531666666323239316138613365656539623639316663316539 -36393630616539623765306464326233313761366533653864316637643766373666326638636134 -35666537666564656131633563646264336332636562356431633835636662366331626365306436 -61363937313335633935626465396638653234656366663630333230333139663234613965613534 -36336237653931636437633631393465373630353634393337316433303533643739303636643464 -62656433333533663334393537343435326637363639643433366633376365363238303238396337 -33343736303935336464316637643637313261333733663764323361326563336562663539616635 -31663935346163343134613162663031346138333635396134653266343539663331343766306334 -34303366316537666338393062653639363763616334363833393134356535303661663663646330 -66633832323032383566343464643636303334363064636436336134393561363836313434636261 -31626432333130636161313831373833326232323733353936333762633362663239626232303730 -63323334323935336161363432323331653839326666626137316433616465326261353731373130 -36663631623436356165373134663466303039346232343530343565326338346462623063643464 -39343333376566663239663064373036343232376634373166643637353035393865636431366363 -37336263653534326465356338633937316337623232653931336264373134383365613565383130 -36353532666564653063646437393266306233363039616237333730633737303734323232303636 -62303634303363316531643365343966333730353631613366376636656666353964643331393831 -63383033663335653732393863633064666630373533333133383335353030663365613639353135 -30663766643135396165383234396132656336363766643665336331666233353135626332653735 -34323130393336616461643538366562323366383064323834636431623530313938303435363335 -66316136386332623234313162313838356238646138616465613236353466616566613134346637 -30643632666339663162383264363561336237656661306331656566393638343434666461636139 -63656266323731383237343732633234623164316639396633373261303764316138623733313065 -34653937393630623364346236346664303864666665376235363163306534313038343062316133 -64666462373265393133333531373132336165653963343139616664376438656361326632353566 -34306334363736666466626136663964636430613734616232383331383235343365663964646438 -33656261616464323564376137373332663435356337653237636133356432663464613132393038 -65373738323061633866343334343366646561396264373164383465343862363163333761313662 -66666461383263376237303461653937356233353236633665636233613732613739356239666335 -31373731666131363166383364373862313666316561303036666265663365626335346166346333 -30376233373138383564366632316137373838356435616338326264653564346633343561363830 -32363235313038643564613939613065383463666566383632356334303835306162643663383736 -30366162353739363432373038626134303266626663393366663833383131656139613738653165 -34373262303233393439373639396566626630363564663765393032383961333030643862363931 -37663863333166356236346463613166613934346133653735613635373734383864366536313330 -32383239666364316635366433653162656132616462396663393263373064383534666363393864 -66663563663566633934313533333766666464383735653139646436616132613634333638383730 -66356466386662393236353931636262633937303537633231636661326332353764356563633833 -62326231353832306466623532613038646464643563366636653964653630333066626431643661 -33646637623661373665643963663835666661326562363336353964643537396663353239666334 -37323031356132663761333830666537313337396263383730363237656232383638643764656633 -32363332386565356631363632313064306431343163323333313262313630383932666632306362 -61393137336464366635666665316636383165313761343339373237323339316565306636626238 -66396537653235353638633330663332396362376561626431323831363337313338313038353933 -66373963333266633532393837353563393761306531366632613161626662306462313631633236 -65386663353263313438333065633466636464616364363464313139353038316139663365636636 -66353130383638306638396135663939643733643665396439333834303339303032393038633338 -63646664353831353666666261343562376462353063626233393466613366386336326132383238 -32336233333063353862333261633730633261613538353439373730363966316532666366666466 -30353032346536393836393661373630653738363462346230306466613962323866306666626132 -63313661363965666335336537326339393062353765386534393338396362326232326363663630 -65616636323931313332333334336233623934386361366162393964353632643232613264623335 -30613266306237333065656239643363643166383839626266336366613562303666343936323063 -34643866353166303434396665613634653731393835373836633237386637653733373133653265 -63376532356461643033313930376536333930353633353433656364663537303261373532303634 -36626132393761626433316333313566396362343939623261653439346533303038366363633138 -32353438333065656166366265393630363534346237326663356132333764333464333133386361 -32306531393430366131616233656630666136373165363337323937653364303864303530636166 -37313537623065636461373638313635636466346365343738363039326566363964633437376662 -62363531346537303562363038303934666331666530326665396632343536393932363032653735 -33656430316531323931636534356334663362316561396630633737333962393066663336353162 -63643537663036306462623763343236333666613333383361623664356563663334666333393232 -32626465303962373530656530303139616538316665386431353338386634333361626166373462 -65363634613163613364666366666631313536303164613566376632343838383866663034346438 -33393136663834356162386432306665373230326666303064323033326235366131383430373764 -31383430383530343838646539326532343366386361656233316131326361666337313731616439 -66373732333531333337373332613735376637303462376662373837303261383462373135386361 -63623731383663346133643939643738333439633765616634653161626339316233646136653532 -30653438636331333839306631346466346230346266643761303465666337343939393966393665 -64656633346431393430663236636433666538346364633339623861633461356535376537316661 -37303435663638323463373637346436323561353939373162386232366330643665306232663237 -64383532666362623537333865386231343561623335353863336666303139643763396130623163 -34613033633338633061613266386165346336323533303463336465633861613136356631633961 -62616131626334633939663965316631386330393534343061333337323739313230353639393464 -31363430643936343138343765363738663937376162323334643966653566323731353261653334 -65393237646633663161313462626366313830393238333566656266353866636162376536366334 -61393962633633663736343764643765633861643434316632616638393561613237666635613766 -30353836353966373836323262323734353438326430623839356536663265623235613032353264 -65666564636235346163616433323465323066643162633033623266626630343834313035393634 -37613765646333396235306565343164373435623432656430383662346636663561643833306332 -61666239363037323037363436306338623637393031363639646262646335653265363035393937 -62663865323236613632663931616564396130303062363738643164333533333164383537653937 -34613963363362336563636163396339656335366466353466363963323536633136333439653936 -65383566366237326137616336366166633839376361303336306536653166393939633636663366 -66306432613034323264363631623063373863653632336331636464303433646535623839626365 -62383330313061383439366461313836646264343436633562623066343865303664336664393264 -33383162366330373031353533353866303936393865303535656232393566376538303565333835 -33363032386331303237366431343235393161623034386163626164623836393765346366333065 -36303831363961356530356136306266396362346337663434363964303936613239316365306162 -66393836333464346338383432336138616330666362326562613239383165393530633734636637 -66373339336133323561306633653233303864333763633137366335356534373334613461356337 -62343835643763636264373639646633373164636562343033346137303234616133313161316533 -35613863633430363936653962303033363836323761373838366665306136353737643938663061 -34356363376538356238306333623265313836666538383062346432316163633733353139646537 -35313866333436623434643830323732623165303862316466373231373564353833316534623234 -31366137366461336163333837353261303131623333643739626634373832376634346262633633 -37653833633733373638616133313933616334646434363834356338303166356562383134376134 -66303366363562646630333062343138353137313734643463383837646562653036643635373136 -31326237646131313633666432346461636137396162663265376166353239306563643865363334 -65343261613565613638326637303432643136373630383761653934636532373661373661383432 -36656131326466326665613364633537633238646332643261316165346534303835653363643533 -65336532306366646438646337303030316466396534656531353230653230333039643935613735 -39663536336163363431666436396364666537653037653134326330366434633239663862366165 -37643864383937343437373964383562633930636230373039363731326536356531633036393137 -39333036353765323638663263316530376231653232356336396534366538643536323734393235 -32353038343163303561343038383532653564393531386264313834336439336134636261386631 -66643533363561326464383565373938646339646534396561333732356136363235376533383736 -39613633333032633539386465386533653932353738313237373766313264353064633837646565 -62396164363234646630326165663666313261316165633631386138643533303037663062326238 -31613337626562643166636239633932663833373065636439373836656663616537326336663131 -39616566393766363764666134353337333938343234333866353566656237326435336464323439 -65646663626238323861353034333564393233323038333563613462336232356133346539336164 -64343662313561653964323566303862353861386663613830366366303463616232376332306565 -30656436653035633239386534393937383635326235633833386264333463653438663563333566 -66303862373536666135326434316331663534356133663533343166363336323034623139303931 -39353830386632396262373537396566656237326161323166626463386337303432306261383830 -38373435663130306565316639613232356261393937626631353336373265326163326163346535 -35613737623565303237356136653338643662663635373263666232326539343264653839363336 -64633737613435623066303664326536356364663231373362376538626262643731313739663639 -65343237353834353131366463333532653739313636666664343239383966623031353033326561 -31316530623830663830666561633832626635643565613966663261646466313131306239313838 -64646538353538646634316231656239633637653330356337376161386635663038383465633765 -38646331623536313861383465386261643333383333356361316130383733336131356238383939 -38343739306337373837363661346430623965646536363361623636313630393537323566626134 -31333737613163366239393065346131653434346135656330346533636137636232383031346330 -39343664323935356238663865393666346462616639613061326464396164336634636463633436 -65636666316365346435336365333661343961386331656264323938666237393830366330633639 -33363732326533313239326130656331656461346661656336356366336434646163376266633563 -34333131643839613662303862316433623934396363616533323261383238643332633036616164 -63346232373233373034636639363139663331323233613936393032623935633230396362663235 -36623864303332623863613036396461376262303165353564613135373735356532616165343032 -38616331303239306239393162333130343164333433613938653663323835353135613433376134 -35313732333438373734363362643031316339613962346466363838343739396230653733376334 -30386139313061323230373464353364313237663061656363333862653661666563653631616262 -31366164363830393662656564633665616331316162303266643163313937343061336135663266 -39626232656362623831303131386662333862396334393162633138376636643632303236666262 -62306436316662303436313962666465353861613031373862393532393336353664613438613938 -33323832326561663861396530316639643038316365313639656236353766623933386335373662 -36663164336531356433383937646166396661313865626263613462643930623735393566343130 -39653765316130626137343564383736366261366566643738313439383138336131373634363362 -38383365646435653331393939616361346364376465656332383536633136343765336330363064 -62353432353430393236346139326135323939616135363763373035313439316563656561623831 -38356232623534303264383239323865353731333464663434643233346339343566363937336636 -30656263663163386462336263663733636264633666386362353735646433363134346436303333 -61303739666431373464306539383765333836303564353736363762383830363161393130366636 -32633865393431383934623765653934643466346436656633386465303134633564336264656430 -30353833376336363835303863333137343235633762393166353566393437343835623537333935 -66653865333334326237656263326139333465613239366434346530306463313061663337326435 -39643065356261643731366138633433626464663965643062636233313338376632306636383231 -39623233663537636137633738626138343466363164656239323432366161323962313764656439 -35373237303931336635386130343962336633323836356439626538333963353937353632633366 -38306236623266343038643439383532653839363161393063393039356565663964636463343238 -66376233326562653934393964353637373537663366613464633038613265393832313832363436 -39396364336164663734316536356234623262653639626530303164303662653531323631653866 -62303863353037326134663565363462363664376362396338326237643633366164303835616165 -30616331376563643134626563663863633561336133313931663761643739366161376637333235 -39303266363537623763653930376164333861356537663264646361613066346534363863643738 -65646331626162313030623032353136633931363666653965623531323437383363363433666265 -63343563323632366539643234366466303137313837376136353434326566666231666633303134 -36373562376639626532313664333631346535386435636233336439666462666265396535363235 -35316339353266623962396262333464613430316362363432626135323763666364383235653934 -33356430366337643964626464353632356133373433653034663664636231616664383832343263 -38386635356335383831653265656162643463373831343531613830393636363066353262636630 -33366532646539373466306461393433393430376464353536323236316362346163306265303764 -38633330666637346133643632316336393435303135313962306237376666346335316135306363 -64636163616466333561666661313834336232663730636636346238623033333235666666633261 -31653431373833383631373364363361326239383535383465366666373862346664386534323331 -65303531663965363133323739306531306563626538623361646663626637326562333130306163 -61353332666236666231656331346235323039646632383830313932616363326339346139373031 -64613563393133376332356561396536613465663832353364396165623438333561633664393938 -61313230666237313765636131306261336238616134356336613434323536633139623735663836 -65623238346362613530333237323135356432376630333436326564616364396235336564633639 -37666161623961376166643431306634316262353630303138306531323432333030343439363438 -32316235323831366632326163386237383163663062383933343763643139613064646330653736 -37343236366234323766393364363333643864313166386165393436613064396638613235643935 -39646536353934636335383731633838626232363464373936373233646230613662373539306131 -63366233363539316134633965333035353031663064393435396334356338376264306533616164 -66393564613231643734363132613964663433353831376536353237333135626164663466656338 -38373835343232646230303430326533633733643231316130326634663239343366396138376534 -30643138353666366166343234323834343631623037313630666262636162316161376235353461 -65383935383230303265656563383035363837656134613165626434623566336531303632386635 -65383139633732623739373436326461393764376461343663613966323363623434663434393664 -38303230396336336636313439316431313835616534656363316230383634616132646465353630 -63663632363039633039636462303537646633616465646132336639353938326539626436343336 -39363235323134633235636339616130333061303464663436613262613032363138336261633938 -64393130313863373166316538353962323035383833353761333966663661306237346135353564 -62313136656263343763306564303533636464613564343934303036663236393463356533626332 -38663664356239656161363464303139376131353162653030396162306335353632333732303636 -66343732316665663964303938303036333063666331383337616264306561636563373961313739 -39356662313330333134646662373334393734363737323964316639353539323933323265663837 -39633234313230383465653235643134636563343032336539613765303239636534383363666166 -66623430346634663665356365346638376661626465353530373735333762656366643733383665 -37646139653963633732306333663363623066323135363135303265376263316163396635313362 -36656133373761643765663139613332376563343666646531303536363865643334336538636333 -63373730343662386235313533653462643164303237623839343037373332363865323165366638 -33336434663639373562613564393838393437363762613430356162653139646433303262633163 -62623965356163643739623466383737306638643533353262656361373339316666396262636462 -36663632613737623636343938633138653766616134636465366263383133343434366639373262 -65373436653230326332336561373962386266323335633864363161376238333062326232613162 -39393131303865623530666637333131386164656131383039663561613931323862376565663936 -64636665373732646339393362666137316139336332316134623061383039613331626630626331 -38623830383331323562393063333561323766633566643064323565356230343439396333316333 -62303238363762323836336533643862376661356235613138633162363364656461653263363031 -37643162343031323662373564373535623234356430393164353638653735323038613130393636 -36346233653639363535303538333732303637643363633037653735306331373639343862323831 -66303833356239666132326132323232626635353431643563383562633463633830643761633366 -33663861333433623064346630363338366438636162663230383561323030666333316230626232 -39656538613034323936306630653266653066333733326137383734666433363730393866343066 -61303236356263323766333665653533373465383630333133653934643861376438316436343766 -33633935353937396633333938326266353766626539326161343434616433356164306335306335 -64376334613866326639653835353061383062303332626230306430353437616635366133326665 -33616664303134346537656562336566666631663835333566376338643632393231646362303039 -62613039303361616339303565313539663630613635303133393632386134636535346665616262 -33373339303532616466613465336636326639643933363030613334623162623661616537336662 -39383862313936626264636630653038646339323635633936633961623863613464623864373638 -35663733326339323032636339666662633064643637336132363266616631366437323439373130 -65383336613162383730373462356433383234633539663332613865653730343430616637303864 -33313536636336636231643532643963313666353836363935373234306134326431393134383438 -32653435383038353532396336343563613666666530663862306265623138333838383539653662 -66343639393734393965326163343733663934386535373463386533316638353563653464663563 -37636630666665393161663930626539393963366339343335313932363961386162346632343432 -63323637396665643532636437626538316533343434326363346231623538393966646334333065 -33353132386532636239643436643766633764316530326561393230313533323064366137316336 -32626334636339373932393065393636306264656461313332613063646239643162663339373035 -32313637386538316665363166633831373332643766646332616434346534656135396237653065 -34356562313839363866643639366435376635336263336265373866313238346130653730313763 -66323865616535393861356666333332393065356431653935646535636431363731643031343562 -63373661396132366566636230623035333031636563373239666633363733386238336661643434 -63313762303537366365626262616537623133373031623036316661626363353639356163333564 -63653636643637303832303838306164343633326534613534623739353038373764326430306332 -35656335373764313464323464323764646439373832643838653464663263656131643937336130 -31666234303839366264333832643334653735366232396232643436356566353432313830666466 -65366638383137666163323737373839666430306465663364313061323561383437626230623433 -63343664363930373331333331656465656433333230333233346433643537626635616533333739 -65386362326533383866636564346337643466363637663133643030333562336230346363303436 -36633234313031303063333438353933663138366666353638653931623062363639313935393534 -62316637646464366665373636396236636466313735643465383666393563646636613665343966 -39623363616336386561353261303339393230323530303164313164363464616166356330323261 -36326566346661313966306661666333643437333639366339356534616166613533366561323265 -63623933623064383234613962353337653838613438653032303863343963303966643732306339 -66343539323238323565303032303534663131333435626664386462303033386535356433326661 -37373265383335383232353534613661393933356366346439613033313563376661653863353733 -33346431353164363834323639363036396464336134616266633739663363346133643365343262 -38366663373139323962633238363866613466306632396666613965383062613861656430653434 -66653931306465343136636136613863653232643136376666616132623837346135303464633833 -63386637353733376336646637336161666234353465663932646332323562343232643861326535 -37303065313534306662356338323935633836653832663233396564326430313662626462373033 -30303833303262323961633633663861616237336332376433326161326134636364623732303265 -62653764633064326335353330306332626365313532346635313033663937636634613637353965 -65326536376362653239383964393866303039343538623737656133343139366536343834646361 -62333330353832356265353034326563383731663038326333303833313761393364316362633639 -38636130303236316537656364326434663636303038326161336238643065323465313937376634 -61613062666339346331616639396564393638623234376565613766346135353061363835656364 -36306466353735646561363436643035316533363365316266323731623765613332613837323436 -30386361363537376165633236623832343536626132623266326265643963373338336235383630 -38306566346163646462383666373436373436323637363466396134626434326433323233323663 -33653135643563303164353066656530646664326433663631336536393962373164626333396636 -36346362346132653861633834613965343762333762336331653739643535383432636332313030 -34376464636532323261383735653630643534316539323662313138363064663866626537663163 -62386233643662663030623264343330623139333337303335393265616662333039663031393039 -37363334623336313663306437623938396264323031326632663663376532333065393431306233 -64663261353732303332316230626233643236353034666537343331646131386437613238643966 -63333132343634613561336364306636316362353134353163663733363539393761643731396164 -63623235343934366630313764353563636563663033303333323434303732666164393066666238 -37313361373733613537613164306539316663343032353135376537306137663634316532323331 -36323739363732616432353833326435333833373538613764343066346136376231383137353836 -32386461633937636131663761366339663238333866613832363937643033643838306332636330 -65303364366136613930353332393362346537626230383031636239313834663631643964306561 -64356333663535636437613637393336613064616236353437656135613562386666656636336430 -37313839393064616331363436633666313930633663303261653865636636613264316332363863 -64616538383332343831343964333833356137366566306435623561613062316132396562333037 -64343963366233316230636435323930653238636165663432323333326536333432356231303234 -35336438393638326239316163383538306265326266333761383037636330363963616435336539 -32643462316435623763363164363264333561613362643330623937396563393837656336393964 -65333065636363616364323832653961636661383233396131626439393939373832303234616137 -35636361353464303736346335613334623936386130383634316538663561303833333966656139 -36373037366633373130353838613764356436336239613835643866633533383164333136303266 -35373830643933303466633431393936616330323164383964613864626134653235653636643164 -64306239373234316431383139386463666132316637633364356230363735343836393532313061 -36393065323236363664653639643436353231643330396333366236663436316361623632626139 -65376538396436343965313533333466663237626639383539323131633737356237313631366533 -65613137356561383734376137346636636339333631316565356233656163356234376166336137 -61656338386536613161373031653732373639613834613638616634336432646432366439356338 -30313532356563356264326430646463663135366637623335626137386230653163643036343435 -62626463333461383664643134313763643664333930336133623531313733373465306132343530 -39616563373737633935363962386233616634366535306261613439636561653036323131333132 -38343633633833393337393832376431353430303930373063333535386139623837383265313264 -38353131653366363964663962623966353933356664323834333164366233653032636634626364 -30666265626130343765333632343939326439393237646330666464373232373066633139353737 -65616363663662346537393632336334653437356232323133386432306536343866653063666461 -32373563313539333763663765336332346437386334646261353261353962386434643531346130 -32353463373436313966653235393831623933343135313031316136626366656263326664386530 -36396530633661346332356364356130636638383363336665363934363430396536653134626233 -66326562656637313261663134613131363363353231306661646666666130623935643536326566 -34656438353136386337623133373030626362316464323765663531626462303061653766636466 -66366238653161396237633261643265643234616633363363303834353633346365383432346564 -30346230373636333664386162326535323633313865663066643065623961356334663632306662 -37663130656163386664373961633630613861343136373939633066343463343663646335656436 -64333539613538336166333862373739656161633063323335393938336633386566366163303938 -65663931306634383237633265376437626636393863383331636361653435653837316332303134 -39613237306234633663366538343135656431623530656661313961616636316330333165623331 -62613538346132303134363762386331356465373466313838373862613931343135643334643063 -38653938376262623364356637386433343530363633303436323263643065393066653631373963 -31643863633432636131373532326138336335346235303139316536666663386635643261376465 -65646234663266323266656634376563333136383663616535356662353961316439326364326564 -39646136356238363364346633373164613437383466663537653836653461646238626632396331 -37346364396262396334363639303963356531396565613438613830306162366439326433313238 -35303337356130656138336530653965353964393337633432333465306530386566336432656166 -38626333616461616337383065643634313635656633386161653861316538386661363937383163 -66323463656561623535616462626239626165316636613639623966633966626462313230353166 -31376330343631386538666561373866653831623939303338356337623430383935346335323836 -36366136633638373461313963313761356230333765396438633634366431623531303536353131 -31366639306634343261333434303264643439363961333566323462333435326237643165336232 -36373935636633343336666362393832326563356666623965623531373233326366356536396237 -65373734623539383939393330653230326235656238346536616335663333396438326231363065 -33643339616137613537643266376533326435303237646564383463623233656432613161313333 -33386639383733353334316662336339323737643364616436323732313134373733666466356137 -37316463656230366637323062356165623732626435376634306361663739303632623633346137 -37323634353338343837303064366364323965663534616466316439353336623238373639616337 -31613364316330363363333761306463303361396136383735626166326161346161323963623062 -33316563623637626364383661636164366631613839386466383032616130623661613764616435 -32356336653565623463363166613237393530636330323236656333323438646439653636353838 -36666561343834353037643731636135353132363430643564373563346531393936373233303336 -37666535323135316133616334363366336162363366383266303737353266613731386663353666 -64386264633563663735653330623766326261336538393034333365396334363135633937616131 -31663565356465303065393636363039393164663535323234616231353531323862633039313865 -39396662353866326335633138306261363263336231643332396130386430653962643134313436 -66366632303561326635636462316263643034333761616561613132626438656366646538633664 -63643038653261643865303531643662306637333062303965363961323761323536326537666265 -32616235656531356463643738306530326664663932366134393362376466383661643430626630 -63643837393461363231633937643866636431636166633237343866653935653263653434616532 -61636631303566396330323537613862333164353234363031616462343566356465373335616461 -62366564623432613466616135633064613665633463353032656337376438303531343862373864 -35396139366362346637626661626563656561656431313737393730636338346137366163383139 -39313937396361366565306135333034633035663234616334333638323237666232333662633335 -62323536643666326633383637323862663566353735386266343335366634663266323439316634 -30326237313565343033393739616533306163363430373566303262393962653431316438633265 -38383433363634316636623932656563356162666364373233323735646366653065313638303636 -64633365373239633030636332323032376339626166653634636330393266613835373237386439 -63326434623863303430386361393263313963373736643538623765333935346432623461646263 -61303665653436383266646263626134363865613665643338386663366139396362343936383566 -62636333663939656636623939396530633161343266376462633234636538383566663663376439 -61396533653662613635646165383561393539306362313835633966613764376663663838323632 -66383436643666383062326138383466656262353531333062323732333365656333333866303436 -36343664336438616431303266373034663238313736316530643361623564363535613130656139 -35363164353832643064356462313761363331396165356134343863383438626333663133616339 -39306534656430353430303966303066623231613833346562376662316435383635616637393636 -65333762336164303133636363613735353262636261616663353837393164646138333264386530 -65653136626166653165656331663335616638353337356539663430386537343734356436323238 -30313966363637376533616232613464333662363839646433616430353034303032396539626663 -65376363303761326235313333303061383965633061656664346531613431363438636563396133 -35396431333033386230636163356261353266353239323663643531353562613466663065323834 -36653638626633666133633830633237333863333162383131626130393036343065646430333236 -61626333613635323233643533363264356438306538626163643264316161336566373439383863 -64376231373163323534623437306336333839386163633739626465373566656537636466396162 -34373032333630663538373964316235666264393061366231626566353438353963313763303030 -36613630316234393130306432316331383365633031326331633664613333613463313065376566 -30356261653263393537306432356335373963366665346265313731363631326533653238653733 -64366539626134313939323034316464626636373663666161386363623939626235356265373462 -63643032653362323939396539636365346335373966623262396264393035343166373737313630 -31656662323733653434633138306339306233653031343137366263363333333761376566663563 -66363734316562613538623932636432343265626663313935313365663236316531356433623438 -66313939323665396564623162313762343934623363373164646630646265616432356337313036 -66303464323563326332333361333035313732623933373838633363656338643139313861386265 -63353331646633316230373635333233326638383737333936316364366634356436393864656236 -36303962613066313738353337383763343732633261653535303632643562323539303632316339 -30666437353438366139366130663366356230626435643566633266613964623730393630343263 -3966 +64383762333163313865336336333839306138353132343136633165346238653965346466643963 +3532376262313134313135363137616331613035376432630a303935653031326464623239346662 +65646532656661663732363861373730323538303330656537303965383738633961373939646637 +6438656532353166650a386566623438336363623563623733343839336339393930373462376138 +35623266666337323461343137353132323430356436323739323035313065613435336531316262 +34353532393735623066313730643365623162623662333733383861623465383938326466633935 +35653530383166613638313866393836393331343337326661396163313639356238383638386338 +31626464356165613564623135336665316436336130666465306530383032346239376136643165 +62346638316463333662613831316265303439396531636164386439626364616261373462343065 +62353635353165613962326530623938306663653961383330316662366565363364616538313439 +39323830613166383062366364616236353561663335636666343165646631663761663561306333 +30316437623563656532663936393336316264643438646163656162323336316533313237656562 +61373032356439613861653163393963333936313730343361306235306164636262623830633431 +66353534326161633065663466636463663961653736333233346363373963653566653166336539 +64386533343765313334343931386634383135663166336539376435303465343231393462376164 +66666666643532643466356234303961313331346439656335356533373534306535306238373636 +66666635343462363661636164386665646166616438306264396562346635626635343539313563 +64636364363165323438636562383032623338306364353536343837323361336163616562313632 +32303135373133313261383164353064656561633562653632626161373165306264373966343936 +32353566306462653066343239636535343735386565386233316239393439653131316336393661 +66303661343234666465643233623737386438626364633763356231643138383033356333346366 +61303164313838616665303033636539623736663536623132613063353531663535386564616234 +37666135623433613264383030613466393031366131326332336433623062633135373664333162 +38363738313365626662653362363135616237333436336137396162613066393335343732663465 +63383834623362616130633239336564363764343436376562373666373334623664346232643065 +33346634343764616537616363646663313165653232386435656630396534336230393065353939 +62623331393036616235316263663532393137663933316435653631663766396639366563313338 +33396363326231666133306632393665383761623434303239303536376164393837386664666238 +30643536613038366535633366643135363538356636333262303964643430353739643864626465 +39353535616432343630626330356234386538343739306439666265313338353763663336333837 +62316635393937626461613137313461353963363438613933623335613433366666363963616237 +66663830336539316661386133623338623462646132393136363930386439633261646462646332 +35656563643234303833376639386464306436333436623737623135303261353936666135353536 +34346364396234393136366666343032386137623836633066376562386366363533303932353565 +32353532643037663539633632313132313064343330373733326664653261396165626530303632 +62353162326563623265393439313165393637343961616566613935613533383663636533646161 +62316639626265336630353565386462343461396564376561346136656162303431646262303631 +34343737363532393738646430353931663334323262356363626366633862343237306339656263 +36353831346265646633653336393139376263346365373266376634396133663264623762353765 +61333634343230383932306636663433663734376638633533336339363066306130353936383239 +35626264323338333161343663353235313236356562343539616162623065663734613962613135 +31663864663261306130666564313230666331353336626137393361356235343339316238396537 +39363564623932613135366462633765656130313438616566633936303737626530663437333366 +62653965633138383762663535363763313662666538306431373034346164616237356264356536 +30626433656266303161323534666364383638636132336338363035343739626634326563623138 +35396339636139616638383264653638663831663964313036343766353864313237393830646636 +33346361373666626161313933316664386131366531656539386636316266313835653830383264 +31333335333061303064613132653432613335646436323038343361343764633131613764373563 +34343163383763323839636638376330333061356165333834303231626239666332666265613330 +33333730633735356630633861643064373238373437346562363838666231333962363535323538 +39653761393631626361666162366161623333636364363732666536636334393861333237326531 +38333165613739326238346237326534323533353264363131373732303163653139613039376165 +38323036383965353935356435653463643438643139626461643930633134656133306165613465 +38653839373433633734643133376164633862346463336430646664653365346663636634346461 +61373562633136393464386462393432613430313235623439396437313233393133356538383239 +37653638326262383231386565303436393363633335343033633830643066646265353633623663 +66666434616362376362396530366539633161646437653836656436623866333364343639653463 +39613932353333333764663539393366383835663630303837333731323066326337316665343535 +37653966613835386136306531393339663336316237613664323338336235643931306637653539 +64383330643438346133396130643434376130633163346665376338343536646339393766356431 +37653831346463326631306439333264393965383066373936643237666337613434353838613032 +39666531356661353932656464613236323432323138313331386534623161656461633936373335 +63363238663237313862383338346162343132616333653666343038646234626163346134303632 +66326362326136633533656334616638356338333834653732653531363061646466666234313137 +31306532333632666566353764373564313466643165636166303266623038336238663638633365 +32326363643638393730643865366165613862643161393334313234313833383336373161323166 +36613039373437323837306334386233333033393234616539316663316663386131383562336263 +34653862616166346362326131363436633733333666396365336461386432346164653330666130 +37333332306361616161623533316466363334326438383038616166653537633136636132326633 +65323534313063346234353861346238306238313732306161646364343633646431656435366236 +38643033393165343434613234663938646233336631366334396532316536313764363662633037 +62633935646235393766303463336437616563373332666130393761326564363331323839363430 +30306639303634613931303931396531323438666661623066323234326635303038303566356533 +61623838323231333065353034366432666163313564646165616139616238346435373631373438 +35643266343539616166633966646364666338373332613862633731653131363965663332653361 +61383261363762383834396361343932386562346435356163323361343734313936646131653637 +39333963373734316135653230306635623431336562663031303465323134303930653364333330 +62316166333738373735363662386438356639386430643061656564626432653239613165383566 +38656236303738626361353035373461396363663237393234633863623433316330396362313161 +33386237643935326165356235393738333234616166353461653834363238636638653164303434 +39613861393066383438313763626634666333383930363761333465396639356638633364323863 +31323938666533636264656437663430356462633935643232323663656262663465386139646162 +33666236316536336565653538346132396261623830356635393430326263613639633563656139 +62353035663335333665613863326236323537376663633133306561346166396231663164353235 +66343063343038386433633761326564646439623238353935316631303935383734353931356136 +38336434653932363262623235656265363239303561623963336564643563373866376363333232 +31383531313330376133343064666136663264653336313738626639623632376665666665623063 +65333365336235626530353464646563366130663531386631613533313736643165626164363364 +35393436366438663161623465323962623536326238303532363634663839326133353566346565 +35343639336437636337373535653535326630323838646662303735633934623737326231353135 +62386333383337396266613337316166303633643837666235626139303063636634363436353830 +38373636353663616130393135626263646566323934376664363837323261386336353332303562 +36613237666163313330306136643763353766376438656234616466626237346334663637353365 +34363265356636366137376366643530326462353331383235363835376562636261663535643032 +32613466643763653363306330333764626630643363316337326339663030623032346138643834 +35396136653333343934656536366564663634373132326163383736623235346465306636363235 +63663631356636326138326234383862366134623533353630663166343662613730666663316534 +61333233646666643132626261346666326233393730383536386239363737636561666233376161 +39393664363561353664343464383035626132613337646338313262663735363365666433386635 +64613063613738643038393038626435313036376538653538623662326266636135643932353339 +32346663363165366161646139613662386662393536386534616565396130313935383530643861 +64313062646339383334653664646336383437393337383366376237633138303736626335636563 +39393239376666623963303230356639656634663566666239663931323939393038316163323139 +66353332613331393166323664373065383765663162366234666264663333333031303664363932 +66373265613365353065313162346237613435666435363038613437313736393832373338343235 +30323963646461386233313333333166343663626336353932376364353434313831376164613766 +65313339616464653933396432386233356166313461323464356231613662373964333437663331 +61316165636537633864353830666262306133313962653765306161373739323834343565316134 +35336533363965336363613032386461653762396166643538616435616362353561643162333365 +61366437353830383865663366393562373464623938326338333030396534636464646135643232 +37303735373136363138626138353062616263663331393236623733356238316531333037326536 +36363065323334356534636464303034323630306433633432376464666566613366383837333430 +34656265666463333133653130666135633931613564623038366239616563613934303761313633 +61356561646238346139333463353163333763643337313538386139623830373862623830343239 +66646139303835323332383661363765366639656562346138396464363437613938373839353533 +33623935333139636661323938393763326362656530336438393462336362386232326335346335 +62333863333333666662336631633339303032306437666361396530643934623933343436363539 +33663566636539346437353564333162303762383961336437623131333133373439633663323332 +36373938303163396233623833663165663564646263333963346236333238663833323863343330 +37313061383037343335336462326239663830313065366437623939376134313338373235666632 +38343339643532303662663161636338623639613137386533366232356664663334333063356138 +66373730343133323731353130643833623336636533633932623531656135313432316639336136 +62333764306631316331383930336366343132386364633730653230306330316136343232636563 +37353232383630653864313335343137393438356162343233633235393764613564616638663736 +63316533323136366136393266316230346130326434626637656238643765363366326266633435 +30363035633634326461373563376337636534333132343635616664346264393538653730646432 +38326637633537313137356262356431336534373338306661356466333933343333333963666335 +31653861623463303265323362646636353135363232623762373263336634303539313136646162 +37383837616137663465613035363234373039383033313233336231613037633762353237333337 +34636434353765303236623033623063353061336430626661323932363039376661633432393938 +61663635663465306331366663353162313238373239306163666563346438383962623232343934 +30666533373035336638313637646233633438373130646438346337326634636230343231623238 +38626533653535636465636239653036346464333865393564396338333761666433623035373761 +61656430393566306361623833366263383431363836643664336364303933316533303130363734 +38343438326235313839356633646232643866633437376463623230656638326462346339613537 +36313766303335306630323939643533666533396633653037356131333732646363393037383734 +35303334336238666362393335303238386235323066363662656239393365343233646232323733 +64643533323062653535393561303334313736633338323037643436333734363661653037616362 +65396465376465623134663764303037346234646462353434613436393837613633356564623333 +39333131313634383339303138636338356566643661653962336266363430643835373362306366 +61343835663532663131636434316264376534343335303964326237616563656665396231396636 +61623232636365376438626263336331303531626533313234666436636530623031613839323137 +34343131663639326539326533306536636638383962343365363865323039376438353562333133 +37613531666239353464633063393365666230363261646534663664653830303837363434386466 +39333536373565656435366230383164393265313636333362643537373061383164623966306263 +39316362313335636362633433363130633933393635613761663237343763383132366334343464 +66653538633630633537373835363433663330366335383032303633346133363965323964363133 +63363064623634316262626562326439363866393261633466646333356638303362353134323964 +39626463333630326361356538656264356663633239383165656631636261626130346630623264 +31633136653765653965393432396263326632303464643233613232623662316436316630653139 +61623033383631313164313961303439336564623061323639636165303861306566653339646636 +36623863653235343730343561303265323064613537306237653565346535653966623639333032 +63303834386631353935623663333839633234613439366664383134383837323338303737663936 +31366534376437363235356535663431363461323732623435316262393839383163366161626230 +32336338666666333735616364633133393166306365363262663234646566626234383861393061 +38643363376330303234353034323438666265343161616336313131353934386631646534343330 +65383134616664353236656261386435363531363563303735356234633163353430373466623132 +39333638353735353439623164383631316366366436343463663863363838336534303363323061 +35643366306238356264383662663738316463383530613464636363323032353238633365336335 +30363537323063353338643466636261393130336533336566343532393434313134613865396564 +66303035626637373861306363306339336638393739666262303038323635323866323035393061 +34633065306265636338353862656238643664343234666537623439646534303465346138633539 +31353439373663633962373239316437303535626431666538333664323165616236633437323631 +61393761363533663031653833666235393261616331663639346538653339666163303730616363 +62363761376438623533363366356530643234303335303666663836343337646537303563353232 +64326337383564363230633666666231616162313735376466636530383165663732323561323631 +34636364376437383938373135646136326230373534333733623763333832373938346334646137 +34336362313133373530376332303366333963306533626436633530346435316165616134386131 +36376430616332333466656434396535666333396639393239366130313863356662613631643132 +62646266303863356630643735386132323638646636626561343039336365393665653532646238 +66663762326534656563316238353432313961656631383266363438303161363535626535363237 +39366365646136343131643335303931393438303133333431643866363935386135306437376565 +61633663636664386432393366376135313761373439383639616363386631376365393531386633 +35613563306230626165363664373837653637396638653139623931393862323466363236336665 +62363337306636663834383535663131663563313864373938346533323333313635343038396436 +30346337326662303039363032633238353439313364373732346633363362666665313431353939 +35623362383661646262613737346535376336303335303735613334316466383334613633656465 +30643362303861633835663031393961346365363363353437346638336239643633356338633364 +38656532376262666366316662373238323634656139653464343332326335323435623330663036 +35333430643061303365323633663062633139656635643866373263373438326638316438313563 +62626562333930333063313436653163633464626163366135323232643530653737363263643938 +65653537363261353566306335643434386539346333393430376661616435656339656564373730 +30616464663665363962366461353765616430666262633465303936633236656331303964386530 +30356562353032366237393132366536636538616636376363626437633864366562633838663861 +35666563313831343037326339346638353136353333313131356635383866356339333461353837 +34613563376130623637346138366434366432303662303934326435376139636430666436653163 +62343730613036373066663366663062323364623739326138666133366534356430613832643061 +32383062643133623331653265333131636234613064366665633932373264363930653366396361 +34636337643761613461613135656238613662623631366439316130393961633231353361316661 +34653636623730393661343161366261373631316336346130653464353235666661353536306539 +34663033646133303132626364316364636338376165653263313165323733643765316466393666 +64323961333165613262373366353866396537636438316666353738633937613535623530353765 +31653232313132626339613666393863336361326563636630303733646265303538633163373961 +35663634613065636466663762353632353631613035336331313031636439623137393665313361 +62373737356161313435376366633462386132646565383138643131396664336264613563623733 +39613139373863393764313963326361623337383534316132643835323630303037303666626564 +63656239646637383064316330376436333661396462363066376436316332303832333065643539 +38363737313732313139353062663263393739643632353130376661336431663331346336653138 +34393332343036613930383138316632623161396437373162663636636636303563336664373464 +65393136623761386638346262616632313762313861663731316635313935383335666161613731 +31306132313361306664313165373735356564306537623866353962323163343765643834626132 +35643062333063313961643864306137393638346538356264613962636262613161326535633836 +32663435353839303465623634366433326466613233326364383462633037313136653835613764 +31376135663235653664313866303433623231313034363832656131343135643763343030656237 +38663435613935316363653364316335343433643938393266636434616335646431663963366133 +34323264373138613639393830613362626435383537373461323264363766313336326130326431 +39366133306232386266623263333234613563303139626665393561353266373861373264353339 +35333231326135396137383434393061626262323332336564613536346137313735386334643363 +66316537356131313364323830646664623537326339663635366134633634323838633263333231 +36343066616234396137643265313462386432636633613263323333663864343534343931623531 +61393032353631613737326133336164616334326335343539613161633861393830353334626163 +31616462643761303736323731336664636662643930366137356338666431396332643435326366 +32653036666131393863386632306363353261306639306362613231663063363938653064626635 +64316132373263666665643162376634643138353730653663393061636134373632306561343639 +34356139376362383366316363393064386564376565363837303538323561333337616431363535 +30623136336363656435306465653262636263343234623137666533613737646164346139333163 +39653535343262396166623163656365626462383231383832613730636665393534646134313138 +64313765633164333232393631326236613062383539616165626333396336656561653035383163 +64333762373437623562333633653733346338313038393632666433316161633466353437303234 +64656133663465623864653538326630306431383032353661663231363161363334343236306436 +32616361643466366132653832656238323037393664663437643862353736616661313566373062 +63306136316330346332326630666333613333363830343631323865616534653266663638316236 +32363730333264306335616362323564353639663033303839633965323931323866383863343139 +66653763313831353935363433353634303631303538346366336537343361666466623938343430 +37393531663262376532336262653435343661323335353335616234653338346532626161633061 +64353334643561656138333534303039376632656334663165613438656334333031326638376334 +31333937353130376261363039313739356361623031373961363762333264616637393962663433 +36623330643534303164663963346466663030666638616532383732326133616234323832323938 +34313932646464326461633839616639633339383837363936356633616338323864363862333861 +30636261386165306264323134363330613438653562613263376461646638366436643038363336 +64323239353230633036343863323539343461343762323336373133383436393531356334316532 +38386535313163383330306432626333666430613963396163373231353364316264646364366438 +39343865353230613132343465653864616336656262333034386630626563326433363839643137 +35623639363732333463363061643264366439646261303333636366303034383336366538356435 +30366533373837383437366132393863366339383132343534373531313032386462376261356366 +32343538666633306163306535656362653761323964376338356466316162656561353032663036 +61376665303136613064663834626633306536663365333234643730333032306137343430356533 +64623734396338343133656539653232656333643030396662623332613432653031353832656430 +37653137613532663138316364383831333532343566653263623366626535376135633034626565 +38666666623062646432343236333962373036643633396466613539623438383031666464613433 +30383963653666373163613432386237623635663333373364386365373738373662643061346262 +66663337386339616264353832333664333333313035376333393438326135353335383034303965 +63613933356432303561313961343130643133663536623534643733313539383564343864356238 +63626537623031363930623761623934386631386236343437303430653030353962353734636237 +62613964333937393465616265656333303566313134613164663962643762613161373132363830 +30393162646530353139323064363037333238303161633137333331383466623736336264326336 +36616536356565316537313931366232316164646265313734643163363330333736346363313766 +39333339393635343362353336616264656537626138633539393866363539656266333732326538 +35666265366163333465663764303837313966653764353264373466383961616666316237633638 +63636330616139306537363037613833386666343931336461366538313432666234356636313664 +30653862343137303365393530376263666238393436616331393735663862383736363161646333 +30393136336666633961656661633836363231616131646363386439323535663736616662663130 +66393132646165313264363730646665353230323634303439336566613231393561626139313331 +37343366376538656430343661646463323933393365366664353133353062636437343066376165 +37623964356366633736323164646330343639336633663938386334323966343562303135356337 +30623439376336393864323433626237323362356130623833643464323233643666386366386265 +64326138383662303130323038363736633731623436383334356361373163333938633833323963 +66646134343638633666666162313334393136373430623935326238646437366131613862613134 +36383139653461396538366231316136623163633462393562613231383663643163623138646139 +34633439393862343265616438633431366436336566623661323235386630386639326263396235 +62333864393238313037656635336433346131396462353239306531643431386130363538623262 +61303931376530343639363934313765646161373337323139653938663738333831333166356130 +30356266633639313939666335393363366163623432373739313639393132643932313333396138 +64383339623133396637393335343939393738316433336463356462643365343165653533306132 +35623665343739363531613734373165366261316663356664636432633938613333336365643364 +37623836316337396430333935616635616138363039303535653661303562613434613262623137 +39623730346463366439363833663265303436306239333733343564343730303862386534346633 +61653439346434313232663630386339666433303033346230343064643339623637656633616136 +66373431393439316432363236363463616364323766666439383438376461306362323330343238 +30616631383233356337626636393466663136323232393262303838346530663461333232616235 +38323134626265333134633732376636396338666437356162336466366365656232656432616330 +36303663613339643530613032633130383338326662316562316632383861616636343936656163 +30393965653663623233313236346164373164613462396438373161346561316261343235663235 +33303732373761616434323639656230303161316561653866313234336438653434303834646431 +38326531393639303964353338623766613361383730646635386432326233313562383635646531 +30353333393235346635666339333039306238313037303338383365656636623266383562306435 +61616466656531336666333564393366303661376337663535656138656430383262666530393833 +63343831626361366166383865666564346334323862383466323239376530363736396466386631 +61356437343766356535623137633838656632396161623833393031303334613166633036616534 +63656235353836653735653437346464616262326639326234383661353530633130303330666139 +36613437653864356238343165386563396464353935663738646530376236393331633238323231 +38386166653032643830633536343363623339623165626664616133356632316338373739653762 +64343736656136633139303564353337316361666530323539626337333165393433376361386266 +63333139336566366333353438623363343266643434306230613163353965663438653137663362 +33653465643435633630393366616131633165393637323230316562363266313631393663373235 +64336633643231633734663263643838366430323361336136383030353864303965393333663538 +64663733396238333732356638643233313366323139656465303462386638656439306231643661 +63633531356635666464616433386663613834663066396266313464313033616631613761663031 +33636437306130653064343235363633303239306132383062323366373233353464636638663032 +39323765396565326163333635366133626262336336373339316535633063393036326135396662 +35353438386431616134313962326331316137353561643361653161373430363939623431316633 +64663266346632643432303630643130323636346565383064353762306435313035376666353764 +32336236643264303531653836393262666333383938383939636435323837666561386539313339 +62333066633766373134633234643835626438633062346235376462663165393830373461353537 +37666537366364336162313036323065343562356531326235353633393039633537346364313837 +63333330373235353763343732333836393731346133643361363339366634653539623531393137 +65346234356266346234616463303563303564643130653339636637623462396637333261663934 +37303333626564626435653134386564663531613032353861333737363864373632333865313333 +36396661623034373162323731643232393166613463323462393730656437383636353032316265 +34626333366239666563623539343439343234613431663332333932396263663466353266343933 +35653536316132646565343364646565633833363965666466353832316631343661616565643733 +33366132333134343966306137346131316134653130373239366466363661396532663361376239 +30613735396563386439376165653465653965633139333032396465323937353461306134383034 +61626334656332383237386566333036376531323066373565613261333837336166353763633862 +35643531363535313263613263656435643934386331636133366164313463386438343763666230 +30613536323136616161613038366437616133396336613631393561653464303336353363666464 +63343134653839306331393031303132303835383539663535376163646139363139646638303730 +62643866333834323762306164313561333164326564346665623563343862316638646434303463 +36613534653665613063373130363035353339336666336539316339656663363365393733326566 +63303865356230333535386435636264663838366132643362323633626431656433393831393538 +31306233373861623039303766396466623033353266363065313337356535313331376635316561 +62346666656133356136343138363666323561366136653638646336306233643662373461356264 +65653137623338656431623638323930613238313362306539623964366532656336393834623166 +32636238313162656264343231393165613636336465383837323265383738333738666134616337 +38343231323938636365303631353232643630626637343839323064323265396138303338376463 +31396130306165623132323038366231313762303361373536626364353139646439666265333138 +37393237613132303231316639646166316331643266313732653731393962303165653734343265 +31386634356231643839643764626637616564646562663936303635653232643363393530346531 +63653062383238306231373761373632663631343437396434623566383339373334623666333335 +64346133313137323564393135643664663966343432376230313632613262653739613138373139 +38633864336132656630656531306266663432613536363135333863386539363235326165346161 +61326536376238633765343632666364666534333732623763323164396530313131663862353336 +33373665616362313138393933373536346138616663653439306365353939303734623831653238 +62643835643763653961346566336536343531333132383165626533616134616230616236636133 +37383532393733646130663262626363343061623462343330376638303631353031396665306634 +30336461336233323262663137633161343163393334626333343532363530386563616138373762 +61373261323165396531383839633138376135303836363664633864306533653436646530656637 +31623766363835383936343237626432336636663962626332356562666239386666616665623963 +31356232323163333836353037326639663635616662383562363930613633393965363366326231 +32326562363731303663396631303633336465326234356436366665376636303234303766323865 +38313563383533386138313162343237636666366266656366353466313864346166353335393461 +38373365343861653361623932623765346333363263313863643437366164613764616439363037 +32663334326361336664376437366330386436363963346239383138303838616463633438316334 +38323739373230613337313561626135666539336165303837363538313865306334663831343333 +35393438646233383562386163393765663364303432383837633966663636333061316138326236 +35666336373762336363316464396538613330353233353637363364613765366265633833326466 +65653765366364656162646637643735366266656365636330383437363166373535343330343330 +63386634313563316331303936393366303530356631616536383734326539613831336136633264 +33643031333462656665653136656163326536646661646637363664333639333935333236356534 +34306666643864653562363162663861386665323433393533616434373632666539353438333563 +39653239333034303061643338633063643034346232346338333735306133616335303066383036 +33643037326266323732663530323838363464316461306331626564623234353938663861653361 +30623234656333663634616635626635323232643437373738623436386337623132383036383435 +39323965643234626566653461633261373862633666386565343836623032616662656238633264 +35313366613930306261343166383531313865653561303634323165323435613161646237313135 +38336236623932303165353637313965623137386266323766356165333762323936363536323565 +66623434363662613531363338646234313166396561366330343435633131303035343462396664 +36333131373634623361633037396432376434623139303464306366376365636236343334386366 +61336563613462346363316239633639396335343330613537353862623062343538363365613634 +63646536613866613039333062396535663164346635393164643164646433373130373136636163 +30373238313830306339663532653763346165646138613765393332386561383031313061333536 +33373261636236653161636165373936353263666536633339396431396637373966343662666466 +64373531313139353362383932646663393835356230313162303837396464633762623631646461 +38333632616233383935393831636232626133613934343234356564333061383764333536383334 +64626462633930303936333935613739326337363933343237393561353466656530613861653030 +65386539306161363231383166313561303366306262613061356663376561656339656161333132 +63346334333933353936633634643363343231343834343739333732663064316465343835333336 +66373939376236356438313763343663353031383566326464306632333166313761363933393164 +38653437356164363737636333633638353137333735656165646533386339353966373130643431 +30613331633637636538356466336633393864346564396130633738653938333932316137646233 +35383034316666663633323962316266333638316561306463383131393331373835643964396437 +33336539663462373931623332663362336663636666633937376162343932313830663639653365 +38623030326162366538666439363539393934323536386664386562393134336436306164356136 +30366630653164336264316336613339393437363037396666613663323037366666366635373238 +39373462383537643139636565333132356165386235306134626239666463393666363635646362 +33396666636637366331386339643065646639323137366134623532666438373233666134346537 +36643866363661633336616439373064386661613961653032656664323735623434336161663330 +30633662363934653261653965393566616433383361316431643230303335326161316136653633 +63633564353032326530386436613564656534663136366665306665666433303032663161353062 +31333831386666383434643734326635333165386631663662633331613433336639383162306266 +64316266633231633662646264666436333936386139623864656165383663333536663565613166 +65336633356538333435373632396138313366316434633231633535313434376235643965666330 +30623261376465303636656235363735623235356530646138383637356363623861623364306465 +65633366636438353066336162373536633337363462393430396361383437376534303033363463 +37666539353963356538643732663836396639333939356265393032383339393038383131343266 +31326532613466336631373137613737313261646366356239616539643762636565316432616235 +32396534623265643065386434643637343135333433316535643462633164363138353834333833 +37383935643938343162386635336239303734393631636161626665623762656339333834643366 +31353439656431643238663563323061313332663364396364346362623866616632363032653930 +63366536376564326434323435653536396530303237323062343462363765616561323633613662 +34306131623036313632353230323539343734336634373438336266366565663037346139623730 +62386161393135613835353336323962383935323538336239646632373932313433363963383030 +64666536373666653035643666366131336634303439376230316231306237646432303136353363 +61313563333430343935663534323737383462353232663931663630343937323763653162376666 +35306237633335636265373165336364653431353832653435663864616466613234343539333235 +37343139363865653036623761353665353233393239386265626135626366353336373361626262 +32653139376335383164373662303531623465326536376432336364623063316365613962323938 +64316632633861396132646137666664393037353238623837616465663836656161613931373763 +61373039313263316364613362326630633638333561313761623362393232316562306532303930 +35376638636233353862396533313938616337373763613839666430343666666266626331323531 +36653733353961646230373462666133653538396562346635323161303936313137303036333238 +61613165633038323264396334333232373138363933323237623631303931343762646165353962 +35613164386634323838356536616230663637373761393035616263393432373566633136383364 +62343366366663396664666432376638353333353764353064343165356363393739396561633837 +34363264623465313134386431643531656563353565663863356532386337393961623632643236 +61336531393733393635396164386234646362393161626436323162346538393132663337373963 +63363566613238346438326531306466623537303230383461663930663865373565666232323834 +37353634393434663165616430303739323638663931383066626632323137663736613431363338 +61356162663239373066393736386635396334663438613938393339386432373462356563663465 +64303436316238363031336265396138663432376361643866653163346465383962653736623566 +31393763393739636262396230366461653630336532656161633331633532313965393337393532 +35666131376133306539313961316661346436393130646132353939303432396163383937396161 +38626136316633653230323336643834653565313935666231613336623139363866636331366366 +36356336363936653762383534393938303863636138323537353432626635653663386261383566 +31306563393536316335646435666534613539656135366461333966303133326362643462643534 +62643234646336383166363236393930623163393764313665386161363930366663653637366466 +36333166666463303033306361343033363365313235633037343837623033303632663665396163 +30623430646565386637343637613637646536656337393131623031363735613434393138636262 +35323962666335626364353539336638326336393430633263613036376331396364333738633633 +37313561306366306435303361613938306632343836623435663131333731343737386435656266 +61653938636661613762376433383062303237396538396336623133633462363731623531623039 +63343538336130373436363664313733353666313937666266393066623737396339656639313263 +37376131616661666337643839386438333932333063323439323631636236366431666533653164 +32323366313930313765643030306435323064633066353062663464363132376465333735616138 +39663062316533653839356539323437383364306337386238306236386234323538326461313832 +37663063383663313338376331313332663863363164306366323835366538613930353566646162 +36633936303733366230343136623130333264373032373939366237646434343633663333626637 +39623639643561363864346561366161633166383935663731313535396331633038623965353537 +62393737393530303230346466616364356262616138346566393162376232623035613466613031 +66363834663231633262363331623534343237653762623465353564623238323866366231663030 +33326538396532633131306539373137616162663936386663376232646630336136316461333539 +36653465323835633963343638356462366361653663346162303662363339653336386463656134 +31656365353838373938373661343739393932343637636231306666663563616162343538323636 +63663039323833323761646236386434623564313863613738343433386237633764343837303136 +32343962663565653438323061326233656461383232343633646137313263666464653764393662 +39613534663562313662343164303136646333376632396339303438663264636632363866643730 +65656433353864373032626363353766313235333262316534343031363831366563646636643737 +36316530376466376436386165396333663262653339343637653461646161666139316132353037 +63393233636632353263323836626339656266636237373563376534306238336139323936646563 +32613936396461656664356364383266306265313330393635303239646662323335656165333665 +39373230383262663339316462353164376330353534663362383639393033636466663631396536 +35393562373030613063346633376363306331626438326263376132653335343263346366656431 +39613362393766653434623563643362333963303636636637333938303235616166643764646365 +34633366626363313937343061396464353239316537376461623266666239386339646666336464 +61643935343861353032656331343635366165373863663332633364313666396538353838363333 +36373030343537366531363432343833653065646431356463343635313939393462366331643431 +65616230383039623334626633356265346430346364363835333736376636656531346261663138 +63646630373638653739343164313433383736633730646530643364353938663837323461326365 +63313365366362633866653166366633663461323836363031306564333965323865663736626665 +33386265653334323461346230393666353263383838613262636432356230646431633832353438 +33626236353365353765343039623862333166383934663038346632303061303737383964373563 +39313433303066663830373930626336393738356564633761653638643834626534616233386437 +64663665613163653431653630313163653130343738636431626439323663303231666231616232 +30313766656334376464306237376130616133633661366165346364656139623335376138316234 +62323566303064396336333137393262373565396534363533626461653036333533333531653738 +35353838373734393536343333336466386466623064343833333537376330356130346162626163 +30366161626631393431653638353138366465613739626434623635626464353034363566633264 +36663364326465333534333162316437353439613639343562623430626139346435366339316435 +35333661316433633835393832656133623439366439356438633335356364313733656665623734 +64666565663436383835316262376432643737333964663039626265626134373736306138646663 +39653838333030353833323531363237643663313865373131363363343530303464643632356435 +32303130396365353139326663316335363733376234316532356566636363343663376238373530 +62316266396135323633383064326162653965336235303336643637333665396134323564313632 +36323635333035306234336161653839343331623738313962363030323830366430303562626333 +37393834303934323836326261303435646563376239643066386336643530333138303431336436 +66653662353639623163333761623161633230653331633639383833636562313639646137343333 +61623639363836646564343630396135356337353834343230626263386265383831313364656138 +38313632363365313064653761393537393164653937656463633338303333313835653433623835 +35646132306464393432643431323937353535336133326136666364643837386430626638313438 +66656434626235313061613039653266343637636437623831646133393235386131613334386231 +65623036353335386639333137663666386436393465646230663532343834326338386537393337 +39303965393836373339613134663134666463646136303161386637316336653465643530626439 +35666237646261346163643936363037666536376465336132323231336235396536336136616437 +63366531633134373432383039386434663166323963616137643261646266643437353839626336 +62303631613236366639333562666262393135393137353234326630633332346630663335383063 +35373431383630623566363863303231613863643964316534356534393537306634313038323464 +62313661326166613433353864393132363266363763333532663665663630353166376461653634 +63343762383766333637343031656339663730633938343663636133626237383364343132626463 +39343066366430356637646532633265663065366633646339333639373265303930323239656261 +38623037383930623933383263643437636231303138393430626333666264376430313361656238 +38333938386639316464363839333437353634613864396336316436333364663364316166653964 +37333638316463666662376436386138646334363031373265316561643237353162333162666165 +65663765353638366666366663626435623533333638306638303734666363346132323330626334 +30383835383339346561363331333863613935323965616235643038343439663832636561633835 +66643161666661636661626463306432313933326535343463326337393463393163346638633438 +33623239623964393064323164303463383036363432336235386661666139363734653362393639 +36313532396265623439346362653138306261343235313062366339616464313965343334323563 +65356133636364303565396362623764393835376235373636356335666136643166373337643936 +35616333616134646162326233346362653666393238326138366266623434353635386362313961 +32346239303663316362366533316166623635653262343536653966383437363339613133663363 +32663030333562383565613763663831333634313963653264326431646663633962303236363363 +63353264353862353634333562643138353630306534323064656131346566363533636461353833 +65636535383233663338636339653464353535316136613333356436396235613735383966373539 +38663661613736353235633138326534663837666337346230663664376231633363343163373232 +39643365326531636135376532373663396466396531633439376261363438633934396462643064 +36666663663533366537616364623665616361363732623264663737383630333639633335326530 +35643436386339363461663331323266636133393466326464343066323637343666323862323639 +37623363646134333431333834366133313137623434623063313739383439633230666637633136 +63356239393138306165343430636534383962316164336161646366303966303034663365633739 +34326266396166346238616464396462663463633366333134316539653865313336323531313765 +38656263653334643239363830366630633736396432396433333065643362343937323333376231 +62303032356530383137316261306137366564306430653031643866393432643737303434613336 +35613638336337346332633730313430356533326534366636313266306561633832626264363334 +33666162306336356264356138393764333164356366346338613966623961356265376537646337 +64643062363732666562373934363233613133653966386461623633343564323066323466633030 +64656261613130343932356437363238623531336136373161613836326664336462656531303931 +37356263383339636531333163666361633737313861353937643162653531666437333966666264 +64323031396664356331666530396464303466383661393539653539336366306237326131346438 +38393836353835336635393632643734353364383366303761383562323631373063373537316161 +62636563356234666536643934356335656665323435613435616461623531336463643030373063 +63653938616536363337306634336434366133336237646633313033396433316136376466393630 +35353939323938656236393233623335313635363139623263656463343934666661333733373366 +34333361626562613937616530343464646339333665663964343165653037323761643366666631 +62643436313961336664336365386333373864626363356531313732663637383862646336323466 +62393933303137346134623664366434303930666438616333646635333138333833373865353533 +33643831646561626463383634626562386637643036303838633933633036313233346434393361 +37646233356132616537636561643561333431643266396134623366646539333239663163383434 +36376433373564643232643935313431613337396139613939663266383034653836623063366530 +39316630356164626263333434303366386536356437613663323739386336613135613033636434 +61396363623332346437363863326363353330353465376436393964393265316464663761383762 +66646232333061383965666137616539643538303764326263666439333839366366373264646466 +31313631393035323962383631636635303438633065373561313034366536636437633465363135 +61353730366362653664313761313962616562393766386262613766636162373761613465636135 +64313131336166343762393039633739656237393731313036333064343366306330303133626566 +61343536316431653164326434343130356262656336613362663230306562316533613362386238 +61343238306232383936633934653564636564363734333430656136376133636132346537303461 +32636633383039396162623732306664333131663332336661653536613830386635366665303737 +65383637343066396137623461313265616537393134653763303837386130303730376531613138 +62303863373465643531643936626135303032316134623138613334303536343231666336383638 +36633232363931643564663734323436306336636337656563393263663830323466363862666339 +35363765633334323763336164646534643634393430646234643262653661333661613938393034 +36313031656264316235343262626562393332643564383630303164316235333534336462643864 +30306531343338363264613766386632323437666666356631393532363363326466373230626661 +63316533356265623333326165373966363730633162666535643436373365313334316432656235 +66303435383138626162626661636130376339613231623666346563616633356635353230373036 +35623237386532353435303461666162616661383834313433643038663038303534303930303239 +32313139633262623732616334353464333432363931633330653261636432373532363763306362 +38366533653838656538323132386538323062343665376437306163393331313332383463383436 +34303838373833333764333261313733383065346136333637363161653262653930326161623866 +39313437393336643263613362663563633438316366633433646665373136336163626131363432 +38303761386363343931333637353238383862373262623664313538376432626266646262616138 +39306136373636313663313462643336333066633333616365613433353563356263346565626637 +63666166613966393461383934613030343738613833376661393664366233373130623138363161 +34643137383366396236623135346665623932613534333534356134363365343537346331323138 +65343933316432323861363836323764643462643030626563646662613061636461363831376130 +35333731653965356162333762663734313637326465313662653433623338373364623263616166 +63376562646131626637613139613966306666656233373434386232326433316335613431663266 +34353733303634653130633565393739623730623738333433643061646264633831336435656536 +34643936646337386565663037633235323437626537656535323232383965656663376431393930 +63643465616333373538396637353431393438383535373832393766383861316366363333636430 +33633461613534366532306132653265353862633936613662326638393464633339613361663162 +31353262313365396135663962306261376337623339656639323331633231306233333762303130 +37383063666533623966376439306162326137313230623630633866393066386265363438643065 +34613265386530636234366237336265303130343239633666373165633531626134383061616366 +65326235613938383166663861393361313965616237303466656638626635666131666364353764 +31613831396161336439396636306365653662616538373037393765666339323162616630336663 +65373762383630663164363335343237373163663039623833633738336539396237626133373163 +62643837313263653236323834393065373764613031376564333865383164383366653537316138 +35633833633664393764363733303533613034333934353237323938336163346165656533663861 +34663836653063616265303439626165393230636566643363313763303233346564363862306666 +39396331396231343833386331663565323934646532386638393235623831346236613961623563 +32373033393264326563376131663362636237616462626666646463333662373262396433643264 +32663464396365653133363831383831636139353963636333623161363465316337313561343263 +34383963376661663137376364626366613438613937623263636466663762666530336237393234 +63383337363530376334393631363964646237363538623531383731616437616432383532613939 +61393637343762373030626664383863383462333363343465623364333235373530343438313338 +64316264643964643364323665376461343636656436356234626432653435666132343963323331 +35326431303864643832613130393063633966373661326231346136333363666230363934376431 +35376434343434626662653962393462316432333735653632366265333135356266663833653966 +30383663663038653733316638353565323063323532623364366161356531306434353738333538 +62343736386538643266653939643139373164313762316635343834363161386365613031346561 +34643062636366353239356330313365313038333132343461653165343161393639303766323331 +62646236303565373532646139343831633638343437373165333938303435633334663831663631 +35393365376334313533323565346435336164366635643165336239663938393438343830613364 +34303565383666306666366232653966616334616236306265363861356437316463346131386166 +37643533363237623636346561313031656265313931386262393862356265616431616365616132 +62343930343661393965613830353164316463383036343331656562386263333165386434653030 +37343437323035313165333531343334303463616134663934636562313063336639353463656364 +61303035306139646337663132343237633564616566353563303130393363643861326136613065 +66356431343234646666366237633837633361306165653765633032623836343531356166303133 +65363332643266643761393065303966346330316365336139363535623730623661643763613335 +65356566373831323663653463653066616433613330356262303266666265663232316566366633 +62323135306530383337326461633136623830313138666664663065383331353038666237623961 +62363233363639386538666161363461306136663637383064343564643639626532623261666562 +33353763633534363861626464323166353466636137643438633164336135616230303437363535 +39376130623963343433636439663934663739663638383633336665623338303162633163663833 +30326339333761313165393230313831376533323131373633303564363431613231656231653361 +36663032363366333035363431323934623133346465343339386464306236353466343233303037 +36393036336166353235303737646439333332653430313362353736623035613865323439353032 +39336366663535313236653332656338353364363163653632303962393830383636613139613364 +65326164346137346534633030623066306332656135346330386161613830616339386438343238 +66636431386361633763653731313831663435373163623234306136656265363864323236323161 +33323263313361616630363663653932643236643763633139393130623535303039353664353439 +35313037346666303963366261613431373663303838656631393638383562346131383164316266 +35363231333861666232316163306266363631383933393631326266343865303330353436336162 +34353361666663386633643236623538356639366131616134636633353666653838323737663237 +30376134613663663038383433353061623562353237666665323935393439313730303837613562 +30376532383761666662373239663137613236656433396430353462646463663661336164376134 +64656336656233386437663965373630393035643662333362303834616362323762353063343363 +36363662313464326330646362636434386438393430653132613166306331333632383436313130 +61363731336464373065393266306438356463656532353063323532343234663065303036336530 +62313134306538626532383335626639396664616634666264326133313434376461646331396337 +65376466646666306362313639386632346539636361646465613062303962356166636564316563 +63343362346538646430306664623636346165303236373264613063336634356162653032326233 +37623630356564383933653965323661623561373465626330373838653838303963336436636439 +65386661663035626635613636383965346135333033366231643034343533613732666339366332 +32316662343038636530653734393533393531616539626432393133633632666635353135383731 +61646432303136393562343161633832353530626164656161613734363336643639393638333065 +32353762303234663234366234373239636139636631383632623733363334346131363763363634 +62303736353637383432356137346663383461343932663063356262383264343131346535653431 +66663336313138353535303332663164353662376133336664323066393062613032626235653562 +64623730653464386230373733666238323930623964333838326135653930313465313537666465 +61346365356439323462356532353831643065623364626461343136653664303432383536366564 +65643130313261336533613665353465653366666138626137396139616135316537643834343438 +32343039333536356464396234643461623530643336633637323662323338663262336633303665 +34376138643336353163386661666364396162666339613839623335313830393563636531333365 +62386562393062663136353634626365613235333331393763353530333234316262383866623931 +66353934393330326337616139626562646138353161356339343936326138356537626235333939 +32353536353939346133386635326337633363323436663637356635326231376230656662626163 +65656332386138653164326634663865656234643933366366663533633738376164343663353362 +62666630333937633333333932636665393665333436336561376664663263306134643864383138 +63363438666535336235373830646633346438393031636331306465623561653930646663333766 +66303332356464303961386138643033373565313864323032333563636462633266623664356363 +38623761333365663933663862663430303565646332376435346134643230663664356561623730 +36656331366238666262383539336462653830626461323834383237363763663034643865376261 +37666464396631663338643537313236386363376164636232363133623531363161656531623331 +38613466633163376263643734623530303964616430306135313562393039303839353063313137 +30653361366536313739376362626436373435326435343061356633323139646663633734313830 +33393334396232613932376138393537376636396464323737613835636431636233646363383962 +33613939333266613433343737366662343039666662623137356261316564613965366462613439 +37353430643035613865666335366166306662343263663166353930666564316139653666363832 +65626531636531326332353036633939396330306662313163383162353937656436363861656334 +62393463383663613433346436373939333365636234393666333035303765663131356131396338 +32306338373363346138656233636264323666333038383237313830633162336134613262343033 +36323135323135313965666363646432313337626438643833323137623530393438613964353362 +33643534363931613232666435376235353636303862373739376332383932303338643836613638 +34333038363836626534646231316465316165366137316237383236316366323664393165326465 +65326164316532646134386263373035386135373664393738363339626432353861363234326236 +62643434663034636338663362306237353839313063633834653539663735653965343965636432 +39393964383864396136306232656366306430313666633866643733643637623733626265666662 +33313130646230323331363432613263346137313365623165613364383939643732616435376238 +37653539396630333164393938663438633132333937333562356665316133643565663263316531 +37666261616231653038643130656464626136363539366363613761613539653037646639303362 +38373931646137313065613333383162656538343263346135646335363965326562316631613463 +39346464343461636535356334616463353934646362356462626633643532373364346164313339 +62303231636431363539306631613232663931313664396163356234363939343336623437363535 +39653865306335353464353265656237623034393764373765653337643266306665653863663530 +61623961313239643736303764376237643432323236383333663464393638613333386166363031 +37646334643265303365653166353134326137613731323135356134666536613231303463323138 +35613964646239363630383763633439633766356234383334656236306638373565306133336662 +63616635323035356135396331396135396461666464353964306132643438613566393936623135 +62633132353135393132343464616566623334333130653932373930316535333066323837316365 +39373632303462316166396334626631316364653032383061383834346362323734626139643564 +63393162663739626337373131303838313333666632373437656464323330623463623837353862 +33333734333536353230326262343264346162356635356162383863346236626361353262623765 +66313834313331306634313064356638386431346664393831373961653031376366343263316261 +34393662303231646434626136336632393532613163336436616466383634616364633331656662 +64306132326163383530386131333865346631656139343862663633353962616230346634303063 +61383239616234636565366132656638366363393061663263333533643133366661366631323736 +63666430396333653731613037616535393635336364636530316139663362373862326334643766 +36343539613832366564613638663331343664616338633636386437666636376666353335376137 +65373332626137636335323131646431656337353262343763393461346333386137653137633138 +36643863323863336637386462386361393730623230376463643363656631373230353566363762 +38653466363964363766333537333461306531353461373866303564393637383664646139643134 +63653963353937336238623463393666623464656361306663396262366633333862313438346232 +36386334633137383865376431663866386232353062343233383235626639343865653332336231 +31363732393730373665333662626437623133383831363634323565373036346330356634383261 +65636537336533353137336265616465303561653233663161316662623238633065336163376361 +39366633356265663634343734313064373833326562386366646138643339316431643331363663 +32376133626334666635383661666230313530643365386636643162363539353865633630376132 +31353634656538393863343261626634383438643232643439643533633236363336323762656364 +38623366333037623661353833396336323065343564343535393535373561303666636265616262 +36613135656338656436306436643535343838353966616536353936333863643034346335356531 +31636132656566623337343837623435373366626530373363353430623162353838396130393732 +66323332306164316336636236353365643262323562396433303863326139653161306435366661 +30663434326465316434393334333635343136363039626139633236343133343031323963653866 +65336566623761623239343862383730346230336562366534303439333536613437643735616630 +33653563383739343031353663326363363433326663643931653730393430376462306266626133 +31363133383437363163306538323934366465316464663835333833626533353263636234663831 +31323934386631313239383863653139613836346231353364323630353734643533336232366439 +64326332646461626131653232373662383131363766336433306566306463626166356364396139 +33396164643433323930303531393563656438366239636366336535383632383538353534336233 +64393439333363646436313935626664626633333237303732346230363861373361366131623161 +35633439336563363733373937316232303334376233383539356661323534386366643933323035 +62313039653133663835613932346464363562333236356266663666643130323833383161396565 +62643732636465613466323065643266633531303139316633316563633263666136333065373337 +33633763666464626237336366303765326462643763643137656561396435396664313833316263 +36316264623231376461373230623332366264363739643337346161323832356438616134313431 +36636538333463316632656130376664356230336266383636643638626634663163373932623338 +33343266393331386437646239646565663737386131353037386230373264376336636438653365 +34356664363736323064326561653039666431353664643565613663356130643439333934643761 +63616562323533363930663366306231333565653433623733616166383035613039303634373933 +30613163663264636261303733333333666461663034313466653937313538666538313566666166 +65363435386665366336623937633663663838366162663261613438613862343935333532353735 +66393739323261343438623964616336613463383664633466303763623035313734623739636166 +62306361353532356535343330366630666430336461336137383834326436396565653262323333 +63386562353462346632373535656462633435663935386636383436336238396134366661333563 +33373064303463653862343065333838383533363661613038343236666461663164326435616538 +39653336616430373531363265393664386139666131343363383465313435376337383631666461 +38616566643065336336636638323036326534323036346333353936616233366332366332656639 +35393330343462326231373239613464393830333031363637323936313732356566623565623833 +66376633326437323666663066383163363035313464393338346365303262643165323061613638 +32643561363435313437333733623832356134316361353363366335613865396135653430623530 +33383464393063333163353333653636623161363738353939623031373634313033323032333262 +61633937336165303864326435303336313065353139373661643339643637393636333264323134 +65376463653364653432393139303361316437666364643938366161633730366237393734663637 +62353533626235653839633835373039636134666532363465623533343464366232653561633465 +35663931346632353933623234323132313663633532366431626633363965656533656138393265 +36316364356664376230333364393464656337346461323034323566633130623234643364333738 +38373036643161383162306261313434356634303732626463356638626333623935376630356137 +61393763653862306261316136633930396363666538623566616662653032393537666561396131 +63303336333661356537383732633430343662653733386138393331356330656134366134323666 +38653339376464353433633334646430333036333063363461353936643532366330316236383936 +62353866616632386633313032366533383532666237313663383462363039323135613435393563 +63373465373162316462396562373930346566623331623662396231653761666238336434396463 +32336133666438366265363438613332346338643438663534336633346465653630373933356530 +32626633613434646136633139346563373636313065316637633230646161373438356530323762 +61653532396365653664656166323463336331346162626339653533333664343236323563346264 +37373736666261353539306537623634303065313162643635663263383637396563373137636261 +39663263336239356236653730636436363561316630363561613862356338636564613932663735 +62353239393637343534343335633632656262383365666463306635643937333865633137303730 +62386165343734353039353766303061623766303939333939623637653964633736363666633761 +62656531346330316463316365313063613066323265643562373539643530373736613365396539 +39356631316462313432613636383565313835623037653835656537366663613066376638626162 +61633366653738616233353762323334613733383633623631343364313664333132363732626339 +64363731646661646666643736356630663361646336633766656235616361326532346636336432 +39306234356237643135656539626462613739636161666264396633313264613039313262366561 +35373132356563306131633535303963333735346364343766386666663866633034343666626637 +33653938326538343730613130383139366563653263393432363432613062303436353635376463 +39326539343931326363363033353430303565656236386265333230656530353337373261373366 +33623363633462666463613936643664623934366261333233623732343636323337323637616261 +66656235653630386561383432623931623238666534626434303530363165633062313265316664 +36316233363061646533393134363437333031616662363233336533346334626361333764663432 +34323330396365393763656436393137633730336366323538333638363333376661656336303263 +66616137316635333165333962326439643732633838656264663735623231336132656266383162 +66626666373330363334323838623764303837656338333431633232383039393334386138396434 +35323464316437366235636435613661313733616331636264316234303661656231656433393234 +34313637353664336137646630383236323835306638643830383139316263643762396537366131 +64646537333662323633623535326334373762323239656633386336373634386364383234646337 +61646361353032393562326330353731333735353531383733616438376632363264356461383537 +62353966313561626361626663656436323230356561316636363463393738306137616630316231 +63623139626234623438663033636466363337626263396263663634656365616634336435666638 +62626234343533666562656335326434613939373461623033323935333534363031316366666331 +64373435373136313635313966646434323137306334393537653761626137323439636632303635 +34393631376565316363646665383266643131626238303637623537386132623962303430346434 +65353566326466613134333662343537616532346639336439353566653466646161653938326536 +35363339636163656432323661653239343464653064316531303038356661313965333464383061 +38396633633562336434626135613565653463633465336139346339343633313062323130306535 +62313263323735356134646361336334373531343265303864346232363832313139396438623936 +63656531336365623636616639623765356666323632653163653234643861363162356634643039 +66343134343237623437396439386365333738653934393337633735393561633634363466613765 +66356139386138633766383061613831666564336530633834336264376137376664353239636262 +33353038366530666534363034326539663164353034623863383164316631336463373663616263 +32616434303336316465333661636238646565633631633265633939666663376235313235303437 +36303961613731333065663337393835323232386539383034313565373431636533613234666533 +31386336303338386265363032356637633866626432633331383661373534663562616339383264 +64333639356135633361313534336433366131353038336234366362373038653335376332636462 +61326636323837396332623330326334386661366363306431386533346662646364613438363336 +37346633663762386438653130336134316233353737623933393335636530383631656365373134 +34333432336265333332623230656635383137353531326334653630373936393265613838373235 +64383463326562663132633836643835373431663965643939656661626564386237363462386137 +33613932306564376565643037356364326464633263636566323436363439393034643564646239 +39316564343037363561303539666130623861626562373036386530353830333037393039356366 +39303433633532616137346336663232373334333531393264663432373231303162636638333536 +34626263663261666237306133303536386536376235636533663561313432363961323265613734 +32363136343335383933633635313439336565376532353134373465346538653765616134326237 +31643130383764623430353866313130656664366663653635396332663334356431623962356538 +34386232386664353937343236613065313739646136646164383266343763646637326366653963 +36633736376263323731653866623134613461636135363930643731323866363363373839343863 +65653234356465343538646666393839633936666661326239323532353762376238373733363330 +34643764336533616232306365393531653638313264333864393462626230393033613565623530 +36376263316336346537633962663236396432663762393730643562303333653033663531306235 +66663432366639653464386233326533653163366133323831346635666430303835333936333665 +35313261613864343962343963356634623431616531303164353166393761303336393665333662 +66393537363765313431663233383462636530376237666162343831393530326361383338333161 +35653734323766623163343736653730383634306130383063343330666635303363623933643434 +30346336643637356332323137636135373966333662616431633332373132396237326362316236 +37323339303738346339316365616465623633303737316431383336623237343839313436386231 +36376331626235643361366638663066373031376131643032626565613231393364376436346364 +33346237646534623463646236393330323237373633613564393934326561653238383761313766 +33343838333863616661653234333865636635653032623831613733323633343033376164383335 +33663932373031633365346236316536323031306639306465656164373262303835326536663061 +32353038343936306534623135353034356665666138343434633763303766393738663436643434 +61626165656664623534333138663062376431336438616464396533313535636535306661366564 +65376261363136313365333038343161326464303762356635383566323835356361613630626631 +37316663326132653930323233313337326364613365393836626236363838303832313137386363 +63313164386464353435396165326634623636656164396237323066646461376339316161663234 +61373361336338346132333336373039636435653132313464653962316635653535343036626336 +63393465333034653366646230393831363135376561616330356264663239343731393162303864 +37646362326534623865613535623737303765373961326434656464326634653662393166363734 +61613964616364343635393863653964303637393963333438313763353364626332333737643739 +62383264653763666234366334373936666262653436336533383237383464396533323330313261 +62636566616536376565623733326137633537313836386239373032323539363735336336306463 +38366539326335663832316331343935376434613231616664353963613837393031633765363565 +62306366656461333636343737616639653538396238393731616566333962323563346563623563 +63336533396266303630313230313364343665333863303630343730356636633066303939633864 +30643965666138646438333438636164623237316161303637323365303532383936393065386363 +37383732343230363766303033643766393737633731346666356639393766353561346264613730 +64336361306361313532393938643638626534356438346161386639353335396633323537643730 +32636331636133313965666666336463656537636363316266346135376464636533343266636364 +35643238383464366531353463636235666330623430326565346136643633343637326161383935 +66323138323838366530623736346438623938666230623233616438623662363933633635633236 +61303339366136343032346636663562396638323463353861316264303732643436336335353462 +35613066636465343265633134613663653634303131636464633636386435643237653163356534 +30666632386138653434303030633835633130326634366661653266356232393866343565343066 +63376263626534353466316265643035653732326236393835373565326466323330643161303137 +65616135386366383862393361363461383464373464323932396131666638653433366561303930 +39343061323533646263613963303338376139623631333461326530633431373134393231393736 +61656564396564613261396133333735633032646564636661633262663765666438636261393530 +61616163353432386335643066343564353232626536356433353030373765653563323231313935 +65346434373636663865366162376433346339343332646664366162383030323936366663613035 +33633762666666613762306563323432393335646237303030626330373634663830643733366630 +35313236333063376365643538393764356463656562303535653634656463616563383530316430 +38376138663665613130653532323966653538666563306537326364323663323437386430343333 +31326466373730623865393833346435313338646561333935646464613862653037353665313265 +63666461666662313839396362396362323365363565633065376164383830643933373334656139 +31623732353131346432636364343639373437366239633139383439303261363331316339343338 +63363864333764346434343534366662643839373335393965353637666562346566666431626563 +37396435666139633231633339333762303563303436666437653834303533346139653663353763 +34323863393866636630373438323165313030376133346462346334326565623337356563663430 +34303863306134353737373735353461383834663436653535356134326139633437663233373537 +31663864383465336431653563666333376331636565366331616438323737663434663233656231 +31613930386439623739363261306463326266343161663931343034393132643333373737366132 +63386134636537393230316239306231316433653532343032343338333138376239386163333663 +63383261376564636336313664623736373264353631343538303536623930653165646232333736 +66346130333736633731616665336430666466313339623438363136353237303733343761646237 +61613432613430663636353139383633333434303162613831663161623539653066383764643436 +63643865363366613765613836306566643633373563363932323735303030623533656562386331 +34366134373464323830346238666237356238616335613735656263373832653836323538316137 +36303733623966626638313238356334653938623866613062373332373339313138646230393835 +38363666376461333832343037303937626533363436383136633230363137386565316566646562 +64353362666464383761363061613237623733346136633933383937306665356139383265616339 +62363536643963383064343833643863363261616366333938346432303336326531356163623235 +32396632643462393038626633623036356163626331306132623561653933393630613336636365 +64316137376364323765303638366662393063356436663330633832383933393931323631373632 +31383939346666333364393963356333623563376430373564333465366130393366346637333261 +66393765613365333163636361346138626638383734346632616666653138303933363263356235 +61643639393136343361646363356362346233396135303464653131383335643437376139376534 +62373733383034653831613166396161356664663662626333363337396633663464633239633035 +36323234313833333233633631303538613732353337666636313237386636623166303266633430 +64333362313731343963333766623162303162346337623734643238343064656233373831306139 +38346466363261313235336533313465386563643034363735613232663735316133303635343034 +31656534383835353832616338363037633962363231653637363038386262386566343466343961 +32613138373064363263306464363237343433373535623331626165616666646164626266636161 +30353336363266316335323135656437366664346231323432633061313730373131376163366562 +38393465636532336166343831376637353735323435303631346431623462383064363035376333 +65616464633163623265666562363264383961353461396533653037613833616130373763626662 +39306436633334303665376139356262303565306338613739653963303235353863626463393063 +61366363353439373030323336383932373836623434383436363439356262376636376564346331 +66333537343332653164656132333936623936353962646362356337353866626438303833306335 +33613439343431616235323263393232363534343561333938636237386134303330313466613930 +63396434343530363534633261613762336165376330623835363738336634363363623762633337 +36643432333739306531343731386636366235393030333234366335633561363332666336396434 +35376235333934346537306134373132646164626131303866636234316533346263343362396162 +33343738663066306536643263613661613933643433363662393065356533383033613338666666 +62616663393938353938393835643565626530643261623866373930386464326164373331373532 +36623337396332643736386166303439636662663334343034626637323664356563383135663265 +66353637393336656333643537623833323464656636643465323436373031363839306334333139 +33623535323234323938643837316163346239343934383632343766373137316235383030396238 +31306264373730383664316132363531616132623633623266303836356432396434336433653462 +32633765653161306338393566353339663834373931613762613337393265616562383166373231 +30373434376437656265303038363634356662666363616534303732646530666461373462663337 +31643663636430633966346431366230613134346134323662646430396130333833653736366464 +34643165323732323366643936353562656663623031353634373733316631376465326534636438 +64366163636634613736353937373865366631623636376630666531663362356361643965326262 +62323035386139373866363432373064376230306630393636623666396663626161373738396538 +36666161363631663664366233613734376465346465383032343131613736343663633838383930 +66386631323763666639613831613939303937303536646263316639613962353834663434363566 +32363730626238306631633562616131363438393533313766653034363261333735333062313162 +63313065356431353538643636326365346637306563663662346236373636623266626564376466 +38386434336135646533653361626435316236396565343264633733373962383262353863303935 +34326235306638323931656561663337396530336134633630623561366539303865326137393763 +35393561383538616231333161323764303732303234373363633438613136376265396362633234 +64633761626632376363306637653463666234643434343932613034306261346663376564313665 +30613033646364616435653965333932363565363335363831323665656638393031373835343263 +34313830393831616133363037323764356564356533323639623839636535346238613135353837 +64303766326264366235313864626261613766303832643466326332623065323138653932636438 +37396532623236613564616439343030656634356531396639356532313366633536346665306237 +32663966653239633861386466386439363565336137333966346139303339313035393363633965 +61353331613631363432626663346664306233396663306265356634663366646434346661393161 +36326665613936316262343530306237313865653766323461323163393831346338313434303664 +37366435326666373833656661663561646134393330653432306332386335366439333066616639 +34623866333035616433376630353965313238653461626236363765633232326632376162316266 +31633638363963373836303865316264616436373031396231316634313666333866646261303863 +39663236663037323761636634393466653162343330613732373566336533396638626163363233 +63313861323836373864393230313666353032653937393134333730353038303762376534323061 +31303663393961343031636661393336306636393765616566356537346665353330353463663839 +36386132376336643066633863623132646433313461356564333132373130323737313739303235 +61333464383732343232323237333231353033336535323232623463356236623064653730383764 +33373537383831376132333839353736386531363739626662366431306636393839663564626132 +63373336333938636630623461313632323564653730356337323738653866373430613930396661 +35306138333463633930633230373734353165383266623964366565323639333437313661343265 +39653561616265663934643762343630633166313861356639376238333138653763646130366464 +63356438386261633132666265343730346335313838346665396262323138386235326366626136 +66393836653235373431316133623830323931386366396461626337323563636230303734623437 +30313362373633663039343534633464636136636234323262663435386434366334353062343338 +30383530343439383862383931656464643838346531383361323839363134386264313131356334 +33313633353966626465356338663965316561323965376130386662373161323139323034616433 +34393864383434393831313463616132626662636566643530323437663961353938326263346132 +62323337333263613231376263383664333235306135333933333065323533366665646162336263 +65346130303561633066323030363932323237613237366135653438366466663761643037656361 +65313233626535356334333236366330666564656665613438646639396563353837386664636339 +62663235366634363265386532313633633238663934383761386331366362353039656633633661 +62636433626633306434343132366538663736336666303637306232373963613065666163653230 +38346665376366333036393735376439346361333565303130613534366638643839666338313637 +30636239326166623835633936613863346537323231623362336636663764613635396537623030 +62353364613631303930613461336132623765626263633238616238376662636432353631383330 +39656438363464653033663637313061656232306562373431313661396236666331376539633639 +37376330333331333639393039303336303738643432663866356365333839653334393837363230 +61653562356136353530386362383234663065353734336366353861666231663138626561343435 +37353763383338346331363464303866373566623061633836383939343834326430313061343633 +33396536383237306365303936373763653231616463383966373935366464336636396539323366 +30353763323838643437346635363461663038393064323761346461326233363236303239316630 +32646435396538613435663130393466356630633863376232306533656137353937363730346430 +37383531306132326633373133393166666161653930326166616632383035656262326139653935 +62636366663131666130346536326431323137353465326134333933373432613232653434376162 +31383430636163626233666530393430353965356234336266313730356161366532623561383366 +32646465346435346465386532323530306235643237633237396265656262393338346165663738 +36613163653030613931326430326364306537386333316365616461343664646536326231623331 +30633330313333306663313236626665356465623636326635333661663135343333346339636637 +36363034653364623634336564363833643434623166666532333039346531373831613263323731 +39636435666439613261623862326432333063616339616562613434393062333038333831666231 +65346633316239663165303439343763616262653538343235373030343561363165383532383664 +33333761386137646230643865653138336461623462666236396165383537383834646437383831 +38373964363465623061346330383965383038633830343531326661633638626561636133343230 +62363464643535643832663461346565613132633161363163323834646562623837633665326232 +62363666333934303635366235626337623432653666613532336537663736333434653164633661 +62363432363631643833653536613733386462356331356564323639343638336261613165353336 +66343761663630393863663938636464356237323962373737663466356132623338383134306563 +34313037666464666266306665666137663331363865363737666262626634336236653263613732 +61623632353036653732356362336263306162643163366234623338346139323432316535353333 +64316233353062643438653933373839393333643765646638636363383361393265613837373130 +30646266376535343139383038333134643866353564613565613864316233383538333862346138 +61323735303731316536663765376165616436316438643632386135373134636165363034306439 +61646461363931303931316332316233646434626630343565346561343336333466383437323339 +62323465336635383832343134653463393333663039346436383838303266333864613933346438 +39356233656132626639376331306233633837313932626438306265643166613961313364363632 +30383639323066396432326664636661613066386364666630313736333533353636623535313133 +37393535346538666434333435353136343763353835303230343236393537326237356332346333 +33643632313238666137356165613061613932376339303664633064383664656136306138373062 +36346162616263366439613438616534386436363665376133366338366234613032333136383566 +30613566336466373337643530306235376662643864623065623165393334326132626635383438 +61343035636362613934376666383463303039336165643737313831666665656330366235303837 +31653661373534326361613865663237353132316436383434663232353561383964663138613733 +38643163633366373630623566663739623939366463393530383736633932613966353665636435 +65666464653435646433666132613936623262383635656433653966653730626434303433633935 +66323234363331643937346437626131353431333162656564613363373330326237623862656663 +37623135643031626130636565346632373938316538663135306561386533336430303462623634 +30303337613966333933643730373734396565646139343838626333643066383865376562636132 +64363633316337646165383734306332363937623962653434393966316331373430343761643064 +33313332393765356231376233353932636635663335323139343331636334383934313830383734 +32383630393337366330386337336535393263643566353534323764646534633136333439383532 +35663935636536326131313364643365346638366139643837326262633639633131623437653838 +62633564303734333031386332326334313036343038636338303231653161383062303333356366 +61313864656136386430366535363238616531393962633962366565346534373737616531333935 +62373939353633373731333538656636646130333234393532366166613331323762373130383530 +33653730653433353862383264323839396537613962613166316662306661316430626262383238 +62633732393539323733376236393235376662316233326531623038393433376131663333653434 +65363762653364333163626464356164663337643533626266646135306461353965313933333835 +33663461313936316336336339343863363331373164663436643364633966346162646164323034 +33626165343439633434393135306337306432363330346134346665626230306661656134616337 +31333836633033306164303666633231346430303662356334323737636165393839633566313436 +65306136343138373561653764336635366164393737303531353533326265386532666332383836 +39323731343132383539343039656331666231613365353166383661366336653231363433346631 +35646230653535626163623839333765633734336563353966346235663231333735326161396161 +34373634636430633662653162303739633065303231383264613631356562623130383838316262 +33663330383664366138313164353338333065363962383731616431313861313566646537333035 +63386464636562396131356537623436636239353632393232373565326566303161316462356131 +61656630393463356663613364613731666362393230333236643830323531383062343061653031 +61313237363533353738393632633936663165663035336661653537626136373930366262333531 +34326538353638643331346638653831353230326661616462303638383566323437666463656663 +65623831373864303031393664626438326466346661316235366565616466663132323131626335 +39643665393234316662316462656430623564663233316564613465363031353963313234393533 +61633061313639663163306664313363386665653431346463646564323230366161386531336466 +38356630636162336164633931343239636664316531623865393932316336343236303836663361 +38643730326336623531323436613634353239323031336664643132626638316133356464616435 +63303334326663626265626361306133616336376238373564623862613639613837626339366138 +62343866653036386133393064353861623365626561393562613934366662366238343766636638 +33333733323533326433626366376339326630633266333366353866356233346166343131326335 +62393932613562396264613237653831306536396664626334656137626339303332633736386433 +61373163363863663365333532343533313864663765636666323762326266653237303730383165 +34313761646562383936373231363761633837636665343364313763633963636136653637333130 +36666630633131326261303434653930626264323663343262633733636232353939393430396436 +33616461316434613533626563353863343530643633316663653165326434373036306261316266 +33663362363338386335626233613137616232653737663932653566326562666461626335303633 +34386438393466343039613436666135366663633534323464653931366637643661656434343362 +65396461636333316662323034373962326430363733363236383766303564333664623435663861 +64656530313363396134393136353732356536323866333266663139613463626136346135306364 +65353938643039376133633433616538643432366439613930373931383537386631326566303463 +33313030646630363262333436663838663437353430613533363162383531326461353362313132 +32326533306238616337396338613032336366346662623037306164643638663935353665643161 +35313664373836383161363462653033343837383437643839323332373733613462303238353036 +34643761633366356137303061386436373762663239333538393939336462623762353536663762 +33323937313037613639393633643033313662363233313235303764666365393732663038326531 +30363663643730653265663564653161343031376533613431396334633530313538623534396461 +62636433343263343936393130633539653164616464306465623765643337376363643635663138 +39356334316161343433396538643638316137316237633362653363343234656336313463643635 +65643333633636333830333739363735343637353061326332656338363064313965376533353365 +36666637386334303261373435313532366634383637636233653665333865626461653133626564 +32383739303534373762626232666365653765643362323035656465653864353663356563393461 +65326264363134363763383862616231336435356535323635393936616237353165383137383833 +66363336633830623639333031393664613837626537663034396661623330333537366563353731 +66363430343264353631663065306461613039383762623837333065316566386463393032383935 +65386666383231663236383161663733613162316434356638623861356363396431366563626362 +31363235613238623664333932333238353461316137303339363935363938343464313235666139 +65333231386335323766323530346662396239616439643538623138643661653134663731353839 +63356631626365363936613739636336326236396530316161363436636239663330386135653864 +35646238313438333635373937666431643761633433653261336466316335623135613963353066 +39343966336238633336353464373130383231396264356265666635653863646633336534366661 +66613336616433356566623739366438393739623836656264366562613330666539396466313339 +32373239373430353832643638643832313863313661383932373532356532373064323535303032 +30656361356433363731346333306462366362663930306233353834343361663766356663666532 +39383464373637373730393933663936336664353730313564643639653463393166323034666263 +37363663323361306430303837313235306164303535646335663835346235323866323761393662 +64373165653065306639623932633062323934316139393038343533326535613839396137666133 +33313762373138383139653531353964663734336234303536376631306336383433643930393437 +65396632393930336161623336643735633234396539333165373964383165313663643133653133 +65633837333537663765613262393736343138343561643166396433633430356336306535303135 +35376133373862363832383434616436653937633335373238656661383933373636653234363938 +32383132623431396535353236343232326562343039303439323763663438336566623532653166 +32353634623831373830386365343834376530626437656531393730333366393665393734356133 +35396635643730316530303333366265666234653361666163666339336431313336373333306430 +38653236306337323736383339356436316266666434346161346666363434326333383037303039 +64616464666635346238333961323365386431626131653533636563646662663766363562326165 +38626533343166653635316633316339393065313363363530653831643738303565343236633966 +31393464366138396433303264383739306237323161623065316637323734646165643937373966 +32336137313735303964373538356537633161343466616564316138383631616137366137326630 +33623531313536636638306435383937646565646636626565306539633563393262656638643166 +62386261633764373438623131356234636262393835303130363564306563353364346330373366 +32386131643566623533643962333463653961313438333432386433333631626336636266313536 +34623831316564303262613032633664666164333934636437343333383133396631643238653439 +63656431356636613862626134386462353064313831616633633736356165623665616465326235 +36656236366564313934313433343338396335326363323065343762346330336633656535616532 +32323763326637386337643837363339666236313161313736323330383766343337363965346462 +33313738333633383137323766316436323933343363623833636663376533383438633562326633 +38306362356263383763616238663135646531633630373164613037383464613963643331303031 +32663364636563303033636135663436626331613039363464396433303335666661663838393465 +65626666623434663266323464633565346163346333383037353037633935313932616337323630 +34316332376161333930666534666264303762653930643539613363336438316466316563653165 +32333765366636653730323130346534656136383233336531666131643333633631666334353330 +31613163633231376362373563393031363835613264663463616136643930663663626464616466 +36383031613539636533346233656638383135336463393937383531373735316261313537333664 +61303465323835653463346137616333653738623237616361326639633230306233663766613962 +33386231393830333634646135356566383462666431653336383434663539663937306166636263 +37386333616530316134383639653361613762636366626461616136363937303362366537633439 +35343163653234623530316634623632396630383230616237363937366136333135366537393632 +66663239383239383633356635656137313039363964366531346434666266303533633431643166 +32636139383339646564653866613733623565613536386363393833373337303335373561663531 +36663532663733346532336631336561373564333039333739653661376236326331643632343138 +64653238646536356234343565653135383532623732333730313739383563323335336436303061 +63663462643230666335393330656361656661656464383165643836363965663731643265646330 +64336534386466623761336461623061336438336338353837653237393939306562333038363138 +33306637366461633230313530343665633564326262653862323534346563343865613731616132 +36396432343863656630646438643162393064323136373862393665633666306166313263626638 +63346639333966376437393737346264393732626265356362633432616261383934303461613539 +36366266376132373237633635316330316335346362376266376663313664373462666561343264 +35613737646265376332643464326333353664643465363663396565653233386333336263613863 +36626430323231313136666164613237663465346338366632326435313134306165626631343432 +39343332613961396136363566616333316466616534626235633666646261383432356662363832 +37653664386365326639663937633731373735383939326135643863343037643231303662336563 +61613433666239376334303133343765363534313865343032613732633463326330646230356664 +34653863623061356131306162373334353636396238613164646563396365636539353961333862 +65353963336238623439356530636137336135653339626633353664623465633566346238623265 +62616564656264393264356430363935376365306631373533646536663138653738363465613462 +35636435623537323561386364373039663335396261303362306262366661343365316264623636 +61663433343234623133633532323538633865326537613838636337316232373534323264353733 +39326331373534613530616136656631336330366361376231663932306535633235663838363363 +37653833656539373931333064666361346530663563623461613030653764356631373362636165 +30646363653733623131313333373231633030613164613139363761363332303730633664316230 +61363632376639363231626433633331343563333863303462383237376566396666376133643537 +33336330373533356461366634336635396631633236613432343237663231323166336239356366 +63633165306632393336373731303333303365303031326465626265626632653033353261656663 +65663937633562353166393434326332383534393637323430306635313233353361376239333563 +35623938383032366532613862383462393366396633646433303662353539363837333937643230 +65313036396362366439663133313061656166386132313533383133316635353832613731356563 +35393033623865313062303535366565383866373138623761303736636565613035343034303936 +33356131643031346332666239306261373635376461623966343438383663366161656662373031 +33313563396463393836663634343033343138363230646534663930396237343465323435633835 +32383938396338323135386263373964363430366266663930313435346464323766323837343534 +65653536623462313030346362396665653266306536626630643634316438643764316434316638 +65376634393737623630336435356631663032326638336464383562396439313935613534646361 +65643234663161373331636665303661666462613561366631363932633534633664623136616130 +35396361313632373835656639333866383262376662303033346137396531616432636264363966 +31653966643637333134303735323239623434366531386431643664623364616136666265373562 +65653134653462373962666264633034353936636636396332633137313833613330313963313465 +33383934386633376661303763363036323439353063353136353337313432313864383539323166 +39643637376466663532303030336439326330303230356261316261323061336338666239303935 +61356661353861306165383337323666316236356565373930636566633966343535616435343465 +66383830333730363733323637346537326536643136356232383163656333373235623435353863 +65613634316261383065613262346230373930346661666366323265623933353435346333326434 +33313361613266396331633566616464663066343831663762376464383035666331653638383135 +32623231393335653132643937626332333830386561366630343030356430363864623936393739 +39333134306436636262623932373737663964363230303931326338326166633161346663323135 +61646564613839376335653833303931373438633963626336303061376431373830366236613432 +30643266313631343132633431633738346466383838613064363661363332373532376431323435 +30666363623036613035666330393162666633633038343832643634643733303637376231626565 +39396235623064373066623334373431303366633936363330643332623539646434373764386633 +34313561633763376235333236363763376332643761666562353032316633316433343636373532 +35623634353932363766643835363461326539363233643035323332353630346539323432643438 +38616262326662663231333730323034323965623434323630616632323336386331363866623134 +34633165616164376231653930303336323538306162623861383636376631366361373330626231 +30333865303138336639386532633437313735396266653339616331636439363661633766323234 +63386464373864663333363733326332646438636362323162313939386434373262636565313438 +61633534313437376565633636656632663331386235616332383231333762303439393765313663 +66303434353861366231663866633531623139643861333539336565633538666536663763666565 +62333435363435346561653730623164626637316632363234346439333331373663363530633735 +36316233613739633734336537383263366433343664393366643430373936323534633061373465 +37393731343033366332663564323330666261303635353466396139323634616465643432323061 +39646564663237653361346561333462396437343864373534363438396233373237656364336166 +38343431363463666135393362373132626365353664323464343230343037363166313837313231 +66323131313534633533623630663733396664363462653261363763643730616131356432356135 +65336430653365393939633938633833633034633264616363303066376232343763646466653139 +31356661393238346239373962333234623530393563376337653561643730353530353461663537 +38303331633735636562396638633461653635333265623764613330616234663164373934383634 +33313330346435363637663933313339396435333761383763386363333239336533363930343030 +31653336373063656239323262326564326465303462303665303735393262353636623938343836 +39656534633630326139326331373839653062323138626435373739353662396562646436353337 +34316238663330636133656437643535633461353762636361636337613532633961313939623130 +32323364383737363662303630356430613833353063313533366235333938636163343439356135 +31633330393432613539303230306265303463646338386530313762306236306163303937663431 +33366334316136343230623161373638393431363436313331616266613137613762373734316136 +38303461333232323530353731333565646464623131363064363037653633656266303163333237 +38656631663739343436663139396263653162626533313331633832643433323431313465326239 +66346536393532613933636431653630616338306166353730333737633366363364383333336166 +35303863616636663365383136393061646338613531313063306638353238363939343238656262 +62313162626561653930636665306137316566323639363166653436376633303731633835323466 +38386663343064663065623439366162306335653530616661323866643661313561323565623734 +35333837643838313337623261623363353164303832626534353334613965386537323336633138 +31343536346664623437663637613331363035393366303664313364643863643333633631343635 +37633434343366643930643037306337613462633161383163613932306261336330613436333434 +39303639636261646662393731393835346532383764633737643534363964346539643765616336 +33663565373039653938386266633032626439623933306264643765346538373030373766353963 +63386263363438666239336339323733363266636166343831653063353337336639623230636562 +63626239643833353065316335633734323464313036323461393662363736366336613835646333 +34373865643236633536356637316331623034653631303065633364663037636562636165326132 +66636531336166633439653866613838336530666330306432353463366132633764633334303266 +32663062663534396132393630656365636132633265663033343361353530633534326434633536 +36373463613233363663323533366462373565323137363930623636613033333537333262653131 +31613632356463663366336531636536636639613936316339363832623631396665666130653036 +30666639646532643134356334633961376132666566326537336163333465653063346265333731 +37656163306164373864643333333138383334343433396434653733643362386235656462333764 +31303562393735366234323963376139643966616535646436346266383734313462653435303934 +39306631656635323163313639393164643465643537386365323461653365383732356337313664 +36376438346533393136323432633330623433666135326431316661613563643735316237363237 +38306237616333343933383532613331376230303134643235363538616230653332653733373363 +64383835383665363931666666313062363438616133303331633563373630653636356531336436 +65313237633232393233313236343035396231386131323438633737333232363336373164666239 +32316161313063636635383765373766636234666262373665333138386266343264326532663062 +61653038643737383362616362633735353331306166366235393834363262643933316439313330 +39663238393636363839643132306366333964373965393130306632383438646135666430653730 +31353931343132653130623933613866356135616631366230653961393862323065656563626639 +64303538663637623833623561633735653530316538363866356265616262616139653962623937 +64316334653635343565613862366366366138653964376631386439313664396230323661613036 +34623232376461366234333262373730643965613730323865343531313465333963663966616536 +32623163656539393132343630383934386534376264303864383430666231373865643461313162 +34336131346332376538353066386161373161373838363762363864626431643435663934333330 +62313063326165323039303032613433383536306136323439353034303930366162343731653962 +66653964313634613666313365383733656535366362656363656662336364376333333066336163 +35386431306166366538366261373163616163303236333639316437353161383061646266303138 +64666264613065373538356566373236373663366331336630653563343065346437613262613937 +37656237306561333465326330336531616533633438666565663533633332396233356563393337 +65396232363435383535333334623431393564636437396437393734373036336164643665663766 +31306363386164646534616230663630653139333431633565623564663663313736636334356462 +62326136333161633661373066386665363436353162613464383366636132666235616537343236 +34613133656237316533313265306538653962383766663730633039383931346137306162653432 +34363138303831383230623431666430333030613361343239306130633530393365343835326337 +36363937396237346536653463636434363563303833626266386434663332666235393839636130 +33396239333063393564363164643063353432383931386634663563376139393836353832613631 +35373761383030386632336564316564383638373531323064373265636333626662616134343138 +30306230353061303764396164343432313330666633636433303438626131633936373139396535 +39636331623833623364363839306332396233333231326363373236653961643430653039313864 +30333135363062653730656637353032323234356537653637313165363337653362353237363862 +30643831623536653862376361343365353665313034303963313935306264666135373332613433 +63643764616632623863366161333465653132386433623234626430616233363437326535623337 +32373935393638336161656136366139376437333761663564643866336231623134303831656135 +31666535373339623538623462333532663465623639303166356538663964633065323230653831 +63333162386665656231326235626237646336666630383063323830663261316333383865366333 +64343863313237356330383561323933353438663138306561646631346637383466386337396138 +64646265336135333339353661306463633364613432366334653461336139333633633263393766 +33643534363336313762303662366139343134343964353239656531616161343966613037623133 +35393334663762376165356238306664383461393735623662346664643536383635363464666639 +62393430663630353735353432323565623666346135303866363866383138366531333834653662 +62646565303630353662633261383532353638393161396262376362643430663730613135663738 +64616266343932613837613636613963336532353032663334363234613239303633376162303838 +63303263366662316631393561373539643166393564333334623061373066356262356134363337 +37333031326562326565646639633063326666323339643765323135623066376338653330383362 +64396363613965343931313234656430613631636261383433633261666432353031613533666338 +30333264633864363136613764653037626635396664326462626530396537326165353733356264 +64303864386532613837643632393735346532393162383934663439393361326336373561396232 +30613336633437333931363939353936336136663435636430393564653938626135363363383336 +38363361333236376563643163643239393866376366353032393533333638346566643832333866 +63316530633262373737336133663336396533313465656163653839613538353762383130613634 +37333662626264323039616333303630396165393164333030313162346330613537313164653936 +30353364363033303762356339663933653835623233356439613666306133316133333466383865 +31653431313763313266643362303032346230633435353334666638333464623637336433363166 +62633232356235396436356634336661323464393435626538373435363831326334373235313963 +38303031306638646635303561313664383338386330373235633566623038306331643638633133 +62653438646332346365303134376664666539383332643461633765626664383531396562346137 +35353064636132363634393336326436613663323435643433663533616265343065323435663864 +63363536356539666630346632633132313963333338646163343035306534326137366432653565 +62386462336331343166643137303939656137353638366639336364643265633835636338386639 +30646631333833626561393864386536353432396465653265666664666435343436346264363761 +66656465623437383835386163376461323432663638343936623232373261633938303735333561 +37646633306566323264333430393563623835646661613861306439373339656564393662363538 +35393033383665323137643735383939653362623336353266396363383761346237306665313133 +66316535653561666232393562366338393965383038353564653230323336663239613035333533 +62633238396262636635326138623735323263346666303564376362343338353536356231383962 +32383637383234643861653836303831646634636135353334303661306339623263653263306232 +66393633653035666130343436653763366530353964373864303730633437323766346135343830 +64323962373235633938353463383862383535663664383234303263623532333463313161386164 +63336365326264646661353638306637363363656662343833343563623431343764313165353034 +33666338613561613764336537636565633838376331363764613563663033343632303162316661 +62356634306231353333636435356332626364366337626261306430323265393739326231666630 +35626161366539643365333232353439613834383830393035363735613837383230323033373139 +31623563343866643862616232656232353134373032663339626263333266313239366435653238 +61313930653166336437383836353332326131663833316662306235323863643736376161333564 +66626233366430303531316262346636306534323362366165643338333337383135343733353733 +64613636356439636532383739313064653961366134623866646263383438653835623333393763 +39666564323966643361323430633732366536366162663730393835623638396666343837333735 +33643436376338326430656338663335333061323766633033623666386637666465626361326363 +63643962363837336664323436303962666239623038326466316631336266643164616635613434 +30643966613638376431633839386131663234663034323366396634396336396138313562363635 +64343038306631383965346638663031356466353837626631633235643331336662393536343332 +61356432376636626566333733376238393166306166323636633564633339636262396466356133 +65613234373362353964666335346563353732373734656434646461663163653435336330613036 +64376533616332653930343131356236613032366233306664303738373961313931363533346464 +36376434393333353465333931343665316432613362376631643633323239376263656662643661 +31393162396438623565616638633037383066343834663365303533363031396437666236363765 +33383966303863343537386531303065323133623530363033613664653935333138333632343634 +35353930313966356664333261626338393636656535616236353334306461366163323261396464 +31613861633065303864346532393235623565666534613864633364613932343536636637336539 +32363130663439643337316562333731343462653831356130363362396435343964646630656636 +64336465363536643636333563346632393839633664303039363866353531633939353264353939 +32386337383830646664336566313538336133336636653862373930623135333866353435626636 +30653135366661396433356435333136616362316466303835663665386433386364626363663664 +37313531346666366433393936343865306437373661626634396564656338376531376662353937 +30323534666334303966663833633836373138346263373631363561633037633331353930386635 +39663162376161656564343334656461663737323566356435323236303937366535653862393637 +31333632363735373337633366326264303831353435373433636236653831353234636537356531 +38626530353135616236386133323635646234323334666634343663313438363065376530336665 +33313664613336613661316335663130386438373039623335303063633534356438376635383739 +66386136353630616330623239616639306463643732346433316662386437656562313661393238 +63613432373662663239333030646233366665323765376637336664656530623639366362323437 +39353034343532303236643164663363346233643737636562666230643865326233396436343431 +34623965336137333363333664613064633465363434393438353531333565336338396639393465 +38396162643439303839353064376632313432346639376332353462316134616236316236346431 +30636534613163313463623861393762396530373535653836383035373134356264303539623066 +39313832396331336230373663633563616663383633386634613535376139396539303733376365 +32663864303035636366656361633234373536323537336230363362333230643636646430373662 +36643637656162616136616164366338313665326530343033653464363033373134313564386233 +35313332333934393564393239663463386163633566376663666230623833613666623964366332 +31636631363234636530636436343133616132376636643864306464336133326138373965326461 +37346365646466373632613765383335636538333030306335323530326362363865393864393665 +66313366393835633539373939666264353366333163386531383161353966316364383330613531 +32613232353634613334643234313061663661303830336431616431626631336439336164366362 +32666461613666653162396561663765333737343537356635626464356338653838313661383766 +35656262613439303434333533383835343265313535626165643038323164393461633034393038 +32646436653132646130393532613733323665636436623234623561386134383931363332623831 +64353639663233663566666164323731343231653536353661653263336537326363306537323561 +62653033376362316335316165393431383961356631386530343034383237336334623933366365 +62626666633039323031306463386534393766336635336434616165386235616363613565386232 +65393164636439353334633031353231666339653961653634326464636563333061363038383130 +64323931376431366664336464363039613461663238633662636335353234376535366331363461 +62333139643531323063356634613833303836373532373263326563333663323661303365333837 +65353031333764306639323830656263623334363939326162346635343431306365326634306533 +65643865333163393937626531653438313161653135303736666463616239343533346531346233 +66633532343235313336373636396265666134343432303765366533373133326563316330326432 +66613164323131393431386663613537396334396133663732323534356136333833643539343537 +39386634333833356631633630386564393833613336393665626665346634373763613262376162 +63396138323463643166333331323630353761663935333136333631626432353330356362613338 +63323638346164326361653861306431376335653661613737636537306633616437356439626364 +35373463353539316133626461316137633436346332623964346137306236626130336231336238 +64386434303466663536643338303533653935323133313138303630666636636536326330626566 +34643932636637663035313139353931333661303762313766663935383834313661396335663765 +33306664366438643330346463353763336339626434643132653434616366663832366264646137 +32613632363136303236356435346230386131343732346230303462383363663533396432666630 +35316562323665323430616264393166313361663438373263363263356234333665313061633232 +65623838383661623130346235333863623138346637393339613236616539373038383139346536 +36616266653264313434386338663062643464656334366531353063303263343736353363383339 +31663538326664343039333664303833333532343366373664326431623634323361396131326662 +38396336306465636532666234646134366133306266386662326230653831633433663665306366 +39303063396566333439663661633237323936326562396663363761386466396131643962623935 +33363039303335373132393766306331356532383037393461373461363866393833343632333731 +37386638613132623931306231393738323534653432663433313334663438363930643935613830 +39613261366137303935346264303862613866316331616663393364333764356136653833663465 +31316539616535343162393466333633343939306231336665653331636463373430346238653364 +35326337376331373333346436316238616563346130313630323336383936353761643264303431 +34646666636431363763373464633236613939333334336262616663363234623434343965316238 +38386165623039326437643131336130656364393536383065336362396638653266653930653733 +35303434383732633339333564616635393636623861373333393862626630323165366263613839 +34366662376539393534393231363931356332633731356131393835303935353831343433353334 +62666162616661383530613266306438656262383337386335343261323338636366663163396333 +32343030333533383863333734653638643031626133663532366139353831353861633931643766 +62343230323437633639373862343131346636396633373566393663353339396666393536313936 +62303561326634393537313935656639383061393531336433333939316637373965326236363162 +32666238303666383865386563336461643463353436363536353566303734306461643333346364 +33653139383330643139626137303462393862323864383634343838333930323937653331636239 +34373161343935333663326530303661343130333063613464333362616435643439636632363433 +33373336666262343132653030653033626537626463333463363431393139333533666134656139 +33343730663065316332313732356633306233323534313930313462386661343531616335326530 +37646462656664366662333365356438333833663533656135316333313239383030366430656363 +66613732303063303832656238333230333462326431626634323764366336326239663233373530 +36646335336262366234623033353430326231663639336533323933663334323262643566613638 +30333561663137396363383264363563616465646466303735636634653464623463656562643963 +63353239393132656234653466313032376431356533323731376533613639613362393138303338 +62663834323332656465333665393739393263663565336564393131326530363138343330663565 +39656233306139613066303633653737633536383165336663323931303130356636663164316564 +63346664633938636330393630376662386536613732333139643161323365346661663834653332 +65613031376361613665613537343530323632336130303731653562363734343535343661386165 +32636364626535663161633038323034633934343137623739326365316539376539373639396564 +30343432303766363433316463323763663564613564326563363834393337623638363432373263 +62373431633235663437386438353366643266316564666137666333623237633437656537346166 +36303332653666303936336131336563393361613032316362363434306631333532356562323337 +64643065343761323439323463313265333866343065333062626266333964313838383335336338 +66363365363832366466623931363838323465386434636565306236303732346565376561613232 +34393462656665663933363637316135383465373961623332326235326334666334303031386361 +39636263663434393062393066663332393137653162643063386239666639613335356363333833 +39633762663865356532643638366237663035626365343761656666656664333762356265643539 +62353865393334303334633064653436386561386634633964633536303731356535383335313161 +36663931366364343835363466653138323639653934383834623466373333396438613730376565 +38643135393262373765383035633737353962666430343834636137383362373936313239643738 +39623335343036653539636664313762373337633063633530326131393932356337386566616363 +35373839646133313135353862383166336633616631363761396262346461376237373565353762 +65653633336162336136353738323866353466393932633333626262653932396633336437643834 +33373535303265326539643162353261323538306462653531623263336338356135626362383533 +61373431616339343164663966326662613538386638663130303835663964616263326131343430 +35643536663735306636366131396563393331346461316138653464306634363634626166363839 +61373666616265303662303935323364373133613738376134363539303738316135356239626632 +32633738626638303965383963633137643166363765353436626663316563663537363939356264 +61393162333330333339663234346631643335383033616562633335663339333962643564343933 +34343738396266386364316339383433666561303333323036633237646432306661313135363761 +32386534366633363465663533353232373439343762623666353166623537656265333339626236 +38666434633062323265373962626437386536633735373037383039373365396163383236306161 +35663430363930626632363236303237353134313132366566353630396363326162356237336162 +31333230323935343938653937323262386632306661383064613034666132393031303331346236 +36366138393563333764303832356562613064323839316461393737303861626437363230353738 +33353966356530613264626233366430346362373739346639366239386330646264303466303236 +62356363613838666263343865336661333036333437626237303932653438323235303836376437 +62623132356238366565663039663431346235646130633031343063336539613364663563393831 +31633666633030613939613264393863656463336232323836363638363831376266623536633632 +30656563626531333036303762303462623437636136303866653366373834323435396237316264 +66343939646331663433623933353038643062616433343264383836323738643464633537666562 +66316639623230353837363536336265303433303166626431646131396366373035396364343737 +61383265353462336235393961386166653965616236343136643364663561656333653532383237 +64313966373734636565363137343965653630363732616461623239313131626533613566653036 +66383035323938323133616261303362323764366538353365393233623732373962306562636261 +35653634646239643666656530313637356265643264383436323561616238343632316233323663 +63346638353532383065343236633066313831313666303435303533623961396634383166343839 +65313134373134333866373930333035333463363937363332363266353330303062663235646232 +61303834366636393837393638666139333632326234336463633531663636376639313434343138 +33366533613337326236383766313264636337623266323761656230326231356639633465396663 +62373033306635383830383736373738376165303265326532636434386232306337376564326239 +35643437333337646239626364656634626631633362333662346263656162313864613666653663 +33643033613534643234323835303738353363353636343139353862346237616234366562663265 +61343730363838313566386437363835373033386164306432316536393633656431643564386537 +66393136636161336330376266353438643633626562326437643331623965633031353862626236 +31336164303666353430636163356138346464383361303332373464653138653665666439643034 +34663837613734303930346664386661346435373664353934356163386237356336363039613939 +31366433616631346531633137613730393630363162303666633965333033343531313335313635 +37316164383664623339353833316464303365326466313465633331646264306230386562646664 +34363430313736633934386661643035656239663962666530373433623966316133633232653336 +32653765313962363366363137313031666366656135303034333835646536326430366233613265 +63646365323536613830666239623632366237343566653663383838336466336337633630376662 +66363861393761663339363535356235626539303830323866363065366566333265336533303637 +30303733306337336231666162663835303138383065366130346538636631363432646263613561 +32643037353738663161316364386661613063303933666532633631363932343831363365373834 +65623437646263633964336231326661613033363136633139363533613162623563366165343735 +61633732343837393831356364626632613862333039356162376335313366643538366339306361 +30643231613761306236316638373839396538613866323566613865646538346438316636626464 +39326164373334323363333134633535656534386263623536336664643536393437333532326537 +35343463633936393836366130333434346165626536383566653262343462623661353932363936 +31643733376630643135613162373530626636396464653662396365326139656532313462393265 +65643035616665623236303764353631383631313364366433323331353333373162613238323731 +33323830613339633962393261383264323839613330663933346337363362396534643762623230 +35646631343734323731313733326336636530333733643630373162633433373939666566653662 +32393732626432386431303964373038626130343862356561376662393135313235353836313830 +31646236333632306237353739333763616431356265373833313061343335323032643638363364 +36386438666332323861393231623639343932363332313765666239616235636662393963366365 +33643062643531363762643630333038633334653632323631313362623231356533343538363533 +65656432346438356463623363633537646538333133643736393332616439303265343630373865 +65656466386163323734383736623366653562313631386339396132313364656631343637366337 +32363130663563393163326166386435333364653165626239366362393966666266343165613365 +34353231626238343433303932396632363562656232393436623537333633633930366630396439 +31373362346336653336373465323964313832353463356465333437373631343331366432333533 +34323936373533643530353135353361353032393538373234333031346537613935386464633339 +39643261663964656539636663646639383434363432666538396537396661613234333966336638 +38333966353862643038306262303830616535333933366439646436646635663537626431393630 +63313034656562633937353336643834336462643963386533386631643133396230343863363665 +30356636346663363634626536326262373863366364626562303235613738313336633237376234 +38386563336231666238376335336162616162626131663933336234643936653438646230633437 +39633230343439323838343430323239386366656436643233353732383663363166376162313331 +37393063626533643232616633643431316530326537363763316436363238313138633665353331 +63303162393234616335336265636261343932396237666562313131623663326532303339623832 +35353232313139626538323762656265666337663635383865306365623966633362666534313433 +36353962613633316435653630313032303135623032306161633432343834633737343739343135 +63306461616337313734666436626432333131373238336134363564633538313730663864356264 +63396166316437376463393133633136626566633962333434333361633633343239366538323066 +38663834393132663465316261626537663638626139613833626465333138303466633838313261 +35623533383430616164613435623938353930356663323461383565343039303364303738383431 +33383239646132373637323232393963343663626562333461393837643262646661623437353666 +33363565633939393263653836333236393339613534636533643637656231313736316466623731 +36343236626338393437306136393063616635303361386665626138633134666635386537343630 +65666565653863626662383366643136623966373332386665393134353939656535306130363832 +34663863666461346538653762323839633534623337316230326665346137626664363434313934 +31326236396631363565656263616638646231643531356565393865363561393865306238326566 +62366338623064646261626238373935636434613232326236393136393339633762353964323462 +64356636343063383064643837383635313364333530386236313466636338646661383432633430 +39623635623761653636303966343736303739633830333731383534316462656534303133376334 +33643438356562303036363861326131306563633035383530363262633033366230393962336131 +31353534336364376639643464616431613337383535373831393033363638303566316132393563 +33653038346235633835653931376130333630623963663233323464373831343834643235653263 +61623734353061303036663761353466326564646335623061333334663536616532653562343136 +39346263643530656532633136623262653033393834663530633965626661613433623935323065 +38633139636661363865323430353139346361623133346232376332616131366138343161356336 +36376230303033396166323638666637633461666337666639633130646262386463646162346137 +38616464306462653938353230343464653433356438353032663336336461373165643439633236 +35306661376433663132613862613934373465626334333138353737666430343138383937626263 +62303965383862303335343365613234373561303162666339653461373166306566373432666334 +66323763366130306261633433343463623063613963343135313863396365666531363731393763 +36353461393937643234356166353835323231393233646564333235633165643034343837613663 +31663961373139386532613630313663333537623734373866343935353437383332623066373563 +30333433643935326166643333363834383335613130306465653432666163666538636339346338 +62323563353038623935643662303764613466393766343164653231646431316462666162626165 +66613830633238666638313031353661656139376236383831363138623731336566663334336338 +38363363343235383537643134653832313866633533363934656536306231653263353937633230 +37346433623138373061323332373433323036396536316365323030363736363634643935653165 +33343764396332343266653161343264646662303032383838656133353337636339373963626633 +32343534623331383761343463363137383364343766663437326438313833333863306664323434 +63386637396231316662393664353237633832303732313362316433396630626232386437663635 +31613732363935626663613232386362303934626235656261623239356437343838313165623938 +35656337626563663235383038633836356238623037326335353863643733376135306634646465 +64326261326537333863656366366462383966633139363834613435646636373535376132393832 +33396136373865326332393137653437666361396630643764636263353931326239633632353635 +33653732353539623361623936376537363232336630666538666436666231306139333563323334 +63386662616630313232623935343565396161623230633332373763386534653763656561323861 +32383461336634653364386333633939373161636534653737366139306634303931343438336539 +39386634656234303330373838356465356665376338303337303937376663313431363932386531 +31356166616263663966353035666330383733393539373235666530656430316233393335636366 +32643633313730633037383034623566633865336362313462616538333361316333363530663235 +38333836663366373938386161316637386439323936323463663138393533616130373464363035 +64373037313138383466643431356264623161336339626463376538313035346366383032623564 +33353966646466346339616331306133623531653134653135393933333532383966336338343436 +32363635346433373161656133313632663039346230353837666533613339666432383639306464 +30646463643865666630343837333236363130393338373066626636373465646563346634643939 +33316634653232393031336235666532333034643165363764396637383364323633376138326532 +36613630386230633533313537373138663136633636663937386137656433323438616364623566 +61643831366262616136653532663936313635393737373135333335663537663735616132393139 +63336135666431663339343765313437613765346634376361313536373237363061373936626339 +38366432323131336439353933663663343631353163393133646162373161636137646535656437 +39313931356631326561666638396333396561613238343464303363356634663834663339313337 +37346530336561326662306437643163306366646361383433386435323133646533646239393663 +64393737363535646562653239323133653039656530393934323638366233353939623632623934 +61383935376434636533633234353636653930316232356339633435613830616335636161616139 +30623830373634346434643362626164353730363230656664633331336331333030646239656461 +38613037633833373362343366373665323365346464643463366138336238333238316565613436 +32373332363062653037356434396533383033336463386466386534333735326430623865653134 +64633737363162323832666538323565633635316332626464653638633031313063626464333732 +39633837313561633361643930636539366531363261656561343535393636653862663738386137 +64323431663364393035373139336133666532666432356461313064373637343161353965376631 +63346666646632636537383533643138373464373964613566386431653361653162323433303430 +34666363373861643130646562666336366364303637643062633165663266373861633538353431 +34306539373034623034666633663863343537393538326336316131656430316233316366336464 +64643035633762396663376133373166323761636366393132636631326135336666313832666364 +39333937663161636439336562386337386666623334363662616635653439353138386562653462 +30653936386536333338333238626161643230623565386137323964653337313766663862313735 +64336461393835616434666538663865366331613333663339663330326364313231383232346364 +32336230353434653564306339333264626536653664353063616236643833646236396131373961 +30376237353335663935353833393163373734383539313462643032336461653038646234373066 +64636365393633613634666266306239373734306536383432663334346237323461393261626437 +34336464373234613736623164343637656332613462363265313633323435656361623038623761 +62386166303034656533663862653633393664623164333766323938376465646262333066323261 +34396233643465663062383666346463396637623461616261383038353738343764613039373362 +33363738633537383934343466386636303739663837633338383965666231633239303937363239 +39353031383366353362643733386338333363383133333065316363353639316435623436346634 +63353233626134383165383464323733616366373631623831386239366162366233356330666464 +61626437333561623432393533383266326233303433336364333533356431366338643230313632 +36616461313138313164316338393033633365353634343437353234643631303465306165616133 +66656466636162396239386630663264383231306264666432666637613833353931663438633238 +32303466333134323364373632623061326537313665323363333961623464396566353731356431 +62626635376230646266363539653232393663646131333035353465326334303435363537343839 +31323461336534643235343539653933316639356265353737386365346638356264323661613136 +62623037356334653165346366373364653934323162373663346334626664646237353636333739 +36343163333463376239396564666533613539316239336163663561363337333433393536663665 +38356461656235623565323037616364383139636463376461343837633962363935343530666533 +33663433623136373664373238363864636639363535323536303134643639333335316535633732 +62323833356464393764653863616361666539626161393262336166303335643536376638363939 +30306364646435666439623730646466346462303938313434386335313562633930316333643832 +34363063303630353638643031353064616430336264386135363661613238313630643638393731 +34313139313332373665373261376632656536326665393136366130363431353437633230343332 +31623436346133386261326337623036643934366163613864333637346539393838653163613333 +31626634363766333265623239363463343036306366333939386237396537346433313734396639 +38663163333933303231663362393633666233633864356663663838666332633561313234386436 +35313535353436333435393731663233616166663564646131303939323832393936643932333537 +30623039663630643665643238326436313636663133373132376435313337336461353536623164 +31333865393637366135353930353763663734646662666337353130613832396233346333386135 +34316561386263643932353836396264646163646135333166366466363737336235636131616164 +36336535623638626265613539643131636234653264613933636466663665613537376430656465 +65366664623163323635613239333864393837326330393438323362643761343933336332336636 +39333437313362633032386234343439303331656639386566613630366533336338643833316431 +66326336316234626264346264633931626162313738616462633137376131386562373765636665 +63396630383963313161363232643634363835633334343334306462353861376463366331663830 +36333638313066666639363862633537643966313561636330623062303831653237336362353233 +39326637663534396164343630663864663062646165656136343933623564353963373762366263 +33613666323133663366356566643833353061393664333134343562336239326664663538323663 +66373936303961383964366461303134396334396339613665613662663664646363346165333933 +31626538393238323966363564643363306439346331303365303963313166633332373862616232 +65623637383965613437656135643634313237353230303835666335386630396362663666653532 +38363565393366333665393231323334343231663632353361336337343738616334353833363064 +66306365363366333337326236306365313336373132666263653736373265343564373336306135 +66393934666364623464333034333963316162373638373739383735623235366664613961323561 +65356362346435633430633538363263626133383963353235303766316661663032313932346265 +30653661316635363738336535323764636438616262343337653361633139643336626638373461 +35363331376663303132663633376331323935326364303138313532303163623638636238666666 +38663839386233396531343161366336363939316362656334373561666363646434613362633333 +66356133663431396633656664383261333839313235346535656466633232666639333538303462 +33666335346531346331613066373964666133363662663835653739383965366461343930633262 +37326465363030656633613139356161643130376437323135653732343362353764623932323764 +63373237613139646662656331663166636336663837663464623433353133396665323438316563 +36333362623431366163333063326433333430303965303736383936393332343432633332396230 +36653530386434663262333364303139393465363432366465326131343633666663336431313833 +30326430386136343463666234643964613264383239383734326632636538363439393364336431 +65613634366330643630336366626162383132643366646562623962343130646338303434313563 +31376439323437323863393438303064323131313536326436383536353332333032303164363065 +32323137613035353936626332336435313630383736366338373430303235663233616466366533 +35303132663633646266653238323335633030346561323334383539373037666334633130316162 +64373334346436373335383962396633313663303366316631613666646365386431383337316562 +63636138623338336133643637653133653533633138306536346563653165656631323939633731 +39303061336335396661656332393662633036613433363937633136386231633736663963316334 +37376262333330663866646335613238343237376530353565323932653366653731396435393539 +38623537373961656434646632303232323131346435623837396666316563643537653563383661 +37353737633832663462353938636434303831656263653837343537363437363162336362663337 +62653833303538383832643934643062643466333863623438366133356665356634396462613331 +34356630623236333737623665346566346566663935633337663961356461623935666138383539 +62346631633062333535613266636131653833336436373538303132663036636266346163383364 +32656163353063303939613437313361646638386462393563643137363536343835306233613462 +32616634613761663630656130386661636532393534383733356364616164643238393738353662 +38343337363237306664393038313066643366363438643462643061663062313966623664333539 +65353438343036333066313761303434373063616433366633633434646134643034386336326161 +38353137343965336533323935383434353838636562373131663561653161313665353839616136 +32663732666461663465326436623465363762356262623561633465396138623532306537343337 +31643833616139613562313964613632306130373733363332623766366630353663313531343634 +62323936326465386332363333633636333635393939363131626635643138353738363839313631 +63636137636533393336636137633130346365663263303833633632656166393935616535386538 +38353335356530326638306533633062323965353931306332313837326631383266613064376536 +35636231363731616331313833613363383965613037393239386430346332636365343630313633 +31303430663433316265313166343264333237343738396137636439356338383336626136623366 +36363862303762366466366134313334643631326438373765646661366135313334656262623035 +64613062353363663364386630653133373666336330396533386330376138636132663765376633 +35363964633634363964653037613261346162663732663030356339636334333738333464323335 +36383231633639303063396162643936613566353239356337346261663861646534326161373533 +63663337663234633230663734396639613361303234626137636431626438656365353363633536 +65363632623534343835366235613135376330313062343431643664363037363837653132666435 +61663737656239326366613564616239383437653130653631396163376534333764333561636238 +37313864383766346562306631646563356332326164383538373531653866346562656364336532 +38303765616166653666323430313037623636353738363434353335373664353861666663333663 +38633934656138363033393261376132366166656137373266373766336631303033643539396135 +32663533343538353635316231666436643838653962643936373462663230343837613632393163 +66306261393432646631386334633131303236333732613734623061313163636235383165303034 +35356631363865313034623431623965373634356164343132613966353733363164383434313534 +32663434343434646637366235633931323034303639643437356533643161396261396239646134 +31333733346330363461333561326332653439323531373137663033393633353564303965636665 +65636130363830656366346435303035663531326661386238376463306534323466663832663236 +35646632316431653765376435383330626231336538326565376539623733313866653632363036 +66353566623435363338313639613766613033313336663637343761303963376561353631346466 +39613061353363623463633236363162303230303461623362383836336432303338363461633738 +31323862356439323464663332316264393031323961383032666665316465306166363536313038 +65303830323434306336316665663530626531363431363761393435363962343134393836353264 +61663839336265663963623830323663613938326233616131636161396638356632663965633832 +39333631646666643764356538613061343762633539616261383265373161633236346162663631 +31653962616165646439386435396539343565396265616637313135383234346338363839616532 +61343037303431663831396531303839653634383166376333303733333337666536366639343062 +65323437383630393731653365356536323232303661623863326636306337333533326164386465 +64386136666661643832383830663765653139633732613962306237343535346466303866643935 +34376664633733613737353436313662666437356335396233643234636539363436653539653730 +36633562633132333862616264643464323532633137646263363537316266323134396336626437 +62643164346262613337626139656438613566316165373063643061336665313963366235306365 +65623665346136666338333864646630616134366465656635303062613264303036626666626532 +30376363396666306337386363653561653431663437626537356564623634363138306432633639 +35646339623435323330353263646134366132323062373635313033373762333538386635323339 +31353435663066356139313536353532653664633761626130353135636631303865373537363432 +64623664306262616366353137653239656435643731303263333838393831356237386231356236 +32333236326633643432303637356366636539393439636262623836356231366633346465353965 +39633564346662636436343935636362643765306537336136633134643436646566336562316261 +39343161353565323136623663656264646639643362343436646364373966383539363433373335 +30643861643463386262643833323666346661636562633336613662356564313434393336653366 +66653263336664346330383434653165316237333235326663613365613136303464663233633538 +64373338646537386635386330616231626431306364353065373738653965623735336630383761 +33343562636338366235313236633732383433653361323866323335343265633962336531373130 +34376439323336383931383664373731343362363334306266633036323964303065316562663835 +39363733313731343030396237386634653165626635323435386139343465336432656235393230 +30313165346261626562623363633335393632306138303436623836343733626662313361373665 +39326664613964633337343865396434346336633033623361393761396334303961343330353261 +61666361353732393537383839333233386637626465373062333064393839336364663335383837 +35343864636335376134356436376433383637613031383038336233636139643731613134646137 +66626535326165663266646135323466373861306536313265313466346261363531393635613536 +35666634623937613235353535613838323238323733363632633861386334363338343065306332 +34306432613364346532613066663734346236383839396531376636666536336364643031303562 +32666564373838633432353263373732346665303664303035623865636137626538366531373166 +36303236663931336438373934306335363031383966646563333637313037656665383262663864 +32363635333862383637366330343834326430356235366139356634376262646239616334613039 +62613638666665656433393338376662303932333135353563366135653063643634386537396263 +36353639643235363533616134366666343064353237316634396165343937636562326633306165 +32343265626665653866303065663632636538383966306538613230373465653836303033643161 +62323334393632383030303331373532643164666265656361623334326439396133616130366431 +65313433313763616238343561346138353761383837333964666534393963653761643366616235 +36666333646462626138643832363364393561616363646366363064356265326131636636316563 +65666232663264633135356235393238383061323963626432636433616365363932326432636464 +37313234326665323535306130396330636161383165623262376261343762646136643639666665 +39343132313339656262383938633833646262633666376364633232383062616565623235393833 +36383039313263363435393032653731343861633731656331373163646138346233623433313438 +35613764636366613839393238366563626531303363613266373835653938316239303137616436 +31383466643464646565356339336561383638316661643933343631643038393631313230653865 +63306538343337396332656161323630666637313732656639653565653432323930373066313030 +65336463656461303436313735313265303536663463353364623864353830316534316430376461 +36343534346263363962326665653361346264646666363432633436633766623232383332396637 +63383534386630323338363034333639376536386662623861333265353766343362316163626636 +66343463653763346431393165363435393937636366666562623333346134323639353939393061 +66336261386338373164653537626638383433356633316564333164636230393766623332366166 +63616430653064636135366465363464326562633838623435613863613133333264393839646365 +31346664363562323939633037666635633832306339613864396635646665303365626339343263 +61346166306130646637373531366166393966623731626631313438623962633663356535306464 +39633030616165316331636539636163396564346535333939626435346662333765323963306439 +38343161323264353335326633333633663633373166396662316533376230303063343031316462 +36323833316566373830343366623435646563373364313135353735653161303161616666656634 +38303230643531393166373632643333303263306536613562316537376366393336356138333731 +62633034323134366533313761336165386632363830313466656566313732303961346135373937 +38666665343162613436323234663931333633366463363966363237313832663764353761656263 +35636532616332333061303862623766386333306463663964373430303565656234643063663034 +38353661396665306234353461616534356161326437323338303632386164653835303634636366 +64323633656638386466663233323362363131663863396330626335623063616164366232656466 +3164 diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index c01a23da5..8ef2950b6 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -1,14 +1,32 @@ ---- -#prod_cidr: 192.168.10.0/24 -#data_cidr: 192.168.100.0/24 -#ipmi_cidr: 192.168.1.0/24 - +data_ips: + c-1: 192.168.100.1 + c-10: 192.168.100.10 + c-11: 192.168.100.11 + c-12: 192.168.100.12 + c-13: 192.168.100.13 + c-14: 192.168.100.14 + c-2: 192.168.100.2 + c-3: 192.168.100.3 + c-4: 192.168.100.4 + c-5: 192.168.100.5 + c-6: 192.168.100.6 + c-7: 192.168.100.7 + c-8: 192.168.100.8 + c-9: 192.168.100.9 + ds1: 192.168.100.15 + ds2: 192.168.100.16 + mds1: 192.168.100.17 + mds2: 192.168.100.18 + srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 - prod_ips: - srcseed: 192.168.10.201 c-1: 192.168.10.1 + c-10: 192.168.10.10 + c-11: 192.168.10.11 + c-12: 192.168.10.12 + c-13: 192.168.10.13 + c-14: 192.168.10.14 c-2: 192.168.10.2 c-3: 192.168.10.3 c-4: 192.168.10.4 @@ -17,25 +35,8 @@ prod_ips: c-7: 192.168.10.7 c-8: 192.168.10.8 c-9: 192.168.10.9 - c-10: 192.168.10.10 - c-11: 192.168.10.11 - c-12: 192.168.10.12 - c-13: 192.168.10.13 - c-14: 192.168.10.14 - -data_ips: - srcseed: 192.168.100.201 - c-1: 192.168.100.1 - c-2: 192.168.100.2 - c-3: 192.168.100.3 - c-4: 192.168.100.4 - c-5: 192.168.100.5 - c-6: 192.168.100.6 - c-7: 192.168.100.7 - c-8: 192.168.100.8 - c-9: 192.168.100.9 - c-10: 192.168.100.10 - c-11: 192.168.100.11 - c-12: 192.168.100.12 - c-13: 192.168.100.13 - c-14: 192.168.100.14 + ds1: 192.168.10.15 + ds2: 192.168.10.16 + mds1: 192.168.10.17 + mds2: 192.168.10.18 + srcseed: 192.168.10.201 From 513b72577dffd91bcd04fdf3989fb19f67d53c8b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 15 Jul 2024 10:52:18 +0100 Subject: [PATCH 028/103] Split compute hosts into two groups --- .../network-interfaces | 0 .../compute_lenovo/network-interfaces | 24 +++++++++++++++ etc/kayobe/inventory/hosts | 30 +++++++++++-------- 3 files changed, 42 insertions(+), 12 deletions(-) rename etc/kayobe/inventory/group_vars/{compute => compute_dell}/network-interfaces (100%) create mode 100644 etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces diff --git a/etc/kayobe/inventory/group_vars/compute/network-interfaces b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces similarity index 100% rename from etc/kayobe/inventory/group_vars/compute/network-interfaces rename to etc/kayobe/inventory/group_vars/compute_dell/network-interfaces diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces b/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces new file mode 100644 index 000000000..eb5f0014f --- /dev/null +++ b/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces @@ -0,0 +1,24 @@ +--- +############################################################################### +# Network interface definitions for the compute group. + +# NOTE: The content of this section is very deployment-specific, since it +# depends on the names and types of networks in the deployment. It should +# define the group-specific attributes of networks. The following example shows +# a basic configuration for a network called "example": +# +# example_interface: eth0 +# +# Global network attributes such as subnet CIDRs are typically configured in +# etc/kayobe/networks.yml. +# +# Further information on the available network attributes is provided in the +# network configuration reference in the Kayobe documentation. + +prod_interface: ens9f0np0 +data_interface: ens3f0np0 +prod_gateway: 192.168.10.201 + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 90efa8205..a12806bb7 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -9,21 +9,27 @@ localhost ansible_connection=local #srcseed c-1 -[compute] +[compute_lenovo] c-1 c-2 -#c-3 -#c-4 -#c-5 -#c-6 -#c-7 +c-3 +c-4 +c-5 +c-6 +c-7 #c-14 -# c-8 -# c-9 -# c-10 -# c-11 -# c-12 -# c-13 + +[compute_dell] +c-8 +c-9 +c-10 +c-11 +c-12 +c-13 + +[compute:children] +compute_lenovo +# compute_dell [storage] #ds1 From ddae898f90f6e047018f47b41176c27396c68a2e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 15 Jul 2024 10:52:59 +0100 Subject: [PATCH 029/103] Import ansible speed ups --- etc/kayobe/ansible.cfg | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 etc/kayobe/ansible.cfg diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg new file mode 100644 index 000000000..b38cb8239 --- /dev/null +++ b/etc/kayobe/ansible.cfg @@ -0,0 +1,19 @@ +[defaults] +forks = 20 +# Use the YAML stdout callback plugin. +stdout_callback = yaml +# Use the stdout_callback when running ad-hoc commands. +bin_ansible_callbacks = True +# Disable fact variable injection to improve performance. +inject_facts_as_vars = False +# Add timing information to output +callbacks_enabled = ansible.posix.profile_tasks +# Silence warning about invalid characters found in group names +force_valid_group_names = ignore + +[inventory] +# Fail when any inventory source cannot be parsed. +any_unparsed_is_failed = True + +[ssh_connection] +pipelining = True From 9db68319f654960aa82a215b65fe6596f75edc60 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 15 Jul 2024 10:55:04 +0100 Subject: [PATCH 030/103] Fix cut and paste error --- etc/kayobe/inventory/group_vars/controllers/network-interfaces | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index eb5f0014f..9899a1e96 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -1,6 +1,6 @@ --- ############################################################################### -# Network interface definitions for the compute group. +# Network interface definitions for the controller group. # NOTE: The content of this section is very deployment-specific, since it # depends on the names and types of networks in the deployment. It should From 5c0be2ee3343b06f49514ad3a725befa6bafb9a0 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 15 Jul 2024 10:56:43 +0100 Subject: [PATCH 031/103] Add TODO in dell interfaces --- etc/kayobe/inventory/group_vars/compute_dell/network-interfaces | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces index eb5f0014f..68a6ff3ee 100644 --- a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces @@ -15,6 +15,7 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. +# TODO update me! prod_interface: ens9f0np0 data_interface: ens3f0np0 prod_gateway: 192.168.10.201 From 0d3112d64d1e00225bdacd47577ec92434d2c394 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 15 Jul 2024 10:58:14 +0100 Subject: [PATCH 032/103] Tidy up the prod gateway --- etc/kayobe/inventory/group_vars/compute_dell/network-interfaces | 1 - .../inventory/group_vars/compute_lenovo/network-interfaces | 1 - etc/kayobe/networks.yml | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces index 68a6ff3ee..8c051009b 100644 --- a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces @@ -18,7 +18,6 @@ # TODO update me! prod_interface: ens9f0np0 data_interface: ens3f0np0 -prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces b/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces index eb5f0014f..704d41237 100644 --- a/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces @@ -17,7 +17,6 @@ prod_interface: ens9f0np0 data_interface: ens3f0np0 -prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 3b0b755db..a1c5a55c5 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -88,7 +88,7 @@ storage_mgmt_net_name: data # network configuration reference in the Kayobe documentation. prod_cidr: 192.168.10.0/24 -#prod_gateway: 192.168.10.201 +prod_gateway: 192.168.10.201 prod_vip_address: 192.168.10.200 data_cidr: 192.168.100.0/24 From e1d3fcf40b938fe58a1106499a8b270a1685cec8 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 15 Jul 2024 11:07:03 +0100 Subject: [PATCH 033/103] Tidy up controller network-interfaces --- etc/kayobe/inventory/group_vars/controllers/network-interfaces | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 9899a1e96..cfc5dd284 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -17,7 +17,6 @@ prod_interface: ens9f0np0 data_interface: ens3f0np0 -prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. From 0bc0f0cc55c15afcaeaa7d495d43279d83af4336 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 15 Jul 2024 10:10:41 +0000 Subject: [PATCH 034/103] Adding Bridge port to controller. --- .../group_vars/controllers/network-interfaces | 5 +++-- etc/kayobe/inventory/hosts | 10 +++++----- etc/kayobe/kolla/globals.yml | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index eb5f0014f..eec2339a0 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -15,10 +15,11 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. -prod_interface: ens9f0np0 +prod_interface: brens9f0np0 data_interface: ens3f0np0 prod_gateway: 192.168.10.201 - +prod_bridge_ports: + - ens9f0np0 ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 90efa8205..844e13357 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -12,11 +12,11 @@ c-1 [compute] c-1 c-2 -#c-3 -#c-4 -#c-5 -#c-6 -#c-7 +c-3 +c-4 +c-5 +c-6 +c-7 #c-14 # c-8 # c-9 diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 733467697..53f816665 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,2 +1,3 @@ --- -# Add extra Kolla global configuration here. \ No newline at end of file +# Add extra Kolla global configuration here. +prechecks_enable_host_ntp_checks: false From ea918b529ea99178db1cf6236742aa39c337c5bd Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 22 Jul 2024 06:32:45 +0000 Subject: [PATCH 035/103] Updated prod & data interface names in network-interfaces, add dell & lenovo in hosts, enabled central logging, grafana & prometheus in kolla.yml. --- .../inventory/group_vars/compute_dell/network-interfaces | 5 ++--- etc/kayobe/inventory/hosts | 2 +- etc/kayobe/kolla.yml | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces index 8c051009b..ca237a006 100644 --- a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces @@ -15,9 +15,8 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. -# TODO update me! -prod_interface: ens9f0np0 -data_interface: ens3f0np0 +prod_interface: eno12399np0 +data_interface: ens2f0np0 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index a12806bb7..bc7e8301e 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -29,7 +29,7 @@ c-13 [compute:children] compute_lenovo -# compute_dell +compute_dell [storage] #ds1 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 2ab22bb8f..28a9aa211 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -282,7 +282,7 @@ #kolla_enable_ceilometer_ipmi: #kolla_enable_ceilometer_prometheus_pushgateway: #kolla_enable_cells: -#kolla_enable_central_logging: +kolla_enable_central_logging: yes #kolla_enable_ceph_rgw: #kolla_enable_ceph_rgw_loadbalancer: #kolla_enable_cinder: @@ -312,7 +312,7 @@ #kolla_enable_glance_image_cache: #kolla_enable_gnocchi: #kolla_enable_gnocchi_statsd: -#kolla_enable_grafana: +kolla_enable_grafana: yes #kolla_enable_grafana_external: #kolla_enable_hacluster: #kolla_enable_haproxy: @@ -403,7 +403,7 @@ #kolla_enable_ovn: #kolla_enable_ovs_dpdk: #kolla_enable_placement: -#kolla_enable_prometheus: +kolla_enable_prometheus: yes #kolla_enable_prometheus_alertmanager: #kolla_enable_prometheus_alertmanager_external: #kolla_enable_prometheus_blackbox_exporter: From 9973f03c8ed7dd16de34d95ed527c23367c157e3 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 22 Jul 2024 10:44:55 +0000 Subject: [PATCH 036/103] Configuring storage. --- etc/kayobe/compute.yml | 13 +++++++++++++ etc/kayobe/controllers.yml | 12 ++++++++++++ .../inventory/group_vars/compute_dell/storage | 10 ++++++++++ etc/kayobe/inventory/group_vars/compute_lenovo/nova | 12 ++++++++++++ .../inventory/group_vars/compute_lenovo/storage | 2 ++ etc/kayobe/inventory/hosts | 10 +++++----- etc/kayobe/kolla.yml | 6 +++++- 7 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 etc/kayobe/inventory/group_vars/compute_dell/storage create mode 100644 etc/kayobe/inventory/group_vars/compute_lenovo/nova create mode 100644 etc/kayobe/inventory/group_vars/compute_lenovo/storage diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index d251291ce..242386b89 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -68,6 +68,19 @@ compute_bootstrap_user: ileigh # format. #compute_lvm_groups: +compute_lvm_groups: + - vgname: nova-compute + disks: + - "{{ compute_nova_disk }}" + create: true + lvnames: + - lvname: nova-compute-0 + size: "100%FREE" + create: true + mount: true + filesystem: ext4 + mntp: /var/lib/nova + # Default list of compute volume groups. See mrlesmithjr.manage_lvm role for # format. #compute_lvm_groups_default: diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index 720ff52b1..77b8b5be5 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -76,6 +76,18 @@ controller_bootstrap_user: ileigh # List of controller volume groups. See mrlesmithjr.manage-lvm role for # format. #controller_lvm_groups: +controller_lvm_groups: + - vgname: nova-compute + disks: + - "{{ compute_nova_disk }}" + create: true + lvnames: + - lvname: nova-compute-0 + size: "100%FREE" + create: true + mount: true + filesystem: ext4 + mntp: /var/lib/nova # Default list of controller volume groups. See mrlesmithjr.manage-lvm role for # format. diff --git a/etc/kayobe/inventory/group_vars/compute_dell/storage b/etc/kayobe/inventory/group_vars/compute_dell/storage new file mode 100644 index 000000000..0cf1a5f9f --- /dev/null +++ b/etc/kayobe/inventory/group_vars/compute_dell/storage @@ -0,0 +1,10 @@ +--- +compute_nova_disk: /dev/md0 +compute_mdadm_arrays: + - name: md0 + devices: + - /dev/nvme2n1 + - /dev/nvme3n1 + - /dev/nvme4n1 + level: '0' + state: present diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/nova b/etc/kayobe/inventory/group_vars/compute_lenovo/nova new file mode 100644 index 000000000..21e26ba9c --- /dev/null +++ b/etc/kayobe/inventory/group_vars/compute_lenovo/nova @@ -0,0 +1,12 @@ +--- +nova_cell_compute_provider_config: + meta: + schema_version: '1.0' + providers: + - identification: + name: $COMPUTE_NODE + traits: + additional: + - 'CUSTOM_SR650' +nova_cpu_dedicated_set: 0-47,^0,^12,^24,^36 +nova_instance_datadir_volume: /var/lib/nova diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/storage b/etc/kayobe/inventory/group_vars/compute_lenovo/storage new file mode 100644 index 000000000..a187f802e --- /dev/null +++ b/etc/kayobe/inventory/group_vars/compute_lenovo/storage @@ -0,0 +1,2 @@ +--- +compute_nova_disk: /dev/sdb diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index bc7e8301e..9b71ab6e5 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -21,11 +21,11 @@ c-7 [compute_dell] c-8 -c-9 -c-10 -c-11 -c-12 -c-13 +#c-9 +#c-10 +#c-11 +#c-12 +#c-13 [compute:children] compute_lenovo diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 28a9aa211..019755539 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -217,7 +217,11 @@ # kolla-ansible. If a variable name is not in this mapping the kayobe name is # used. #kolla_overcloud_inventory_pass_through_host_vars_map_extra: - +kolla_overcloud_inventory_pass_through_host_vars_map_extra: + - "nova_cpu_dedicated_set" + - "nova_cpu_shared_set" + - "nova_cell_compute_provider_config" + - "nova_instance_datadir_volume" # Dict mapping names of variables in # kolla_overcloud_inventory_pass_through_host_vars to the variable to use in # kolla-ansible. If a variable name is not in this mapping the kayobe name is From 01f6e06741f0c9878ac637c9ca91804caf9de266 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 22 Jul 2024 11:15:44 +0000 Subject: [PATCH 037/103] Fixed up host files --- etc/kayobe/kolla.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 019755539..29974196c 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -217,7 +217,7 @@ # kolla-ansible. If a variable name is not in this mapping the kayobe name is # used. #kolla_overcloud_inventory_pass_through_host_vars_map_extra: -kolla_overcloud_inventory_pass_through_host_vars_map_extra: +kolla_overcloud_inventory_pass_through_host_vars_extra: - "nova_cpu_dedicated_set" - "nova_cpu_shared_set" - "nova_cell_compute_provider_config" From 08792755d4dbd214c6f42fb8b94a1bd206bc209f Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 22 Jul 2024 11:30:10 +0000 Subject: [PATCH 038/103] Adding c-1 overrides --- etc/kayobe/inventory/host_vars/c-1 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 etc/kayobe/inventory/host_vars/c-1 diff --git a/etc/kayobe/inventory/host_vars/c-1 b/etc/kayobe/inventory/host_vars/c-1 new file mode 100644 index 000000000..3e0ab17f7 --- /dev/null +++ b/etc/kayobe/inventory/host_vars/c-1 @@ -0,0 +1,12 @@ +nova_cell_compute_provider_config: + meta: + schema_version: '1.0' + providers: + - identification: + name: $COMPUTE_NODE + traits: + additional: + - 'CUSTOM_GENERAL' +nova_cpu_shared_set: 0-47,^0,^12,^24,^36 +nova_cpu_dedicated_set: "" +nova_instance_datadir_volume: /var/lib/nova From 4f743776b212cad4f7c5eb61a804997adb72a2a5 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 29 Jul 2024 08:33:24 +0000 Subject: [PATCH 039/103] Configuring hugepages --- etc/kayobe/ansible/hugepages.yml | 14 ++++++++++++++ etc/kayobe/ansible/requirements.yml | 4 ++++ .../post.d/10-hugepages.yml | 1 + etc/kayobe/inventory/group_vars/compute_dell/nova | 12 ++++++++++++ .../inventory/group_vars/compute_lenovo/hugepages | 4 ++++ 5 files changed, 35 insertions(+) create mode 100644 etc/kayobe/ansible/hugepages.yml create mode 100644 etc/kayobe/ansible/requirements.yml create mode 120000 etc/kayobe/hooks/overcloud-host-configure/post.d/10-hugepages.yml create mode 100644 etc/kayobe/inventory/group_vars/compute_dell/nova create mode 100644 etc/kayobe/inventory/group_vars/compute_lenovo/hugepages diff --git a/etc/kayobe/ansible/hugepages.yml b/etc/kayobe/ansible/hugepages.yml new file mode 100644 index 000000000..9b0157d3e --- /dev/null +++ b/etc/kayobe/ansible/hugepages.yml @@ -0,0 +1,14 @@ +--- +- name: Configure compute hypervisors with performance optimisations + hosts: + - controllers,compute + roles: + - name: stackhpc.linux.grubcmdline + kernel_cmdline: + - default_hugepagesz={{ perf_tuning_default_hugepagesz }} + - hugepagesz={{ perf_tuning_hugepagesz }} + - hugepages={{ perf_tuning_hugepages }} + handlers: + - name: reboot + debug: + msg: "please reboot" diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml new file mode 100644 index 000000000..2ab92dea9 --- /dev/null +++ b/etc/kayobe/ansible/requirements.yml @@ -0,0 +1,4 @@ +--- +collections: + - name: stackhpc.linux + version: 1.2.3 diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/10-hugepages.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/10-hugepages.yml new file mode 120000 index 000000000..7abf7a7ff --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-configure/post.d/10-hugepages.yml @@ -0,0 +1 @@ +../../../ansible/hugepages.yml \ No newline at end of file diff --git a/etc/kayobe/inventory/group_vars/compute_dell/nova b/etc/kayobe/inventory/group_vars/compute_dell/nova new file mode 100644 index 000000000..ed589c935 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/compute_dell/nova @@ -0,0 +1,12 @@ +--- +nova_cell_compute_provider_config: + meta: + schema_version: '1.0' + providers: + - identification: + name: $COMPUTE_NODE + traits: + additional: + - 'CUSTOM_R660' +nova_cpu_dedicated_set: 0-63,^0,^16,^32,^48 +nova_instance_datadir_volume: /var/lib/nova diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/hugepages b/etc/kayobe/inventory/group_vars/compute_lenovo/hugepages new file mode 100644 index 000000000..2b9434f00 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/compute_lenovo/hugepages @@ -0,0 +1,4 @@ +--- +perf_tuning_default_hugepagesz: "1G" +perf_tuning_hugepagesz: "1G" +perf_tuning_hugepages: 1408 From c8e4bf11f6464dc2953cde677cef7ac478f3dde0 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 29 Jul 2024 08:35:58 +0000 Subject: [PATCH 040/103] Ignore ansible galaxy roles --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 713cead5c..2fd3e0fea 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,7 @@ etc/kayobe/inventory/group_vars/seed/ansible-host # Ignore kolla configuration. etc/kolla + +# Ansible Galaxy roles & collections +etc/kayobe/ansible/roles/*\.*/ +etc/kayobe/ansible/collections/ From 803e47d0c0f36859b94f79e6a567f823632c1b5c Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 29 Jul 2024 08:58:19 +0000 Subject: [PATCH 041/103] Add ceph nodes into inventory --- .../group_vars/storage/network-interfaces | 4 +--- etc/kayobe/inventory/hosts | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/storage/network-interfaces b/etc/kayobe/inventory/group_vars/storage/network-interfaces index 89f9f419d..d461f565c 100644 --- a/etc/kayobe/inventory/group_vars/storage/network-interfaces +++ b/etc/kayobe/inventory/group_vars/storage/network-interfaces @@ -15,10 +15,8 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. -prod_interface: eno8303 -# TODO - check this! +prod_interface: enp177s0 data_interface: ens2f0np0 -prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 9b71ab6e5..6d0145ae1 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -32,7 +32,17 @@ compute_lenovo compute_dell [storage] -#ds1 -#ds2 +ds1 +ds2 #mds1 -#mds2 +mds2 + +[mons:children] +storage + +[mgrs:children] +storage + +[osds:children] +storage + From 30851cf70a0530980e65323aaf688900a285de62 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 29 Jul 2024 10:07:52 +0100 Subject: [PATCH 042/103] Try to stop hugepages going onto c-1 All controlelrs are doing vcpu hypervisors and we don't want hugepages there. --- etc/kayobe/ansible/hugepages.yml | 2 +- etc/kayobe/inventory/group_vars/controllers/storage | 2 ++ etc/kayobe/inventory/hosts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 etc/kayobe/inventory/group_vars/controllers/storage diff --git a/etc/kayobe/ansible/hugepages.yml b/etc/kayobe/ansible/hugepages.yml index 9b0157d3e..d6151f8ad 100644 --- a/etc/kayobe/ansible/hugepages.yml +++ b/etc/kayobe/ansible/hugepages.yml @@ -1,7 +1,7 @@ --- - name: Configure compute hypervisors with performance optimisations hosts: - - controllers,compute + - compute roles: - name: stackhpc.linux.grubcmdline kernel_cmdline: diff --git a/etc/kayobe/inventory/group_vars/controllers/storage b/etc/kayobe/inventory/group_vars/controllers/storage new file mode 100644 index 000000000..a187f802e --- /dev/null +++ b/etc/kayobe/inventory/group_vars/controllers/storage @@ -0,0 +1,2 @@ +--- +compute_nova_disk: /dev/sdb diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 6d0145ae1..9e0e0b61a 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -10,7 +10,6 @@ localhost ansible_connection=local c-1 [compute_lenovo] -c-1 c-2 c-3 c-4 @@ -30,6 +29,7 @@ c-8 [compute:children] compute_lenovo compute_dell +controllers [storage] ds1 From 9e30baf88e14485bc5526bff2843dd7da81172ed Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 29 Jul 2024 09:57:45 +0100 Subject: [PATCH 043/103] Import cephadm support --- etc/kayobe/ansible/cephadm-commands-post.yml | 13 + etc/kayobe/ansible/cephadm-commands-pre.yml | 13 + etc/kayobe/ansible/cephadm-crush-rules.yml | 11 + etc/kayobe/ansible/cephadm-deploy.yml | 11 + etc/kayobe/ansible/cephadm-ec-profiles.yml | 11 + etc/kayobe/ansible/cephadm-gather-keys.yml | 83 +++ etc/kayobe/ansible/cephadm-keys.yml | 11 + etc/kayobe/ansible/cephadm-pools.yml | 11 + etc/kayobe/ansible/cephadm.yml | 9 + etc/kayobe/ansible/cephadm_readme.rst | 537 +++++++++++++++++++ etc/kayobe/ansible/requirements.yml | 2 + etc/kayobe/cephadm.yml | 142 +++++ etc/kayobe/inventory/groups | 15 + 13 files changed, 869 insertions(+) create mode 100644 etc/kayobe/ansible/cephadm-commands-post.yml create mode 100644 etc/kayobe/ansible/cephadm-commands-pre.yml create mode 100644 etc/kayobe/ansible/cephadm-crush-rules.yml create mode 100644 etc/kayobe/ansible/cephadm-deploy.yml create mode 100644 etc/kayobe/ansible/cephadm-ec-profiles.yml create mode 100644 etc/kayobe/ansible/cephadm-gather-keys.yml create mode 100644 etc/kayobe/ansible/cephadm-keys.yml create mode 100644 etc/kayobe/ansible/cephadm-pools.yml create mode 100644 etc/kayobe/ansible/cephadm.yml create mode 100644 etc/kayobe/ansible/cephadm_readme.rst create mode 100644 etc/kayobe/cephadm.yml diff --git a/etc/kayobe/ansible/cephadm-commands-post.yml b/etc/kayobe/ansible/cephadm-commands-post.yml new file mode 100644 index 000000000..d60a135a3 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-commands-post.yml @@ -0,0 +1,13 @@ +--- +- name: Ensure additional Ceph commands are run after post-deployment configuration + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-commands + tasks: + - import_role: + name: stackhpc.cephadm.commands + vars: + cephadm_commands: "{{ cephadm_commands_post | default([]) }}" diff --git a/etc/kayobe/ansible/cephadm-commands-pre.yml b/etc/kayobe/ansible/cephadm-commands-pre.yml new file mode 100644 index 000000000..abe58caa5 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-commands-pre.yml @@ -0,0 +1,13 @@ +--- +- name: Ensure additional Ceph commands are run before post-deployment configuration + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-commands + tasks: + - import_role: + name: stackhpc.cephadm.commands + vars: + cephadm_commands: "{{ cephadm_commands_pre | default([]) }}" diff --git a/etc/kayobe/ansible/cephadm-crush-rules.yml b/etc/kayobe/ansible/cephadm-crush-rules.yml new file mode 100644 index 000000000..b95b9f960 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-crush-rules.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph CRUSH rules + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-crush-rules + tasks: + - import_role: + name: stackhpc.cephadm.crush_rules diff --git a/etc/kayobe/ansible/cephadm-deploy.yml b/etc/kayobe/ansible/cephadm-deploy.yml new file mode 100644 index 000000000..9cb0f712b --- /dev/null +++ b/etc/kayobe/ansible/cephadm-deploy.yml @@ -0,0 +1,11 @@ +--- +- name: Deploy Ceph using Cephadm + any_errors_fatal: true + gather_facts: true + hosts: ceph + tags: + - cephadm + - cephadm-deploy + tasks: + - import_role: + name: stackhpc.cephadm.cephadm diff --git a/etc/kayobe/ansible/cephadm-ec-profiles.yml b/etc/kayobe/ansible/cephadm-ec-profiles.yml new file mode 100644 index 000000000..5fb0f3bdc --- /dev/null +++ b/etc/kayobe/ansible/cephadm-ec-profiles.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph EC profiles + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-ec-profiles + tasks: + - import_role: + name: stackhpc.cephadm.ec_profiles diff --git a/etc/kayobe/ansible/cephadm-gather-keys.yml b/etc/kayobe/ansible/cephadm-gather-keys.yml new file mode 100644 index 000000000..d2acf92d0 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-gather-keys.yml @@ -0,0 +1,83 @@ +--- +- name: Gather Ceph configuration and keys and populate kayobe-config + gather_facts: false + hosts: mons + run_once: true + tags: + - cephadm + - cephadm-gather-keys + vars: + # Map from an OpenStack service to the directory in which to store Ceph keys for it. + kolla_service_to_key_dir: + cinder-backup: cinder/cinder-backup + cinder-volume: cinder/cinder-volume + glance: glance + manila: manila + nova: nova + # Map from an OpenStack service to the directory in which to store Ceph configuration for it. + kolla_service_to_conf_dir: + cinder-backup: cinder + cinder-volume: cinder + glance: glance + manila: manila + nova: nova + tasks: + - name: Get Ceph keys + stackhpc.cephadm.cephadm_key: + name: "{{ item }}" + output_format: plain + state: info + register: cephadm_key_info + become: true + loop: "{{ kolla_ceph_services | selectattr('required') | map(attribute='keys') | flatten | unique }}" + + - name: Generate ceph.conf + command: "cephadm shell -- ceph config generate-minimal-conf" + become: true + register: cephadm_ceph_conf + changed_when: false + + - name: Ensure Kolla config directories are present + file: + state: directory + path: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.name] }}" + loop: "{{ kolla_ceph_services | selectattr('required') }}" + loop_control: + label: "{{ item.name }}" + delegate_to: localhost + + - name: Save Ceph keys to Kayobe configuration + vars: + key_info: "{{ cephadm_key_info.results | selectattr('item', 'equalto', item.1) | first }}" + cephadm_key: "{{ key_info.stdout }}" + cephadm_user: "{{ item.1 }}" + copy: + # Include a trailing newline. + content: | + {{ cephadm_key }} + dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.0.name] }}/ceph.{{ cephadm_user }}.keyring" + loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" + loop_control: + label: "{{ item.0.name }}: {{ item.1 }}" + delegate_to: localhost + notify: Please add and commit the Kayobe configuration + + - name: Save ceph.conf to Kayobe configuration + copy: + # Include a trailing newline. + # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf. + content: | + {{ cephadm_ceph_conf.stdout | regex_replace('\t') }} + dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf" + loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" + loop_control: + label: "{{ item.0.name }}: {{ item.1 }}" + delegate_to: localhost + notify: Please add and commit the Kayobe configuration + + handlers: + - name: Please add and commit the Kayobe configuration + debug: + msg: >- + Please add and commit the Ceph configuration files and keys in Kayobe + configuration. Remember to encrypt the keys using Ansible Vault. diff --git a/etc/kayobe/ansible/cephadm-keys.yml b/etc/kayobe/ansible/cephadm-keys.yml new file mode 100644 index 000000000..6f632f4ca --- /dev/null +++ b/etc/kayobe/ansible/cephadm-keys.yml @@ -0,0 +1,11 @@ +--- +- name: Generate Ceph keys + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-keys + tasks: + - import_role: + name: stackhpc.cephadm.keys diff --git a/etc/kayobe/ansible/cephadm-pools.yml b/etc/kayobe/ansible/cephadm-pools.yml new file mode 100644 index 000000000..938e6de19 --- /dev/null +++ b/etc/kayobe/ansible/cephadm-pools.yml @@ -0,0 +1,11 @@ +--- +- name: Define Ceph pools + gather_facts: false + hosts: mons + become: true + tags: + - cephadm + - cephadm-keys + tasks: + - import_role: + name: stackhpc.cephadm.pools diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml new file mode 100644 index 000000000..b699df153 --- /dev/null +++ b/etc/kayobe/ansible/cephadm.yml @@ -0,0 +1,9 @@ +--- +# Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. +- import_playbook: cephadm-deploy.yml +- import_playbook: cephadm-commands-pre.yml +- import_playbook: cephadm-ec-profiles.yml +- import_playbook: cephadm-crush-rules.yml +- import_playbook: cephadm-pools.yml +- import_playbook: cephadm-keys.yml +- import_playbook: cephadm-commands-post.yml diff --git a/etc/kayobe/ansible/cephadm_readme.rst b/etc/kayobe/ansible/cephadm_readme.rst new file mode 100644 index 000000000..35a6616c3 --- /dev/null +++ b/etc/kayobe/ansible/cephadm_readme.rst @@ -0,0 +1,537 @@ +==== +Ceph +==== + +This section describes how to use the Cephadm integration included in StackHPC +Kayobe configuration to deploy Ceph. + +The Cephadm integration takes the form of custom playbooks that wrap +around the Ansible `stackhpc.cephadm collection +`_ and provide a means to +create or modify Ceph cluster deployments. Supported features are: + +- creating a new cluster from scratch (RedHat/Debian family distros + supported) +- creating pools, users, CRUSH rules and EC profiles +- modifying the OSD spec after initial deployment +- destroying the cluster + +Resources +========= + +- https://docs.ceph.com/en/quincy/cephadm/index.html +- https://docs.ceph.com/en/quincy/ +- https://docs.ceph.com/en/reef/cephadm/index.html +- https://docs.ceph.com/en/reef/ +- https://github.com/stackhpc/ansible-collection-cephadm + +Configuration +============= + +Inventory +--------- + +The collection assumes a set of group entries in Ansible’s inventory. +The following groups are defined in the Kayobe base configuration +inventory ``groups`` file: + +- ``ceph`` (parent for all Ceph nodes) +- ``mons`` +- ``mgrs`` +- ``osds`` +- ``rgws`` (optional) + +Ceph hosts should be added to these groups as appropriate. Typically at +least the ``mons``, ``mgrs``, and ``osds`` groups should be populated. + +Example: Separate monitors +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a system with separate monitor hosts, the following could be added +to ``etc/kayobe/environments//inventory/groups``, to define two +top-level Ceph host groups: + +.. code:: ini + + [ceph-mons] + [ceph-osds] + + [mons:children] + ceph-mons + + [mgrs:children] + ceph-mons + + [osds:children] + ceph-osds + +Then, populate the ``ceph-mons`` and ``ceph-osds`` groups with the +necessary hosts, e.g. in +``etc/kayobe/environments//inventory/hosts``. + +Example: Colocated monitors +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a system with only colocated monitor and OSD hosts, the following +might be appropriate: + +NOTE: we are using ``storage`` rather than ``ceph``, since ``ceph`` +is already required by the cephadm collection, and redefining it would +introduce a circular dependency between groups. + +.. code:: ini + + [storage] + + [mons:children] + storage + + [mgrs:children] + storage + + [osds:children] + storage + +Then populate the ``storage`` group with the necessary hosts, +e.g. in ``etc/kayobe/environments//inventory/hosts``. + +Ceph deployment configuration +----------------------------- + +Default variables for configuring Ceph are provided in +``etc/kayobe/cephadm.yml``. Many of these defaults will be sufficient, +but you will likely need to set ``cephadm_osd_spec`` to define the OSD +specification. + +Ceph release +~~~~~~~~~~~~ + +The Ceph release series is not strictly dependent upon the StackHPC OpenStack +release, however this configuration does define a default Ceph release series +and container image tag. The default release series is currently |ceph_series|. + +If you wish to use a different Ceph release series, set +``cephadm_ceph_release``. + +If you wish to use different Ceph container image tags, set the following +variables: + +* ``cephadm_image_tag`` +* ``cephadm_haproxy_image_tag`` +* ``cephadm_keepalived_image_tag`` + +OSD specification +~~~~~~~~~~~~~~~~~ + +The following example is a basic OSD spec that adds OSDs for all +available disks with encryption at rest: + +.. code:: yaml + + cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: "*" + data_devices: + all: true + encrypted: true + +More information about OSD service placement is available +`here `__. + +Container image +~~~~~~~~~~~~~~~ + +The container image to be deployed by Cephadm is defined by +``cephadm_image``, and the tag by ``cephadm_image_tag``. The StackHPC +Kayobe configuration provides defaults for both of these. + +Firewalld +~~~~~~~~~ + +If the Ceph storage hosts are running firewalld, it may be helpful to +set ``cephadm_enable_firewalld`` to ``true`` to enable configuration of +firewall rules for Ceph services. + +Ceph post-deployment configuration +---------------------------------- + +The ``stackhpc.cephadm`` collection also provides roles for +post-deployment configuration of pools, users, CRUSH rules and EC +profiles. + +EC profiles +~~~~~~~~~~~ + +An Erasure Coding (EC) profile is required in order to use Erasure Coded +storage pools. Example EC profile: + +.. code:: yaml + + # List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role + # for format. + cephadm_ec_profiles: + - name: ec_4_2_hdd + k: 4 + m: 2 + crush_device_class: hdd + +CRUSH rules +~~~~~~~~~~~ + +CRUSH rules may not be required in a simple setup with a homogeneous +pool of storage. They are useful when there are different tiers of +storage. The following example CRUSH rules define separate tiers for +Hard Disk Drives (HDDs) and Solid State Drives (SSDs). + +.. code:: yaml + + # List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. + cephadm_crush_rules: + - name: replicated_hdd + bucket_root: default + bucket_type: host + device_class: hdd + rule_type: replicated + state: present + - name: replicated_ssd + bucket_root: default + bucket_type: host + device_class: ssd + rule_type: replicated + state: present + +Pools +~~~~~ + +The following example pools should be sufficient to work with the +default `external Ceph +configuration `__ +for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. + +.. code:: yaml + + # List of Ceph pools. See stackhpc.cephadm.pools role for format. + cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: vms + application: rbd + state: present + +If a pool needs to use a particular CRUSH rule, this can be defined via +``rule_name: ``. + +Keys +~~~~ + +The following example keys should be sufficient to work with the default +`external Ceph +configuration `__ +for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. + +.. code:: yaml + + # List of Cephx keys. See stackhpc.cephadm.keys role for format. + cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present + +Ceph Commands +~~~~~~~~~~~~~ + +It is possible to run an arbitrary list of commands against the cluster after +deployment by setting the ``cephadm_commands_pre`` and ``cephadm_commands_post`` +variables. Each should be a list of commands to pass to ``cephadm shell -- +ceph``. For example: + +.. code:: yaml + + # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands + # for format. + cephadm_commands_pre: + # Configure Prometheus exporter to listen on a specific interface. The default + # is to listen on all interfaces. + - "config set mgr mgr/prometheus/server_addr 10.0.0.1" + +Both variables have the same format, however commands in the +``cephadm_commands_pre`` list are executed before the rest of the Ceph +post-deployment configuration is applied. Commands in the +``cephadm_commands_post`` list are executed after the rest of the Ceph +post-deployment configuration is applied. + +Messenger v2 encryption in transit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Messenger v2 is the default on-wire protocol since the Nautilus release. It +supports `encryption of data in transit +`_, +but this is not used by default. It may be enabled as follows: + +.. code:: yaml + + # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands + # for format. + cephadm_commands_pre: + # Enable messenger v2 encryption in transit. + - "config set global ms_cluster_mode secure" + - "config set global ms_service_mode secure" + - "config set global ms_client_mode secure" + +Manila & CephFS +~~~~~~~~~~~~~~~ + +Using Manila with the CephFS backend requires the configuration of additional +resources. + +A Manila key should be added to cephadm_keys: + +.. code:: yaml + + # Append the following to cephadm_keys: + - name: client.manila + caps: + mon: "allow r" + mgr: "allow rw" + state: present + +A CephFS filesystem requires two pools, one for metadata and one for data: + +.. code:: yaml + + # Append the following to cephadm_pools: + - name: cephfs_data + application: cephfs + state: present + - name: cephfs_metadata + application: cephfs + state: present + +Finally, the CephFS filesystem itself should be created: + +.. code:: yaml + + # Append the following to cephadm_commands_post: + - "fs new manila-cephfs cephfs_metadata cephfs_data" + - "orch apply mds manila-cephfs" + +In this example, the filesystem is named ``manila-cephfs``. This name +should be used in the Kolla Manila configuration e.g.: + +.. code:: yaml + + manila_cephfs_filesystem_name: manila-cephfs + +RADOS Gateways +-------------- + +RADOS Gateways (RGWs) are defined with the following: + +.. code:: yaml + + cephadm_radosgw_services: + - id: myrgw + count_per_host: 1 + spec: + rgw_frontend_port: 8100 + +The port chosen must not conflict with any other processes running on the Ceph +hosts. Port 8100 does not conflict with our default suite of services. + +Ceph RGWs require additional configuration to: + + * Support both S3 and Swift APIs. + + * Authenticate user access via Keystone. + + * Allow cross-project and public object access. + +The set of commands below configure all of these. + +.. code:: yaml + + # Append the following to cephadm_commands_post: + - "config set client.rgw rgw_content_length_compat true" + - "config set client.rgw rgw_enable_apis 's3, swift, swift_auth, admin'" + - "config set client.rgw rgw_enforce_swift_acls true" + - "config set client.rgw rgw_keystone_accepted_admin_roles 'admin'" + - "config set client.rgw rgw_keystone_accepted_roles 'member, Member, _member_, admin'" + - "config set client.rgw rgw_keystone_admin_domain Default" + - "config set client.rgw rgw_keystone_admin_password {{ secrets_ceph_rgw_keystone_password }}" + - "config set client.rgw rgw_keystone_admin_project service" + - "config set client.rgw rgw_keystone_admin_user 'ceph_rgw'" + - "config set client.rgw rgw_keystone_api_version '3'" + - "config set client.rgw rgw_keystone_token_cache_size '10000'" + - "config set client.rgw rgw_keystone_url https://{{ kolla_internal_fqdn }}:5000" + - "config set client.rgw rgw_keystone_verify_ssl false" + - "config set client.rgw rgw_max_attr_name_len '1000'" + - "config set client.rgw rgw_max_attr_size '1000'" + - "config set client.rgw rgw_max_attrs_num_in_req '1000'" + - "config set client.rgw rgw_s3_auth_use_keystone true" + - "config set client.rgw rgw_swift_account_in_url true" + - "config set client.rgw rgw_swift_versioning_enabled true" + +As we have configured Ceph to respond to Swift APIs, you will need to tell +Kolla to account for this when registering Swift endpoints with Keystone. Also, +when ``rgw_swift_account_in_url`` is set, the equivalent Kolla variable should +be set in Kolla ``globals.yml`` too: + +.. code:: yaml + + ceph_rgw_swift_compatibility: false + ceph_rgw_swift_account_in_url: true + +``secrets_ceph_rgw_keystone_password`` should be stored in the Kayobe +``secrets.yml``, and set to the same value as ``ceph_rgw_keystone_password`` in +the Kolla ``passwords.yml``. As such, you will need to configure Keystone +before deploying the RADOS gateways. If you are using the Kolla load balancer +(see :ref:`RGWs-with-hyper-converged-Ceph` for more info), you can specify the +``haproxy`` and ``loadbalancer`` tags here too. + +.. code:: yaml + + kayobe overcloud service deploy -kt ceph-rgw,keystone,haproxy,loadbalancer + + +.. _RGWs-with-hyper-converged-Ceph: + +RGWs with hyper-converged Ceph +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using a hyper-converged Ceph setup (i.e. your OpenStack controllers +and Ceph storage nodes share the same hosts), you should double-check that +``rgw_frontend_port`` does not conflict with any processes on the controllers. +For example, port 80 (and 443) will be bound to the Kolla-deployed haproxy. You +should choose a custom port that does not conflict with any OpenStack endpoints +too (``openstack endpoint list``). + +You may also want to use the Kolla-deployed haproxy to load balance your RGWs. +This means you will not need to define any Ceph ingress services. Instead, you +add definitions of your Ceph hosts to Kolla ``globals.yml``: + +.. code:: yaml + + ceph_rgw_hosts: + - host: controller1 + ip: + port: 8100 + - host: controller2 + ip: + port: 8100 + - host: controller3 + ip: + port: 8100 + +HA with Ingress services +~~~~~~~~~~~~~~~~~~~~~~~~ + +Ingress services are defined with the following. ``id`` should match the name +(not id) of the RGW service to which ingress will point to. ``spec`` is a +service specification required by Cephadm to deploy the ingress (haproxy + +keepalived pair). + +Note that the ``virtual_ip`` here must be different than the Kolla VIP. The +choice of subnet will be dependent on your deployment, and can be outside +of any Ceph networks. + +.. code:: yaml + + cephadm_ingress_services: + - id: rgw.myrgw + spec: + frontend_port: 443 + monitor_port: 1967 + virtual_ip: 10.66.0.1/24 + ssl_cert: {example_certificate_chain} + +When using ingress services, you will need to stop Kolla from configuring your +RGWs to use the Kolla-deployed haproxy. Set the following in Kolla +``globals.yml``: + +.. code:: yaml + + enable_ceph_rgw_loadbalancer: false + +Deployment +========== + +Host configuration +------------------ + +Configure the Ceph hosts: + +.. code:: bash + + kayobe overcloud host configure --limit storage + +Ceph deployment +--------------- + +.. + **FIXME**: Wait for Ceph to come up, so that we can just run cephadm.yml + +Deploy the Ceph services: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml + +You can check the status of Ceph via Cephadm on the storage nodes: + +.. code:: bash + + sudo cephadm shell -- ceph -s + +Once the Ceph cluster has finished initialising, run the full +cephadm.yml playbook to perform post-deployment configuration: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml + +The ``cephadm.yml`` playbook imports various other playbooks, which may also be +run individually to perform specific tasks. Note that if you want to deploy +additional services (such as RGWs or ingress) after an initial deployment, you +will need to set ``cephadm_bootstrap`` to true. For example: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml -e cephadm_bootstrap=true + +Configuration generation +------------------------ + +Generate keys and configuration for Kolla Ansible: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml + +This will generate Ceph keys and configuration under +``etc/kayobe/environments//kolla/config/``, which should be +committed to the configuration. + +This configuration will be used during +``kayobe overcloud service deploy``. diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 2ab92dea9..9e561c058 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -2,3 +2,5 @@ collections: - name: stackhpc.linux version: 1.2.3 + - name: stackhpc.cephadm + version: 1.15.1 diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml new file mode 100644 index 000000000..cc067467d --- /dev/null +++ b/etc/kayobe/cephadm.yml @@ -0,0 +1,142 @@ +--- +############################################################################### +# Cephadm deployment configuration. + +# Ceph release name. +cephadm_ceph_release: "quincy" + +# TODO placeholders +stackhpc_docker_registry: "" +stackhpc_sync_ceph_images: false + +# Ceph FSID. +#cephadm_fsid: + +# Ceph container image. +cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}" + +# Ceph container image tag. +cephadm_image_tag: "v17.2.7" + +# Ceph custom repo workaround for Ubuntu Jammy as there are no official ceph repos for jammy. +cephadm_custom_repos: "{{ ansible_facts['distribution_release'] == 'jammy' }}" + +# HAProxy container image. +cephadm_haproxy_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/haproxy:{{ cephadm_haproxy_image_tag }}" + +# HAProxy container image tag. +cephadm_haproxy_image_tag: "2.3" + +# Keepalived container image. +cephadm_keepalived_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/keepalived:{{ cephadm_keepalived_image_tag }}" + +# Keepalived container image tag. +cephadm_keepalived_image_tag: "2.1.5" + +# Ceph container image registry URL. +cephadm_registry_url: "{{ stackhpc_docker_registry }}" + +# Ceph container image registry username. +cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" + +# Ceph container image registry password. +cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" + +# Cephadm admin network interface. +cephadm_admin_interface: "{{ admin_oc_net_name | net_interface }}" + +# Ceph public network interface. +cephadm_public_interface: "{{ storage_net_name | net_interface }}" + +# Ceph public network subnet CIDR. +cephadm_public_network: "{{ storage_net_name | net_cidr }}" + +# Ceph cluster network interface. +cephadm_cluster_interface: "{{ storage_mgmt_net_name | net_interface }}" + +# Ceph cluster network subnet CIDR. +cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" + +# Whether to enable firewalld for Ceph storage hosts. +#cephadm_enable_firewalld: + +# List of /dev/device paths to use, e.g. for multipath devices that can't be +# used with an OSD spec. +#cephadm_osd_devices: + +# Ceph OSD specification. +#cephadm_osd_spec: + +# Additional arguments to pass to `cephadm bootstrap`. +#cephadm_bootstrap_additional_parameters: + +############################################################################### +# Ceph post-deployment configuration. + +# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role +# for format. +#cephadm_ec_profiles: + +# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. +#cephadm_crush_rules: + +# List of Ceph pools. See stackhpc.cephadm.pools role for format. +#cephadm_pools: + +# List of Cephx keys. See stackhpc.cephadm.keys role for format. +#cephadm_keys: + +# A list of commands to pass to cephadm shell -- ceph. See +# stackhpc.cephadm.commands for format. Pre commands run before the rest of the +# post-deployment configuration, post commands run after the rest of the +# post-deployment configuration. +cephadm_commands_pre: "{{ cephadm_commands_pre_default + cephadm_commands_pre_extra }}" +cephadm_commands_post: "{{ cephadm_commands_post_default + cephadm_commands_post_extra }}" + +cephadm_commands_pre_default: [] +cephadm_commands_pre_extra: [] + +cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | default(False) | bool else [] }}" +cephadm_commands_post_extra: [] + +############################################################################### +# Kolla Ceph auto-configuration. + +# List of Kolla Ansible services that require Ceph configuration files. +kolla_ceph_services: + - name: cinder-backup + keys: + - client.cinder + - client.cinder-backup + required: "{{ kolla_ceph_cinder_backup_required | bool }}" + - name: cinder-volume + keys: + - client.cinder + required: "{{ kolla_ceph_cinder_volume_required | bool }}" + - name: glance + keys: + - client.glance + required: "{{ kolla_ceph_glance_required | bool }}" + - name: manila + keys: + - client.manila + required: "{{ kolla_ceph_manila_required | bool }}" + - name: nova + keys: + - client.cinder + required: "{{ kolla_ceph_nova_required | bool }}" + +# Whether to generate Ceph configuration for Cinder volume. +kolla_ceph_cinder_volume_required: "{{ kolla_enable_cinder | bool }}" + +# Whether to generate Ceph configuration for Cinder backup. +kolla_ceph_cinder_backup_required: "{{ kolla_enable_cinder_backup | default(true) | bool }}" + +# Whether to generate Ceph configuration for Glance. +kolla_ceph_glance_required: "{{ kolla_enable_glance | bool }}" + +# Whether to generate Ceph configuration for Manila. +kolla_ceph_manila_required: "{{ kolla_enable_manila | bool }}" + +# Whether to generate Ceph configuration for Nova. +kolla_ceph_nova_required: "{{ kolla_enable_nova | bool }}" diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index 16619fed8..e51195309 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -97,3 +97,18 @@ overcloud mgmt-switches ctl-switches hs-switches + +############################################################################### +# Ceph groups. + +[ceph:children] +mons +mgrs +osds +rgws + +# Empty groups to provide declaration of groups required for Cephadm. +[mons] +[mgrs] +[osds] +[rgws] From fade0e526eb4021fc863c9aefffdadb0436d2a04 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 29 Jul 2024 10:12:10 +0100 Subject: [PATCH 044/103] Just doing hugepages on lenovo compute for now --- etc/kayobe/ansible/hugepages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/hugepages.yml b/etc/kayobe/ansible/hugepages.yml index d6151f8ad..161235ef7 100644 --- a/etc/kayobe/ansible/hugepages.yml +++ b/etc/kayobe/ansible/hugepages.yml @@ -1,7 +1,7 @@ --- - name: Configure compute hypervisors with performance optimisations hosts: - - compute + - compute_lenovo roles: - name: stackhpc.linux.grubcmdline kernel_cmdline: From 2d8ff204f7bf67adfa8c3f8b00ca0e49d46ca01a Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 29 Jul 2024 09:45:41 +0000 Subject: [PATCH 045/103] Fix ceph inventory --- etc/kayobe/network-allocation.yml | 16 ++++++++-------- etc/kayobe/storage.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index 8ef2950b6..f1345608b 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -13,10 +13,10 @@ data_ips: c-7: 192.168.100.7 c-8: 192.168.100.8 c-9: 192.168.100.9 - ds1: 192.168.100.15 - ds2: 192.168.100.16 - mds1: 192.168.100.17 - mds2: 192.168.100.18 + ds1: 192.168.100.202 + ds2: 192.168.100.203 + mds1: 192.168.100.204 + mds2: 192.168.100.205 srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 @@ -35,8 +35,8 @@ prod_ips: c-7: 192.168.10.7 c-8: 192.168.10.8 c-9: 192.168.10.9 - ds1: 192.168.10.15 - ds2: 192.168.10.16 - mds1: 192.168.10.17 - mds2: 192.168.10.18 + ds1: 192.168.10.202 + ds2: 192.168.10.203 + mds1: 192.168.10.204 + mds2: 192.168.10.205 srcseed: 192.168.10.201 diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index e9e52dfe6..c689fddb4 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -4,7 +4,7 @@ # User with which to access the storages via SSH during bootstrap, in order # to setup the Kayobe user account. Default is {{ os_distribution }}. -#storage_bootstrap_user: +storage_bootstrap_user: ileigh ############################################################################### # Storage network interface configuration. From 2115aa57cd5b69151546be26dc836850533bb4d7 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Mon, 29 Jul 2024 10:53:10 +0100 Subject: [PATCH 046/103] Move to ovn --- etc/kayobe/kolla.yml | 6 +++--- etc/kayobe/kolla/globals.yml | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 29974196c..96fe95df1 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -395,8 +395,8 @@ kolla_enable_grafana: yes #kolla_enable_nova_libvirt_container: #kolla_enable_nova_serialconsole_proxy: #kolla_enable_nova_ssh: -#kolla_enable_octavia: -#kolla_enable_octavia_driver_agent: +kolla_enable_octavia: yes +kolla_enable_octavia_driver_agent: ovn #kolla_enable_opensearch: #kolla_enable_opensearch_dashboards: #kolla_enable_opensearch_dashboards_external: @@ -404,7 +404,7 @@ kolla_enable_grafana: yes #kolla_enable_openvswitch: #kolla_enable_osprofiler: #kolla_enable_outward_rabbitmq: -#kolla_enable_ovn: +kolla_enable_ovn: yes #kolla_enable_ovs_dpdk: #kolla_enable_placement: kolla_enable_prometheus: yes diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 53f816665..a421f4d1e 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,3 +1,9 @@ --- # Add extra Kolla global configuration here. prechecks_enable_host_ntp_checks: false + +# Just use ovn octavia +octavia_auto_configure: "no" +octavia_provider_drivers: "ovn:OVN provider" +octavia_provider_agents: "ovn" +enable_horizon_octavia: false From 1ff5129a3b2b3020345024da615edadd1cf13303 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 29 Jul 2024 10:25:27 +0000 Subject: [PATCH 047/103] Adding intial ceph config --- etc/kayobe/cephadm.yml | 60 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index cc067467d..6127ad879 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -65,7 +65,16 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" #cephadm_osd_devices: # Ceph OSD specification. -#cephadm_osd_spec: +cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: '*' + spec: + data_devices: + size: '2TB:' + db_devices: + size: ':2TB' # Additional arguments to pass to `cephadm bootstrap`. #cephadm_bootstrap_additional_parameters: @@ -82,9 +91,47 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" # List of Ceph pools. See stackhpc.cephadm.pools role for format. #cephadm_pools: +cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: cephfs_data + application: cephfs + state: present + - name: cephfs_metadata + application: cephfs + state: present # List of Cephx keys. See stackhpc.cephadm.keys role for format. #cephadm_keys: +cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present + - name: client.manila + caps: + mon: "allow r" + mgr: "allow rw" + state: present # A list of commands to pass to cephadm shell -- ceph. See # stackhpc.cephadm.commands for format. Pre commands run before the rest of the @@ -94,11 +141,14 @@ cephadm_commands_pre: "{{ cephadm_commands_pre_default + cephadm_commands_pre_ex cephadm_commands_post: "{{ cephadm_commands_post_default + cephadm_commands_post_extra }}" cephadm_commands_pre_default: [] -cephadm_commands_pre_extra: [] - +cephadm_commands_pre_extra: + - "config set global ms_cluster_mode secure" + - "config set global ms_service_mode secure" + - "config set global ms_client_mode secure" cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | default(False) | bool else [] }}" -cephadm_commands_post_extra: [] - +cephadm_commands_post_extra: + - "fs new manila-cephfs cephfs_metadata cephfs_data" + - "orch apply mds manila-cephfs" ############################################################################### # Kolla Ceph auto-configuration. From 673d21d375f3da6872131004e05f65e04dc8a0e1 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 19 Aug 2024 08:44:40 +0000 Subject: [PATCH 048/103] Adding Ceph --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/config/cinder/ceph.conf | 4 ++++ .../ceph.client.cinder-backup.keyring | 17 +++++++++++++++++ .../cinder-backup/ceph.client.cinder.keyring | 15 +++++++++++++++ .../config/glance/ceph.client.glance.keyring | 14 ++++++++++++++ etc/kayobe/kolla/config/glance/ceph.conf | 4 ++++ .../config/nova/ceph.client.cinder.keyring | 15 +++++++++++++++ etc/kayobe/kolla/config/nova/ceph.conf | 4 ++++ etc/kayobe/kolla/globals.yml | 3 +++ 9 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/kolla/config/cinder/ceph.conf create mode 100644 etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring create mode 100644 etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring create mode 100644 etc/kayobe/kolla/config/glance/ceph.client.glance.keyring create mode 100644 etc/kayobe/kolla/config/glance/ceph.conf create mode 100644 etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring create mode 100644 etc/kayobe/kolla/config/nova/ceph.conf diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 96fe95df1..d8d44c306 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -289,7 +289,7 @@ kolla_overcloud_inventory_pass_through_host_vars_extra: kolla_enable_central_logging: yes #kolla_enable_ceph_rgw: #kolla_enable_ceph_rgw_loadbalancer: -#kolla_enable_cinder: +kolla_enable_cinder: yes #kolla_enable_cinder_backend_hnas_nfs: #kolla_enable_cinder_backend_iscsi: #kolla_enable_cinder_backend_lvm: diff --git a/etc/kayobe/kolla/config/cinder/ceph.conf b/etc/kayobe/kolla/config/cinder/ceph.conf new file mode 100644 index 000000000..461f51fc9 --- /dev/null +++ b/etc/kayobe/kolla/config/cinder/ceph.conf @@ -0,0 +1,4 @@ +# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +[global] +fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] diff --git a/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring new file mode 100644 index 000000000..107d93b12 --- /dev/null +++ b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring @@ -0,0 +1,17 @@ +$ANSIBLE_VAULT;1.1;AES256 +32343030363536653566636631643630646563623061633133376165653031636361346361386535 +3434636436323663313763653061313464646466636164370a636137346132616664656663353461 +39336335633237616234363766313039366661616133326638643030343330303531623838663633 +3664643337656130390a383530613166393131373735643539346162336365356634393263393934 +66376238656538643334386530386332336336353036303339363235333362623261663336323934 +66323736333032313239656162616131663264316265633133343733333430666430656130643165 +39383061373165356436366434333537396437373931666131663735666365616337636633333732 +39383137643336663133326261666335363036666564333865376566323330373532373631626331 +64643365636162353530383664323665613631363362653665613530353130323764323161366531 +30353231623464653131366566383561663737396466326532643131666330343630356333363032 +33356635376366343932333262373063633233323131373436646235333834663365356566636139 +31316438343035323337316463303562623834366236356463376263353966366364623465363232 +37623537373062333834333431643863613136633466303330636462316139633666346462316230 +30373435313666616366336435313464613537666537383731646466613933363234303238383630 +35663838623761313638623665643439613662383731343563613364363533393165616562613766 +31303733636332363332 diff --git a/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring new file mode 100644 index 000000000..0755939f2 --- /dev/null +++ b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring @@ -0,0 +1,15 @@ +$ANSIBLE_VAULT;1.1;AES256 +61333337303062373834616466393336356566363838343931393764666238656265623461323531 +6536633237336366663131613038613161383135643663360a646366383635353462346261636438 +34613937666666333238633065353537333632353664333965383836613661653965306137393434 +3061363366656234380a323534313766636336386635346633373436363738396430636465373230 +66336666333339613430396338383061616362613535306638616162303834373361636535616336 +64386136363363383132336436636630333933373331663037636136363163333234653934633433 +36613337623031376234643461643736336236376664336237636262343735353966666665373364 +38343835653630343437326666646561663332303438393431303235346434306239623233363831 +35383733613766653834313539633532623836613134663132353630346235363661636230373764 +33633866376131633163666365616238643565396461323336666365303336613334343431363738 +63363164616232643764613561326163336636666465663636343334643235663839656266343964 +30343366383939383562366466323163373739323737623737366661643330333862323632323333 +35366531356662303431306231663665663465613163663935613365633134396334323935613466 +3832663233346334653462313131616330306161313464383031 diff --git a/etc/kayobe/kolla/config/glance/ceph.client.glance.keyring b/etc/kayobe/kolla/config/glance/ceph.client.glance.keyring new file mode 100644 index 000000000..4e7dfdd06 --- /dev/null +++ b/etc/kayobe/kolla/config/glance/ceph.client.glance.keyring @@ -0,0 +1,14 @@ +$ANSIBLE_VAULT;1.1;AES256 +31326335316637383363383335383035326562323936646166343238323735343137363666333932 +3530313933373765353963643062636235326435633733380a316531316261616564633732306465 +33653631616263616633366461373234353237343632623136663632666665343536636434316461 +3631636532396130370a656664333939376237376637333165643465346437656235396161616538 +36373732343232333563336331366336323936336638613166376465316636663234623565653035 +32366334633062383834663038366437383737326331633462663230616330333361663139373863 +64333665646134353032646134393733303537663039313966396232323031363466323930363036 +66386438653265393035646637373034633663353833613462363735386465376465373566346332 +63356330653061393839613330386635343738383663393664346535663336343061353865313834 +61323461396636326265326432646363373062643065396161643035613937303934303436353038 +31333634663032653064626338353531376239323335353262336161626637313437653961373334 +31663138323862346238346435616134383733323733373462383731353038396630323733343366 +3362 diff --git a/etc/kayobe/kolla/config/glance/ceph.conf b/etc/kayobe/kolla/config/glance/ceph.conf new file mode 100644 index 000000000..461f51fc9 --- /dev/null +++ b/etc/kayobe/kolla/config/glance/ceph.conf @@ -0,0 +1,4 @@ +# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +[global] +fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] diff --git a/etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring b/etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring new file mode 100644 index 000000000..4226fa4f2 --- /dev/null +++ b/etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring @@ -0,0 +1,15 @@ +$ANSIBLE_VAULT;1.1;AES256 +64306563353463633066363230393232353939303234666166376634363764633535636662323032 +3662376265643233636261613463363764646331333062310a363732616238373461336263336139 +39616137343132336233653232626233386630626264623832613463326461623439373038656230 +3534633163353461320a383366353534633236353732323761333834313332376635383539306463 +63336338303135353938393765313233666230376130376230663631303330643264656261336532 +37353437646262313732373034363266386632383264303838343933656661373135336233663637 +32313239663939343534393137343434346133366264663434376663333566336439343637313632 +64653937636131326165616439393766313635393230326464633135666631336434616332306163 +66306365383964626530316665633730363764303338623034366164663965656664653331346630 +61613364666361653765666235653363623032633732363563613735333961393839623339666266 +66323564616438383234396430383832373231333239626565636433626465343962393438363732 +62653864396464363966663663383233373538656661386631343735313463313830623330643164 +37333531346637613734633834376566326335343834333366316265636134656438613562646234 +3732393862376361316465353863396161323065346130313863 diff --git a/etc/kayobe/kolla/config/nova/ceph.conf b/etc/kayobe/kolla/config/nova/ceph.conf new file mode 100644 index 000000000..461f51fc9 --- /dev/null +++ b/etc/kayobe/kolla/config/nova/ceph.conf @@ -0,0 +1,4 @@ +# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +[global] +fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index a421f4d1e..d8ff0b13f 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -7,3 +7,6 @@ octavia_auto_configure: "no" octavia_provider_drivers: "ovn:OVN provider" octavia_provider_agents: "ovn" enable_horizon_octavia: false +# Enabling Ceph +cinder_backend_ceph: true +glance_backend_ceph: true From b0eae7e91f2be6dc30bfb33d42192917b469b280 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 19 Aug 2024 08:47:24 +0000 Subject: [PATCH 049/103] Updates to cephadm config --- etc/kayobe/ansible/multipath-setup.yml | 29 ++++++++++++++ etc/kayobe/cephadm.yml | 40 +++++++++++++++---- .../group_vars/compute_dell/hugepages | 4 ++ 3 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 etc/kayobe/ansible/multipath-setup.yml create mode 100644 etc/kayobe/inventory/group_vars/compute_dell/hugepages diff --git a/etc/kayobe/ansible/multipath-setup.yml b/etc/kayobe/ansible/multipath-setup.yml new file mode 100644 index 000000000..93e947049 --- /dev/null +++ b/etc/kayobe/ansible/multipath-setup.yml @@ -0,0 +1,29 @@ +--- +- hosts: storage + become: True + gather_facts: False + tags: + - multipath + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + tasks: + - name: Ensure device-mapper-multipath package is installed + ansible.builtin.command: + cmd: "dnf install -y device-mapper-multipath" + creates: /usr/sbin/multipathd + - name: Ensure dm_multipath module is loaded + community.general.modprobe: + name: dm_multipath + state: present + - name: Create /etc/multipath.conf + ansible.builtin.command: + cmd: "mpathconf --enable --user_friendly_names n --option path_grouping_policy:multibus" + creates: /etc/multipath.conf + - name: Ensure multipathd is enabled and running + ansible.builtin.service: + name: multipathd + enabled: yes + state: started diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 6127ad879..428b7292c 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -34,13 +34,13 @@ cephadm_keepalived_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_ima cephadm_keepalived_image_tag: "2.1.5" # Ceph container image registry URL. -cephadm_registry_url: "{{ stackhpc_docker_registry }}" +# cephadm_registry_url: "{{ stackhpc_docker_registry }}" # Ceph container image registry username. -cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" +# cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" # Ceph container image registry password. -cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" +# cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" # Cephadm admin network interface. cephadm_admin_interface: "{{ admin_oc_net_name | net_interface }}" @@ -72,9 +72,17 @@ cephadm_osd_spec: host_pattern: '*' spec: data_devices: - size: '2TB:' - db_devices: - size: ':2TB' + paths: + - /dev/mapper/mpatha + - /dev/mapper/mpathb + - /dev/mapper/mpathc + - /dev/mapper/mpathd + - /dev/mapper/mpathe + - /dev/mapper/mpathf + - /dev/mapper/mpathg + - /dev/mapper/mpathh + - /dev/mapper/mpathi + - /dev/mapper/mpathj # Additional arguments to pass to `cephadm bootstrap`. #cephadm_bootstrap_additional_parameters: @@ -85,6 +93,11 @@ cephadm_osd_spec: # List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role # for format. #cephadm_ec_profiles: +cephadm_ec_profiles: + - name: ec_2_1_hdd + k: 2 + m: 1 + crush_device_class: hdd # List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. #cephadm_crush_rules: @@ -95,18 +108,27 @@ cephadm_pools: - name: backups application: rbd state: present + pool_type: erasure + erasure_profile: ec_2_1_hdd - name: images application: rbd state: present + pool_type: erasure + erasure_profile: ec_2_1_hdd - name: volumes application: rbd state: present + pool_type: erasure + erasure_profile: ec_2_1_hdd - name: cephfs_data application: cephfs state: present + pool_type: erasure + erasure_profile: ec_2_1_hdd - name: cephfs_metadata application: cephfs state: present + size: 3 # List of Cephx keys. See stackhpc.cephadm.keys role for format. #cephadm_keys: @@ -114,13 +136,15 @@ cephadm_keys: - name: client.cinder caps: mon: "profile rbd" - osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" - mgr: "profile rbd pool=volumes, profile rbd pool=vms" + osd: "profile rbd pool=volumes, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes" + state: present - name: client.cinder-backup caps: mon: "profile rbd" osd: "profile rbd pool=volumes, profile rbd pool=backups" mgr: "profile rbd pool=volumes, profile rbd pool=backups" + state: present - name: client.glance caps: mon: "profile rbd" diff --git a/etc/kayobe/inventory/group_vars/compute_dell/hugepages b/etc/kayobe/inventory/group_vars/compute_dell/hugepages new file mode 100644 index 000000000..36952a70b --- /dev/null +++ b/etc/kayobe/inventory/group_vars/compute_dell/hugepages @@ -0,0 +1,4 @@ +--- +perf_tuning_default_hugepagesz: "1G" +perf_tuning_hugepagesz: "1G" +perf_tuning_hugepages: 1984 From 624622b94baff464986f9c18bdb6ef965ec68364 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 19 Aug 2024 08:48:47 +0000 Subject: [PATCH 050/103] Bringing in hypervisors --- etc/kayobe/ansible/hugepages.yml | 2 +- etc/kayobe/globals.yml | 2 +- .../inventory/group_vars/compute_dell/nova | 2 +- etc/kayobe/inventory/hosts | 18 +++++++++--------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/etc/kayobe/ansible/hugepages.yml b/etc/kayobe/ansible/hugepages.yml index 161235ef7..f0c2013e2 100644 --- a/etc/kayobe/ansible/hugepages.yml +++ b/etc/kayobe/ansible/hugepages.yml @@ -1,7 +1,7 @@ --- - name: Configure compute hypervisors with performance optimisations hosts: - - compute_lenovo + - compute_lenovo,compute_dell roles: - name: stackhpc.linux.grubcmdline kernel_cmdline: diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index 5680720e1..c15f4d485 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -66,7 +66,7 @@ os_release: 9 # to not specify a gather subset. #kayobe_ansible_setup_gather_subset: -selinux_do_reboot: false +selinux_do_reboot: true ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/compute_dell/nova b/etc/kayobe/inventory/group_vars/compute_dell/nova index ed589c935..0411a9290 100644 --- a/etc/kayobe/inventory/group_vars/compute_dell/nova +++ b/etc/kayobe/inventory/group_vars/compute_dell/nova @@ -8,5 +8,5 @@ nova_cell_compute_provider_config: traits: additional: - 'CUSTOM_R660' -nova_cpu_dedicated_set: 0-63,^0,^16,^32,^48 +nova_cpu_dedicated_set: 0-127,^0,^32,^64,^96 nova_instance_datadir_volume: /var/lib/nova diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 9e0e0b61a..c05dbecf2 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -11,20 +11,20 @@ c-1 [compute_lenovo] c-2 -c-3 -c-4 +#c-3 OS on sdb +#c-4 OS on sdb c-5 c-6 c-7 -#c-14 +# c-14 networking broken [compute_dell] c-8 -#c-9 -#c-10 -#c-11 -#c-12 -#c-13 +c-9 +c-10 +#c-11 storage broken +c-12 +c-13 [compute:children] compute_lenovo @@ -34,7 +34,7 @@ controllers [storage] ds1 ds2 -#mds1 +mds1 mds2 [mons:children] From 481d10e3a05d718ce1eb70c46ce12942a8db8114 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 19 Aug 2024 09:31:11 +0000 Subject: [PATCH 051/103] Moving cinder volume --- etc/kayobe/kolla.yml | 1 + .../kolla/inventory/overcloud-service.j2 | 570 ++++++++++++++++++ 2 files changed, 571 insertions(+) create mode 100644 etc/kayobe/kolla/inventory/overcloud-service.j2 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index d8d44c306..5a33a6600 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -175,6 +175,7 @@ # Custom overcloud inventory containing a mapping from services to components. #kolla_overcloud_inventory_custom_services: +kolla_overcloud_inventory_custom_services: "{{ lookup('template', kayobe_config_path ~ '/kolla/inventory/overcloud-service.j2') }}" # Full custom overcloud inventory contents. By default this will be the # concatenation of the top level, component, and service inventories. diff --git a/etc/kayobe/kolla/inventory/overcloud-service.j2 b/etc/kayobe/kolla/inventory/overcloud-service.j2 new file mode 100644 index 000000000..36e67da13 --- /dev/null +++ b/etc/kayobe/kolla/inventory/overcloud-service.j2 @@ -0,0 +1,570 @@ +# This inventory section provides a mapping of services to components. +# +# Top level groups define the roles of hosts, e.g. controller or compute. +# Components define groups of services, e.g. nova or ironic. +# Services define single containers, e.g. nova-compute or ironic-api. + +# Additional control implemented here. These groups allow you to control which +# services run on which hosts at a per-service level. +# +# Word of caution: Some services are required to run on the same host to +# function appropriately. For example, neutron-metadata-agent must run on the +# same host as the l3-agent and (depending on configuration) the dhcp-agent. + +# Common +[cron:children] +common + +[fluentd:children] +common + +[kolla-logs:children] +common + +[kolla-toolbox:children] +common + +[opensearch:children] +control + +# Opensearch dashboards +[opensearch-dashboards:children] +opensearch + +# Glance +[glance-api:children] +glance + +# Nova +[nova-api:children] +nova + +[nova-conductor:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +[nova-super-conductor:children] +nova + +[nova-novncproxy:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +[nova-scheduler:children] +nova + +[nova-spicehtml5proxy:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +# NOTE: HA for nova-compute services with ironic is still an experimental +# feature. Provide the option to use a single compute host, even when multiple +# controllers are in use. +{% if kolla_nova_compute_ironic_host is not none %} +[nova-compute-ironic] +{{ kolla_nova_compute_ironic_host }} +{% else %} +[nova-compute-ironic:children] +nova +{% endif %} + +[nova-serialproxy:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +# Neutron +[neutron-server:children] +control + +[neutron-dhcp-agent:children] +neutron + +[neutron-l3-agent:children] +neutron + +[neutron-metadata-agent:children] +neutron + +[neutron-ovn-metadata-agent:children] +compute +network + +[neutron-bgp-dragent:children] +neutron + +[neutron-infoblox-ipam-agent:children] +neutron + +[neutron-metering-agent:children] +neutron + +[ironic-neutron-agent:children] +neutron + +[neutron-ovn-agent:children] +compute +network + +# Cinder +[cinder-api:children] +cinder + +[cinder-backup:children] +cinder + +[cinder-scheduler:children] +cinder + +[cinder-volume:children] +cinder + +# Cloudkitty +[cloudkitty-api:children] +cloudkitty + +[cloudkitty-processor:children] +cloudkitty + +# Freezer +[freezer-api:children] +freezer + +[freezer-scheduler:children] +freezer + +# iSCSI +[iscsid:children] +compute +storage +ironic + +[tgtd:children] +storage + +# Manila +[manila-api:children] +manila + +[manila-scheduler:children] +manila + +[manila-share:children] +network + +[manila-data:children] +manila + +# Swift +[swift-proxy-server:children] +swift + +[swift-account-server:children] +storage + +[swift-container-server:children] +storage + +[swift-object-server:children] +storage + +# Barbican +[barbican-api:children] +barbican + +[barbican-keystone-listener:children] +barbican + +[barbican-worker:children] +barbican + +# Heat +[heat-api:children] +heat + +[heat-api-cfn:children] +heat + +[heat-engine:children] +heat + +# Murano +[murano-api:children] +murano + +[murano-engine:children] +murano + +# Monasca +[monasca-agent-collector:children] +monasca-agent + +[monasca-agent-forwarder:children] +monasca-agent + +[monasca-agent-statsd:children] +monasca-agent + +[monasca-api:children] +monasca + +[monasca-log-persister:children] +monasca + +[monasca-log-metrics:children] +monasca + +[monasca-thresh:children] +monasca + +[monasca-notification:children] +monasca + +[monasca-persister:children] +monasca + +# Storm +[storm-worker:children] +storm + +[storm-nimbus:children] +storm + +# Ironic +[ironic-api:children] +ironic + +[ironic-conductor:children] +ironic + +#[ironic-inspector:children] +#ironic + +[ironic-inspector] +# FIXME: Ideally we wouldn't reference controllers in here directly, but only +# one inspector service should exist, and groups can't be indexed in an +# inventory (e.g. ironic[0]). +{% if groups.get('controllers', []) | length > 0 %} +{{ groups['controllers'][0] }} +{% endif %} + +[ironic-tftp:children] +ironic + +[ironic-http:children] +ironic + +# Magnum +[magnum-api:children] +magnum + +[magnum-conductor:children] +magnum + +# Sahara +[sahara-api:children] +sahara + +[sahara-engine:children] +sahara + +# Solum +[solum-api:children] +solum + +[solum-worker:children] +solum + +[solum-deployer:children] +solum + +[solum-conductor:children] +solum + +[solum-application-deployment:children] +solum + +[solum-image-builder:children] +solum + +# Mistral +[mistral-api:children] +mistral + +[mistral-executor:children] +mistral + +[mistral-engine:children] +mistral + +[mistral-event-engine:children] +mistral + +# Ceilometer +[ceilometer-central:children] +ceilometer + +[ceilometer-notification:children] +ceilometer + +[ceilometer-compute:children] +compute + +[ceilometer-ipmi:children] +compute + +# Aodh +[aodh-api:children] +aodh + +[aodh-evaluator:children] +aodh + +[aodh-listener:children] +aodh + +[aodh-notifier:children] +aodh + +# Cyborg +[cyborg-api:children] +cyborg + +[cyborg-agent:children] +compute + +[cyborg-conductor:children] +cyborg + +# Gnocchi +[gnocchi-api:children] +gnocchi + +[gnocchi-statsd:children] +gnocchi + +[gnocchi-metricd:children] +gnocchi + +# Trove +[trove-api:children] +trove + +[trove-conductor:children] +trove + +[trove-taskmanager:children] +trove + +# Multipathd +[multipathd:children] +compute +storage + +# Watcher +[watcher-api:children] +watcher + +[watcher-engine:children] +watcher + +[watcher-applier:children] +watcher + +# Senlin +[senlin-api:children] +senlin + +[senlin-conductor:children] +senlin + +[senlin-engine:children] +senlin + +[senlin-health-manager:children] +senlin + +# Octavia +[octavia-api:children] +octavia + +[octavia-driver-agent:children] +octavia + +[octavia-health-manager:children] +octavia + +[octavia-housekeeping:children] +octavia + +[octavia-worker:children] +octavia + +# Designate +[designate-api:children] +designate + +[designate-central:children] +designate + +[designate-producer:children] +designate + +[designate-mdns:children] +network + +[designate-worker:children] +designate + +[designate-sink:children] +designate + +[designate-backend-bind9:children] +designate + +# Placement +[placement-api:children] +placement + +# Zun +[zun-api:children] +zun + +[zun-wsproxy:children] +zun + +[zun-compute:children] +compute + +[zun-cni-daemon:children] +compute + +# Skyline +[skyline-apiserver:children] +skyline + +[skyline-console:children] +skyline + +# Tacker +[tacker-server:children] +tacker + +[tacker-conductor:children] +tacker + +# Vitrage +[vitrage-api:children] +vitrage + +[vitrage-notifier:children] +vitrage + +[vitrage-graph:children] +vitrage + +[vitrage-ml:children] +vitrage + +[vitrage-persistor:children] +vitrage + +# Blazar +[blazar-api:children] +blazar + +[blazar-manager:children] +blazar + +# Prometheus +[prometheus-node-exporter:children] +monitoring +control +compute +network +storage + +[prometheus-mysqld-exporter:children] +mariadb + +[prometheus-haproxy-exporter:children] +loadbalancer + +[prometheus-memcached-exporter:children] +memcached + +[prometheus-cadvisor:children] +monitoring +control +compute +network +storage + +[prometheus-alertmanager:children] +monitoring + +[prometheus-openstack-exporter:children] +monitoring + +[prometheus-elasticsearch-exporter:children] +opensearch + +[prometheus-blackbox-exporter:children] +monitoring + +[prometheus-libvirt-exporter:children] +compute + +[prometheus-msteams:children] +prometheus-alertmanager + +[masakari-api:children] +control + +[masakari-engine:children] +control + +[masakari-hostmonitor:children] +control + +[masakari-instancemonitor:children] +compute + +[ovn-controller:children] +ovn-controller-compute +ovn-controller-network + +[ovn-controller-compute:children] +compute + +[ovn-controller-network:children] +network + +[ovn-database:children] +control + +[ovn-northd:children] +ovn-database + +[ovn-nb-db:children] +ovn-database + +[ovn-sb-db:children] +ovn-database + +[venus-api:children] +venus + +[venus-manager:children] +venus From f7a8a509cfe01b72ebf805379da906b87b3baf91 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 19 Aug 2024 13:11:23 +0000 Subject: [PATCH 052/103] Fixing Ceph pools --- etc/kayobe/ansible/requirements.yml | 2 +- etc/kayobe/cephadm.yml | 12 +++++------- .../cinder-volume/ceph.client.cinder.keyring | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 9e561c058..7aafbba73 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -3,4 +3,4 @@ collections: - name: stackhpc.linux version: 1.2.3 - name: stackhpc.cephadm - version: 1.15.1 + version: 1.16.0 diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 428b7292c..36ae9c959 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -108,23 +108,21 @@ cephadm_pools: - name: backups application: rbd state: present - pool_type: erasure - erasure_profile: ec_2_1_hdd + size: 3 - name: images application: rbd state: present - pool_type: erasure - erasure_profile: ec_2_1_hdd + size: 3 - name: volumes application: rbd state: present - pool_type: erasure - erasure_profile: ec_2_1_hdd + size: 3 - name: cephfs_data application: cephfs state: present pool_type: erasure erasure_profile: ec_2_1_hdd + allow_ec_overwrites: true - name: cephfs_metadata application: cephfs state: present @@ -171,7 +169,7 @@ cephadm_commands_pre_extra: - "config set global ms_client_mode secure" cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | default(False) | bool else [] }}" cephadm_commands_post_extra: - - "fs new manila-cephfs cephfs_metadata cephfs_data" + - "fs new manila-cephfs cephfs_metadata cephfs_data --force" - "orch apply mds manila-cephfs" ############################################################################### # Kolla Ceph auto-configuration. diff --git a/etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring b/etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring new file mode 100644 index 000000000..0755939f2 --- /dev/null +++ b/etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring @@ -0,0 +1,15 @@ +$ANSIBLE_VAULT;1.1;AES256 +61333337303062373834616466393336356566363838343931393764666238656265623461323531 +6536633237336366663131613038613161383135643663360a646366383635353462346261636438 +34613937666666333238633065353537333632353664333965383836613661653965306137393434 +3061363366656234380a323534313766636336386635346633373436363738396430636465373230 +66336666333339613430396338383061616362613535306638616162303834373361636535616336 +64386136363363383132336436636630333933373331663037636136363163333234653934633433 +36613337623031376234643461643736336236376664336237636262343735353966666665373364 +38343835653630343437326666646561663332303438393431303235346434306239623233363831 +35383733613766653834313539633532623836613134663132353630346235363661636230373764 +33633866376131633163666365616238643565396461323336666365303336613334343431363738 +63363164616232643764613561326163336636666465663636343334643235663839656266343964 +30343366383939383562366466323163373739323737623737366661643330333862323632323333 +35366531356662303431306231663665663465613163663935613365633134396334323935613466 +3832663233346334653462313131616330306161313464383031 From e449baf74dc17d15e9d941fc37cddb81319bce9f Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Wed, 28 Aug 2024 14:48:31 +0100 Subject: [PATCH 053/103] Fix Octavia driver agent evaluating to false --- etc/kayobe/kolla.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 5a33a6600..a415e42e8 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -397,7 +397,7 @@ kolla_enable_grafana: yes #kolla_enable_nova_serialconsole_proxy: #kolla_enable_nova_ssh: kolla_enable_octavia: yes -kolla_enable_octavia_driver_agent: ovn +#kolla_enable_octavia_driver_agent: #kolla_enable_opensearch: #kolla_enable_opensearch_dashboards: #kolla_enable_opensearch_dashboards_external: From 1cafb2e86bb4b792f6b21de2feb81c14d0689d93 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 6 Sep 2024 07:22:35 +0000 Subject: [PATCH 054/103] updating manila and ceph config --- etc/kayobe/kolla.yml | 4 ++-- .../kolla/config/manila/ceph.client.manila.keyring | 9 +++++++++ etc/kayobe/kolla/config/manila/ceph.conf | 7 +++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 etc/kayobe/kolla/config/manila/ceph.client.manila.keyring create mode 100644 etc/kayobe/kolla/config/manila/ceph.conf diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 5a33a6600..ae00dc2fc 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -358,8 +358,8 @@ kolla_enable_grafana: yes #kolla_enable_kuryr: #kolla_enable_loadbalancer: #kolla_enable_magnum: -#kolla_enable_manila: -#kolla_enable_manila_backend_cephfs_native: +kolla_enable_manila: yes +kolla_enable_manila_backend_cephfs_native: yes #kolla_enable_manila_backend_cephfs_nfs: #kolla_enable_manila_backend_generic: #kolla_enable_manila_backend_glusterfs_nfs: diff --git a/etc/kayobe/kolla/config/manila/ceph.client.manila.keyring b/etc/kayobe/kolla/config/manila/ceph.client.manila.keyring new file mode 100644 index 000000000..6e1d7072d --- /dev/null +++ b/etc/kayobe/kolla/config/manila/ceph.client.manila.keyring @@ -0,0 +1,9 @@ +$ANSIBLE_VAULT;1.1;AES256 +61343063333633393830353031646336643935643536353332386133303562643162353633343438 +3132323162393263623264623236373062343732656139630a353063646335333961303937363533 +31653030306366666461653664626631343833323033663164343266616465326335663939626466 +3536356361396231300a623862646234656338633432303234333134613237643834646361666437 +34356138336235356162653161613930316635623130373037396234643965303733323939376565 +62663138363133336162303731663463353865346635653962383439383039383634336633613661 +35306230356464653639393137353938613462363534623039363766353666656131336265653261 +62386639316661353261 diff --git a/etc/kayobe/kolla/config/manila/ceph.conf b/etc/kayobe/kolla/config/manila/ceph.conf new file mode 100644 index 000000000..aedf6f27e --- /dev/null +++ b/etc/kayobe/kolla/config/manila/ceph.conf @@ -0,0 +1,7 @@ +# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +[global] +fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] +auth_cluster_required = cephx +auth_service_required = cephx +auth_client_required = cephx From 56d21e121434d5a3352db77ab0377fd2ed8a3cfd Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Mon, 16 Sep 2024 18:39:48 +0200 Subject: [PATCH 055/103] [2024.1-only] Synchronise with latest kayobe Change-Id: Ice2551c023db9674d6fd6596209bda3af8df60d6 --- etc/kayobe/infra-vms.yml | 16 ++++++++++++---- etc/kayobe/seed-vm.yml | 14 +++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index a8f1fd9b2..e48004103 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -31,10 +31,12 @@ # Base image for the infra VM root volume. Default is # "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" -# when os_distribution is "ubuntu", or -# https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 -# when os_distribution is "rocky", -# or +# when os_distribution is "ubuntu", +# "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" +# when os_distribution is "rocky" and seed_vm_boot_firmware is "efi", +# "https://dl.rockylinux.org/vault/rocky/9.3/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" +# when os_distribution is "rocky" and seed_vm_boot_firmware is not "efi" +# (default is "bios"), or # "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20221206.0.x86_64.qcow2" # otherwise. #infra_vm_root_image: @@ -58,6 +60,12 @@ # OS family. Needed for config drive generation. #infra_vm_os_family: +# Boot firmware. Possible values are 'bios' or 'efi'. Default is 'bios'. +#infra_vm_boot_firmware: + +# Machine type. Libvirt default configuration is used. +#infra_vm_machine: + ############################################################################### # Infrastructure VM node configuration. diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml index f10fbaa19..42aea8da5 100644 --- a/etc/kayobe/seed-vm.yml +++ b/etc/kayobe/seed-vm.yml @@ -32,9 +32,11 @@ # Base image for the seed VM root volume. Default is # "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" # when os_distribution is "ubuntu", -# https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2 -# when os_distribution is "rocky", -# or +# "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" +# when os_distribution is "rocky" and seed_vm_boot_firmware is "efi", +# "https://dl.rockylinux.org/vault/rocky/9.3/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" +# when os_distribution is "rocky" and seed_vm_boot_firmware is not "efi" +# (default is "bios"), or # "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20221206.0.x86_64.qcow2" # otherwise. #seed_vm_root_image: @@ -61,6 +63,12 @@ # #seed_vm_interfaces: +# Boot firmware. Possible values are 'bios' or 'efi'. Default is 'bios'. +#seed_vm_boot_firmware: + +# Machine type. Libvirt default configuration is used. +#seed_vm_machine: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes From 029f035ae4365a8165b7e963126b54a9376e36b4 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 7 Jan 2025 10:49:37 +0000 Subject: [PATCH 056/103] Fixed OS on sdb, uncommented c-3 & c-4 --- etc/kayobe/inventory/hosts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index c05dbecf2..a9a85674c 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -11,8 +11,8 @@ c-1 [compute_lenovo] c-2 -#c-3 OS on sdb -#c-4 OS on sdb +c-3 +c-4 c-5 c-6 c-7 From 8d531d1181107376bbf0e7ba7fe37f73ef93041e Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 27 May 2025 09:22:22 +0100 Subject: [PATCH 057/103] Add new ceph hosts --- .../network-interfaces | 0 .../storage_r760/network-interfaces | 23 +++++++++++++++++++ etc/kayobe/inventory/hosts | 15 +++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) rename etc/kayobe/inventory/group_vars/{storage => storage_me5}/network-interfaces (100%) create mode 100644 etc/kayobe/inventory/group_vars/storage_r760/network-interfaces diff --git a/etc/kayobe/inventory/group_vars/storage/network-interfaces b/etc/kayobe/inventory/group_vars/storage_me5/network-interfaces similarity index 100% rename from etc/kayobe/inventory/group_vars/storage/network-interfaces rename to etc/kayobe/inventory/group_vars/storage_me5/network-interfaces diff --git a/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces b/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces new file mode 100644 index 000000000..bf8b429dd --- /dev/null +++ b/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces @@ -0,0 +1,23 @@ +--- +############################################################################### +# Network interface definitions for the storage group. + +# NOTE: The content of this section is very deployment-specific, since it +# depends on the names and types of networks in the deployment. It should +# define the group-specific attributes of networks. The following example shows +# a basic configuration for a network called "example": +# +# example_interface: eth0 +# +# Global network attributes such as subnet CIDRs are typically configured in +# etc/kayobe/networks.yml. +# +# Further information on the available network attributes is provided in the +# network configuration reference in the Kayobe documentation. + +prod_interface: ens1f0 +data_interface: ens4f0np0 + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index a9a85674c..04d1d0787 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -31,12 +31,25 @@ compute_lenovo compute_dell controllers -[storage] +[storage_me5] ds1 ds2 mds1 mds2 +[storage_r760] +ds3 +ds4 +ds5 +ds6 +ds7 +ds8 + +[storage] +[storage:children] +storage_me5 +storage_760xd2 + [mons:children] storage From a0dc8a2efae0b2a6b09938c0ca19fe1b02979cc2 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 27 May 2025 09:22:49 +0000 Subject: [PATCH 058/103] Added new ceph nodes --- etc/kayobe/cephadm.yml | 32 ++++++++++++++++++++++--------- etc/kayobe/inventory/hosts | 2 +- etc/kayobe/network-allocation.yml | 12 ++++++++++++ 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 36ae9c959..8d98c480d 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -65,11 +65,11 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" #cephadm_osd_devices: # Ceph OSD specification. -cephadm_osd_spec: +cephadm_osd_spec: | service_type: osd service_id: osd_spec_default placement: - host_pattern: '*' + host_pattern: 'ds[1-2]|mds[1-2]' spec: data_devices: paths: @@ -83,6 +83,14 @@ cephadm_osd_spec: - /dev/mapper/mpathh - /dev/mapper/mpathi - /dev/mapper/mpathj + --- + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: 'ds[3-8]' + spec: + data_devices: + size: '>15T' # Additional arguments to pass to `cephadm bootstrap`. #cephadm_bootstrap_additional_parameters: @@ -94,9 +102,9 @@ cephadm_osd_spec: # for format. #cephadm_ec_profiles: cephadm_ec_profiles: - - name: ec_2_1_hdd - k: 2 - m: 1 + - name: ec_4_2_hdd + k: 4 + m: 2 crush_device_class: hdd # List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. @@ -108,20 +116,26 @@ cephadm_pools: - name: backups application: rbd state: present - size: 3 + pool_type: erasure + erasure_profile: ec_4_2_hdd + allow_ec_overwrites: true - name: images application: rbd state: present - size: 3 + pool_type: erasure + erasure_profile: ec_4_2_hdd + allow_ec_overwrites: true - name: volumes application: rbd state: present - size: 3 + pool_type: erasure + erasure_profile: ec_4_2_hdd + allow_ec_overwrites: true - name: cephfs_data application: cephfs state: present pool_type: erasure - erasure_profile: ec_2_1_hdd + erasure_profile: ec_4_2_hdd allow_ec_overwrites: true - name: cephfs_metadata application: cephfs diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 04d1d0787..846396bbe 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -48,7 +48,7 @@ ds8 [storage] [storage:children] storage_me5 -storage_760xd2 +storage_r760 [mons:children] storage diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index f1345608b..7e7f771da 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -15,6 +15,12 @@ data_ips: c-9: 192.168.100.9 ds1: 192.168.100.202 ds2: 192.168.100.203 + ds3: 192.168.100.206 + ds4: 192.168.100.207 + ds5: 192.168.100.208 + ds6: 192.168.100.209 + ds7: 192.168.100.210 + ds8: 192.168.100.211 mds1: 192.168.100.204 mds2: 192.168.100.205 srcseed: 192.168.100.201 @@ -37,6 +43,12 @@ prod_ips: c-9: 192.168.10.9 ds1: 192.168.10.202 ds2: 192.168.10.203 + ds3: 192.168.10.206 + ds4: 192.168.10.207 + ds5: 192.168.10.208 + ds6: 192.168.10.209 + ds7: 192.168.10.210 + ds8: 192.168.10.211 mds1: 192.168.10.204 mds2: 192.168.10.205 srcseed: 192.168.10.201 From 30057615f146c00aac9e6e037fabbede22b3b01b Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 27 May 2025 10:37:03 +0100 Subject: [PATCH 059/103] Add new JBO network for public_api --- .../group_vars/controllers/network-interfaces | 4 ++++ etc/kayobe/network-allocation.yml | 2 ++ etc/kayobe/networks.yml | 12 ++++++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 0e184fa16..e427094e1 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -20,6 +20,10 @@ data_interface: ens3f0np0 prod_bridge_ports: - ens9f0np0 +jbo_interface: breno8303 +jbo_bridge_ports: + - eno8303 + ############################################################################## # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index 7e7f771da..91152ac8b 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -26,6 +26,8 @@ data_ips: srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 +jbo_ips: + srcseed: 130.88.9.201 prod_ips: c-1: 192.168.10.1 c-10: 192.168.10.10 diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index a1c5a55c5..8ae85b51f 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -35,11 +35,14 @@ internal_net_name: data # containing one item, external_net_name. #external_net_names: external_net_names: +# physnet1 - prod +# physnet2 +- jbo # Name of the network used to expose the public OpenStack API endpoints. #public_net_name: -public_net_name: prod +public_net_name: jbo # Name of the network used by Neutron to carry tenant overlay network traffic. #tunnel_net_name: @@ -89,7 +92,12 @@ storage_mgmt_net_name: data prod_cidr: 192.168.10.0/24 prod_gateway: 192.168.10.201 -prod_vip_address: 192.168.10.200 + +# *.openstack.man.uksrc.org +jbo_gateway: 130.88.9.125 +jbo_cidr: 130.88.9.0/24 +jbo_vip_address: 130.88.9.4 +jbo_fqdn: api.openstack.man.uksrc.org data_cidr: 192.168.100.0/24 data_vip_address: 192.168.100.200 From b721eaaca75abfd0bec3ae386c78d7b88a039ca7 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 27 May 2025 10:43:15 +0100 Subject: [PATCH 060/103] Move prod gateway into group vars --- .../inventory/group_vars/compute_dell/network-interfaces | 1 + .../inventory/group_vars/compute_lenovo/network-interfaces | 1 + .../inventory/group_vars/storage_me5/network-interfaces | 1 + .../inventory/group_vars/storage_r760/network-interfaces | 1 + etc/kayobe/networks.yml | 4 +++- 5 files changed, 7 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces index ca237a006..c3983ba45 100644 --- a/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute_dell/network-interfaces @@ -17,6 +17,7 @@ prod_interface: eno12399np0 data_interface: ens2f0np0 +prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces b/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces index 704d41237..eb5f0014f 100644 --- a/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces +++ b/etc/kayobe/inventory/group_vars/compute_lenovo/network-interfaces @@ -17,6 +17,7 @@ prod_interface: ens9f0np0 data_interface: ens3f0np0 +prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/storage_me5/network-interfaces b/etc/kayobe/inventory/group_vars/storage_me5/network-interfaces index d461f565c..265a07206 100644 --- a/etc/kayobe/inventory/group_vars/storage_me5/network-interfaces +++ b/etc/kayobe/inventory/group_vars/storage_me5/network-interfaces @@ -17,6 +17,7 @@ prod_interface: enp177s0 data_interface: ens2f0np0 +prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces b/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces index bf8b429dd..5767d5d7d 100644 --- a/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces +++ b/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces @@ -17,6 +17,7 @@ prod_interface: ens1f0 data_interface: ens4f0np0 +prod_gateway: 192.168.10.201 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 8ae85b51f..416da32a3 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -91,7 +91,9 @@ storage_mgmt_net_name: data # network configuration reference in the Kayobe documentation. prod_cidr: 192.168.10.0/24 -prod_gateway: 192.168.10.201 +# Gateway is set in group vars +# so controllers are not confused +# prod_gateway: 192.168.10.201 # *.openstack.man.uksrc.org jbo_gateway: 130.88.9.125 From 015f2a5e641ce34cb4dba516c86d07e3299cabcd Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 27 May 2025 10:46:49 +0100 Subject: [PATCH 061/103] Upgrade Ceph to reef --- etc/kayobe/cephadm.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 8d98c480d..8c9239ed4 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -3,7 +3,7 @@ # Cephadm deployment configuration. # Ceph release name. -cephadm_ceph_release: "quincy" +cephadm_ceph_release: "reef" # TODO placeholders stackhpc_docker_registry: "" @@ -16,7 +16,7 @@ stackhpc_sync_ceph_images: false cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}" # Ceph container image tag. -cephadm_image_tag: "v17.2.7" +cephadm_image_tag: "v18.2.7" # Ceph custom repo workaround for Ubuntu Jammy as there are no official ceph repos for jammy. cephadm_custom_repos: "{{ ansible_facts['distribution_release'] == 'jammy' }}" @@ -25,13 +25,13 @@ cephadm_custom_repos: "{{ ansible_facts['distribution_release'] == 'jammy' }}" cephadm_haproxy_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/haproxy:{{ cephadm_haproxy_image_tag }}" # HAProxy container image tag. -cephadm_haproxy_image_tag: "2.3" +cephadm_haproxy_image_tag: "2.6" # Keepalived container image. cephadm_keepalived_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/keepalived:{{ cephadm_keepalived_image_tag }}" # Keepalived container image tag. -cephadm_keepalived_image_tag: "2.1.5" +cephadm_keepalived_image_tag: "2.2.4" # Ceph container image registry URL. # cephadm_registry_url: "{{ stackhpc_docker_registry }}" From 9061d129332e60f4c809d3fb5d930dcc7146c142 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 27 May 2025 10:13:30 +0000 Subject: [PATCH 062/103] Updated Data & Prod interface names --- .../inventory/group_vars/controllers/network-interfaces | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index e427094e1..3ad1f9a4b 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -15,10 +15,10 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. -prod_interface: brens9f0np0 -data_interface: ens3f0np0 +prod_interface: breno12399 +data_interface: ens2f0np0 prod_bridge_ports: - - ens9f0np0 + - eno12399 jbo_interface: breno8303 jbo_bridge_ports: From 973b2f07ec917396cc00a8b39768bfa4de970f8c Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Tue, 27 May 2025 14:31:24 +0100 Subject: [PATCH 063/103] Move to unmaintained/2023.1 requirements --- etc/kayobe/pip.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/pip.yml b/etc/kayobe/pip.yml index 563c3ce08..10a4817a3 100644 --- a/etc/kayobe/pip.yml +++ b/etc/kayobe/pip.yml @@ -3,7 +3,7 @@ # Upper constraints file for installation of python packages. # Default value is # "https://releases.openstack.org/constraints/upper/{{ openstack_release }}" -#pip_upper_constraints_file: +pip_upper_constraints_file: "https://opendev.org/openstack/requirements/raw/branch/unmaintained/2023.1/upper-constraints.txt" # Use a local PyPi mirror for installing Pip packages #pip_local_mirror: false From 47313b283e52673b32377b29c5cbfc7d38a84a94 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Wed, 28 May 2025 08:38:14 +0000 Subject: [PATCH 064/103] Fixed hosts --- etc/kayobe/inventory/hosts | 10 +++++----- etc/kayobe/networks.yml | 3 --- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 846396bbe..104d1e4b0 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -6,13 +6,14 @@ localhost ansible_connection=local #srcseed [controllers] -#srcseed -c-1 +srcseed +#c-1 [compute_lenovo] +c-1 c-2 -c-3 -c-4 +# c-3 sdb needs fixing +# c-4 sdb needs fixing c-5 c-6 c-7 @@ -29,7 +30,6 @@ c-13 [compute:children] compute_lenovo compute_dell -controllers [storage_me5] ds1 diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 416da32a3..97eb96483 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -35,9 +35,6 @@ internal_net_name: data # containing one item, external_net_name. #external_net_names: external_net_names: -# physnet1 -- prod -# physnet2 - jbo # Name of the network used to expose the public OpenStack API endpoints. From 107004130b07c4be8d12489d67a05767ddc1290d Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Wed, 28 May 2025 11:41:00 +0000 Subject: [PATCH 065/103] Fix gateway and cephadm --- etc/kayobe/cephadm.yml | 14 ++++++++++++-- .../inventory/group_vars/controllers/storage | 2 -- etc/kayobe/networks.yml | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) delete mode 100644 etc/kayobe/inventory/group_vars/controllers/storage diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 8c9239ed4..5848c3480 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -69,7 +69,11 @@ cephadm_osd_spec: | service_type: osd service_id: osd_spec_default placement: - host_pattern: 'ds[1-2]|mds[1-2]' + hosts: + - ds1 + - ds2 + - mds1 + - mds2 spec: data_devices: paths: @@ -87,7 +91,13 @@ cephadm_osd_spec: | service_type: osd service_id: osd_spec_default placement: - host_pattern: 'ds[3-8]' + hosts: + - ds3 + - ds4 + - ds5 + - ds6 + - ds7 + - ds8 spec: data_devices: size: '>15T' diff --git a/etc/kayobe/inventory/group_vars/controllers/storage b/etc/kayobe/inventory/group_vars/controllers/storage deleted file mode 100644 index a187f802e..000000000 --- a/etc/kayobe/inventory/group_vars/controllers/storage +++ /dev/null @@ -1,2 +0,0 @@ ---- -compute_nova_disk: /dev/sdb diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 97eb96483..a9c2546ac 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -93,7 +93,7 @@ prod_cidr: 192.168.10.0/24 # prod_gateway: 192.168.10.201 # *.openstack.man.uksrc.org -jbo_gateway: 130.88.9.125 +jbo_gateway: 130.88.9.250 jbo_cidr: 130.88.9.0/24 jbo_vip_address: 130.88.9.4 jbo_fqdn: api.openstack.man.uksrc.org From b3898ad8d89f8024491d0ac7ccf5599441220845 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Wed, 28 May 2025 13:07:25 +0100 Subject: [PATCH 066/103] Hashivault --- etc/kayobe/ansible/requirements.yml | 2 + etc/kayobe/ansible/vault-deploy-barbican.yml | 107 ++++++++++++++++ etc/kayobe/ansible/vault-deploy-overcloud.yml | 118 ++++++++++++++++++ etc/kayobe/ansible/vault-deploy-seed.yml | 83 ++++++++++++ .../ansible/vault-generate-backend-tls.yml | 82 ++++++++++++ .../ansible/vault-generate-internal-tls.yml | 56 +++++++++ .../vault-generate-test-external-tls.yml | 57 +++++++++ etc/kayobe/ansible/vault-unseal-overcloud.yml | 37 ++++++ etc/kayobe/ansible/vault-unseal-seed.yml | 34 +++++ etc/kayobe/inventory/group_vars/all/vault | 85 +++++++++++++ 10 files changed, 661 insertions(+) create mode 100644 etc/kayobe/ansible/vault-deploy-barbican.yml create mode 100644 etc/kayobe/ansible/vault-deploy-overcloud.yml create mode 100644 etc/kayobe/ansible/vault-deploy-seed.yml create mode 100644 etc/kayobe/ansible/vault-generate-backend-tls.yml create mode 100644 etc/kayobe/ansible/vault-generate-internal-tls.yml create mode 100644 etc/kayobe/ansible/vault-generate-test-external-tls.yml create mode 100644 etc/kayobe/ansible/vault-unseal-overcloud.yml create mode 100644 etc/kayobe/ansible/vault-unseal-seed.yml create mode 100644 etc/kayobe/inventory/group_vars/all/vault diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 7aafbba73..793b3816e 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -4,3 +4,5 @@ collections: version: 1.2.3 - name: stackhpc.cephadm version: 1.16.0 + - name: stackhpc.hashicorp + version: 2.6.1 diff --git a/etc/kayobe/ansible/vault-deploy-barbican.yml b/etc/kayobe/ansible/vault-deploy-barbican.yml new file mode 100644 index 000000000..54f17bb35 --- /dev/null +++ b/etc/kayobe/ansible/vault-deploy-barbican.yml @@ -0,0 +1,107 @@ +--- +- name: Configure AppRole + any_errors_fatal: true + gather_facts: true + hosts: controllers[0] + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + tasks: + - name: Assert that secrets_barbican_approle_secret_id is defined + ansible.builtin.assert: + that: + - secrets_barbican_approle_secret_id is defined + fail_msg: Please define secrets_barbican_approle_secret_id in your secrets.yml + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + name: vault_keys + + - name: Ensure hvac is installed + ansible.builtin.pip: + name: hvac + state: present + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv_path }}/kayobe" + + - name: Ensure AppRole is configured + environment: + https_proxy: "" + block: + - name: Enable AppRole auth module + hashivault_auth_method: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ vault_ca_cert }}" + token: "{{ vault_keys.root_token }}" + method_type: approle + state: enabled + + - name: Enable barbican kv store + hashivault_secret_engine: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ vault_ca_cert }}" + token: "{{ vault_keys.root_token }}" + name: barbican + backend: kv + description: Barbican kv store + + - name: Ensure barbican policy is defined + hashivault_policy: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ vault_ca_cert }}" + token: "{{ vault_keys.root_token }}" + name: barbican-policy + state: present + rules: | + path "barbican/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } + + - name: Ensure barbican AppRole is defined + hashivault_approle_role: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ vault_ca_cert }}" + token: "{{ vault_keys.root_token }}" + bind_secret_id: true + secret_id_bound_cidrs: "{{ internal_net_name | net_cidr }}" + secret_id_ttl: 0 + token_policies: barbican-policy + name: barbican + + - name: Get barbican Approle ID + hashivault_approle_role_id: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ vault_ca_cert }}" + token: "{{ vault_keys.root_token }}" + name: barbican + register: barbican_role_id + + - name: Print barbican Approle ID + ansible.builtin.debug: + msg: barbican role id is {{ barbican_role_id.id }} + + - name: Write barbican Approle ID to file if requested + delegate_to: localhost + ansible.builtin.copy: + content: "{{ barbican_role_id.id }}" + dest: "{{ stackhpc_barbican_role_id_file_path | default('~/barbican-role-id') }}" + when: stackhpc_write_barbican_role_id_to_file | default(false) | bool + + - name: Check if barbican Approle Secret ID is defined + hashivault_approle_role_secret_get: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ vault_ca_cert }}" + token: "{{ vault_keys.root_token }}" + secret: "{{ secrets_barbican_approle_secret_id }}" + name: barbican + register: barbican_approle_secret_get + + - name: Ensure barbican AppRole Secret ID is defined + hashivault_approle_role_secret: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ vault_ca_cert }}" + token: "{{ vault_keys.root_token }}" + secret: "{{ secrets_barbican_approle_secret_id }}" + name: barbican + when: barbican_approle_secret_get.status == "absent" diff --git a/etc/kayobe/ansible/vault-deploy-overcloud.yml b/etc/kayobe/ansible/vault-deploy-overcloud.yml new file mode 100644 index 000000000..31ccf2d2e --- /dev/null +++ b/etc/kayobe/ansible/vault-deploy-overcloud.yml @@ -0,0 +1,118 @@ +--- +# Required for uri module to work with self-signed certificates and for systems to trust +# the self-signed CA +- name: Install CA on controllers + hosts: controllers + tasks: + - name: Copy the intermediate CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem" + dest: "{{ '/etc/pki/ca-trust/source/anchors/OS-TLS-ROOT.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' + }}" + mode: "0644" + become: true + + - name: Update system CA + become: true + ansible.builtin.command: "{{ 'update-ca-trust' if ansible_facts.os_family == 'RedHat' else 'update-ca-certificates' }}" +- name: Deploy HashiCorp Vault on the overcloud + any_errors_fatal: true + gather_facts: true + hosts: controllers + vars: + consul_bind_interface: "{{ internal_net_name | net_interface }}" + vault_bind_address: "{{ internal_net_name | net_ip }}" + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure /opt/kayobe/vault exists + ansible.builtin.file: + path: /opt/kayobe/vault + state: directory + + - name: Template out TLS key and cert + ansible.builtin.copy: + # Within the Hashicorp Vault container these uids & gids map to the vault user + src: "{{ kayobe_env_config_path }}/vault/{{ item }}" + dest: /opt/kayobe/vault/{{ item }} + owner: 100 + group: 1000 + mode: "0600" + loop: + - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" + - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + become: true + + - name: Apply vault role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault + vars: + hashicorp_registry_url: "{{ overcloud_hashicorp_registry_url }}" + hashicorp_registry_username: "{{ overcloud_hashicorp_registry_username }}" + hashicorp_registry_password: "{{ overcloud_hashicorp_registry_password }}" + consul_docker_image: "{{ overcloud_consul_docker_image }}" + consul_docker_tag: "{{ overcloud_consul_docker_tag }}" + vault_config_dir: /opt/kayobe/vault + vault_cluster_name: overcloud + vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_docker_image: "{{ overcloud_vault_docker_image }}" + vault_docker_tag: "{{ overcloud_vault_docker_tag }}" + vault_tls_cert: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" + vault_tls_key: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + copy_self_signed_ca: true + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_write_keys_file: true + vault_write_keys_file_path: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + name: vault_keys + + - name: Apply vault unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ vault_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ vault_keys.keys_base64 }}" + environment: + https_proxy: "" + +- name: Configure PKI + any_errors_fatal: true + gather_facts: true + hosts: controllers[0] + tasks: + - name: Apply vault pki role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_pki + vars: + vault_token: "{{ vault_keys.root_token }}" + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_pki_root_create: false + vault_pki_intermediate_import: true + vault_pki_intermediate_ca_name: OS-TLS-INT + vault_pki_intermediate_ca_bundle: "{{ lookup('file', kayobe_env_config_path + '/vault/OS-TLS-INT.pem') }}" + vault_pki_intermediate_ca_cert: "{{ lookup('file', kayobe_env_config_path + '/vault/OS-TLS-INT.crt') }}" + vault_pki_intermediate_roles: "{{ overcloud_vault_pki_roles }}" + vault_pki_write_certificate_files: true + vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/vault" + environment: + https_proxy: "" diff --git a/etc/kayobe/ansible/vault-deploy-seed.yml b/etc/kayobe/ansible/vault-deploy-seed.yml new file mode 100644 index 000000000..9617aa905 --- /dev/null +++ b/etc/kayobe/ansible/vault-deploy-seed.yml @@ -0,0 +1,83 @@ +--- +- name: Deploy Hashicorp Vault on the seed + any_errors_fatal: true + gather_facts: true + hosts: seed + vars: + consul_bind_interface: lo + vault_bind_address: "{{ ansible_facts[consul_bind_interface].ipv4.address }}" + vault_api_addr: http://{{ vault_bind_address }}:8200 + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python PyYAML and hvac modules are installed + ansible.builtin.pip: + name: + - PyYAML + - hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure vault directory exists in Kayobe configuration + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/vault/" + state: directory + delegate_to: localhost + run_once: true + + - name: Apply vault role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault + vars: + hashicorp_registry_url: "{{ seed_hashicorp_registry_url }}" + hashicorp_registry_username: "{{ seed_hashicorp_registry_username }}" + hashicorp_registry_password: "{{ seed_hashicorp_registry_password }}" + consul_docker_image: "{{ seed_consul_docker_image }}" + consul_docker_tag: "{{ seed_consul_docker_tag }}" + vault_config_dir: /opt/kayobe/vault + vault_cluster_name: seed + vault_docker_image: "{{ seed_vault_docker_image }}" + vault_docker_tag: "{{ seed_vault_docker_tag }}" + vault_write_keys_file: true + vault_write_keys_file_path: "{{ kayobe_env_config_path }}/vault/seed-vault-keys.json" + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/seed-vault-keys.json" + name: vault_keys + + - name: Apply vault unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_unseal_keys: "{{ vault_keys.keys_base64 }}" + + - name: Apply vault PKI role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_pki + vars: + vault_token: "{{ vault_keys.root_token }}" + vault_pki_root_ca_name: OS-TLS-ROOT + vault_pki_write_root_ca_to_file: true + vault_pki_intermediate_ca_name: OS-TLS-INT + vault_pki_intermediate_export: true + vault_pki_intermediate_roles: "{{ seed_vault_pki_roles }}" + vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/vault" + vault_pki_generate_certificates: true + vault_pki_write_certificates: true + vault_pki_certificate_subject: + - common_name: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}" + role: "{{ seed_vault_pki_role_name }}" + extra_params: + ip_sans: "{% for host in groups['controllers'] %}{{ internal_net_name | net_ip(host) }}{% if not loop.last %},{% endif %}{% endfor %},{{ kolla_internal_vip_address }}" + vault_pki_write_certificate_files: true + vault_pki_write_pem_bundle: false + vault_pki_write_int_ca_to_file: true diff --git a/etc/kayobe/ansible/vault-generate-backend-tls.yml b/etc/kayobe/ansible/vault-generate-backend-tls.yml new file mode 100644 index 000000000..32e502fbe --- /dev/null +++ b/etc/kayobe/ansible/vault-generate-backend-tls.yml @@ -0,0 +1,82 @@ +--- +# Required for uri module to work with self-signed certificates and for systems to trust +# the self-signed CA +- name: Install CA + hosts: controllers:network + tasks: + - name: Copy the intermediate CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem" + dest: "{{ '/etc/pki/ca-trust/source/anchors/OS-TLS-ROOT.crt' if ansible_facts.os_family == 'RedHat' \ + else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + mode: "0644" + become: true + + - name: Update system CA + become: true + ansible.builtin.command: "{{ 'update-ca-trust' if ansible_facts.os_family == 'RedHat' else 'update-ca-certificates' }}" +- name: Generate backend API certificates + hosts: controllers:network + vars: + vault_api_addr: https://{{ internal_net_name | net_ip(groups['controllers'][0]) }}:8200 + vault_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + name: vault_keys + + - name: Issue a certificate for backend TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ vault_keys.root_token }}" + mount_point: "{{ vault_intermediate_ca_name }}" + role: "{{ overcloud_vault_pki_backend_tls_role_name }}" + common_name: "" + extra_params: + ip_sans: "{{ internal_net_name | net_ip }}" + register: backend_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Copy backend cert + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/{{ inventory_hostname }}-cert.pem" + content: | + {{ backend_cert.data.certificate }} + {{ backend_cert.data.issuing_ca }} + mode: "0600" + delegate_to: localhost + + - name: Copy backend key + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/{{ inventory_hostname }}-key.pem" + content: | + {{ backend_cert.data.private_key }} + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/vault-generate-internal-tls.yml b/etc/kayobe/ansible/vault-generate-internal-tls.yml new file mode 100644 index 000000000..a585d1bc9 --- /dev/null +++ b/etc/kayobe/ansible/vault-generate-internal-tls.yml @@ -0,0 +1,56 @@ +--- +- name: Generate internal API certificate + hosts: controllers + run_once: true + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + name: vault_keys + + - name: Issue a certificate for internal TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ vault_keys.root_token }}" + mount_point: "{{ vault_intermediate_ca_name }}" + role: "{{ overcloud_vault_pki_internal_tls_role_name }}" + common_name: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% endif %}" + extra_params: + ip_sans: "{{ kolla_internal_vip_address }}" + register: internal_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Ensure CA certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates/ca" + state: directory + delegate_to: localhost + + - name: Copy internal API PEM bundle + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy-internal.pem" + content: | + {{ internal_cert.data.certificate }} + {{ internal_cert.data.issuing_ca }} + {{ internal_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + + - name: Copy root CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/vault-generate-test-external-tls.yml b/etc/kayobe/ansible/vault-generate-test-external-tls.yml new file mode 100644 index 000000000..de02ddb03 --- /dev/null +++ b/etc/kayobe/ansible/vault-generate-test-external-tls.yml @@ -0,0 +1,57 @@ +--- +- name: Generate external API certificate (for testing only) + hosts: controllers + run_once: true + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + # NOTE: Using the same CA as internal TLS. + vault_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + name: vault_keys + + - name: Issue a certificate for external TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ vault_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ vault_keys.root_token }}" + mount_point: "{{ vault_intermediate_ca_name }}" + role: "{{ overcloud_vault_pki_external_tls_role_name }}" + common_name: "{% if kolla_external_fqdn != kolla_external_vip_address %}{{ kolla_external_fqdn }}{% endif %}" + extra_params: + ip_sans: "{{ kolla_external_vip_address }}" + register: external_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Ensure CA certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates/ca" + state: directory + delegate_to: localhost + + - name: Copy external API PEM bundle + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy.pem" + content: | + {{ external_cert.data.certificate }} + {{ external_cert.data.issuing_ca }} + {{ external_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + + - name: Copy root CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/vault-unseal-overcloud.yml b/etc/kayobe/ansible/vault-unseal-overcloud.yml new file mode 100644 index 000000000..d5d38678e --- /dev/null +++ b/etc/kayobe/ansible/vault-unseal-overcloud.yml @@ -0,0 +1,37 @@ +--- +- name: Deploy HashiCorp Vault on the overcloud + any_errors_fatal: true + gather_facts: true + hosts: controllers + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + name: vault_keys + + - name: Apply vault unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ vault_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ vault_keys.keys_base64 }}" + environment: + https_proxy: "" diff --git a/etc/kayobe/ansible/vault-unseal-seed.yml b/etc/kayobe/ansible/vault-unseal-seed.yml new file mode 100644 index 000000000..3c6428b44 --- /dev/null +++ b/etc/kayobe/ansible/vault-unseal-seed.yml @@ -0,0 +1,34 @@ +--- +- name: Deploy Hashicorp Vault on the seed + any_errors_fatal: true + gather_facts: true + hosts: seed + vars: + vault_api_addr: http://127.0.0.1:8200 + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/seed-vault-keys.json" + name: vault_keys + + - name: Apply vault unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_unseal_keys: "{{ vault_keys.keys_base64 }}" diff --git a/etc/kayobe/inventory/group_vars/all/vault b/etc/kayobe/inventory/group_vars/all/vault new file mode 100644 index 000000000..e5f1967ab --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/vault @@ -0,0 +1,85 @@ +############################################################################### +# Hashicorp Vault deployment configuration. + +# Registry information for seed. +#seed_hashicorp_registry_url: "{{ stackhpc_docker_registry if stackhpc_sync_hashicorp_images | bool else '' }}" +#seed_hashicorp_registry_username: "{{ stackhpc_docker_registry_username if stackhpc_sync_hashicorp_images | bool else '' }}" +#seed_hashicorp_registry_password: "{{ stackhpc_docker_registry_password if stackhpc_sync_hashicorp_images | bool else '' }}" + +# Seed Consul container image. +seed_consul_docker_image: "" + +# Seed Consul container image tag. +seed_consul_docker_tag: "1.16.4" + +# Seed Vault container image. +seed_vault_docker_image: "hashicorp/vault" + +# Seed Vault container image tag. +seed_vault_docker_tag: "1.14.8" + +# Seed Vault PKI Role name +seed_vault_pki_role_name: "ServerCert" + +# Seed Vault PKI Roles definition +seed_vault_pki_roles: + - name: "{{ seed_vault_pki_role_name }}" + config: + max_ttl: 8760h + ttl: 8760h + allow_any_name: true + allow_ip_sans: true + require_cn: false + server_flag: true + key_type: rsa + key_bits: 4096 + country: ["UK"] + locality: ["Bristol"] + organization: ["StackHPC"] + ou: ["OpenStack"] + +# Registry information for overcloud. +#overcloud_hashicorp_registry_url: "{{ stackhpc_docker_registry if stackhpc_sync_hashicorp_images | bool else '' }}" +#overcloud_hashicorp_registry_username: "{{ stackhpc_docker_registry_username if stackhpc_sync_hashicorp_images | bool else '' }}" +#overcloud_hashicorp_registry_password: "{{ stackhpc_docker_registry_password if stackhpc_sync_hashicorp_images | bool else '' }}" + +# Overcloud Consul container image. +overcloud_consul_docker_image: "hashicorp/consul" + +# Overcloud Consul container image tag. +overcloud_consul_docker_tag: "1.16.4" + +# Overcloud Vault container image. +overcloud_vault_docker_image: "hashicorp/vault" + +# Overcloud Vault container image tag. +overcloud_vault_docker_tag: "1.14.8" + +# Overcloud Vault PKI Default Role name +overcloud_vault_pki_default_role_name: "ServerCert" + +# Overcloud Vault PKI Internal TLS Role name +overcloud_vault_pki_internal_tls_role_name: "{{ overcloud_vault_pki_default_role_name }}" + +# Overcloud Vault PKI Backend TLS Role name +overcloud_vault_pki_backend_tls_role_name: "{{ overcloud_vault_pki_default_role_name }}" + +# Overcloud Vault PKI External TLS Role name (for testing only) +overcloud_vault_pki_external_tls_role_name: "{{ overcloud_vault_pki_default_role_name }}" + +# Overcloud Vault PKI Roles definition +overcloud_vault_pki_roles: + - name: "{{ overcloud_vault_pki_default_role_name }}" + config: + max_ttl: 8760h + ttl: 8760h + allow_any_name: true + allow_ip_sans: true + require_cn: false + server_flag: true + key_type: rsa + key_bits: 4096 + country: ["UK"] + locality: ["Bristol"] + organization: ["StackHPC"] + ou: ["OpenStack"] From 4dc87e9e6948420c0ce749307935fad6b7dec15b Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Wed, 28 May 2025 13:34:20 +0100 Subject: [PATCH 067/103] StackHPC Firewall configuration --- etc/kayobe/compute.yml | 20 +- etc/kayobe/controllers.yml | 20 +- etc/kayobe/inventory/group_vars/all/firewall | 330 +++++++++++++++++++ etc/kayobe/storage.yml | 20 +- 4 files changed, 363 insertions(+), 27 deletions(-) create mode 100644 etc/kayobe/inventory/group_vars/all/firewall diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index 691e0a672..69a47b7eb 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -138,22 +138,24 @@ compute_lvm_groups: # Compute node firewalld configuration. # Whether to install and enable firewalld. -#compute_firewalld_enabled: +compute_firewalld_enabled: true # A list of zones to create. Each item is a dict containing a 'zone' item. -#compute_firewalld_zones: +compute_firewalld_zones: "{{ stackhpc_firewalld_zones }}" -# A firewalld zone to set as the default. Default is unset, in which case the -# default zone will not be changed. -#compute_firewalld_default_zone: +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +compute_firewalld_default_zone: trusted -# A list of firewall rules to apply. Each item is a dict containing arguments -# to pass to the firewalld module. Arguments are omitted if not provided, with -# the following exceptions: +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: # - offline: true # - permanent: true # - state: enabled -#compute_firewalld_rules: +compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" ############################################################################### # Compute node host libvirt configuration. diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index abd21f457..e8679a657 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -166,22 +166,24 @@ controller_lvm_groups: # Controller node firewalld configuration. # Whether to install and enable firewalld. -#controller_firewalld_enabled: +controller_firewalld_enabled: true # A list of zones to create. Each item is a dict containing a 'zone' item. -#controller_firewalld_zones: +controller_firewalld_zones: "{{ stackhpc_firewalld_zones }}" -# A firewalld zone to set as the default. Default is unset, in which case the -# default zone will not be changed. -#controller_firewalld_default_zone: +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +controller_firewalld_default_zone: trusted -# A list of firewall rules to apply. Each item is a dict containing arguments -# to pass to the firewalld module. Arguments are omitted if not provided, with -# the following exceptions: +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: # - offline: true # - permanent: true # - state: enabled -#controller_firewalld_rules: +controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/all/firewall b/etc/kayobe/inventory/group_vars/all/firewall new file mode 100644 index 000000000..488e95b65 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/firewall @@ -0,0 +1,330 @@ +--- +######################################### +# StackHPC default firewall configuration +######################################### +# This file contains the reference StackHPC firewalld rule implementation. It +# is designed to cover as many services and use cases as possible, however may +# not be compatible with all deployments. For more information, see the +# firewall page in the configuration section of the SKC docs. + +# A templated list of firewalld zones, according to the host's network +# interfaces. +# This variable can be used to set any *_firewalld_zones kayobe variable e.g. +# compute_firewalld_zones: "{{ stackhpc_firewalld_zones }}" in compute.yml +stackhpc_firewalld_zones: | + {% set network_zones = [] %} + {% for network in network_interfaces %} + {% if network | net_zone %} + {% set _ = network_zones.append({'zone': network | net_zone }) %} + {% endif %} + {% endfor %} + {{ network_zones | unique | list }} + +# A templated list of firewalld rules, according to the enabled services, +# host's group membership, and host's network configuration. +# This variable can be used to set any *_firewalld_rules kayobe variable e.g. +# compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" in compute.yml +stackhpc_firewalld_rules: | + {% set stackhpc_firewalld_rules_verified = stackhpc_firewalld_rules_extra %} + {% for rule in stackhpc_firewalld_rules_default | unique %} + {% if rule | ansible.utils.remove_keys('state') not in stackhpc_firewalld_rules_verified | map('ansible.utils.remove_keys', 'state') | default([]) %} + {% if rule.network is not defined %} + {% set _ = stackhpc_firewalld_rules_verified.append(rule) %} + {% elif rule.network in network_interfaces and rule.network | net_zone %} + {% set _ = stackhpc_firewalld_rules_verified.append(rule) %} + {% endif %} + {% endif %} + {% endfor %} + {{ stackhpc_firewalld_rules_verified | list }} + +# A single list of default firewall rules, combining the enabled rules from +# stackhpc_firewalld_rules_template. This variable is designed to fail to +# template if any conflicting rules are found. +stackhpc_firewalld_rules_default: | + {% set stackhpc_firewalld_rules_formatted = [] %} + {% for ruleset in stackhpc_firewalld_rules_template %} + {% if ruleset.enabled | bool %} + {% for rule in ruleset.rules %} + {% if rule.zone is not defined %} + {% set rule = rule | combine({'zone': rule.network | net_zone }) %} + {% endif %} + {% if rule not in stackhpc_firewalld_rules_formatted %} + {% if rule | ansible.utils.remove_keys('state') in stackhpc_firewalld_rules_formatted | map('ansible.utils.remove_keys', 'state') %} + {% set _ = stackhpc_firewalld_rules_formatted.append({'state':'failure'}) %} + {% elif rule.network is not defined %} + {% set _ = stackhpc_firewalld_rules_formatted.append(rule) %} + {% elif rule.network in network_interfaces and rule.network | net_zone %} + {% set _ = stackhpc_firewalld_rules_formatted.append(rule) %} + {% endif %} + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {{ undef(hint='ERROR: Conflicting firewall rules found') if ({'state':'failure'} in stackhpc_firewalld_rules_formatted) else stackhpc_firewalld_rules_formatted }} + +stackhpc_firewalld_rules_template: | + {{ stackhpc_common_firewalld_rules_template + + (stackhpc_controller_firewalld_rules_template if 'controllers' in group_names else []) + + (stackhpc_compute_firewalld_rules_template if 'compute' in group_names else []) + + (stackhpc_ceph_firewalld_rules_template if 'ceph' in group_names else []) + + (stackhpc_monitoring_firewalld_rules_template if 'monitoring' in group_names else []) + + (stackhpc_seed_firewalld_rules_template if 'seed' in group_names else []) + + (stackhpc_seed_hypervisor_firewalld_rules_template if 'seed-hypervisor' in group_names else []) + + (stackhpc_wazuh_manager_infra_vm_firewalld_rules_template if 'wazuh-manager' in group_names else []) + + (stackhpc_ansible_control_infra_vm_firewalld_rules_template if inventory_hostname == 'localhost' else []) }} + +############################################################################### +# Debug Vars + +# This variable is not applied anywhere. It exists for debugging purpouses +# only. Print it with: +# kayobe configuration dump --var-name stackhpc_firewalld_rules_debug +stackhpc_firewalld_rules_debug: | + {% set stackhpc_firewalld_services_debug = [] %} + {% for rule in stackhpc_firewalld_rules %} + {% if rule.service is defined %} + {% set _ = stackhpc_firewalld_services_debug.append(rule.service + ' ' + rule.state + ' ' + rule.zone | default()) %} + {% else %} + {% set _ = stackhpc_firewalld_services_debug.append(rule.port + ' ' + rule.state + ' ' + rule.zone | default()) %} + {% endif %} + {% endfor %} + {{ stackhpc_firewalld_services_debug | list }} + +############################################################################### +# Extra firewalld rules + +# Extra firewalld rules. Each item is a dict containing arguments to pass to +# the firewalld module. +# These rules have higher precidence than the +# default rules and are not validated before being applied. Use with caution. +# NOTE: The format of this variable is NOT the same as the +# stackhpc_*_firewalld_rules_template variables found elsewhere in this file. +stackhpc_firewalld_rules_extra: [] + +############################################################################### +# Common firewalld rules + +stackhpc_common_firewalld_rules_template: + # Common + - rules: + - service: ssh + network: "{{ admin_oc_net_name }}" + state: enabled + enabled: true + # Cockpit, dhcpv6-client, and SSH are enabled by default in firewalld + - rules: + - service: cockpit + zone: public + state: disabled + - service: dhcpv6-client + zone: public + state: disabled + - service: ssh + zone: public + state: disabled + enabled: "{{ admin_oc_net_name | net_zone != 'public' }}" + +############################################################################### +# Controller firewalld rules + +stackhpc_controller_firewalld_rules_template: + # Overcloud Ironic + - rules: + # Ironic inspector API + - port: 5050/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + # Ironic API + - port: 6385/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 8089/tcp + network: "{{ provision_wl_net_name }}" + state: enabled + - service: dhcp + network: "{{ provision_wl_net_name }}" + state: enabled + - service: ntp + network: "{{ provision_wl_net_name }}" + state: enabled + - service: tftp + network: "{{ provision_wl_net_name }}" + state: enabled + enabled: "{{ kolla_enable_ironic | bool }}" + # Designate + - rules: + - port: 53/tcp + network: "{{ public_net_name }}" + state: enabled + - port: 53/udp + network: "{{ public_net_name }}" + state: enabled + - port: 5354/tcp + network: "{{ public_net_name }}" + state: enabled + - port: 5354/udp + network: "{{ public_net_name }}" + state: enabled + enabled: "{{ kolla_enable_designate | bool }}" + # GENEVE + - rules: + - port: 6081/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ 'geneve' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types) and 'network' in group_names }}" + # VXLAN + - rules: + - port: 4789/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ 'vxlan' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types) and 'network' in group_names }}" + # Octavia + - rules: + - port: 5555/udp + network: "{{ octavia_net_name | default() }}" + state: enabled + enabled: "{{ kolla_enable_octavia | bool and octavia_net_name is not none }}" + # DHCP, Was required to create VMs at a customer site, need to work out condition + - rules: + - port: 67/udp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: true + +############################################################################### +# Compute firewalld rules + +stackhpc_compute_firewalld_rules_template: + # GENEVE + - rules: + - port: 6081/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ ('geneve' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types)) | bool }}" + # VXLAN + - rules: + - port: 4789/udp + network: "{{ tunnel_net_name }}" + state: enabled + enabled: "{{ ('vxlan' in (kolla_neutron_ml2_type_drivers + kolla_neutron_ml2_tenant_network_types)) | bool }}" + +############################################################################### +# Ceph firewalld rules + +# Port on which radosgw is exposed. +# See: https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-2024.1/configuration/cephadm.html#rados-gateways +stackhpc_ceph_firewalld_radosgw_port: 8100 + +stackhpc_ceph_firewalld_rules_template: + # Ceph Prometheus exporter + - rules: + - port: 9283/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ kolla_enable_prometheus_ceph_mgr_exporter | default(false) | bool and 'mgrs' in group_names }}" + # Ceph + - rules: + - service: ceph + network: "{{ storage_net_name }}" + state: enabled + - service: ceph + network: "{{ storage_mgmt_net_name }}" + state: enabled + - service: ceph-mon + network: "{{ storage_net_name }}" + state: "{{ 'enabled' if 'mons' in group_names else 'disabled' }}" + - port: "{{ stackhpc_ceph_firewalld_radosgw_port }}/tcp" + network: "{{ storage_net_name }}" + state: "{{ 'enabled' if 'rgws' in group_names else 'disabled' }}" + enabled: "{{ 'ceph' in group_names }}" + +############################################################################### +# Monitoring firewalld rules + +stackhpc_monitoring_firewalld_rules_template: [] + +############################################################################### +# Infra VM firewalld rules (Wazuh Manager) + +stackhpc_wazuh_manager_infra_vm_firewalld_rules_template: + - rules: + - port: 1514/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 1514/udp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 1515/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 443/tcp + network: "{{ wazuh_dashboard_net_name | default(provision_oc_net_name) }}" + state: enabled + - port: 9200/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 9300-9400/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - port: 55000/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: true + +############################################################################### +# Infra VM firewalld rules (Ansible Control) + +stackhpc_ansible_control_infra_vm_firewalld_rules_template: [] + +############################################################################### +# Seed firewalld rules + +stackhpc_seed_firewalld_rules_template: + # Pulp server + - rules: + - port: "{{ pulp_port }}/tcp" + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ seed_pulp_container_enabled | bool }}" + # Squid proxy + - rules: + - service: squid + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ seed_squid_container_enabled | bool }}" + # Ironic + - rules: + # nginx + - port: 8080/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + # Ironic inspector API + - port: 5050/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + # Ironic API + - port: 6385/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + - service: dhcp + network: "{{ provision_oc_net_name }}" + state: enabled + - service: tftp + network: "{{ provision_oc_net_name }}" + state: enabled + - service: ntp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: true #FIXME: Make rules conditional on Bifrost deployment + # Redfish exporter + - rules: + - port: 9610/tcp + network: "{{ provision_oc_net_name }}" + state: enabled + enabled: "{{ stackhpc_enable_redfish_exporter | default(false) }}" + # TODO: Rules if SNAT enabled on seed + +############################################################################### +# Seed Hypervisor firewalld rules + +stackhpc_seed_hypervisor_firewalld_rules_template: [] diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index 8d425ec55..ac7b30ff1 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -129,22 +129,24 @@ storage_bootstrap_user: ileigh # Storage node firewalld configuration. # Whether to install and enable firewalld. -#storage_firewalld_enabled: +storage_firewalld_enabled: true # A list of zones to create. Each item is a dict containing a 'zone' item. -#storage_firewalld_zones: +storage_firewalld_zones: "{{ stackhpc_firewalld_zones }}" -# A firewalld zone to set as the default. Default is unset, in which case the -# default zone will not be changed. -#storage_firewalld_default_zone: +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +storage_firewalld_default_zone: trusted -# A list of firewall rules to apply. Each item is a dict containing arguments -# to pass to the firewalld module. Arguments are omitted if not provided, with -# the following exceptions: +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: # - offline: true # - permanent: true # - state: enabled -#storage_firewalld_rules: +storage_firewalld_rules: "{{ stackhpc_firewalld_rules }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. From da91623af19eeb247f2307759009900ddc6812a7 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Wed, 28 May 2025 14:01:37 +0000 Subject: [PATCH 068/103] Caracal upgrade --- etc/kayobe/ansible/vault-deploy-overcloud.yml | 2 - etc/kayobe/ansible/vault-deploy-seed.yml | 2 - etc/kayobe/inventory/group_vars/all/firewall | 2 +- etc/kayobe/inventory/group_vars/all/vault | 4 +- etc/kayobe/inventory/hosts | 4 +- .../kolla/inventory/overcloud-service.j2 | 105 +- etc/kayobe/kolla/passwords.yml | 3474 +++++++++-------- etc/kayobe/vault/OS-TLS-INT.crt | 30 + etc/kayobe/vault/OS-TLS-INT.pem | 354 ++ etc/kayobe/vault/OS-TLS-ROOT.pem | 30 + etc/kayobe/vault/overcloud.crt | 62 + etc/kayobe/vault/overcloud.key | 167 + etc/kayobe/vault/seed-vault-keys.json | 46 + requirements.txt | 1 + 14 files changed, 2549 insertions(+), 1734 deletions(-) create mode 100644 etc/kayobe/vault/OS-TLS-INT.crt create mode 100644 etc/kayobe/vault/OS-TLS-INT.pem create mode 100644 etc/kayobe/vault/OS-TLS-ROOT.pem create mode 100644 etc/kayobe/vault/overcloud.crt create mode 100644 etc/kayobe/vault/overcloud.key create mode 100644 etc/kayobe/vault/seed-vault-keys.json create mode 100644 requirements.txt diff --git a/etc/kayobe/ansible/vault-deploy-overcloud.yml b/etc/kayobe/ansible/vault-deploy-overcloud.yml index 31ccf2d2e..d1ad7eaff 100644 --- a/etc/kayobe/ansible/vault-deploy-overcloud.yml +++ b/etc/kayobe/ansible/vault-deploy-overcloud.yml @@ -62,8 +62,6 @@ name: stackhpc.hashicorp.vault vars: hashicorp_registry_url: "{{ overcloud_hashicorp_registry_url }}" - hashicorp_registry_username: "{{ overcloud_hashicorp_registry_username }}" - hashicorp_registry_password: "{{ overcloud_hashicorp_registry_password }}" consul_docker_image: "{{ overcloud_consul_docker_image }}" consul_docker_tag: "{{ overcloud_consul_docker_tag }}" vault_config_dir: /opt/kayobe/vault diff --git a/etc/kayobe/ansible/vault-deploy-seed.yml b/etc/kayobe/ansible/vault-deploy-seed.yml index 9617aa905..356ea1970 100644 --- a/etc/kayobe/ansible/vault-deploy-seed.yml +++ b/etc/kayobe/ansible/vault-deploy-seed.yml @@ -38,8 +38,6 @@ name: stackhpc.hashicorp.vault vars: hashicorp_registry_url: "{{ seed_hashicorp_registry_url }}" - hashicorp_registry_username: "{{ seed_hashicorp_registry_username }}" - hashicorp_registry_password: "{{ seed_hashicorp_registry_password }}" consul_docker_image: "{{ seed_consul_docker_image }}" consul_docker_tag: "{{ seed_consul_docker_tag }}" vault_config_dir: /opt/kayobe/vault diff --git a/etc/kayobe/inventory/group_vars/all/firewall b/etc/kayobe/inventory/group_vars/all/firewall index 488e95b65..3be48df50 100644 --- a/etc/kayobe/inventory/group_vars/all/firewall +++ b/etc/kayobe/inventory/group_vars/all/firewall @@ -121,7 +121,7 @@ stackhpc_common_firewalld_rules_template: state: disabled - service: ssh zone: public - state: disabled + state: enabled enabled: "{{ admin_oc_net_name | net_zone != 'public' }}" ############################################################################### diff --git a/etc/kayobe/inventory/group_vars/all/vault b/etc/kayobe/inventory/group_vars/all/vault index e5f1967ab..e3da5a39f 100644 --- a/etc/kayobe/inventory/group_vars/all/vault +++ b/etc/kayobe/inventory/group_vars/all/vault @@ -2,12 +2,12 @@ # Hashicorp Vault deployment configuration. # Registry information for seed. -#seed_hashicorp_registry_url: "{{ stackhpc_docker_registry if stackhpc_sync_hashicorp_images | bool else '' }}" +seed_hashicorp_registry_url: "https://registry.docker.io" #seed_hashicorp_registry_username: "{{ stackhpc_docker_registry_username if stackhpc_sync_hashicorp_images | bool else '' }}" #seed_hashicorp_registry_password: "{{ stackhpc_docker_registry_password if stackhpc_sync_hashicorp_images | bool else '' }}" # Seed Consul container image. -seed_consul_docker_image: "" +seed_consul_docker_image: "hashicorp/consul" # Seed Consul container image tag. seed_consul_docker_tag: "1.16.4" diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 104d1e4b0..dcce7ed34 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -2,8 +2,8 @@ # localhost. localhost ansible_connection=local -#[seed] -#srcseed +[seed] +srcseed [controllers] srcseed diff --git a/etc/kayobe/kolla/inventory/overcloud-service.j2 b/etc/kayobe/kolla/inventory/overcloud-service.j2 index 36e67da13..5c805f9d2 100644 --- a/etc/kayobe/kolla/inventory/overcloud-service.j2 +++ b/etc/kayobe/kolla/inventory/overcloud-service.j2 @@ -1,3 +1,4 @@ + # This inventory section provides a mapping of services to components. # # Top level groups define the roles of hosts, e.g. controller or compute. @@ -137,13 +138,6 @@ cloudkitty [cloudkitty-processor:children] cloudkitty -# Freezer -[freezer-api:children] -freezer - -[freezer-scheduler:children] -freezer - # iSCSI [iscsid:children] compute @@ -199,48 +193,6 @@ heat [heat-engine:children] heat -# Murano -[murano-api:children] -murano - -[murano-engine:children] -murano - -# Monasca -[monasca-agent-collector:children] -monasca-agent - -[monasca-agent-forwarder:children] -monasca-agent - -[monasca-agent-statsd:children] -monasca-agent - -[monasca-api:children] -monasca - -[monasca-log-persister:children] -monasca - -[monasca-log-metrics:children] -monasca - -[monasca-thresh:children] -monasca - -[monasca-notification:children] -monasca - -[monasca-persister:children] -monasca - -# Storm -[storm-worker:children] -storm - -[storm-nimbus:children] -storm - # Ironic [ironic-api:children] ironic @@ -272,32 +224,6 @@ magnum [magnum-conductor:children] magnum -# Sahara -[sahara-api:children] -sahara - -[sahara-engine:children] -sahara - -# Solum -[solum-api:children] -solum - -[solum-worker:children] -solum - -[solum-deployer:children] -solum - -[solum-conductor:children] -solum - -[solum-application-deployment:children] -solum - -[solum-image-builder:children] -solum - # Mistral [mistral-api:children] mistral @@ -382,19 +308,6 @@ watcher [watcher-applier:children] watcher -# Senlin -[senlin-api:children] -senlin - -[senlin-conductor:children] -senlin - -[senlin-engine:children] -senlin - -[senlin-health-manager:children] -senlin - # Octavia [octavia-api:children] octavia @@ -464,22 +377,6 @@ tacker [tacker-conductor:children] tacker -# Vitrage -[vitrage-api:children] -vitrage - -[vitrage-notifier:children] -vitrage - -[vitrage-graph:children] -vitrage - -[vitrage-ml:children] -vitrage - -[vitrage-persistor:children] -vitrage - # Blazar [blazar-api:children] blazar diff --git a/etc/kayobe/kolla/passwords.yml b/etc/kayobe/kolla/passwords.yml index 17b62042a..42fef590c 100644 --- a/etc/kayobe/kolla/passwords.yml +++ b/etc/kayobe/kolla/passwords.yml @@ -1,1622 +1,1854 @@ $ANSIBLE_VAULT;1.1;AES256 -64383762333163313865336336333839306138353132343136633165346238653965346466643963 -3532376262313134313135363137616331613035376432630a303935653031326464623239346662 -65646532656661663732363861373730323538303330656537303965383738633961373939646637 -6438656532353166650a386566623438336363623563623733343839336339393930373462376138 -35623266666337323461343137353132323430356436323739323035313065613435336531316262 -34353532393735623066313730643365623162623662333733383861623465383938326466633935 -35653530383166613638313866393836393331343337326661396163313639356238383638386338 -31626464356165613564623135336665316436336130666465306530383032346239376136643165 -62346638316463333662613831316265303439396531636164386439626364616261373462343065 -62353635353165613962326530623938306663653961383330316662366565363364616538313439 -39323830613166383062366364616236353561663335636666343165646631663761663561306333 -30316437623563656532663936393336316264643438646163656162323336316533313237656562 -61373032356439613861653163393963333936313730343361306235306164636262623830633431 -66353534326161633065663466636463663961653736333233346363373963653566653166336539 -64386533343765313334343931386634383135663166336539376435303465343231393462376164 -66666666643532643466356234303961313331346439656335356533373534306535306238373636 -66666635343462363661636164386665646166616438306264396562346635626635343539313563 -64636364363165323438636562383032623338306364353536343837323361336163616562313632 -32303135373133313261383164353064656561633562653632626161373165306264373966343936 -32353566306462653066343239636535343735386565386233316239393439653131316336393661 -66303661343234666465643233623737386438626364633763356231643138383033356333346366 -61303164313838616665303033636539623736663536623132613063353531663535386564616234 -37666135623433613264383030613466393031366131326332336433623062633135373664333162 -38363738313365626662653362363135616237333436336137396162613066393335343732663465 -63383834623362616130633239336564363764343436376562373666373334623664346232643065 -33346634343764616537616363646663313165653232386435656630396534336230393065353939 -62623331393036616235316263663532393137663933316435653631663766396639366563313338 -33396363326231666133306632393665383761623434303239303536376164393837386664666238 -30643536613038366535633366643135363538356636333262303964643430353739643864626465 -39353535616432343630626330356234386538343739306439666265313338353763663336333837 -62316635393937626461613137313461353963363438613933623335613433366666363963616237 -66663830336539316661386133623338623462646132393136363930386439633261646462646332 -35656563643234303833376639386464306436333436623737623135303261353936666135353536 -34346364396234393136366666343032386137623836633066376562386366363533303932353565 -32353532643037663539633632313132313064343330373733326664653261396165626530303632 -62353162326563623265393439313165393637343961616566613935613533383663636533646161 -62316639626265336630353565386462343461396564376561346136656162303431646262303631 -34343737363532393738646430353931663334323262356363626366633862343237306339656263 -36353831346265646633653336393139376263346365373266376634396133663264623762353765 -61333634343230383932306636663433663734376638633533336339363066306130353936383239 -35626264323338333161343663353235313236356562343539616162623065663734613962613135 -31663864663261306130666564313230666331353336626137393361356235343339316238396537 -39363564623932613135366462633765656130313438616566633936303737626530663437333366 -62653965633138383762663535363763313662666538306431373034346164616237356264356536 -30626433656266303161323534666364383638636132336338363035343739626634326563623138 -35396339636139616638383264653638663831663964313036343766353864313237393830646636 -33346361373666626161313933316664386131366531656539386636316266313835653830383264 -31333335333061303064613132653432613335646436323038343361343764633131613764373563 -34343163383763323839636638376330333061356165333834303231626239666332666265613330 -33333730633735356630633861643064373238373437346562363838666231333962363535323538 -39653761393631626361666162366161623333636364363732666536636334393861333237326531 -38333165613739326238346237326534323533353264363131373732303163653139613039376165 -38323036383965353935356435653463643438643139626461643930633134656133306165613465 -38653839373433633734643133376164633862346463336430646664653365346663636634346461 -61373562633136393464386462393432613430313235623439396437313233393133356538383239 -37653638326262383231386565303436393363633335343033633830643066646265353633623663 -66666434616362376362396530366539633161646437653836656436623866333364343639653463 -39613932353333333764663539393366383835663630303837333731323066326337316665343535 -37653966613835386136306531393339663336316237613664323338336235643931306637653539 -64383330643438346133396130643434376130633163346665376338343536646339393766356431 -37653831346463326631306439333264393965383066373936643237666337613434353838613032 -39666531356661353932656464613236323432323138313331386534623161656461633936373335 -63363238663237313862383338346162343132616333653666343038646234626163346134303632 -66326362326136633533656334616638356338333834653732653531363061646466666234313137 -31306532333632666566353764373564313466643165636166303266623038336238663638633365 -32326363643638393730643865366165613862643161393334313234313833383336373161323166 -36613039373437323837306334386233333033393234616539316663316663386131383562336263 -34653862616166346362326131363436633733333666396365336461386432346164653330666130 -37333332306361616161623533316466363334326438383038616166653537633136636132326633 -65323534313063346234353861346238306238313732306161646364343633646431656435366236 -38643033393165343434613234663938646233336631366334396532316536313764363662633037 -62633935646235393766303463336437616563373332666130393761326564363331323839363430 -30306639303634613931303931396531323438666661623066323234326635303038303566356533 -61623838323231333065353034366432666163313564646165616139616238346435373631373438 -35643266343539616166633966646364666338373332613862633731653131363965663332653361 -61383261363762383834396361343932386562346435356163323361343734313936646131653637 -39333963373734316135653230306635623431336562663031303465323134303930653364333330 -62316166333738373735363662386438356639386430643061656564626432653239613165383566 -38656236303738626361353035373461396363663237393234633863623433316330396362313161 -33386237643935326165356235393738333234616166353461653834363238636638653164303434 -39613861393066383438313763626634666333383930363761333465396639356638633364323863 -31323938666533636264656437663430356462633935643232323663656262663465386139646162 -33666236316536336565653538346132396261623830356635393430326263613639633563656139 -62353035663335333665613863326236323537376663633133306561346166396231663164353235 -66343063343038386433633761326564646439623238353935316631303935383734353931356136 -38336434653932363262623235656265363239303561623963336564643563373866376363333232 -31383531313330376133343064666136663264653336313738626639623632376665666665623063 -65333365336235626530353464646563366130663531386631613533313736643165626164363364 -35393436366438663161623465323962623536326238303532363634663839326133353566346565 -35343639336437636337373535653535326630323838646662303735633934623737326231353135 -62386333383337396266613337316166303633643837666235626139303063636634363436353830 -38373636353663616130393135626263646566323934376664363837323261386336353332303562 -36613237666163313330306136643763353766376438656234616466626237346334663637353365 -34363265356636366137376366643530326462353331383235363835376562636261663535643032 -32613466643763653363306330333764626630643363316337326339663030623032346138643834 -35396136653333343934656536366564663634373132326163383736623235346465306636363235 -63663631356636326138326234383862366134623533353630663166343662613730666663316534 -61333233646666643132626261346666326233393730383536386239363737636561666233376161 -39393664363561353664343464383035626132613337646338313262663735363365666433386635 -64613063613738643038393038626435313036376538653538623662326266636135643932353339 -32346663363165366161646139613662386662393536386534616565396130313935383530643861 -64313062646339383334653664646336383437393337383366376237633138303736626335636563 -39393239376666623963303230356639656634663566666239663931323939393038316163323139 -66353332613331393166323664373065383765663162366234666264663333333031303664363932 -66373265613365353065313162346237613435666435363038613437313736393832373338343235 -30323963646461386233313333333166343663626336353932376364353434313831376164613766 -65313339616464653933396432386233356166313461323464356231613662373964333437663331 -61316165636537633864353830666262306133313962653765306161373739323834343565316134 -35336533363965336363613032386461653762396166643538616435616362353561643162333365 -61366437353830383865663366393562373464623938326338333030396534636464646135643232 -37303735373136363138626138353062616263663331393236623733356238316531333037326536 -36363065323334356534636464303034323630306433633432376464666566613366383837333430 -34656265666463333133653130666135633931613564623038366239616563613934303761313633 -61356561646238346139333463353163333763643337313538386139623830373862623830343239 -66646139303835323332383661363765366639656562346138396464363437613938373839353533 -33623935333139636661323938393763326362656530336438393462336362386232326335346335 -62333863333333666662336631633339303032306437666361396530643934623933343436363539 -33663566636539346437353564333162303762383961336437623131333133373439633663323332 -36373938303163396233623833663165663564646263333963346236333238663833323863343330 -37313061383037343335336462326239663830313065366437623939376134313338373235666632 -38343339643532303662663161636338623639613137386533366232356664663334333063356138 -66373730343133323731353130643833623336636533633932623531656135313432316639336136 -62333764306631316331383930336366343132386364633730653230306330316136343232636563 -37353232383630653864313335343137393438356162343233633235393764613564616638663736 -63316533323136366136393266316230346130326434626637656238643765363366326266633435 -30363035633634326461373563376337636534333132343635616664346264393538653730646432 -38326637633537313137356262356431336534373338306661356466333933343333333963666335 -31653861623463303265323362646636353135363232623762373263336634303539313136646162 -37383837616137663465613035363234373039383033313233336231613037633762353237333337 -34636434353765303236623033623063353061336430626661323932363039376661633432393938 -61663635663465306331366663353162313238373239306163666563346438383962623232343934 -30666533373035336638313637646233633438373130646438346337326634636230343231623238 -38626533653535636465636239653036346464333865393564396338333761666433623035373761 -61656430393566306361623833366263383431363836643664336364303933316533303130363734 -38343438326235313839356633646232643866633437376463623230656638326462346339613537 -36313766303335306630323939643533666533396633653037356131333732646363393037383734 -35303334336238666362393335303238386235323066363662656239393365343233646232323733 -64643533323062653535393561303334313736633338323037643436333734363661653037616362 -65396465376465623134663764303037346234646462353434613436393837613633356564623333 -39333131313634383339303138636338356566643661653962336266363430643835373362306366 -61343835663532663131636434316264376534343335303964326237616563656665396231396636 -61623232636365376438626263336331303531626533313234666436636530623031613839323137 -34343131663639326539326533306536636638383962343365363865323039376438353562333133 -37613531666239353464633063393365666230363261646534663664653830303837363434386466 -39333536373565656435366230383164393265313636333362643537373061383164623966306263 -39316362313335636362633433363130633933393635613761663237343763383132366334343464 -66653538633630633537373835363433663330366335383032303633346133363965323964363133 -63363064623634316262626562326439363866393261633466646333356638303362353134323964 -39626463333630326361356538656264356663633239383165656631636261626130346630623264 -31633136653765653965393432396263326632303464643233613232623662316436316630653139 -61623033383631313164313961303439336564623061323639636165303861306566653339646636 -36623863653235343730343561303265323064613537306237653565346535653966623639333032 -63303834386631353935623663333839633234613439366664383134383837323338303737663936 -31366534376437363235356535663431363461323732623435316262393839383163366161626230 -32336338666666333735616364633133393166306365363262663234646566626234383861393061 -38643363376330303234353034323438666265343161616336313131353934386631646534343330 -65383134616664353236656261386435363531363563303735356234633163353430373466623132 -39333638353735353439623164383631316366366436343463663863363838336534303363323061 -35643366306238356264383662663738316463383530613464636363323032353238633365336335 -30363537323063353338643466636261393130336533336566343532393434313134613865396564 -66303035626637373861306363306339336638393739666262303038323635323866323035393061 -34633065306265636338353862656238643664343234666537623439646534303465346138633539 -31353439373663633962373239316437303535626431666538333664323165616236633437323631 -61393761363533663031653833666235393261616331663639346538653339666163303730616363 -62363761376438623533363366356530643234303335303666663836343337646537303563353232 -64326337383564363230633666666231616162313735376466636530383165663732323561323631 -34636364376437383938373135646136326230373534333733623763333832373938346334646137 -34336362313133373530376332303366333963306533626436633530346435316165616134386131 -36376430616332333466656434396535666333396639393239366130313863356662613631643132 -62646266303863356630643735386132323638646636626561343039336365393665653532646238 -66663762326534656563316238353432313961656631383266363438303161363535626535363237 -39366365646136343131643335303931393438303133333431643866363935386135306437376565 -61633663636664386432393366376135313761373439383639616363386631376365393531386633 -35613563306230626165363664373837653637396638653139623931393862323466363236336665 -62363337306636663834383535663131663563313864373938346533323333313635343038396436 -30346337326662303039363032633238353439313364373732346633363362666665313431353939 -35623362383661646262613737346535376336303335303735613334316466383334613633656465 -30643362303861633835663031393961346365363363353437346638336239643633356338633364 -38656532376262666366316662373238323634656139653464343332326335323435623330663036 -35333430643061303365323633663062633139656635643866373263373438326638316438313563 -62626562333930333063313436653163633464626163366135323232643530653737363263643938 -65653537363261353566306335643434386539346333393430376661616435656339656564373730 -30616464663665363962366461353765616430666262633465303936633236656331303964386530 -30356562353032366237393132366536636538616636376363626437633864366562633838663861 -35666563313831343037326339346638353136353333313131356635383866356339333461353837 -34613563376130623637346138366434366432303662303934326435376139636430666436653163 -62343730613036373066663366663062323364623739326138666133366534356430613832643061 -32383062643133623331653265333131636234613064366665633932373264363930653366396361 -34636337643761613461613135656238613662623631366439316130393961633231353361316661 -34653636623730393661343161366261373631316336346130653464353235666661353536306539 -34663033646133303132626364316364636338376165653263313165323733643765316466393666 -64323961333165613262373366353866396537636438316666353738633937613535623530353765 -31653232313132626339613666393863336361326563636630303733646265303538633163373961 -35663634613065636466663762353632353631613035336331313031636439623137393665313361 -62373737356161313435376366633462386132646565383138643131396664336264613563623733 -39613139373863393764313963326361623337383534316132643835323630303037303666626564 -63656239646637383064316330376436333661396462363066376436316332303832333065643539 -38363737313732313139353062663263393739643632353130376661336431663331346336653138 -34393332343036613930383138316632623161396437373162663636636636303563336664373464 -65393136623761386638346262616632313762313861663731316635313935383335666161613731 -31306132313361306664313165373735356564306537623866353962323163343765643834626132 -35643062333063313961643864306137393638346538356264613962636262613161326535633836 -32663435353839303465623634366433326466613233326364383462633037313136653835613764 -31376135663235653664313866303433623231313034363832656131343135643763343030656237 -38663435613935316363653364316335343433643938393266636434616335646431663963366133 -34323264373138613639393830613362626435383537373461323264363766313336326130326431 -39366133306232386266623263333234613563303139626665393561353266373861373264353339 -35333231326135396137383434393061626262323332336564613536346137313735386334643363 -66316537356131313364323830646664623537326339663635366134633634323838633263333231 -36343066616234396137643265313462386432636633613263323333663864343534343931623531 -61393032353631613737326133336164616334326335343539613161633861393830353334626163 -31616462643761303736323731336664636662643930366137356338666431396332643435326366 -32653036666131393863386632306363353261306639306362613231663063363938653064626635 -64316132373263666665643162376634643138353730653663393061636134373632306561343639 -34356139376362383366316363393064386564376565363837303538323561333337616431363535 -30623136336363656435306465653262636263343234623137666533613737646164346139333163 -39653535343262396166623163656365626462383231383832613730636665393534646134313138 -64313765633164333232393631326236613062383539616165626333396336656561653035383163 -64333762373437623562333633653733346338313038393632666433316161633466353437303234 -64656133663465623864653538326630306431383032353661663231363161363334343236306436 -32616361643466366132653832656238323037393664663437643862353736616661313566373062 -63306136316330346332326630666333613333363830343631323865616534653266663638316236 -32363730333264306335616362323564353639663033303839633965323931323866383863343139 -66653763313831353935363433353634303631303538346366336537343361666466623938343430 -37393531663262376532336262653435343661323335353335616234653338346532626161633061 -64353334643561656138333534303039376632656334663165613438656334333031326638376334 -31333937353130376261363039313739356361623031373961363762333264616637393962663433 -36623330643534303164663963346466663030666638616532383732326133616234323832323938 -34313932646464326461633839616639633339383837363936356633616338323864363862333861 -30636261386165306264323134363330613438653562613263376461646638366436643038363336 -64323239353230633036343863323539343461343762323336373133383436393531356334316532 -38386535313163383330306432626333666430613963396163373231353364316264646364366438 -39343865353230613132343465653864616336656262333034386630626563326433363839643137 -35623639363732333463363061643264366439646261303333636366303034383336366538356435 -30366533373837383437366132393863366339383132343534373531313032386462376261356366 -32343538666633306163306535656362653761323964376338356466316162656561353032663036 -61376665303136613064663834626633306536663365333234643730333032306137343430356533 -64623734396338343133656539653232656333643030396662623332613432653031353832656430 -37653137613532663138316364383831333532343566653263623366626535376135633034626565 -38666666623062646432343236333962373036643633396466613539623438383031666464613433 -30383963653666373163613432386237623635663333373364386365373738373662643061346262 -66663337386339616264353832333664333333313035376333393438326135353335383034303965 -63613933356432303561313961343130643133663536623534643733313539383564343864356238 -63626537623031363930623761623934386631386236343437303430653030353962353734636237 -62613964333937393465616265656333303566313134613164663962643762613161373132363830 -30393162646530353139323064363037333238303161633137333331383466623736336264326336 -36616536356565316537313931366232316164646265313734643163363330333736346363313766 -39333339393635343362353336616264656537626138633539393866363539656266333732326538 -35666265366163333465663764303837313966653764353264373466383961616666316237633638 -63636330616139306537363037613833386666343931336461366538313432666234356636313664 -30653862343137303365393530376263666238393436616331393735663862383736363161646333 -30393136336666633961656661633836363231616131646363386439323535663736616662663130 -66393132646165313264363730646665353230323634303439336566613231393561626139313331 -37343366376538656430343661646463323933393365366664353133353062636437343066376165 -37623964356366633736323164646330343639336633663938386334323966343562303135356337 -30623439376336393864323433626237323362356130623833643464323233643666386366386265 -64326138383662303130323038363736633731623436383334356361373163333938633833323963 -66646134343638633666666162313334393136373430623935326238646437366131613862613134 -36383139653461396538366231316136623163633462393562613231383663643163623138646139 -34633439393862343265616438633431366436336566623661323235386630386639326263396235 -62333864393238313037656635336433346131396462353239306531643431386130363538623262 -61303931376530343639363934313765646161373337323139653938663738333831333166356130 -30356266633639313939666335393363366163623432373739313639393132643932313333396138 -64383339623133396637393335343939393738316433336463356462643365343165653533306132 -35623665343739363531613734373165366261316663356664636432633938613333336365643364 -37623836316337396430333935616635616138363039303535653661303562613434613262623137 -39623730346463366439363833663265303436306239333733343564343730303862386534346633 -61653439346434313232663630386339666433303033346230343064643339623637656633616136 -66373431393439316432363236363463616364323766666439383438376461306362323330343238 -30616631383233356337626636393466663136323232393262303838346530663461333232616235 -38323134626265333134633732376636396338666437356162336466366365656232656432616330 -36303663613339643530613032633130383338326662316562316632383861616636343936656163 -30393965653663623233313236346164373164613462396438373161346561316261343235663235 -33303732373761616434323639656230303161316561653866313234336438653434303834646431 -38326531393639303964353338623766613361383730646635386432326233313562383635646531 -30353333393235346635666339333039306238313037303338383365656636623266383562306435 -61616466656531336666333564393366303661376337663535656138656430383262666530393833 -63343831626361366166383865666564346334323862383466323239376530363736396466386631 -61356437343766356535623137633838656632396161623833393031303334613166633036616534 -63656235353836653735653437346464616262326639326234383661353530633130303330666139 -36613437653864356238343165386563396464353935663738646530376236393331633238323231 -38386166653032643830633536343363623339623165626664616133356632316338373739653762 -64343736656136633139303564353337316361666530323539626337333165393433376361386266 -63333139336566366333353438623363343266643434306230613163353965663438653137663362 -33653465643435633630393366616131633165393637323230316562363266313631393663373235 -64336633643231633734663263643838366430323361336136383030353864303965393333663538 -64663733396238333732356638643233313366323139656465303462386638656439306231643661 -63633531356635666464616433386663613834663066396266313464313033616631613761663031 -33636437306130653064343235363633303239306132383062323366373233353464636638663032 -39323765396565326163333635366133626262336336373339316535633063393036326135396662 -35353438386431616134313962326331316137353561643361653161373430363939623431316633 -64663266346632643432303630643130323636346565383064353762306435313035376666353764 -32336236643264303531653836393262666333383938383939636435323837666561386539313339 -62333066633766373134633234643835626438633062346235376462663165393830373461353537 -37666537366364336162313036323065343562356531326235353633393039633537346364313837 -63333330373235353763343732333836393731346133643361363339366634653539623531393137 -65346234356266346234616463303563303564643130653339636637623462396637333261663934 -37303333626564626435653134386564663531613032353861333737363864373632333865313333 -36396661623034373162323731643232393166613463323462393730656437383636353032316265 -34626333366239666563623539343439343234613431663332333932396263663466353266343933 -35653536316132646565343364646565633833363965666466353832316631343661616565643733 -33366132333134343966306137346131316134653130373239366466363661396532663361376239 -30613735396563386439376165653465653965633139333032396465323937353461306134383034 -61626334656332383237386566333036376531323066373565613261333837336166353763633862 -35643531363535313263613263656435643934386331636133366164313463386438343763666230 -30613536323136616161613038366437616133396336613631393561653464303336353363666464 -63343134653839306331393031303132303835383539663535376163646139363139646638303730 -62643866333834323762306164313561333164326564346665623563343862316638646434303463 -36613534653665613063373130363035353339336666336539316339656663363365393733326566 -63303865356230333535386435636264663838366132643362323633626431656433393831393538 -31306233373861623039303766396466623033353266363065313337356535313331376635316561 -62346666656133356136343138363666323561366136653638646336306233643662373461356264 -65653137623338656431623638323930613238313362306539623964366532656336393834623166 -32636238313162656264343231393165613636336465383837323265383738333738666134616337 -38343231323938636365303631353232643630626637343839323064323265396138303338376463 -31396130306165623132323038366231313762303361373536626364353139646439666265333138 -37393237613132303231316639646166316331643266313732653731393962303165653734343265 -31386634356231643839643764626637616564646562663936303635653232643363393530346531 -63653062383238306231373761373632663631343437396434623566383339373334623666333335 -64346133313137323564393135643664663966343432376230313632613262653739613138373139 -38633864336132656630656531306266663432613536363135333863386539363235326165346161 -61326536376238633765343632666364666534333732623763323164396530313131663862353336 -33373665616362313138393933373536346138616663653439306365353939303734623831653238 -62643835643763653961346566336536343531333132383165626533616134616230616236636133 -37383532393733646130663262626363343061623462343330376638303631353031396665306634 -30336461336233323262663137633161343163393334626333343532363530386563616138373762 -61373261323165396531383839633138376135303836363664633864306533653436646530656637 -31623766363835383936343237626432336636663962626332356562666239386666616665623963 -31356232323163333836353037326639663635616662383562363930613633393965363366326231 -32326562363731303663396631303633336465326234356436366665376636303234303766323865 -38313563383533386138313162343237636666366266656366353466313864346166353335393461 -38373365343861653361623932623765346333363263313863643437366164613764616439363037 -32663334326361336664376437366330386436363963346239383138303838616463633438316334 -38323739373230613337313561626135666539336165303837363538313865306334663831343333 -35393438646233383562386163393765663364303432383837633966663636333061316138326236 -35666336373762336363316464396538613330353233353637363364613765366265633833326466 -65653765366364656162646637643735366266656365636330383437363166373535343330343330 -63386634313563316331303936393366303530356631616536383734326539613831336136633264 -33643031333462656665653136656163326536646661646637363664333639333935333236356534 -34306666643864653562363162663861386665323433393533616434373632666539353438333563 -39653239333034303061643338633063643034346232346338333735306133616335303066383036 -33643037326266323732663530323838363464316461306331626564623234353938663861653361 -30623234656333663634616635626635323232643437373738623436386337623132383036383435 -39323965643234626566653461633261373862633666386565343836623032616662656238633264 -35313366613930306261343166383531313865653561303634323165323435613161646237313135 -38336236623932303165353637313965623137386266323766356165333762323936363536323565 -66623434363662613531363338646234313166396561366330343435633131303035343462396664 -36333131373634623361633037396432376434623139303464306366376365636236343334386366 -61336563613462346363316239633639396335343330613537353862623062343538363365613634 -63646536613866613039333062396535663164346635393164643164646433373130373136636163 -30373238313830306339663532653763346165646138613765393332386561383031313061333536 -33373261636236653161636165373936353263666536633339396431396637373966343662666466 -64373531313139353362383932646663393835356230313162303837396464633762623631646461 -38333632616233383935393831636232626133613934343234356564333061383764333536383334 -64626462633930303936333935613739326337363933343237393561353466656530613861653030 -65386539306161363231383166313561303366306262613061356663376561656339656161333132 -63346334333933353936633634643363343231343834343739333732663064316465343835333336 -66373939376236356438313763343663353031383566326464306632333166313761363933393164 -38653437356164363737636333633638353137333735656165646533386339353966373130643431 -30613331633637636538356466336633393864346564396130633738653938333932316137646233 -35383034316666663633323962316266333638316561306463383131393331373835643964396437 -33336539663462373931623332663362336663636666633937376162343932313830663639653365 -38623030326162366538666439363539393934323536386664386562393134336436306164356136 -30366630653164336264316336613339393437363037396666613663323037366666366635373238 -39373462383537643139636565333132356165386235306134626239666463393666363635646362 -33396666636637366331386339643065646639323137366134623532666438373233666134346537 -36643866363661633336616439373064386661613961653032656664323735623434336161663330 -30633662363934653261653965393566616433383361316431643230303335326161316136653633 -63633564353032326530386436613564656534663136366665306665666433303032663161353062 -31333831386666383434643734326635333165386631663662633331613433336639383162306266 -64316266633231633662646264666436333936386139623864656165383663333536663565613166 -65336633356538333435373632396138313366316434633231633535313434376235643965666330 -30623261376465303636656235363735623235356530646138383637356363623861623364306465 -65633366636438353066336162373536633337363462393430396361383437376534303033363463 -37666539353963356538643732663836396639333939356265393032383339393038383131343266 -31326532613466336631373137613737313261646366356239616539643762636565316432616235 -32396534623265643065386434643637343135333433316535643462633164363138353834333833 -37383935643938343162386635336239303734393631636161626665623762656339333834643366 -31353439656431643238663563323061313332663364396364346362623866616632363032653930 -63366536376564326434323435653536396530303237323062343462363765616561323633613662 -34306131623036313632353230323539343734336634373438336266366565663037346139623730 -62386161393135613835353336323962383935323538336239646632373932313433363963383030 -64666536373666653035643666366131336634303439376230316231306237646432303136353363 -61313563333430343935663534323737383462353232663931663630343937323763653162376666 -35306237633335636265373165336364653431353832653435663864616466613234343539333235 -37343139363865653036623761353665353233393239386265626135626366353336373361626262 -32653139376335383164373662303531623465326536376432336364623063316365613962323938 -64316632633861396132646137666664393037353238623837616465663836656161613931373763 -61373039313263316364613362326630633638333561313761623362393232316562306532303930 -35376638636233353862396533313938616337373763613839666430343666666266626331323531 -36653733353961646230373462666133653538396562346635323161303936313137303036333238 -61613165633038323264396334333232373138363933323237623631303931343762646165353962 -35613164386634323838356536616230663637373761393035616263393432373566633136383364 -62343366366663396664666432376638353333353764353064343165356363393739396561633837 -34363264623465313134386431643531656563353565663863356532386337393961623632643236 -61336531393733393635396164386234646362393161626436323162346538393132663337373963 -63363566613238346438326531306466623537303230383461663930663865373565666232323834 -37353634393434663165616430303739323638663931383066626632323137663736613431363338 -61356162663239373066393736386635396334663438613938393339386432373462356563663465 -64303436316238363031336265396138663432376361643866653163346465383962653736623566 -31393763393739636262396230366461653630336532656161633331633532313965393337393532 -35666131376133306539313961316661346436393130646132353939303432396163383937396161 -38626136316633653230323336643834653565313935666231613336623139363866636331366366 -36356336363936653762383534393938303863636138323537353432626635653663386261383566 -31306563393536316335646435666534613539656135366461333966303133326362643462643534 -62643234646336383166363236393930623163393764313665386161363930366663653637366466 -36333166666463303033306361343033363365313235633037343837623033303632663665396163 -30623430646565386637343637613637646536656337393131623031363735613434393138636262 -35323962666335626364353539336638326336393430633263613036376331396364333738633633 -37313561306366306435303361613938306632343836623435663131333731343737386435656266 -61653938636661613762376433383062303237396538396336623133633462363731623531623039 -63343538336130373436363664313733353666313937666266393066623737396339656639313263 -37376131616661666337643839386438333932333063323439323631636236366431666533653164 -32323366313930313765643030306435323064633066353062663464363132376465333735616138 -39663062316533653839356539323437383364306337386238306236386234323538326461313832 -37663063383663313338376331313332663863363164306366323835366538613930353566646162 -36633936303733366230343136623130333264373032373939366237646434343633663333626637 -39623639643561363864346561366161633166383935663731313535396331633038623965353537 -62393737393530303230346466616364356262616138346566393162376232623035613466613031 -66363834663231633262363331623534343237653762623465353564623238323866366231663030 -33326538396532633131306539373137616162663936386663376232646630336136316461333539 -36653465323835633963343638356462366361653663346162303662363339653336386463656134 -31656365353838373938373661343739393932343637636231306666663563616162343538323636 -63663039323833323761646236386434623564313863613738343433386237633764343837303136 -32343962663565653438323061326233656461383232343633646137313263666464653764393662 -39613534663562313662343164303136646333376632396339303438663264636632363866643730 -65656433353864373032626363353766313235333262316534343031363831366563646636643737 -36316530376466376436386165396333663262653339343637653461646161666139316132353037 -63393233636632353263323836626339656266636237373563376534306238336139323936646563 -32613936396461656664356364383266306265313330393635303239646662323335656165333665 -39373230383262663339316462353164376330353534663362383639393033636466663631396536 -35393562373030613063346633376363306331626438326263376132653335343263346366656431 -39613362393766653434623563643362333963303636636637333938303235616166643764646365 -34633366626363313937343061396464353239316537376461623266666239386339646666336464 -61643935343861353032656331343635366165373863663332633364313666396538353838363333 -36373030343537366531363432343833653065646431356463343635313939393462366331643431 -65616230383039623334626633356265346430346364363835333736376636656531346261663138 -63646630373638653739343164313433383736633730646530643364353938663837323461326365 -63313365366362633866653166366633663461323836363031306564333965323865663736626665 -33386265653334323461346230393666353263383838613262636432356230646431633832353438 -33626236353365353765343039623862333166383934663038346632303061303737383964373563 -39313433303066663830373930626336393738356564633761653638643834626534616233386437 -64663665613163653431653630313163653130343738636431626439323663303231666231616232 -30313766656334376464306237376130616133633661366165346364656139623335376138316234 -62323566303064396336333137393262373565396534363533626461653036333533333531653738 -35353838373734393536343333336466386466623064343833333537376330356130346162626163 -30366161626631393431653638353138366465613739626434623635626464353034363566633264 -36663364326465333534333162316437353439613639343562623430626139346435366339316435 -35333661316433633835393832656133623439366439356438633335356364313733656665623734 -64666565663436383835316262376432643737333964663039626265626134373736306138646663 -39653838333030353833323531363237643663313865373131363363343530303464643632356435 -32303130396365353139326663316335363733376234316532356566636363343663376238373530 -62316266396135323633383064326162653965336235303336643637333665396134323564313632 -36323635333035306234336161653839343331623738313962363030323830366430303562626333 -37393834303934323836326261303435646563376239643066386336643530333138303431336436 -66653662353639623163333761623161633230653331633639383833636562313639646137343333 -61623639363836646564343630396135356337353834343230626263386265383831313364656138 -38313632363365313064653761393537393164653937656463633338303333313835653433623835 -35646132306464393432643431323937353535336133326136666364643837386430626638313438 -66656434626235313061613039653266343637636437623831646133393235386131613334386231 -65623036353335386639333137663666386436393465646230663532343834326338386537393337 -39303965393836373339613134663134666463646136303161386637316336653465643530626439 -35666237646261346163643936363037666536376465336132323231336235396536336136616437 -63366531633134373432383039386434663166323963616137643261646266643437353839626336 -62303631613236366639333562666262393135393137353234326630633332346630663335383063 -35373431383630623566363863303231613863643964316534356534393537306634313038323464 -62313661326166613433353864393132363266363763333532663665663630353166376461653634 -63343762383766333637343031656339663730633938343663636133626237383364343132626463 -39343066366430356637646532633265663065366633646339333639373265303930323239656261 -38623037383930623933383263643437636231303138393430626333666264376430313361656238 -38333938386639316464363839333437353634613864396336316436333364663364316166653964 -37333638316463666662376436386138646334363031373265316561643237353162333162666165 -65663765353638366666366663626435623533333638306638303734666363346132323330626334 -30383835383339346561363331333863613935323965616235643038343439663832636561633835 -66643161666661636661626463306432313933326535343463326337393463393163346638633438 -33623239623964393064323164303463383036363432336235386661666139363734653362393639 -36313532396265623439346362653138306261343235313062366339616464313965343334323563 -65356133636364303565396362623764393835376235373636356335666136643166373337643936 -35616333616134646162326233346362653666393238326138366266623434353635386362313961 -32346239303663316362366533316166623635653262343536653966383437363339613133663363 -32663030333562383565613763663831333634313963653264326431646663633962303236363363 -63353264353862353634333562643138353630306534323064656131346566363533636461353833 -65636535383233663338636339653464353535316136613333356436396235613735383966373539 -38663661613736353235633138326534663837666337346230663664376231633363343163373232 -39643365326531636135376532373663396466396531633439376261363438633934396462643064 -36666663663533366537616364623665616361363732623264663737383630333639633335326530 -35643436386339363461663331323266636133393466326464343066323637343666323862323639 -37623363646134333431333834366133313137623434623063313739383439633230666637633136 -63356239393138306165343430636534383962316164336161646366303966303034663365633739 -34326266396166346238616464396462663463633366333134316539653865313336323531313765 -38656263653334643239363830366630633736396432396433333065643362343937323333376231 -62303032356530383137316261306137366564306430653031643866393432643737303434613336 -35613638336337346332633730313430356533326534366636313266306561633832626264363334 -33666162306336356264356138393764333164356366346338613966623961356265376537646337 -64643062363732666562373934363233613133653966386461623633343564323066323466633030 -64656261613130343932356437363238623531336136373161613836326664336462656531303931 -37356263383339636531333163666361633737313861353937643162653531666437333966666264 -64323031396664356331666530396464303466383661393539653539336366306237326131346438 -38393836353835336635393632643734353364383366303761383562323631373063373537316161 -62636563356234666536643934356335656665323435613435616461623531336463643030373063 -63653938616536363337306634336434366133336237646633313033396433316136376466393630 -35353939323938656236393233623335313635363139623263656463343934666661333733373366 -34333361626562613937616530343464646339333665663964343165653037323761643366666631 -62643436313961336664336365386333373864626363356531313732663637383862646336323466 -62393933303137346134623664366434303930666438616333646635333138333833373865353533 -33643831646561626463383634626562386637643036303838633933633036313233346434393361 -37646233356132616537636561643561333431643266396134623366646539333239663163383434 -36376433373564643232643935313431613337396139613939663266383034653836623063366530 -39316630356164626263333434303366386536356437613663323739386336613135613033636434 -61396363623332346437363863326363353330353465376436393964393265316464663761383762 -66646232333061383965666137616539643538303764326263666439333839366366373264646466 -31313631393035323962383631636635303438633065373561313034366536636437633465363135 -61353730366362653664313761313962616562393766386262613766636162373761613465636135 -64313131336166343762393039633739656237393731313036333064343366306330303133626566 -61343536316431653164326434343130356262656336613362663230306562316533613362386238 -61343238306232383936633934653564636564363734333430656136376133636132346537303461 -32636633383039396162623732306664333131663332336661653536613830386635366665303737 -65383637343066396137623461313265616537393134653763303837386130303730376531613138 -62303863373465643531643936626135303032316134623138613334303536343231666336383638 -36633232363931643564663734323436306336636337656563393263663830323466363862666339 -35363765633334323763336164646534643634393430646234643262653661333661613938393034 -36313031656264316235343262626562393332643564383630303164316235333534336462643864 -30306531343338363264613766386632323437666666356631393532363363326466373230626661 -63316533356265623333326165373966363730633162666535643436373365313334316432656235 -66303435383138626162626661636130376339613231623666346563616633356635353230373036 -35623237386532353435303461666162616661383834313433643038663038303534303930303239 -32313139633262623732616334353464333432363931633330653261636432373532363763306362 -38366533653838656538323132386538323062343665376437306163393331313332383463383436 -34303838373833333764333261313733383065346136333637363161653262653930326161623866 -39313437393336643263613362663563633438316366633433646665373136336163626131363432 -38303761386363343931333637353238383862373262623664313538376432626266646262616138 -39306136373636313663313462643336333066633333616365613433353563356263346565626637 -63666166613966393461383934613030343738613833376661393664366233373130623138363161 -34643137383366396236623135346665623932613534333534356134363365343537346331323138 -65343933316432323861363836323764643462643030626563646662613061636461363831376130 -35333731653965356162333762663734313637326465313662653433623338373364623263616166 -63376562646131626637613139613966306666656233373434386232326433316335613431663266 -34353733303634653130633565393739623730623738333433643061646264633831336435656536 -34643936646337386565663037633235323437626537656535323232383965656663376431393930 -63643465616333373538396637353431393438383535373832393766383861316366363333636430 -33633461613534366532306132653265353862633936613662326638393464633339613361663162 -31353262313365396135663962306261376337623339656639323331633231306233333762303130 -37383063666533623966376439306162326137313230623630633866393066386265363438643065 -34613265386530636234366237336265303130343239633666373165633531626134383061616366 -65326235613938383166663861393361313965616237303466656638626635666131666364353764 -31613831396161336439396636306365653662616538373037393765666339323162616630336663 -65373762383630663164363335343237373163663039623833633738336539396237626133373163 -62643837313263653236323834393065373764613031376564333865383164383366653537316138 -35633833633664393764363733303533613034333934353237323938336163346165656533663861 -34663836653063616265303439626165393230636566643363313763303233346564363862306666 -39396331396231343833386331663565323934646532386638393235623831346236613961623563 -32373033393264326563376131663362636237616462626666646463333662373262396433643264 -32663464396365653133363831383831636139353963636333623161363465316337313561343263 -34383963376661663137376364626366613438613937623263636466663762666530336237393234 -63383337363530376334393631363964646237363538623531383731616437616432383532613939 -61393637343762373030626664383863383462333363343465623364333235373530343438313338 -64316264643964643364323665376461343636656436356234626432653435666132343963323331 -35326431303864643832613130393063633966373661326231346136333363666230363934376431 -35376434343434626662653962393462316432333735653632366265333135356266663833653966 -30383663663038653733316638353565323063323532623364366161356531306434353738333538 -62343736386538643266653939643139373164313762316635343834363161386365613031346561 -34643062636366353239356330313365313038333132343461653165343161393639303766323331 -62646236303565373532646139343831633638343437373165333938303435633334663831663631 -35393365376334313533323565346435336164366635643165336239663938393438343830613364 -34303565383666306666366232653966616334616236306265363861356437316463346131386166 -37643533363237623636346561313031656265313931386262393862356265616431616365616132 -62343930343661393965613830353164316463383036343331656562386263333165386434653030 -37343437323035313165333531343334303463616134663934636562313063336639353463656364 -61303035306139646337663132343237633564616566353563303130393363643861326136613065 -66356431343234646666366237633837633361306165653765633032623836343531356166303133 -65363332643266643761393065303966346330316365336139363535623730623661643763613335 -65356566373831323663653463653066616433613330356262303266666265663232316566366633 -62323135306530383337326461633136623830313138666664663065383331353038666237623961 -62363233363639386538666161363461306136663637383064343564643639626532623261666562 -33353763633534363861626464323166353466636137643438633164336135616230303437363535 -39376130623963343433636439663934663739663638383633336665623338303162633163663833 -30326339333761313165393230313831376533323131373633303564363431613231656231653361 -36663032363366333035363431323934623133346465343339386464306236353466343233303037 -36393036336166353235303737646439333332653430313362353736623035613865323439353032 -39336366663535313236653332656338353364363163653632303962393830383636613139613364 -65326164346137346534633030623066306332656135346330386161613830616339386438343238 -66636431386361633763653731313831663435373163623234306136656265363864323236323161 -33323263313361616630363663653932643236643763633139393130623535303039353664353439 -35313037346666303963366261613431373663303838656631393638383562346131383164316266 -35363231333861666232316163306266363631383933393631326266343865303330353436336162 -34353361666663386633643236623538356639366131616134636633353666653838323737663237 -30376134613663663038383433353061623562353237666665323935393439313730303837613562 -30376532383761666662373239663137613236656433396430353462646463663661336164376134 -64656336656233386437663965373630393035643662333362303834616362323762353063343363 -36363662313464326330646362636434386438393430653132613166306331333632383436313130 -61363731336464373065393266306438356463656532353063323532343234663065303036336530 -62313134306538626532383335626639396664616634666264326133313434376461646331396337 -65376466646666306362313639386632346539636361646465613062303962356166636564316563 -63343362346538646430306664623636346165303236373264613063336634356162653032326233 -37623630356564383933653965323661623561373465626330373838653838303963336436636439 -65386661663035626635613636383965346135333033366231643034343533613732666339366332 -32316662343038636530653734393533393531616539626432393133633632666635353135383731 -61646432303136393562343161633832353530626164656161613734363336643639393638333065 -32353762303234663234366234373239636139636631383632623733363334346131363763363634 -62303736353637383432356137346663383461343932663063356262383264343131346535653431 -66663336313138353535303332663164353662376133336664323066393062613032626235653562 -64623730653464386230373733666238323930623964333838326135653930313465313537666465 -61346365356439323462356532353831643065623364626461343136653664303432383536366564 -65643130313261336533613665353465653366666138626137396139616135316537643834343438 -32343039333536356464396234643461623530643336633637323662323338663262336633303665 -34376138643336353163386661666364396162666339613839623335313830393563636531333365 -62386562393062663136353634626365613235333331393763353530333234316262383866623931 -66353934393330326337616139626562646138353161356339343936326138356537626235333939 -32353536353939346133386635326337633363323436663637356635326231376230656662626163 -65656332386138653164326634663865656234643933366366663533633738376164343663353362 -62666630333937633333333932636665393665333436336561376664663263306134643864383138 -63363438666535336235373830646633346438393031636331306465623561653930646663333766 -66303332356464303961386138643033373565313864323032333563636462633266623664356363 -38623761333365663933663862663430303565646332376435346134643230663664356561623730 -36656331366238666262383539336462653830626461323834383237363763663034643865376261 -37666464396631663338643537313236386363376164636232363133623531363161656531623331 -38613466633163376263643734623530303964616430306135313562393039303839353063313137 -30653361366536313739376362626436373435326435343061356633323139646663633734313830 -33393334396232613932376138393537376636396464323737613835636431636233646363383962 -33613939333266613433343737366662343039666662623137356261316564613965366462613439 -37353430643035613865666335366166306662343263663166353930666564316139653666363832 -65626531636531326332353036633939396330306662313163383162353937656436363861656334 -62393463383663613433346436373939333365636234393666333035303765663131356131396338 -32306338373363346138656233636264323666333038383237313830633162336134613262343033 -36323135323135313965666363646432313337626438643833323137623530393438613964353362 -33643534363931613232666435376235353636303862373739376332383932303338643836613638 -34333038363836626534646231316465316165366137316237383236316366323664393165326465 -65326164316532646134386263373035386135373664393738363339626432353861363234326236 -62643434663034636338663362306237353839313063633834653539663735653965343965636432 -39393964383864396136306232656366306430313666633866643733643637623733626265666662 -33313130646230323331363432613263346137313365623165613364383939643732616435376238 -37653539396630333164393938663438633132333937333562356665316133643565663263316531 -37666261616231653038643130656464626136363539366363613761613539653037646639303362 -38373931646137313065613333383162656538343263346135646335363965326562316631613463 -39346464343461636535356334616463353934646362356462626633643532373364346164313339 -62303231636431363539306631613232663931313664396163356234363939343336623437363535 -39653865306335353464353265656237623034393764373765653337643266306665653863663530 -61623961313239643736303764376237643432323236383333663464393638613333386166363031 -37646334643265303365653166353134326137613731323135356134666536613231303463323138 -35613964646239363630383763633439633766356234383334656236306638373565306133336662 -63616635323035356135396331396135396461666464353964306132643438613566393936623135 -62633132353135393132343464616566623334333130653932373930316535333066323837316365 -39373632303462316166396334626631316364653032383061383834346362323734626139643564 -63393162663739626337373131303838313333666632373437656464323330623463623837353862 -33333734333536353230326262343264346162356635356162383863346236626361353262623765 -66313834313331306634313064356638386431346664393831373961653031376366343263316261 -34393662303231646434626136336632393532613163336436616466383634616364633331656662 -64306132326163383530386131333865346631656139343862663633353962616230346634303063 -61383239616234636565366132656638366363393061663263333533643133366661366631323736 -63666430396333653731613037616535393635336364636530316139663362373862326334643766 -36343539613832366564613638663331343664616338633636386437666636376666353335376137 -65373332626137636335323131646431656337353262343763393461346333386137653137633138 -36643863323863336637386462386361393730623230376463643363656631373230353566363762 -38653466363964363766333537333461306531353461373866303564393637383664646139643134 -63653963353937336238623463393666623464656361306663396262366633333862313438346232 -36386334633137383865376431663866386232353062343233383235626639343865653332336231 -31363732393730373665333662626437623133383831363634323565373036346330356634383261 -65636537336533353137336265616465303561653233663161316662623238633065336163376361 -39366633356265663634343734313064373833326562386366646138643339316431643331363663 -32376133626334666635383661666230313530643365386636643162363539353865633630376132 -31353634656538393863343261626634383438643232643439643533633236363336323762656364 -38623366333037623661353833396336323065343564343535393535373561303666636265616262 -36613135656338656436306436643535343838353966616536353936333863643034346335356531 -31636132656566623337343837623435373366626530373363353430623162353838396130393732 -66323332306164316336636236353365643262323562396433303863326139653161306435366661 -30663434326465316434393334333635343136363039626139633236343133343031323963653866 -65336566623761623239343862383730346230336562366534303439333536613437643735616630 -33653563383739343031353663326363363433326663643931653730393430376462306266626133 -31363133383437363163306538323934366465316464663835333833626533353263636234663831 -31323934386631313239383863653139613836346231353364323630353734643533336232366439 -64326332646461626131653232373662383131363766336433306566306463626166356364396139 -33396164643433323930303531393563656438366239636366336535383632383538353534336233 -64393439333363646436313935626664626633333237303732346230363861373361366131623161 -35633439336563363733373937316232303334376233383539356661323534386366643933323035 -62313039653133663835613932346464363562333236356266663666643130323833383161396565 -62643732636465613466323065643266633531303139316633316563633263666136333065373337 -33633763666464626237336366303765326462643763643137656561396435396664313833316263 -36316264623231376461373230623332366264363739643337346161323832356438616134313431 -36636538333463316632656130376664356230336266383636643638626634663163373932623338 -33343266393331386437646239646565663737386131353037386230373264376336636438653365 -34356664363736323064326561653039666431353664643565613663356130643439333934643761 -63616562323533363930663366306231333565653433623733616166383035613039303634373933 -30613163663264636261303733333333666461663034313466653937313538666538313566666166 -65363435386665366336623937633663663838366162663261613438613862343935333532353735 -66393739323261343438623964616336613463383664633466303763623035313734623739636166 -62306361353532356535343330366630666430336461336137383834326436396565653262323333 -63386562353462346632373535656462633435663935386636383436336238396134366661333563 -33373064303463653862343065333838383533363661613038343236666461663164326435616538 -39653336616430373531363265393664386139666131343363383465313435376337383631666461 -38616566643065336336636638323036326534323036346333353936616233366332366332656639 -35393330343462326231373239613464393830333031363637323936313732356566623565623833 -66376633326437323666663066383163363035313464393338346365303262643165323061613638 -32643561363435313437333733623832356134316361353363366335613865396135653430623530 -33383464393063333163353333653636623161363738353939623031373634313033323032333262 -61633937336165303864326435303336313065353139373661643339643637393636333264323134 -65376463653364653432393139303361316437666364643938366161633730366237393734663637 -62353533626235653839633835373039636134666532363465623533343464366232653561633465 -35663931346632353933623234323132313663633532366431626633363965656533656138393265 -36316364356664376230333364393464656337346461323034323566633130623234643364333738 -38373036643161383162306261313434356634303732626463356638626333623935376630356137 -61393763653862306261316136633930396363666538623566616662653032393537666561396131 -63303336333661356537383732633430343662653733386138393331356330656134366134323666 -38653339376464353433633334646430333036333063363461353936643532366330316236383936 -62353866616632386633313032366533383532666237313663383462363039323135613435393563 -63373465373162316462396562373930346566623331623662396231653761666238336434396463 -32336133666438366265363438613332346338643438663534336633346465653630373933356530 -32626633613434646136633139346563373636313065316637633230646161373438356530323762 -61653532396365653664656166323463336331346162626339653533333664343236323563346264 -37373736666261353539306537623634303065313162643635663263383637396563373137636261 -39663263336239356236653730636436363561316630363561613862356338636564613932663735 -62353239393637343534343335633632656262383365666463306635643937333865633137303730 -62386165343734353039353766303061623766303939333939623637653964633736363666633761 -62656531346330316463316365313063613066323265643562373539643530373736613365396539 -39356631316462313432613636383565313835623037653835656537366663613066376638626162 -61633366653738616233353762323334613733383633623631343364313664333132363732626339 -64363731646661646666643736356630663361646336633766656235616361326532346636336432 -39306234356237643135656539626462613739636161666264396633313264613039313262366561 -35373132356563306131633535303963333735346364343766386666663866633034343666626637 -33653938326538343730613130383139366563653263393432363432613062303436353635376463 -39326539343931326363363033353430303565656236386265333230656530353337373261373366 -33623363633462666463613936643664623934366261333233623732343636323337323637616261 -66656235653630386561383432623931623238666534626434303530363165633062313265316664 -36316233363061646533393134363437333031616662363233336533346334626361333764663432 -34323330396365393763656436393137633730336366323538333638363333376661656336303263 -66616137316635333165333962326439643732633838656264663735623231336132656266383162 -66626666373330363334323838623764303837656338333431633232383039393334386138396434 -35323464316437366235636435613661313733616331636264316234303661656231656433393234 -34313637353664336137646630383236323835306638643830383139316263643762396537366131 -64646537333662323633623535326334373762323239656633386336373634386364383234646337 -61646361353032393562326330353731333735353531383733616438376632363264356461383537 -62353966313561626361626663656436323230356561316636363463393738306137616630316231 -63623139626234623438663033636466363337626263396263663634656365616634336435666638 -62626234343533666562656335326434613939373461623033323935333534363031316366666331 -64373435373136313635313966646434323137306334393537653761626137323439636632303635 -34393631376565316363646665383266643131626238303637623537386132623962303430346434 -65353566326466613134333662343537616532346639336439353566653466646161653938326536 -35363339636163656432323661653239343464653064316531303038356661313965333464383061 -38396633633562336434626135613565653463633465336139346339343633313062323130306535 -62313263323735356134646361336334373531343265303864346232363832313139396438623936 -63656531336365623636616639623765356666323632653163653234643861363162356634643039 -66343134343237623437396439386365333738653934393337633735393561633634363466613765 -66356139386138633766383061613831666564336530633834336264376137376664353239636262 -33353038366530666534363034326539663164353034623863383164316631336463373663616263 -32616434303336316465333661636238646565633631633265633939666663376235313235303437 -36303961613731333065663337393835323232386539383034313565373431636533613234666533 -31386336303338386265363032356637633866626432633331383661373534663562616339383264 -64333639356135633361313534336433366131353038336234366362373038653335376332636462 -61326636323837396332623330326334386661366363306431386533346662646364613438363336 -37346633663762386438653130336134316233353737623933393335636530383631656365373134 -34333432336265333332623230656635383137353531326334653630373936393265613838373235 -64383463326562663132633836643835373431663965643939656661626564386237363462386137 -33613932306564376565643037356364326464633263636566323436363439393034643564646239 -39316564343037363561303539666130623861626562373036386530353830333037393039356366 -39303433633532616137346336663232373334333531393264663432373231303162636638333536 -34626263663261666237306133303536386536376235636533663561313432363961323265613734 -32363136343335383933633635313439336565376532353134373465346538653765616134326237 -31643130383764623430353866313130656664366663653635396332663334356431623962356538 -34386232386664353937343236613065313739646136646164383266343763646637326366653963 -36633736376263323731653866623134613461636135363930643731323866363363373839343863 -65653234356465343538646666393839633936666661326239323532353762376238373733363330 -34643764336533616232306365393531653638313264333864393462626230393033613565623530 -36376263316336346537633962663236396432663762393730643562303333653033663531306235 -66663432366639653464386233326533653163366133323831346635666430303835333936333665 -35313261613864343962343963356634623431616531303164353166393761303336393665333662 -66393537363765313431663233383462636530376237666162343831393530326361383338333161 -35653734323766623163343736653730383634306130383063343330666635303363623933643434 -30346336643637356332323137636135373966333662616431633332373132396237326362316236 -37323339303738346339316365616465623633303737316431383336623237343839313436386231 -36376331626235643361366638663066373031376131643032626565613231393364376436346364 -33346237646534623463646236393330323237373633613564393934326561653238383761313766 -33343838333863616661653234333865636635653032623831613733323633343033376164383335 -33663932373031633365346236316536323031306639306465656164373262303835326536663061 -32353038343936306534623135353034356665666138343434633763303766393738663436643434 -61626165656664623534333138663062376431336438616464396533313535636535306661366564 -65376261363136313365333038343161326464303762356635383566323835356361613630626631 -37316663326132653930323233313337326364613365393836626236363838303832313137386363 -63313164386464353435396165326634623636656164396237323066646461376339316161663234 -61373361336338346132333336373039636435653132313464653962316635653535343036626336 -63393465333034653366646230393831363135376561616330356264663239343731393162303864 -37646362326534623865613535623737303765373961326434656464326634653662393166363734 -61613964616364343635393863653964303637393963333438313763353364626332333737643739 -62383264653763666234366334373936666262653436336533383237383464396533323330313261 -62636566616536376565623733326137633537313836386239373032323539363735336336306463 -38366539326335663832316331343935376434613231616664353963613837393031633765363565 -62306366656461333636343737616639653538396238393731616566333962323563346563623563 -63336533396266303630313230313364343665333863303630343730356636633066303939633864 -30643965666138646438333438636164623237316161303637323365303532383936393065386363 -37383732343230363766303033643766393737633731346666356639393766353561346264613730 -64336361306361313532393938643638626534356438346161386639353335396633323537643730 -32636331636133313965666666336463656537636363316266346135376464636533343266636364 -35643238383464366531353463636235666330623430326565346136643633343637326161383935 -66323138323838366530623736346438623938666230623233616438623662363933633635633236 -61303339366136343032346636663562396638323463353861316264303732643436336335353462 -35613066636465343265633134613663653634303131636464633636386435643237653163356534 -30666632386138653434303030633835633130326634366661653266356232393866343565343066 -63376263626534353466316265643035653732326236393835373565326466323330643161303137 -65616135386366383862393361363461383464373464323932396131666638653433366561303930 -39343061323533646263613963303338376139623631333461326530633431373134393231393736 -61656564396564613261396133333735633032646564636661633262663765666438636261393530 -61616163353432386335643066343564353232626536356433353030373765653563323231313935 -65346434373636663865366162376433346339343332646664366162383030323936366663613035 -33633762666666613762306563323432393335646237303030626330373634663830643733366630 -35313236333063376365643538393764356463656562303535653634656463616563383530316430 -38376138663665613130653532323966653538666563306537326364323663323437386430343333 -31326466373730623865393833346435313338646561333935646464613862653037353665313265 -63666461666662313839396362396362323365363565633065376164383830643933373334656139 -31623732353131346432636364343639373437366239633139383439303261363331316339343338 -63363864333764346434343534366662643839373335393965353637666562346566666431626563 -37396435666139633231633339333762303563303436666437653834303533346139653663353763 -34323863393866636630373438323165313030376133346462346334326565623337356563663430 -34303863306134353737373735353461383834663436653535356134326139633437663233373537 -31663864383465336431653563666333376331636565366331616438323737663434663233656231 -31613930386439623739363261306463326266343161663931343034393132643333373737366132 -63386134636537393230316239306231316433653532343032343338333138376239386163333663 -63383261376564636336313664623736373264353631343538303536623930653165646232333736 -66346130333736633731616665336430666466313339623438363136353237303733343761646237 -61613432613430663636353139383633333434303162613831663161623539653066383764643436 -63643865363366613765613836306566643633373563363932323735303030623533656562386331 -34366134373464323830346238666237356238616335613735656263373832653836323538316137 -36303733623966626638313238356334653938623866613062373332373339313138646230393835 -38363666376461333832343037303937626533363436383136633230363137386565316566646562 -64353362666464383761363061613237623733346136633933383937306665356139383265616339 -62363536643963383064343833643863363261616366333938346432303336326531356163623235 -32396632643462393038626633623036356163626331306132623561653933393630613336636365 -64316137376364323765303638366662393063356436663330633832383933393931323631373632 -31383939346666333364393963356333623563376430373564333465366130393366346637333261 -66393765613365333163636361346138626638383734346632616666653138303933363263356235 -61643639393136343361646363356362346233396135303464653131383335643437376139376534 -62373733383034653831613166396161356664663662626333363337396633663464633239633035 -36323234313833333233633631303538613732353337666636313237386636623166303266633430 -64333362313731343963333766623162303162346337623734643238343064656233373831306139 -38346466363261313235336533313465386563643034363735613232663735316133303635343034 -31656534383835353832616338363037633962363231653637363038386262386566343466343961 -32613138373064363263306464363237343433373535623331626165616666646164626266636161 -30353336363266316335323135656437366664346231323432633061313730373131376163366562 -38393465636532336166343831376637353735323435303631346431623462383064363035376333 -65616464633163623265666562363264383961353461396533653037613833616130373763626662 -39306436633334303665376139356262303565306338613739653963303235353863626463393063 -61366363353439373030323336383932373836623434383436363439356262376636376564346331 -66333537343332653164656132333936623936353962646362356337353866626438303833306335 -33613439343431616235323263393232363534343561333938636237386134303330313466613930 -63396434343530363534633261613762336165376330623835363738336634363363623762633337 -36643432333739306531343731386636366235393030333234366335633561363332666336396434 -35376235333934346537306134373132646164626131303866636234316533346263343362396162 -33343738663066306536643263613661613933643433363662393065356533383033613338666666 -62616663393938353938393835643565626530643261623866373930386464326164373331373532 -36623337396332643736386166303439636662663334343034626637323664356563383135663265 -66353637393336656333643537623833323464656636643465323436373031363839306334333139 -33623535323234323938643837316163346239343934383632343766373137316235383030396238 -31306264373730383664316132363531616132623633623266303836356432396434336433653462 -32633765653161306338393566353339663834373931613762613337393265616562383166373231 -30373434376437656265303038363634356662666363616534303732646530666461373462663337 -31643663636430633966346431366230613134346134323662646430396130333833653736366464 -34643165323732323366643936353562656663623031353634373733316631376465326534636438 -64366163636634613736353937373865366631623636376630666531663362356361643965326262 -62323035386139373866363432373064376230306630393636623666396663626161373738396538 -36666161363631663664366233613734376465346465383032343131613736343663633838383930 -66386631323763666639613831613939303937303536646263316639613962353834663434363566 -32363730626238306631633562616131363438393533313766653034363261333735333062313162 -63313065356431353538643636326365346637306563663662346236373636623266626564376466 -38386434336135646533653361626435316236396565343264633733373962383262353863303935 -34326235306638323931656561663337396530336134633630623561366539303865326137393763 -35393561383538616231333161323764303732303234373363633438613136376265396362633234 -64633761626632376363306637653463666234643434343932613034306261346663376564313665 -30613033646364616435653965333932363565363335363831323665656638393031373835343263 -34313830393831616133363037323764356564356533323639623839636535346238613135353837 -64303766326264366235313864626261613766303832643466326332623065323138653932636438 -37396532623236613564616439343030656634356531396639356532313366633536346665306237 -32663966653239633861386466386439363565336137333966346139303339313035393363633965 -61353331613631363432626663346664306233396663306265356634663366646434346661393161 -36326665613936316262343530306237313865653766323461323163393831346338313434303664 -37366435326666373833656661663561646134393330653432306332386335366439333066616639 -34623866333035616433376630353965313238653461626236363765633232326632376162316266 -31633638363963373836303865316264616436373031396231316634313666333866646261303863 -39663236663037323761636634393466653162343330613732373566336533396638626163363233 -63313861323836373864393230313666353032653937393134333730353038303762376534323061 -31303663393961343031636661393336306636393765616566356537346665353330353463663839 -36386132376336643066633863623132646433313461356564333132373130323737313739303235 -61333464383732343232323237333231353033336535323232623463356236623064653730383764 -33373537383831376132333839353736386531363739626662366431306636393839663564626132 -63373336333938636630623461313632323564653730356337323738653866373430613930396661 -35306138333463633930633230373734353165383266623964366565323639333437313661343265 -39653561616265663934643762343630633166313861356639376238333138653763646130366464 -63356438386261633132666265343730346335313838346665396262323138386235326366626136 -66393836653235373431316133623830323931386366396461626337323563636230303734623437 -30313362373633663039343534633464636136636234323262663435386434366334353062343338 -30383530343439383862383931656464643838346531383361323839363134386264313131356334 -33313633353966626465356338663965316561323965376130386662373161323139323034616433 -34393864383434393831313463616132626662636566643530323437663961353938326263346132 -62323337333263613231376263383664333235306135333933333065323533366665646162336263 -65346130303561633066323030363932323237613237366135653438366466663761643037656361 -65313233626535356334333236366330666564656665613438646639396563353837386664636339 -62663235366634363265386532313633633238663934383761386331366362353039656633633661 -62636433626633306434343132366538663736336666303637306232373963613065666163653230 -38346665376366333036393735376439346361333565303130613534366638643839666338313637 -30636239326166623835633936613863346537323231623362336636663764613635396537623030 -62353364613631303930613461336132623765626263633238616238376662636432353631383330 -39656438363464653033663637313061656232306562373431313661396236666331376539633639 -37376330333331333639393039303336303738643432663866356365333839653334393837363230 -61653562356136353530386362383234663065353734336366353861666231663138626561343435 -37353763383338346331363464303866373566623061633836383939343834326430313061343633 -33396536383237306365303936373763653231616463383966373935366464336636396539323366 -30353763323838643437346635363461663038393064323761346461326233363236303239316630 -32646435396538613435663130393466356630633863376232306533656137353937363730346430 -37383531306132326633373133393166666161653930326166616632383035656262326139653935 -62636366663131666130346536326431323137353465326134333933373432613232653434376162 -31383430636163626233666530393430353965356234336266313730356161366532623561383366 -32646465346435346465386532323530306235643237633237396265656262393338346165663738 -36613163653030613931326430326364306537386333316365616461343664646536326231623331 -30633330313333306663313236626665356465623636326635333661663135343333346339636637 -36363034653364623634336564363833643434623166666532333039346531373831613263323731 -39636435666439613261623862326432333063616339616562613434393062333038333831666231 -65346633316239663165303439343763616262653538343235373030343561363165383532383664 -33333761386137646230643865653138336461623462666236396165383537383834646437383831 -38373964363465623061346330383965383038633830343531326661633638626561636133343230 -62363464643535643832663461346565613132633161363163323834646562623837633665326232 -62363666333934303635366235626337623432653666613532336537663736333434653164633661 -62363432363631643833653536613733386462356331356564323639343638336261613165353336 -66343761663630393863663938636464356237323962373737663466356132623338383134306563 -34313037666464666266306665666137663331363865363737666262626634336236653263613732 -61623632353036653732356362336263306162643163366234623338346139323432316535353333 -64316233353062643438653933373839393333643765646638636363383361393265613837373130 -30646266376535343139383038333134643866353564613565613864316233383538333862346138 -61323735303731316536663765376165616436316438643632386135373134636165363034306439 -61646461363931303931316332316233646434626630343565346561343336333466383437323339 -62323465336635383832343134653463393333663039346436383838303266333864613933346438 -39356233656132626639376331306233633837313932626438306265643166613961313364363632 -30383639323066396432326664636661613066386364666630313736333533353636623535313133 -37393535346538666434333435353136343763353835303230343236393537326237356332346333 -33643632313238666137356165613061613932376339303664633064383664656136306138373062 -36346162616263366439613438616534386436363665376133366338366234613032333136383566 -30613566336466373337643530306235376662643864623065623165393334326132626635383438 -61343035636362613934376666383463303039336165643737313831666665656330366235303837 -31653661373534326361613865663237353132316436383434663232353561383964663138613733 -38643163633366373630623566663739623939366463393530383736633932613966353665636435 -65666464653435646433666132613936623262383635656433653966653730626434303433633935 -66323234363331643937346437626131353431333162656564613363373330326237623862656663 -37623135643031626130636565346632373938316538663135306561386533336430303462623634 -30303337613966333933643730373734396565646139343838626333643066383865376562636132 -64363633316337646165383734306332363937623962653434393966316331373430343761643064 -33313332393765356231376233353932636635663335323139343331636334383934313830383734 -32383630393337366330386337336535393263643566353534323764646534633136333439383532 -35663935636536326131313364643365346638366139643837326262633639633131623437653838 -62633564303734333031386332326334313036343038636338303231653161383062303333356366 -61313864656136386430366535363238616531393962633962366565346534373737616531333935 -62373939353633373731333538656636646130333234393532366166613331323762373130383530 -33653730653433353862383264323839396537613962613166316662306661316430626262383238 -62633732393539323733376236393235376662316233326531623038393433376131663333653434 -65363762653364333163626464356164663337643533626266646135306461353965313933333835 -33663461313936316336336339343863363331373164663436643364633966346162646164323034 -33626165343439633434393135306337306432363330346134346665626230306661656134616337 -31333836633033306164303666633231346430303662356334323737636165393839633566313436 -65306136343138373561653764336635366164393737303531353533326265386532666332383836 -39323731343132383539343039656331666231613365353166383661366336653231363433346631 -35646230653535626163623839333765633734336563353966346235663231333735326161396161 -34373634636430633662653162303739633065303231383264613631356562623130383838316262 -33663330383664366138313164353338333065363962383731616431313861313566646537333035 -63386464636562396131356537623436636239353632393232373565326566303161316462356131 -61656630393463356663613364613731666362393230333236643830323531383062343061653031 -61313237363533353738393632633936663165663035336661653537626136373930366262333531 -34326538353638643331346638653831353230326661616462303638383566323437666463656663 -65623831373864303031393664626438326466346661316235366565616466663132323131626335 -39643665393234316662316462656430623564663233316564613465363031353963313234393533 -61633061313639663163306664313363386665653431346463646564323230366161386531336466 -38356630636162336164633931343239636664316531623865393932316336343236303836663361 -38643730326336623531323436613634353239323031336664643132626638316133356464616435 -63303334326663626265626361306133616336376238373564623862613639613837626339366138 -62343866653036386133393064353861623365626561393562613934366662366238343766636638 -33333733323533326433626366376339326630633266333366353866356233346166343131326335 -62393932613562396264613237653831306536396664626334656137626339303332633736386433 -61373163363863663365333532343533313864663765636666323762326266653237303730383165 -34313761646562383936373231363761633837636665343364313763633963636136653637333130 -36666630633131326261303434653930626264323663343262633733636232353939393430396436 -33616461316434613533626563353863343530643633316663653165326434373036306261316266 -33663362363338386335626233613137616232653737663932653566326562666461626335303633 -34386438393466343039613436666135366663633534323464653931366637643661656434343362 -65396461636333316662323034373962326430363733363236383766303564333664623435663861 -64656530313363396134393136353732356536323866333266663139613463626136346135306364 -65353938643039376133633433616538643432366439613930373931383537386631326566303463 -33313030646630363262333436663838663437353430613533363162383531326461353362313132 -32326533306238616337396338613032336366346662623037306164643638663935353665643161 -35313664373836383161363462653033343837383437643839323332373733613462303238353036 -34643761633366356137303061386436373762663239333538393939336462623762353536663762 -33323937313037613639393633643033313662363233313235303764666365393732663038326531 -30363663643730653265663564653161343031376533613431396334633530313538623534396461 -62636433343263343936393130633539653164616464306465623765643337376363643635663138 -39356334316161343433396538643638316137316237633362653363343234656336313463643635 -65643333633636333830333739363735343637353061326332656338363064313965376533353365 -36666637386334303261373435313532366634383637636233653665333865626461653133626564 -32383739303534373762626232666365653765643362323035656465653864353663356563393461 -65326264363134363763383862616231336435356535323635393936616237353165383137383833 -66363336633830623639333031393664613837626537663034396661623330333537366563353731 -66363430343264353631663065306461613039383762623837333065316566386463393032383935 -65386666383231663236383161663733613162316434356638623861356363396431366563626362 -31363235613238623664333932333238353461316137303339363935363938343464313235666139 -65333231386335323766323530346662396239616439643538623138643661653134663731353839 -63356631626365363936613739636336326236396530316161363436636239663330386135653864 -35646238313438333635373937666431643761633433653261336466316335623135613963353066 -39343966336238633336353464373130383231396264356265666635653863646633336534366661 -66613336616433356566623739366438393739623836656264366562613330666539396466313339 -32373239373430353832643638643832313863313661383932373532356532373064323535303032 -30656361356433363731346333306462366362663930306233353834343361663766356663666532 -39383464373637373730393933663936336664353730313564643639653463393166323034666263 -37363663323361306430303837313235306164303535646335663835346235323866323761393662 -64373165653065306639623932633062323934316139393038343533326535613839396137666133 -33313762373138383139653531353964663734336234303536376631306336383433643930393437 -65396632393930336161623336643735633234396539333165373964383165313663643133653133 -65633837333537663765613262393736343138343561643166396433633430356336306535303135 -35376133373862363832383434616436653937633335373238656661383933373636653234363938 -32383132623431396535353236343232326562343039303439323763663438336566623532653166 -32353634623831373830386365343834376530626437656531393730333366393665393734356133 -35396635643730316530303333366265666234653361666163666339336431313336373333306430 -38653236306337323736383339356436316266666434346161346666363434326333383037303039 -64616464666635346238333961323365386431626131653533636563646662663766363562326165 -38626533343166653635316633316339393065313363363530653831643738303565343236633966 -31393464366138396433303264383739306237323161623065316637323734646165643937373966 -32336137313735303964373538356537633161343466616564316138383631616137366137326630 -33623531313536636638306435383937646565646636626565306539633563393262656638643166 -62386261633764373438623131356234636262393835303130363564306563353364346330373366 -32386131643566623533643962333463653961313438333432386433333631626336636266313536 -34623831316564303262613032633664666164333934636437343333383133396631643238653439 -63656431356636613862626134386462353064313831616633633736356165623665616465326235 -36656236366564313934313433343338396335326363323065343762346330336633656535616532 -32323763326637386337643837363339666236313161313736323330383766343337363965346462 -33313738333633383137323766316436323933343363623833636663376533383438633562326633 -38306362356263383763616238663135646531633630373164613037383464613963643331303031 -32663364636563303033636135663436626331613039363464396433303335666661663838393465 -65626666623434663266323464633565346163346333383037353037633935313932616337323630 -34316332376161333930666534666264303762653930643539613363336438316466316563653165 -32333765366636653730323130346534656136383233336531666131643333633631666334353330 -31613163633231376362373563393031363835613264663463616136643930663663626464616466 -36383031613539636533346233656638383135336463393937383531373735316261313537333664 -61303465323835653463346137616333653738623237616361326639633230306233663766613962 -33386231393830333634646135356566383462666431653336383434663539663937306166636263 -37386333616530316134383639653361613762636366626461616136363937303362366537633439 -35343163653234623530316634623632396630383230616237363937366136333135366537393632 -66663239383239383633356635656137313039363964366531346434666266303533633431643166 -32636139383339646564653866613733623565613536386363393833373337303335373561663531 -36663532663733346532336631336561373564333039333739653661376236326331643632343138 -64653238646536356234343565653135383532623732333730313739383563323335336436303061 -63663462643230666335393330656361656661656464383165643836363965663731643265646330 -64336534386466623761336461623061336438336338353837653237393939306562333038363138 -33306637366461633230313530343665633564326262653862323534346563343865613731616132 -36396432343863656630646438643162393064323136373862393665633666306166313263626638 -63346639333966376437393737346264393732626265356362633432616261383934303461613539 -36366266376132373237633635316330316335346362376266376663313664373462666561343264 -35613737646265376332643464326333353664643465363663396565653233386333336263613863 -36626430323231313136666164613237663465346338366632326435313134306165626631343432 -39343332613961396136363566616333316466616534626235633666646261383432356662363832 -37653664386365326639663937633731373735383939326135643863343037643231303662336563 -61613433666239376334303133343765363534313865343032613732633463326330646230356664 -34653863623061356131306162373334353636396238613164646563396365636539353961333862 -65353963336238623439356530636137336135653339626633353664623465633566346238623265 -62616564656264393264356430363935376365306631373533646536663138653738363465613462 -35636435623537323561386364373039663335396261303362306262366661343365316264623636 -61663433343234623133633532323538633865326537613838636337316232373534323264353733 -39326331373534613530616136656631336330366361376231663932306535633235663838363363 -37653833656539373931333064666361346530663563623461613030653764356631373362636165 -30646363653733623131313333373231633030613164613139363761363332303730633664316230 -61363632376639363231626433633331343563333863303462383237376566396666376133643537 -33336330373533356461366634336635396631633236613432343237663231323166336239356366 -63633165306632393336373731303333303365303031326465626265626632653033353261656663 -65663937633562353166393434326332383534393637323430306635313233353361376239333563 -35623938383032366532613862383462393366396633646433303662353539363837333937643230 -65313036396362366439663133313061656166386132313533383133316635353832613731356563 -35393033623865313062303535366565383866373138623761303736636565613035343034303936 -33356131643031346332666239306261373635376461623966343438383663366161656662373031 -33313563396463393836663634343033343138363230646534663930396237343465323435633835 -32383938396338323135386263373964363430366266663930313435346464323766323837343534 -65653536623462313030346362396665653266306536626630643634316438643764316434316638 -65376634393737623630336435356631663032326638336464383562396439313935613534646361 -65643234663161373331636665303661666462613561366631363932633534633664623136616130 -35396361313632373835656639333866383262376662303033346137396531616432636264363966 -31653966643637333134303735323239623434366531386431643664623364616136666265373562 -65653134653462373962666264633034353936636636396332633137313833613330313963313465 -33383934386633376661303763363036323439353063353136353337313432313864383539323166 -39643637376466663532303030336439326330303230356261316261323061336338666239303935 -61356661353861306165383337323666316236356565373930636566633966343535616435343465 -66383830333730363733323637346537326536643136356232383163656333373235623435353863 -65613634316261383065613262346230373930346661666366323265623933353435346333326434 -33313361613266396331633566616464663066343831663762376464383035666331653638383135 -32623231393335653132643937626332333830386561366630343030356430363864623936393739 -39333134306436636262623932373737663964363230303931326338326166633161346663323135 -61646564613839376335653833303931373438633963626336303061376431373830366236613432 -30643266313631343132633431633738346466383838613064363661363332373532376431323435 -30666363623036613035666330393162666633633038343832643634643733303637376231626565 -39396235623064373066623334373431303366633936363330643332623539646434373764386633 -34313561633763376235333236363763376332643761666562353032316633316433343636373532 -35623634353932363766643835363461326539363233643035323332353630346539323432643438 -38616262326662663231333730323034323965623434323630616632323336386331363866623134 -34633165616164376231653930303336323538306162623861383636376631366361373330626231 -30333865303138336639386532633437313735396266653339616331636439363661633766323234 -63386464373864663333363733326332646438636362323162313939386434373262636565313438 -61633534313437376565633636656632663331386235616332383231333762303439393765313663 -66303434353861366231663866633531623139643861333539336565633538666536663763666565 -62333435363435346561653730623164626637316632363234346439333331373663363530633735 -36316233613739633734336537383263366433343664393366643430373936323534633061373465 -37393731343033366332663564323330666261303635353466396139323634616465643432323061 -39646564663237653361346561333462396437343864373534363438396233373237656364336166 -38343431363463666135393362373132626365353664323464343230343037363166313837313231 -66323131313534633533623630663733396664363462653261363763643730616131356432356135 -65336430653365393939633938633833633034633264616363303066376232343763646466653139 -31356661393238346239373962333234623530393563376337653561643730353530353461663537 -38303331633735636562396638633461653635333265623764613330616234663164373934383634 -33313330346435363637663933313339396435333761383763386363333239336533363930343030 -31653336373063656239323262326564326465303462303665303735393262353636623938343836 -39656534633630326139326331373839653062323138626435373739353662396562646436353337 -34316238663330636133656437643535633461353762636361636337613532633961313939623130 -32323364383737363662303630356430613833353063313533366235333938636163343439356135 -31633330393432613539303230306265303463646338386530313762306236306163303937663431 -33366334316136343230623161373638393431363436313331616266613137613762373734316136 -38303461333232323530353731333565646464623131363064363037653633656266303163333237 -38656631663739343436663139396263653162626533313331633832643433323431313465326239 -66346536393532613933636431653630616338306166353730333737633366363364383333336166 -35303863616636663365383136393061646338613531313063306638353238363939343238656262 -62313162626561653930636665306137316566323639363166653436376633303731633835323466 -38386663343064663065623439366162306335653530616661323866643661313561323565623734 -35333837643838313337623261623363353164303832626534353334613965386537323336633138 -31343536346664623437663637613331363035393366303664313364643863643333633631343635 -37633434343366643930643037306337613462633161383163613932306261336330613436333434 -39303639636261646662393731393835346532383764633737643534363964346539643765616336 -33663565373039653938386266633032626439623933306264643765346538373030373766353963 -63386263363438666239336339323733363266636166343831653063353337336639623230636562 -63626239643833353065316335633734323464313036323461393662363736366336613835646333 -34373865643236633536356637316331623034653631303065633364663037636562636165326132 -66636531336166633439653866613838336530666330306432353463366132633764633334303266 -32663062663534396132393630656365636132633265663033343361353530633534326434633536 -36373463613233363663323533366462373565323137363930623636613033333537333262653131 -31613632356463663366336531636536636639613936316339363832623631396665666130653036 -30666639646532643134356334633961376132666566326537336163333465653063346265333731 -37656163306164373864643333333138383334343433396434653733643362386235656462333764 -31303562393735366234323963376139643966616535646436346266383734313462653435303934 -39306631656635323163313639393164643465643537386365323461653365383732356337313664 -36376438346533393136323432633330623433666135326431316661613563643735316237363237 -38306237616333343933383532613331376230303134643235363538616230653332653733373363 -64383835383665363931666666313062363438616133303331633563373630653636356531336436 -65313237633232393233313236343035396231386131323438633737333232363336373164666239 -32316161313063636635383765373766636234666262373665333138386266343264326532663062 -61653038643737383362616362633735353331306166366235393834363262643933316439313330 -39663238393636363839643132306366333964373965393130306632383438646135666430653730 -31353931343132653130623933613866356135616631366230653961393862323065656563626639 -64303538663637623833623561633735653530316538363866356265616262616139653962623937 -64316334653635343565613862366366366138653964376631386439313664396230323661613036 -34623232376461366234333262373730643965613730323865343531313465333963663966616536 -32623163656539393132343630383934386534376264303864383430666231373865643461313162 -34336131346332376538353066386161373161373838363762363864626431643435663934333330 -62313063326165323039303032613433383536306136323439353034303930366162343731653962 -66653964313634613666313365383733656535366362656363656662336364376333333066336163 -35386431306166366538366261373163616163303236333639316437353161383061646266303138 -64666264613065373538356566373236373663366331336630653563343065346437613262613937 -37656237306561333465326330336531616533633438666565663533633332396233356563393337 -65396232363435383535333334623431393564636437396437393734373036336164643665663766 -31306363386164646534616230663630653139333431633565623564663663313736636334356462 -62326136333161633661373066386665363436353162613464383366636132666235616537343236 -34613133656237316533313265306538653962383766663730633039383931346137306162653432 -34363138303831383230623431666430333030613361343239306130633530393365343835326337 -36363937396237346536653463636434363563303833626266386434663332666235393839636130 -33396239333063393564363164643063353432383931386634663563376139393836353832613631 -35373761383030386632336564316564383638373531323064373265636333626662616134343138 -30306230353061303764396164343432313330666633636433303438626131633936373139396535 -39636331623833623364363839306332396233333231326363373236653961643430653039313864 -30333135363062653730656637353032323234356537653637313165363337653362353237363862 -30643831623536653862376361343365353665313034303963313935306264666135373332613433 -63643764616632623863366161333465653132386433623234626430616233363437326535623337 -32373935393638336161656136366139376437333761663564643866336231623134303831656135 -31666535373339623538623462333532663465623639303166356538663964633065323230653831 -63333162386665656231326235626237646336666630383063323830663261316333383865366333 -64343863313237356330383561323933353438663138306561646631346637383466386337396138 -64646265336135333339353661306463633364613432366334653461336139333633633263393766 -33643534363336313762303662366139343134343964353239656531616161343966613037623133 -35393334663762376165356238306664383461393735623662346664643536383635363464666639 -62393430663630353735353432323565623666346135303866363866383138366531333834653662 -62646565303630353662633261383532353638393161396262376362643430663730613135663738 -64616266343932613837613636613963336532353032663334363234613239303633376162303838 -63303263366662316631393561373539643166393564333334623061373066356262356134363337 -37333031326562326565646639633063326666323339643765323135623066376338653330383362 -64396363613965343931313234656430613631636261383433633261666432353031613533666338 -30333264633864363136613764653037626635396664326462626530396537326165353733356264 -64303864386532613837643632393735346532393162383934663439393361326336373561396232 -30613336633437333931363939353936336136663435636430393564653938626135363363383336 -38363361333236376563643163643239393866376366353032393533333638346566643832333866 -63316530633262373737336133663336396533313465656163653839613538353762383130613634 -37333662626264323039616333303630396165393164333030313162346330613537313164653936 -30353364363033303762356339663933653835623233356439613666306133316133333466383865 -31653431313763313266643362303032346230633435353334666638333464623637336433363166 -62633232356235396436356634336661323464393435626538373435363831326334373235313963 -38303031306638646635303561313664383338386330373235633566623038306331643638633133 -62653438646332346365303134376664666539383332643461633765626664383531396562346137 -35353064636132363634393336326436613663323435643433663533616265343065323435663864 -63363536356539666630346632633132313963333338646163343035306534326137366432653565 -62386462336331343166643137303939656137353638366639336364643265633835636338386639 -30646631333833626561393864386536353432396465653265666664666435343436346264363761 -66656465623437383835386163376461323432663638343936623232373261633938303735333561 -37646633306566323264333430393563623835646661613861306439373339656564393662363538 -35393033383665323137643735383939653362623336353266396363383761346237306665313133 -66316535653561666232393562366338393965383038353564653230323336663239613035333533 -62633238396262636635326138623735323263346666303564376362343338353536356231383962 -32383637383234643861653836303831646634636135353334303661306339623263653263306232 -66393633653035666130343436653763366530353964373864303730633437323766346135343830 -64323962373235633938353463383862383535663664383234303263623532333463313161386164 -63336365326264646661353638306637363363656662343833343563623431343764313165353034 -33666338613561613764336537636565633838376331363764613563663033343632303162316661 -62356634306231353333636435356332626364366337626261306430323265393739326231666630 -35626161366539643365333232353439613834383830393035363735613837383230323033373139 -31623563343866643862616232656232353134373032663339626263333266313239366435653238 -61313930653166336437383836353332326131663833316662306235323863643736376161333564 -66626233366430303531316262346636306534323362366165643338333337383135343733353733 -64613636356439636532383739313064653961366134623866646263383438653835623333393763 -39666564323966643361323430633732366536366162663730393835623638396666343837333735 -33643436376338326430656338663335333061323766633033623666386637666465626361326363 -63643962363837336664323436303962666239623038326466316631336266643164616635613434 -30643966613638376431633839386131663234663034323366396634396336396138313562363635 -64343038306631383965346638663031356466353837626631633235643331336662393536343332 -61356432376636626566333733376238393166306166323636633564633339636262396466356133 -65613234373362353964666335346563353732373734656434646461663163653435336330613036 -64376533616332653930343131356236613032366233306664303738373961313931363533346464 -36376434393333353465333931343665316432613362376631643633323239376263656662643661 -31393162396438623565616638633037383066343834663365303533363031396437666236363765 -33383966303863343537386531303065323133623530363033613664653935333138333632343634 -35353930313966356664333261626338393636656535616236353334306461366163323261396464 -31613861633065303864346532393235623565666534613864633364613932343536636637336539 -32363130663439643337316562333731343462653831356130363362396435343964646630656636 -64336465363536643636333563346632393839633664303039363866353531633939353264353939 -32386337383830646664336566313538336133336636653862373930623135333866353435626636 -30653135366661396433356435333136616362316466303835663665386433386364626363663664 -37313531346666366433393936343865306437373661626634396564656338376531376662353937 -30323534666334303966663833633836373138346263373631363561633037633331353930386635 -39663162376161656564343334656461663737323566356435323236303937366535653862393637 -31333632363735373337633366326264303831353435373433636236653831353234636537356531 -38626530353135616236386133323635646234323334666634343663313438363065376530336665 -33313664613336613661316335663130386438373039623335303063633534356438376635383739 -66386136353630616330623239616639306463643732346433316662386437656562313661393238 -63613432373662663239333030646233366665323765376637336664656530623639366362323437 -39353034343532303236643164663363346233643737636562666230643865326233396436343431 -34623965336137333363333664613064633465363434393438353531333565336338396639393465 -38396162643439303839353064376632313432346639376332353462316134616236316236346431 -30636534613163313463623861393762396530373535653836383035373134356264303539623066 -39313832396331336230373663633563616663383633386634613535376139396539303733376365 -32663864303035636366656361633234373536323537336230363362333230643636646430373662 -36643637656162616136616164366338313665326530343033653464363033373134313564386233 -35313332333934393564393239663463386163633566376663666230623833613666623964366332 -31636631363234636530636436343133616132376636643864306464336133326138373965326461 -37346365646466373632613765383335636538333030306335323530326362363865393864393665 -66313366393835633539373939666264353366333163386531383161353966316364383330613531 -32613232353634613334643234313061663661303830336431616431626631336439336164366362 -32666461613666653162396561663765333737343537356635626464356338653838313661383766 -35656262613439303434333533383835343265313535626165643038323164393461633034393038 -32646436653132646130393532613733323665636436623234623561386134383931363332623831 -64353639663233663566666164323731343231653536353661653263336537326363306537323561 -62653033376362316335316165393431383961356631386530343034383237336334623933366365 -62626666633039323031306463386534393766336635336434616165386235616363613565386232 -65393164636439353334633031353231666339653961653634326464636563333061363038383130 -64323931376431366664336464363039613461663238633662636335353234376535366331363461 -62333139643531323063356634613833303836373532373263326563333663323661303365333837 -65353031333764306639323830656263623334363939326162346635343431306365326634306533 -65643865333163393937626531653438313161653135303736666463616239343533346531346233 -66633532343235313336373636396265666134343432303765366533373133326563316330326432 -66613164323131393431386663613537396334396133663732323534356136333833643539343537 -39386634333833356631633630386564393833613336393665626665346634373763613262376162 -63396138323463643166333331323630353761663935333136333631626432353330356362613338 -63323638346164326361653861306431376335653661613737636537306633616437356439626364 -35373463353539316133626461316137633436346332623964346137306236626130336231336238 -64386434303466663536643338303533653935323133313138303630666636636536326330626566 -34643932636637663035313139353931333661303762313766663935383834313661396335663765 -33306664366438643330346463353763336339626434643132653434616366663832366264646137 -32613632363136303236356435346230386131343732346230303462383363663533396432666630 -35316562323665323430616264393166313361663438373263363263356234333665313061633232 -65623838383661623130346235333863623138346637393339613236616539373038383139346536 -36616266653264313434386338663062643464656334366531353063303263343736353363383339 -31663538326664343039333664303833333532343366373664326431623634323361396131326662 -38396336306465636532666234646134366133306266386662326230653831633433663665306366 -39303063396566333439663661633237323936326562396663363761386466396131643962623935 -33363039303335373132393766306331356532383037393461373461363866393833343632333731 -37386638613132623931306231393738323534653432663433313334663438363930643935613830 -39613261366137303935346264303862613866316331616663393364333764356136653833663465 -31316539616535343162393466333633343939306231336665653331636463373430346238653364 -35326337376331373333346436316238616563346130313630323336383936353761643264303431 -34646666636431363763373464633236613939333334336262616663363234623434343965316238 -38386165623039326437643131336130656364393536383065336362396638653266653930653733 -35303434383732633339333564616635393636623861373333393862626630323165366263613839 -34366662376539393534393231363931356332633731356131393835303935353831343433353334 -62666162616661383530613266306438656262383337386335343261323338636366663163396333 -32343030333533383863333734653638643031626133663532366139353831353861633931643766 -62343230323437633639373862343131346636396633373566393663353339396666393536313936 -62303561326634393537313935656639383061393531336433333939316637373965326236363162 -32666238303666383865386563336461643463353436363536353566303734306461643333346364 -33653139383330643139626137303462393862323864383634343838333930323937653331636239 -34373161343935333663326530303661343130333063613464333362616435643439636632363433 -33373336666262343132653030653033626537626463333463363431393139333533666134656139 -33343730663065316332313732356633306233323534313930313462386661343531616335326530 -37646462656664366662333365356438333833663533656135316333313239383030366430656363 -66613732303063303832656238333230333462326431626634323764366336326239663233373530 -36646335336262366234623033353430326231663639336533323933663334323262643566613638 -30333561663137396363383264363563616465646466303735636634653464623463656562643963 -63353239393132656234653466313032376431356533323731376533613639613362393138303338 -62663834323332656465333665393739393263663565336564393131326530363138343330663565 -39656233306139613066303633653737633536383165336663323931303130356636663164316564 -63346664633938636330393630376662386536613732333139643161323365346661663834653332 -65613031376361613665613537343530323632336130303731653562363734343535343661386165 -32636364626535663161633038323034633934343137623739326365316539376539373639396564 -30343432303766363433316463323763663564613564326563363834393337623638363432373263 -62373431633235663437386438353366643266316564666137666333623237633437656537346166 -36303332653666303936336131336563393361613032316362363434306631333532356562323337 -64643065343761323439323463313265333866343065333062626266333964313838383335336338 -66363365363832366466623931363838323465386434636565306236303732346565376561613232 -34393462656665663933363637316135383465373961623332326235326334666334303031386361 -39636263663434393062393066663332393137653162643063386239666639613335356363333833 -39633762663865356532643638366237663035626365343761656666656664333762356265643539 -62353865393334303334633064653436386561386634633964633536303731356535383335313161 -36663931366364343835363466653138323639653934383834623466373333396438613730376565 -38643135393262373765383035633737353962666430343834636137383362373936313239643738 -39623335343036653539636664313762373337633063633530326131393932356337386566616363 -35373839646133313135353862383166336633616631363761396262346461376237373565353762 -65653633336162336136353738323866353466393932633333626262653932396633336437643834 -33373535303265326539643162353261323538306462653531623263336338356135626362383533 -61373431616339343164663966326662613538386638663130303835663964616263326131343430 -35643536663735306636366131396563393331346461316138653464306634363634626166363839 -61373666616265303662303935323364373133613738376134363539303738316135356239626632 -32633738626638303965383963633137643166363765353436626663316563663537363939356264 -61393162333330333339663234346631643335383033616562633335663339333962643564343933 -34343738396266386364316339383433666561303333323036633237646432306661313135363761 -32386534366633363465663533353232373439343762623666353166623537656265333339626236 -38666434633062323265373962626437386536633735373037383039373365396163383236306161 -35663430363930626632363236303237353134313132366566353630396363326162356237336162 -31333230323935343938653937323262386632306661383064613034666132393031303331346236 -36366138393563333764303832356562613064323839316461393737303861626437363230353738 -33353966356530613264626233366430346362373739346639366239386330646264303466303236 -62356363613838666263343865336661333036333437626237303932653438323235303836376437 -62623132356238366565663039663431346235646130633031343063336539613364663563393831 -31633666633030613939613264393863656463336232323836363638363831376266623536633632 -30656563626531333036303762303462623437636136303866653366373834323435396237316264 -66343939646331663433623933353038643062616433343264383836323738643464633537666562 -66316639623230353837363536336265303433303166626431646131396366373035396364343737 -61383265353462336235393961386166653965616236343136643364663561656333653532383237 -64313966373734636565363137343965653630363732616461623239313131626533613566653036 -66383035323938323133616261303362323764366538353365393233623732373962306562636261 -35653634646239643666656530313637356265643264383436323561616238343632316233323663 -63346638353532383065343236633066313831313666303435303533623961396634383166343839 -65313134373134333866373930333035333463363937363332363266353330303062663235646232 -61303834366636393837393638666139333632326234336463633531663636376639313434343138 -33366533613337326236383766313264636337623266323761656230326231356639633465396663 -62373033306635383830383736373738376165303265326532636434386232306337376564326239 -35643437333337646239626364656634626631633362333662346263656162313864613666653663 -33643033613534643234323835303738353363353636343139353862346237616234366562663265 -61343730363838313566386437363835373033386164306432316536393633656431643564386537 -66393136636161336330376266353438643633626562326437643331623965633031353862626236 -31336164303666353430636163356138346464383361303332373464653138653665666439643034 -34663837613734303930346664386661346435373664353934356163386237356336363039613939 -31366433616631346531633137613730393630363162303666633965333033343531313335313635 -37316164383664623339353833316464303365326466313465633331646264306230386562646664 -34363430313736633934386661643035656239663962666530373433623966316133633232653336 -32653765313962363366363137313031666366656135303034333835646536326430366233613265 -63646365323536613830666239623632366237343566653663383838336466336337633630376662 -66363861393761663339363535356235626539303830323866363065366566333265336533303637 -30303733306337336231666162663835303138383065366130346538636631363432646263613561 -32643037353738663161316364386661613063303933666532633631363932343831363365373834 -65623437646263633964336231326661613033363136633139363533613162623563366165343735 -61633732343837393831356364626632613862333039356162376335313366643538366339306361 -30643231613761306236316638373839396538613866323566613865646538346438316636626464 -39326164373334323363333134633535656534386263623536336664643536393437333532326537 -35343463633936393836366130333434346165626536383566653262343462623661353932363936 -31643733376630643135613162373530626636396464653662396365326139656532313462393265 -65643035616665623236303764353631383631313364366433323331353333373162613238323731 -33323830613339633962393261383264323839613330663933346337363362396534643762623230 -35646631343734323731313733326336636530333733643630373162633433373939666566653662 -32393732626432386431303964373038626130343862356561376662393135313235353836313830 -31646236333632306237353739333763616431356265373833313061343335323032643638363364 -36386438666332323861393231623639343932363332313765666239616235636662393963366365 -33643062643531363762643630333038633334653632323631313362623231356533343538363533 -65656432346438356463623363633537646538333133643736393332616439303265343630373865 -65656466386163323734383736623366653562313631386339396132313364656631343637366337 -32363130663563393163326166386435333364653165626239366362393966666266343165613365 -34353231626238343433303932396632363562656232393436623537333633633930366630396439 -31373362346336653336373465323964313832353463356465333437373631343331366432333533 -34323936373533643530353135353361353032393538373234333031346537613935386464633339 -39643261663964656539636663646639383434363432666538396537396661613234333966336638 -38333966353862643038306262303830616535333933366439646436646635663537626431393630 -63313034656562633937353336643834336462643963386533386631643133396230343863363665 -30356636346663363634626536326262373863366364626562303235613738313336633237376234 -38386563336231666238376335336162616162626131663933336234643936653438646230633437 -39633230343439323838343430323239386366656436643233353732383663363166376162313331 -37393063626533643232616633643431316530326537363763316436363238313138633665353331 -63303162393234616335336265636261343932396237666562313131623663326532303339623832 -35353232313139626538323762656265666337663635383865306365623966633362666534313433 -36353962613633316435653630313032303135623032306161633432343834633737343739343135 -63306461616337313734666436626432333131373238336134363564633538313730663864356264 -63396166316437376463393133633136626566633962333434333361633633343239366538323066 -38663834393132663465316261626537663638626139613833626465333138303466633838313261 -35623533383430616164613435623938353930356663323461383565343039303364303738383431 -33383239646132373637323232393963343663626562333461393837643262646661623437353666 -33363565633939393263653836333236393339613534636533643637656231313736316466623731 -36343236626338393437306136393063616635303361386665626138633134666635386537343630 -65666565653863626662383366643136623966373332386665393134353939656535306130363832 -34663863666461346538653762323839633534623337316230326665346137626664363434313934 -31326236396631363565656263616638646231643531356565393865363561393865306238326566 -62366338623064646261626238373935636434613232326236393136393339633762353964323462 -64356636343063383064643837383635313364333530386236313466636338646661383432633430 -39623635623761653636303966343736303739633830333731383534316462656534303133376334 -33643438356562303036363861326131306563633035383530363262633033366230393962336131 -31353534336364376639643464616431613337383535373831393033363638303566316132393563 -33653038346235633835653931376130333630623963663233323464373831343834643235653263 -61623734353061303036663761353466326564646335623061333334663536616532653562343136 -39346263643530656532633136623262653033393834663530633965626661613433623935323065 -38633139636661363865323430353139346361623133346232376332616131366138343161356336 -36376230303033396166323638666637633461666337666639633130646262386463646162346137 -38616464306462653938353230343464653433356438353032663336336461373165643439633236 -35306661376433663132613862613934373465626334333138353737666430343138383937626263 -62303965383862303335343365613234373561303162666339653461373166306566373432666334 -66323763366130306261633433343463623063613963343135313863396365666531363731393763 -36353461393937643234356166353835323231393233646564333235633165643034343837613663 -31663961373139386532613630313663333537623734373866343935353437383332623066373563 -30333433643935326166643333363834383335613130306465653432666163666538636339346338 -62323563353038623935643662303764613466393766343164653231646431316462666162626165 -66613830633238666638313031353661656139376236383831363138623731336566663334336338 -38363363343235383537643134653832313866633533363934656536306231653263353937633230 -37346433623138373061323332373433323036396536316365323030363736363634643935653165 -33343764396332343266653161343264646662303032383838656133353337636339373963626633 -32343534623331383761343463363137383364343766663437326438313833333863306664323434 -63386637396231316662393664353237633832303732313362316433396630626232386437663635 -31613732363935626663613232386362303934626235656261623239356437343838313165623938 -35656337626563663235383038633836356238623037326335353863643733376135306634646465 -64326261326537333863656366366462383966633139363834613435646636373535376132393832 -33396136373865326332393137653437666361396630643764636263353931326239633632353635 -33653732353539623361623936376537363232336630666538666436666231306139333563323334 -63386662616630313232623935343565396161623230633332373763386534653763656561323861 -32383461336634653364386333633939373161636534653737366139306634303931343438336539 -39386634656234303330373838356465356665376338303337303937376663313431363932386531 -31356166616263663966353035666330383733393539373235666530656430316233393335636366 -32643633313730633037383034623566633865336362313462616538333361316333363530663235 -38333836663366373938386161316637386439323936323463663138393533616130373464363035 -64373037313138383466643431356264623161336339626463376538313035346366383032623564 -33353966646466346339616331306133623531653134653135393933333532383966336338343436 -32363635346433373161656133313632663039346230353837666533613339666432383639306464 -30646463643865666630343837333236363130393338373066626636373465646563346634643939 -33316634653232393031336235666532333034643165363764396637383364323633376138326532 -36613630386230633533313537373138663136633636663937386137656433323438616364623566 -61643831366262616136653532663936313635393737373135333335663537663735616132393139 -63336135666431663339343765313437613765346634376361313536373237363061373936626339 -38366432323131336439353933663663343631353163393133646162373161636137646535656437 -39313931356631326561666638396333396561613238343464303363356634663834663339313337 -37346530336561326662306437643163306366646361383433386435323133646533646239393663 -64393737363535646562653239323133653039656530393934323638366233353939623632623934 -61383935376434636533633234353636653930316232356339633435613830616335636161616139 -30623830373634346434643362626164353730363230656664633331336331333030646239656461 -38613037633833373362343366373665323365346464643463366138336238333238316565613436 -32373332363062653037356434396533383033336463386466386534333735326430623865653134 -64633737363162323832666538323565633635316332626464653638633031313063626464333732 -39633837313561633361643930636539366531363261656561343535393636653862663738386137 -64323431663364393035373139336133666532666432356461313064373637343161353965376631 -63346666646632636537383533643138373464373964613566386431653361653162323433303430 -34666363373861643130646562666336366364303637643062633165663266373861633538353431 -34306539373034623034666633663863343537393538326336316131656430316233316366336464 -64643035633762396663376133373166323761636366393132636631326135336666313832666364 -39333937663161636439336562386337386666623334363662616635653439353138386562653462 -30653936386536333338333238626161643230623565386137323964653337313766663862313735 -64336461393835616434666538663865366331613333663339663330326364313231383232346364 -32336230353434653564306339333264626536653664353063616236643833646236396131373961 -30376237353335663935353833393163373734383539313462643032336461653038646234373066 -64636365393633613634666266306239373734306536383432663334346237323461393261626437 -34336464373234613736623164343637656332613462363265313633323435656361623038623761 -62386166303034656533663862653633393664623164333766323938376465646262333066323261 -34396233643465663062383666346463396637623461616261383038353738343764613039373362 -33363738633537383934343466386636303739663837633338383965666231633239303937363239 -39353031383366353362643733386338333363383133333065316363353639316435623436346634 -63353233626134383165383464323733616366373631623831386239366162366233356330666464 -61626437333561623432393533383266326233303433336364333533356431366338643230313632 -36616461313138313164316338393033633365353634343437353234643631303465306165616133 -66656466636162396239386630663264383231306264666432666637613833353931663438633238 -32303466333134323364373632623061326537313665323363333961623464396566353731356431 -62626635376230646266363539653232393663646131333035353465326334303435363537343839 -31323461336534643235343539653933316639356265353737386365346638356264323661613136 -62623037356334653165346366373364653934323162373663346334626664646237353636333739 -36343163333463376239396564666533613539316239336163663561363337333433393536663665 -38356461656235623565323037616364383139636463376461343837633962363935343530666533 -33663433623136373664373238363864636639363535323536303134643639333335316535633732 -62323833356464393764653863616361666539626161393262336166303335643536376638363939 -30306364646435666439623730646466346462303938313434386335313562633930316333643832 -34363063303630353638643031353064616430336264386135363661613238313630643638393731 -34313139313332373665373261376632656536326665393136366130363431353437633230343332 -31623436346133386261326337623036643934366163613864333637346539393838653163613333 -31626634363766333265623239363463343036306366333939386237396537346433313734396639 -38663163333933303231663362393633666233633864356663663838666332633561313234386436 -35313535353436333435393731663233616166663564646131303939323832393936643932333537 -30623039663630643665643238326436313636663133373132376435313337336461353536623164 -31333865393637366135353930353763663734646662666337353130613832396233346333386135 -34316561386263643932353836396264646163646135333166366466363737336235636131616164 -36336535623638626265613539643131636234653264613933636466663665613537376430656465 -65366664623163323635613239333864393837326330393438323362643761343933336332336636 -39333437313362633032386234343439303331656639386566613630366533336338643833316431 -66326336316234626264346264633931626162313738616462633137376131386562373765636665 -63396630383963313161363232643634363835633334343334306462353861376463366331663830 -36333638313066666639363862633537643966313561636330623062303831653237336362353233 -39326637663534396164343630663864663062646165656136343933623564353963373762366263 -33613666323133663366356566643833353061393664333134343562336239326664663538323663 -66373936303961383964366461303134396334396339613665613662663664646363346165333933 -31626538393238323966363564643363306439346331303365303963313166633332373862616232 -65623637383965613437656135643634313237353230303835666335386630396362663666653532 -38363565393366333665393231323334343231663632353361336337343738616334353833363064 -66306365363366333337326236306365313336373132666263653736373265343564373336306135 -66393934666364623464333034333963316162373638373739383735623235366664613961323561 -65356362346435633430633538363263626133383963353235303766316661663032313932346265 -30653661316635363738336535323764636438616262343337653361633139643336626638373461 -35363331376663303132663633376331323935326364303138313532303163623638636238666666 -38663839386233396531343161366336363939316362656334373561666363646434613362633333 -66356133663431396633656664383261333839313235346535656466633232666639333538303462 -33666335346531346331613066373964666133363662663835653739383965366461343930633262 -37326465363030656633613139356161643130376437323135653732343362353764623932323764 -63373237613139646662656331663166636336663837663464623433353133396665323438316563 -36333362623431366163333063326433333430303965303736383936393332343432633332396230 -36653530386434663262333364303139393465363432366465326131343633666663336431313833 -30326430386136343463666234643964613264383239383734326632636538363439393364336431 -65613634366330643630336366626162383132643366646562623962343130646338303434313563 -31376439323437323863393438303064323131313536326436383536353332333032303164363065 -32323137613035353936626332336435313630383736366338373430303235663233616466366533 -35303132663633646266653238323335633030346561323334383539373037666334633130316162 -64373334346436373335383962396633313663303366316631613666646365386431383337316562 -63636138623338336133643637653133653533633138306536346563653165656631323939633731 -39303061336335396661656332393662633036613433363937633136386231633736663963316334 -37376262333330663866646335613238343237376530353565323932653366653731396435393539 -38623537373961656434646632303232323131346435623837396666316563643537653563383661 -37353737633832663462353938636434303831656263653837343537363437363162336362663337 -62653833303538383832643934643062643466333863623438366133356665356634396462613331 -34356630623236333737623665346566346566663935633337663961356461623935666138383539 -62346631633062333535613266636131653833336436373538303132663036636266346163383364 -32656163353063303939613437313361646638386462393563643137363536343835306233613462 -32616634613761663630656130386661636532393534383733356364616164643238393738353662 -38343337363237306664393038313066643366363438643462643061663062313966623664333539 -65353438343036333066313761303434373063616433366633633434646134643034386336326161 -38353137343965336533323935383434353838636562373131663561653161313665353839616136 -32663732666461663465326436623465363762356262623561633465396138623532306537343337 -31643833616139613562313964613632306130373733363332623766366630353663313531343634 -62323936326465386332363333633636333635393939363131626635643138353738363839313631 -63636137636533393336636137633130346365663263303833633632656166393935616535386538 -38353335356530326638306533633062323965353931306332313837326631383266613064376536 -35636231363731616331313833613363383965613037393239386430346332636365343630313633 -31303430663433316265313166343264333237343738396137636439356338383336626136623366 -36363862303762366466366134313334643631326438373765646661366135313334656262623035 -64613062353363663364386630653133373666336330396533386330376138636132663765376633 -35363964633634363964653037613261346162663732663030356339636334333738333464323335 -36383231633639303063396162643936613566353239356337346261663861646534326161373533 -63663337663234633230663734396639613361303234626137636431626438656365353363633536 -65363632623534343835366235613135376330313062343431643664363037363837653132666435 -61663737656239326366613564616239383437653130653631396163376534333764333561636238 -37313864383766346562306631646563356332326164383538373531653866346562656364336532 -38303765616166653666323430313037623636353738363434353335373664353861666663333663 -38633934656138363033393261376132366166656137373266373766336631303033643539396135 -32663533343538353635316231666436643838653962643936373462663230343837613632393163 -66306261393432646631386334633131303236333732613734623061313163636235383165303034 -35356631363865313034623431623965373634356164343132613966353733363164383434313534 -32663434343434646637366235633931323034303639643437356533643161396261396239646134 -31333733346330363461333561326332653439323531373137663033393633353564303965636665 -65636130363830656366346435303035663531326661386238376463306534323466663832663236 -35646632316431653765376435383330626231336538326565376539623733313866653632363036 -66353566623435363338313639613766613033313336663637343761303963376561353631346466 -39613061353363623463633236363162303230303461623362383836336432303338363461633738 -31323862356439323464663332316264393031323961383032666665316465306166363536313038 -65303830323434306336316665663530626531363431363761393435363962343134393836353264 -61663839336265663963623830323663613938326233616131636161396638356632663965633832 -39333631646666643764356538613061343762633539616261383265373161633236346162663631 -31653962616165646439386435396539343565396265616637313135383234346338363839616532 -61343037303431663831396531303839653634383166376333303733333337666536366639343062 -65323437383630393731653365356536323232303661623863326636306337333533326164386465 -64386136666661643832383830663765653139633732613962306237343535346466303866643935 -34376664633733613737353436313662666437356335396233643234636539363436653539653730 -36633562633132333862616264643464323532633137646263363537316266323134396336626437 -62643164346262613337626139656438613566316165373063643061336665313963366235306365 -65623665346136666338333864646630616134366465656635303062613264303036626666626532 -30376363396666306337386363653561653431663437626537356564623634363138306432633639 -35646339623435323330353263646134366132323062373635313033373762333538386635323339 -31353435663066356139313536353532653664633761626130353135636631303865373537363432 -64623664306262616366353137653239656435643731303263333838393831356237386231356236 -32333236326633643432303637356366636539393439636262623836356231366633346465353965 -39633564346662636436343935636362643765306537336136633134643436646566336562316261 -39343161353565323136623663656264646639643362343436646364373966383539363433373335 -30643861643463386262643833323666346661636562633336613662356564313434393336653366 -66653263336664346330383434653165316237333235326663613365613136303464663233633538 -64373338646537386635386330616231626431306364353065373738653965623735336630383761 -33343562636338366235313236633732383433653361323866323335343265633962336531373130 -34376439323336383931383664373731343362363334306266633036323964303065316562663835 -39363733313731343030396237386634653165626635323435386139343465336432656235393230 -30313165346261626562623363633335393632306138303436623836343733626662313361373665 -39326664613964633337343865396434346336633033623361393761396334303961343330353261 -61666361353732393537383839333233386637626465373062333064393839336364663335383837 -35343864636335376134356436376433383637613031383038336233636139643731613134646137 -66626535326165663266646135323466373861306536313265313466346261363531393635613536 -35666634623937613235353535613838323238323733363632633861386334363338343065306332 -34306432613364346532613066663734346236383839396531376636666536336364643031303562 -32666564373838633432353263373732346665303664303035623865636137626538366531373166 -36303236663931336438373934306335363031383966646563333637313037656665383262663864 -32363635333862383637366330343834326430356235366139356634376262646239616334613039 -62613638666665656433393338376662303932333135353563366135653063643634386537396263 -36353639643235363533616134366666343064353237316634396165343937636562326633306165 -32343265626665653866303065663632636538383966306538613230373465653836303033643161 -62323334393632383030303331373532643164666265656361623334326439396133616130366431 -65313433313763616238343561346138353761383837333964666534393963653761643366616235 -36666333646462626138643832363364393561616363646366363064356265326131636636316563 -65666232663264633135356235393238383061323963626432636433616365363932326432636464 -37313234326665323535306130396330636161383165623262376261343762646136643639666665 -39343132313339656262383938633833646262633666376364633232383062616565623235393833 -36383039313263363435393032653731343861633731656331373163646138346233623433313438 -35613764636366613839393238366563626531303363613266373835653938316239303137616436 -31383466643464646565356339336561383638316661643933343631643038393631313230653865 -63306538343337396332656161323630666637313732656639653565653432323930373066313030 -65336463656461303436313735313265303536663463353364623864353830316534316430376461 -36343534346263363962326665653361346264646666363432633436633766623232383332396637 -63383534386630323338363034333639376536386662623861333265353766343362316163626636 -66343463653763346431393165363435393937636366666562623333346134323639353939393061 -66336261386338373164653537626638383433356633316564333164636230393766623332366166 -63616430653064636135366465363464326562633838623435613863613133333264393839646365 -31346664363562323939633037666635633832306339613864396635646665303365626339343263 -61346166306130646637373531366166393966623731626631313438623962633663356535306464 -39633030616165316331636539636163396564346535333939626435346662333765323963306439 -38343161323264353335326633333633663633373166396662316533376230303063343031316462 -36323833316566373830343366623435646563373364313135353735653161303161616666656634 -38303230643531393166373632643333303263306536613562316537376366393336356138333731 -62633034323134366533313761336165386632363830313466656566313732303961346135373937 -38666665343162613436323234663931333633366463363966363237313832663764353761656263 -35636532616332333061303862623766386333306463663964373430303565656234643063663034 -38353661396665306234353461616534356161326437323338303632386164653835303634636366 -64323633656638386466663233323362363131663863396330626335623063616164366232656466 -3164 +37333931303262363465653966383132373063363633643335666334666338376533353136663734 +3434343834313764636433356366336337643234303365300a636233393961653137343061656430 +35316161353331663231656631326561353164636438613265396236646464383734323035333337 +3765386262383064380a666132653131343163396431386461656137333530646266336635346434 +66383035366438356433393632623163376231376238666437313461653033613263363839323266 +64346239363237636465626666303234383930323265373931613366373936343564613039353639 +39386231323930366139323430336630643464623766633364383439623665386237306264653535 +32643137326333376433313834383931643764356537353233623763386366333863303138613061 +39633438376531316237333836656363366262313730646138363939393237306361336364646263 +66333538633332353539333035616536663262623362356336323937373936623830356539616638 +38636434663037376233356633393865373136626536616332366439316235303639373034306532 +62323561333338616337663332623263636534623331663264346336623739373262363965653937 +38643064343135313732666432653361323337626135663666363863366437303862353230396566 +35626662623536323337393564616565313164663565326239616633643563636466333730393233 +36306130363035353066306530626366383763623161653531333764326637346366646336306535 +63623531313933326137336630663532666363623738376638343963393136303537616664643832 +63663134356133323338623134303266323234396535346235633833396439376635373033616334 +65393039363236323735326532663339306662623134613530393263366263363031323262306262 +35343164623665653438366135363064386431323835666636336664383262633135303433356333 +34653633653639636530633638646661386533316630656332666363366135366263633236626135 +66616238356533613535326538383639343763656263623231373838666430636530343936613637 +63373964393463643063333131336664383336646634396336303037336432633330373036633963 +63656236623336373336373161326236396330363830346264353933633266343761666166646232 +61313237353361636236303661666132633433633033306133663866386636623437643764363832 +34626162383165633966393836386236393939653363626466383466353132373639643164643737 +39613535336339333233323962366635313233343136303830316564323061333566393439376163 +66313632363763343532383036383638373466333037666436656531346638333362366562306335 +35366234623232333461663739653866356238393966306663646163373734623562363530633434 +65316434373264303266613637623032316531383132343734663961383134386364303236643030 +66653233356337626632396436626230666166313633343638656665343062303331656361356134 +36376634373564646134663836656337646131323832613334616265373638336339383132623931 +35666137343530323537393830663537346666626634386561333664336131313939663464316335 +64323039623965656463653431363530373839666336623464663537333539633465383564393265 +34643437343165376266363033623237396362336638323165363239636561643630326337323962 +65393266396536666562366132356638386232303536353535626132636534383766306137366435 +30383936366533336636666636393466336661633537373666626264383465626466383430613035 +63343162333935336130343037626566303434396535303238623666646332356237653434663864 +32336333376266333265646136626339333065333333386335633939383034613036386432633134 +61393037353933646432353031306136373962613135666266336434373033376434373066353666 +38333033316262343837383939653861656238613034623832326636303437323332303830306430 +32313231613562363437363739636462303538396339653431646137356635636465623335613237 +66666439343839333663643634643165623163366562646636316239306237613835306233623739 +34613136393234666232313934303635313664386465376664303961623365316663343433313335 +37623236323430303030643661363937333237663731613366363334356438663937313938346230 +64636266666131666632346634656365373762616534383638353937313032653833636165373364 +35613864393231623436313030613064373633653231353731646366336666396430633039613363 +34393763626330623334653234616139663065613265633164383337666331333964356535356138 +62626363323237323264396663393830306462646136313437656532333563346238633362613561 +37353866633164623364376563646638656263316361623531643835616330636139346266333562 +34383735386234653766393864363335656439393137356137313234386633306639343131643535 +33343931653736666565653337373866643562386464663635376237303165383761636334336562 +62613438616337316431643866353034363533633339386435303631656663356633643163316134 +64356336613066323762326435383233333937343363376261353536313763626535323633643138 +38376163633866613836303462343537623664346130363830663838626431376464346539333233 +31363430393566653936626463613934636635313363366430326233656131343532366566346535 +37666333653964323437373266666135653833623832623362383466366235636433646233663039 +35363263396261333766656133353032386138353037623431626263383364353136303566353938 +66376134346631383136343637626337373134303730653032393833306434366264656565643132 +35393934646266363763303330626133383766303266396333356364386339393261616435666361 +64343761376436353861363964303061616139316432316361373535303337613438313535333861 +30326263653534313237313264616334366632636461356535636138303039343538643362336638 +61323934383631633036343033646362343364653032623934386462653138616263303034656365 +32383864393435376639306563656566656332306534656537316166653839316234373264386533 +62373833656638663236636265623766373935363632373139306239316435316131383966663338 +36623062326431353563326564323766653035376539323133323761336235616262306539396137 +35316462363039313863383035646133353263336231613862303862313365653164623232613365 +64393566633738616433346431663266363961343031376231626364336238386631396430643532 +37396363323234653835383532323661643032393639613635323231623435306533326162393063 +62326435393630363630646162666336393365336361363465316333626266383037303632303834 +36393161343736626238353030653838356532376266306266636536633162363464373561333366 +64393235373831323164626664643962326131373065363663623765393938386439666139613636 +62636338663863623564653766666665396439336161373730646663653031666161653538373366 +64613331663165336164346463633337646262346630616631626232643039643138646161333037 +66393465626137383534366433346363376663643339306161363361343462323731386332323632 +37353663393735373164376465636439643737383163316236343263396339663861626536303765 +33356135393336356638316433323033306563653563653033343963613465356462616233616665 +33333265616532346162646237633965346363353738383939623037623538646265363932356634 +61326434663830316534333037383962326165373034386335643634373738333766373138393966 +65636439376337313631646639643861613762396164643434666534383435646262383666303061 +62616430363735313837336237633861626133363832366134343638316462376163393038616436 +31343335393262663334363066663266303864613632383166643739363831616565323065636663 +36323831656433623361326236356635366464316334393065363762613363643963653535643332 +37616335343161626366343834663832386239353763373537363364666365363963373764336139 +61656137356438373436633238653265636136376337363031373061333964623830326562646130 +33326139366163646437386361383932393934666639663737363961653831333834316262636533 +32366638636261363865656137363461653066366466333931393432633730626636333065336230 +34383833356235353233643331333961303130373162633266633539633430383639316539353238 +37613438326263386662373433306565346439306432323031613130613630393964626463393032 +36336265646331316639613064393261353235393638363436646233383232656432373164646163 +61656534336666623132376637636534646338386261663163396363363863313564316434633934 +33316437646537366163633561356438396632326437323033353830613166316661663738373139 +35346134316230333332373336333361366439326239646333353035353938663439343739633636 +63653630323137393639336433323438336335313863306137316435316135643535313932366463 +64666366376133616236383066373464376161393236663065646335626364333035356232353137 +38613865656230343735393764653461363932626362636661336630613434656261663836623362 +66383765656233343738383166353766383064303230353630316563396335633236636562616565 +38313830616465336133336337356663393636623138363162656661346531366635376633333962 +63363266343739333961343938353534346632376163383733626135653564383132636462643433 +62613561336530343430633630316339386231303331303539646563633736316636656238656232 +38626364363736616263633763383561353661303634653438663334616262656537643938663361 +33643733333064386534663439633239376534623533373065363836373562306565346262623333 +37323566323161633964643135623032613036363036623766303338653331646436623830313231 +34343662626161383031663430306362663765373933643166323966333437376234643264653266 +39366663366634626639356663663231353835323866323339396639346339353437613362613731 +32323535633137633535303165346164656665393161636131313135646138336437346435666661 +32653230626230393131353764623535363532366330303939373465383330363437386164623331 +31313363326131346463343766613264633534336532393965663734343766363166656334653134 +61643432356234333231323330346363323530313333393933623733636230313531383435646332 +33616464323233346634396537393061336161656632313636303430353931656532643639333865 +61303835366136383331363964353262653062646261353766393065613537636631326632356463 +38636532616631333834666235306237396566666333356464656266313430303435363562323664 +32616138346230396565393164623333636564363066383865323164663762353236623362356366 +62613533613335656139653139666331303538653663326534323833653862323538613664383735 +38383362666565663364616130376636323932383639306135656337363436323161633833303730 +35663363373332393966356337633038393761313638303838306234626330316266333465313632 +63393132363263316139313233346630383333643439613331386334373965383331326133633363 +62636261323165393133633738656139623261303664393061323063613035323463646262343361 +63616661663237663966336331386134393036613130396366643035613336343538656231313437 +34356436333465653732623963323665346239396531326262383234633436373337623030343835 +39336464613833363861303638656239616566633439663661306637313063326631343865616363 +35663639343761623264663835376436663563656533326531643637383837336635393462383266 +64333437363765626532336331363937353834366464313337343836353962306235366365356139 +63363863383236376433646561306566346636333333323461646165326635373935633139303530 +65396432666563653436663062613663353839636334373334386466383335656665666434333530 +35616464643536303537303066633238333364633738303764656332366332656438376231636563 +63316235643832366133303833303366386266363939616466626263613530386362353761383064 +66363837626465376438323466323363343463666334653433326166616565656439613062636164 +37613736636566343032663262366664303239383634336532373933303761646663613763346463 +66633163623163323437346232663061333736616435353261626164643866336235346364626135 +35366134623833646465363631633261623530346336313738313932353863653963326464393735 +36363761356264336263343764376637306463636361633133653764626564343734623036346166 +34353766386564343732653066393637353035363165653239316235636561346431616538633235 +33353566393936356239393236663264663234386237653334393239333361373037643430326339 +33373664623462646432623831633239376539376263366430653436353734616364383130396331 +34376133666665343663653761366136356663323031643539633034303230373832313030613131 +62663033636336653566353033373738356563636233363432313062613437326564613839383538 +39636138323566663634373861623534363234633939303539323933323030333131333136316139 +38376561643066343131666135336461356462666333343637663632303431383664303731636439 +66333863336162333365373539623039343438643064306333363735363034383937363032306133 +32376132376638643130373132623435373437303635343033376430623530643633356536333538 +32363935376164653638346466353966333166393631393365323465316331656331303833633165 +36373430386331313363353939353633346164643564653362383034396564333062333566663736 +30326165393630353230656337616266366136396566326530383531323138376564323934616534 +34303963616138663338353635313935623536633039616131316531643031376132393732623234 +30653964643832633061616238353834356437393662666333343739613331616134353665636161 +31633835366663313864333633623038636633373737616334353834376461393731663238396565 +35336361646165356266363636363633656133623735363565653738363434626162376266386238 +34366231636136346435626464373562633136653830613661326264623138643330636139353162 +66383865373432663237336435656635356235333765353863633332653933623539373633326237 +65313431346230386566346439333430663463333434633134326266633664663162363030366233 +62336636666538336439383563303863356361393564336539313530343063343634303437306365 +33613833336462353637363738333361353565363461666230303165303434656633656437313730 +61376566663632333130383231353133333161626561363562653837623862306438346639333961 +35383564643539336236376433643165363166356230363633336365376637306539346266393337 +66623934323765353063303866373239636636353763313864616662313564646237316162313733 +34386233663362646562333134326233303239363030376362303665656134656265373934656336 +33343936633864393166653366643965333561306238346533386438653738366238316335356433 +63313536663235336135346166383163636437633637366532393661646338383339633835626138 +34396463346665333561346161633363356261666132356161336334656562393464326531653365 +39363338366135343837633332316164383562363964633264346431613064393262373066623934 +32646436333664353435633464303334323431643231333933616633313239303537323230376333 +35633839613563656461333738333737633737636364313932323232326163356238656334656333 +38386338623138316664663632333835393537356632313433666465343339383162346230653161 +34383466343630336239623964303339303832313431306133316335326330383766636230353631 +36613963303330656631653661343436336564396466386464663735353162353133333865376235 +39386161666366636237366665336435656636373533336564373965313038363866316534366536 +61343333336139346361323363313132383231343530386234333565323232616332666231613264 +34396664316262643863653934326635653633616362393333376166663738373561633862323063 +35666531383563313036613135613538323430353133316262623863313266316231373632656638 +65386464623562366366363536383636353538366434333065326136383834363963353863393964 +35643339313133613430373332353861313533316663626438303064356365626161643836643861 +66396237336134646365313037366433353232623933616234343034636163376463303866323339 +32396634373936326334623131316335653633373332656466353663666436373138653434623566 +36306337303734313937363438333239643262623932333334353865303232316461646662393731 +66643934343338663931353337626333656363366537656161303466666433316263376361326166 +33303630303639623165643165333836636334353765386162626639383630326133613934363032 +36326462626237336262626566616664376337646366663632363339343938636136623862666135 +32653463393036343461633632333264386431313139626632336663383965613763383433366439 +36643434623338363362663437393161656430373434613732343331643936383462316637373235 +63303235306131376636353065663962326430376132643764633435653236356535346337363831 +62306561353932623538376330313233623863346438646662313961343461323264333532303037 +32376636396138636664383533653437346631386437663138663830346162343639393133626263 +34316631376431326634386639616364353437666232646436663764343539616539646165303634 +61643465303934633933323962316533643034383331346266396231306565303663323437626264 +65316233316532643133616230376463353635333031383530643466646663386135306661376264 +33323135303965303235316666623130356561336138316539323137333265306430383234383636 +39633737313735623338353139393737613436613739386462323036383563323830613437356664 +38353130393066666464616361623636393736383833353433633766353566343163636634616134 +38313136623766393534383632343537636538643464333439323034363939366235346565626139 +30373965623333663830666437666533646166366264626335616535656162363030336139663462 +30363733326238383932386630333663323238653431656337643631333330663837643230633866 +37323335333737336236626635323232653130353837646236653033343432383435326338343331 +63623263653138386361383838623265383733313665396339333836343538336666663332303865 +39626133656339353166363166306263336538656163396638396337316130663835376637663861 +39363239333535313633613133393434663532393665346365376537396461346238313831353435 +65353538633431373933336538666438643638383066386566663766353035653530306363663338 +65383264663336383738326266336262643933613435666236383335326632383434656539353763 +30363732333130616333306563303562306439363638656562346563653065306663653161333231 +65336461363961343732336334643037363039396532613863396466313137653231646131643461 +64373166313932666362313536646662653634353766643136343261313139393666383033666464 +34653237326339303533333232333235383265653939636162303334643366303538353836386464 +61393537366663633465323962646335656166343834323631303964313533313866326132626536 +61386635653030303236633134356564313662303735666163636664323738303633633665623232 +38343230633331326633383434626465333634646333363866623630343537386431663030316638 +64623937346266313835643338393066353636393739616664356666303137363636306231666330 +33653835353366646665613862616563643436636132633135306566633464643431636633666362 +30353437663139376663623038653866633661656536663836643839306465323939353261336331 +39633930643664306534343030373266623864616363333836613665363835616432666361383839 +35323934343332313163666562626533336561363735303036336533623933653337653937363939 +36623861383936653133323866613038363866623434353161313835383836383633333862303334 +36623137336632356163323837613735643232336136383630396439636562613032326135636162 +37353763376134346639316339633831313331396439336364333938386232623138323465346162 +63376465326630396232636233623239653838376461326361623864383664323063613033653630 +66373630613532333136626333636466646131646235613961653732656230326130366665343033 +61373936366665653737323733336533353763353439623833656665346435366362336262666166 +33656366316131363363613632623864623861396430623566373735653530393632323832363033 +63323730313939393631323637626233623233393265353239353862613033663163363861643530 +61616538623434343262353964343964306333366561613636616634633037613733343165356530 +39336631396636313763306464626234326636363261663832333733643061653661336538646434 +65613666333330306334636131613030626230363434616262346235343530373239396666343739 +36633161653838643430336439326136646364626630343436303762376334656564383030396531 +62666132346131356232353164336161653335326432326238323931626461643531663164643238 +36663036663532643962353430353433653536663565363636343530666433323332306233356536 +65636133626438373433366136313639643266623635326361633063663164613665643538396565 +34383236623861346235303735636363653134653537653766653031633938383330343437356364 +39613136393731353464376234316330373064633936663664356661366531363934623230333065 +37396430393132366434633863323065316132656638363366393065353861343138393632323634 +33346433353166363733353739666234313864633936303463663533646264656235336532613666 +63383735363938373234333662653031316466616531383431373930343766373163303238313936 +32333538626532363736366233383332623139663561356236333864626136613232386139383363 +61346533643133366563383032376430643039663931613637646163656539646362666366653866 +32366239613230623732666532646237633635386361663733316130363832663638653837313239 +39393239303865346637653136393531376462393461396230643334353137616635336537666633 +37353939366231373864643534353433353032326337306130386530366338396338666533373035 +35633838376362326536383435653838636338653335646438366433346264313133323132346437 +64383533346336393965353361323335613661343435336566646435383762663230666366613233 +62656466633365666535323264333762323964336662333261326537356134303534373135626230 +38663532306132623464336562313937613534343837623539623263326139346336643364373436 +31613663343239393835386535353265313936343665633663356334356639666635396264613233 +39316365376634373266323737376636393132333965396534666436626262393561663237656261 +61333131333731316132636534626431356137616138656638623933383336623661313661386430 +61326139633930646139356633653963336465363931393330373939343137643466393238316430 +37393230343464666433383263623364656132303534643830393035313964393439663739386339 +30623466393133343861346439393964363337393062366333316632663734653636346165303837 +36396133333738653461663931343861306565363764333234366437343366323763663039643539 +35326664636662386561356435653736386436346539626437626635336531303335613766313534 +37383666653339363464323832386330373234346137396465366334323637326137653635623062 +64316264643363646563333338396466393564636365626365663230323032303831373037326430 +35316338333130326461346666663133353337363763343735323562663930343665393431306139 +63393930643066353863326662623361613931613964616138316365613832313533643333373939 +63636362663862333033326136366639633166613364316362336266343439656238646531336565 +32663531356637303636633334323938623435343535613439333564396665336364396364353130 +63336431306466646636616165646339336230333831353966653163323636663966343665396635 +64623038356430363834313030626131346437356331313937323533366231303961626438386637 +37363037626365373039363839306664626337643730313133646562366266663339623633386334 +32333438346233313136633132336265653632623164363039376566613939333661613561306561 +31373562303763393239623464396438653330376338303864653438356433623363373330623763 +65386163343366636361633530656338373366386337396632656436613666316465353663653931 +37343737373137303136303362363264646236316633653338353563613639643934343930343033 +32333436313062323262373065326164346231393166343031646537363433663365333666643432 +37663465356231623263663739613864303264643332643765663735353737346664616334346261 +65333364393330326633373462623536643530303163643033393533396239303263656236623834 +31323964333666636436316361376331326436326232396234313238356338393439633933363834 +37646264373535353438616563333562373561386165363836636365383665633732343764373734 +37376632376130363366666466346434646430633964356334306264303664353134333431343239 +33346361316437616463373330626637613734326338626366343137643965663131356333353833 +36656664343734306165326438393038316137626165316662313961643136313639386266656561 +62633935643337643436313266333861323431346434653833313835333637323063643633323964 +33313936366231646662353765643136666439316634613134643635316338663161643336383062 +37643938663863653630356339353561623231333264343461616263633663326335633935353037 +65386232376162653230356633346131376237303837313637376566343539346666303835653263 +36636138313362666462303432393739353237353139343437663035633764626365386538666435 +34303861653731666136393332393739353531646234626336613237653336356563616562303030 +33346263336533356134313938313434386533353661643831643738613836353732363534633766 +37386463353533303535643666653761626165666133643365626337306239623463373064356638 +66353730646237656532353264366663653231363463613733653763383739323433303864333866 +36303436613863616533363332393262623533636334313431633961386361643639326636373962 +36343331666237643663383332656163613333356361303963653864333330336539363139306239 +37636161346563396433356365366638363165626161636531343665333231633566633938393563 +35663265313936343632303666646465373931353232646261616232343835323763303538323961 +31653862623738353132396432356238396663653836613465356632366535633630303730383735 +39363038323836313836323861636631333363333762353833323431356461363937343964356139 +36353937376361623331636639346536633061313262316437636265396339643039666466663336 +38313730356166393662653962636338353131366332373064343633303333626266636636366537 +66333733336331303337656531656364303438633132643732623332663936316365383936356364 +62383336333638663831663034303433626636616264366534626235323865376339303530636439 +62616331613737373633316666343236393531353266383634303761353430356161623665323039 +62373534393230626561396535353039346430633563626338396532636431643762363139383664 +32333530323436376534336466343235636432383765666365313833393562313536343434383866 +33643732393266623366366365336363323030353232613361343635386166633635653637343465 +35333430633761346637333335376632633165313731343332366263363435353165346239393030 +38336666653532373031666265373139303738303037356333636636363537393739373164626130 +34373635303965663563663831336630373335643061643633326232653132633831376164386130 +30646238646161663633356632623833303461636365383434333261613431356166646637396161 +65643934366566303463393461383061623363333364656531316539393435316236386662623836 +61666363373336326535363634363866656561363633646330613065613764656162613464636565 +35353664363964386535323362376462346465323561306437333461356536633266323035643230 +39346439356439306163323462326162326438623431666139636331643865646237356437343635 +61643862653363633730396132313232613631623561363365373964346532353063643161613937 +33666631636231626262306436313966393863383535656434363131306439636464303034313734 +63623232656136316562663165613038643432626537313164343733666637656565373464366562 +65326636616532376631363061623634353432323630373733643330633363303430383535376563 +38623966353835386531643533333437316261393837396134623738353762343661376562366632 +30656263653264323639303264336131653063653632316638363765383839303763396462643933 +35396565363936653139353939333136363231303763316335626461303232323134323864323136 +36643261333639333463333331343139316539306666313332653164316439656462366664363330 +36363162303431643963653235643437643939353935366466336434663535373263386532326265 +37653530346130633636313564636630396266393264653439356361613865363032653132666235 +30643539323964353661333935353839643137643037636433303633633532326630353064393964 +32613934383865356134393437613133623861313962346263313739623834613239653039636537 +66313039623336643136333731346666366138376461626637383134326363656266326262636339 +39353734343030323263353061333934346465396263316230333635393339363861653934663238 +62643137376537396333633663616630386364616537613732633666616338613032323562363434 +31323263356230613861626137663733633436333666386137633739343536653662663839313561 +39616438613862383530313366653562663265663231653138303439653438386539303937313538 +39353334363437333331636162323134653233356233653966383638373832343339613636346236 +34373432376435393538336261323663613164303338303732356464663038623537626262663139 +39613731633666306533323065633363643332646165663738643235323863623265386331336234 +31356230313232653363666334653766633561343330386630303534383863623531633666383036 +37356338396134326432343230323232386233366639633663393562646331656536313161313634 +39393162343339326565623439383361653434346663336233636538636165386137336465666466 +37316464376339643234663035633032393763356537333134323064306237326539373334653666 +38366239623736343833396238323665316639666635613836383036303436366138346166326564 +32366438313239343131303666333839393265383932376630366634346139363235353636396165 +36393135346535363430363863306134396261613332616335616464343962343964323838353834 +63306434656161333434633432366463313362396461633838366533616636633563653661663838 +39393931653233623033353130333236613834303637316437363833393062666633633630393462 +32396163306139346333383561636264643330663335316537653463656235306435633464323466 +31636135666532646665343335616564623332396563343265373839316662386565333764663534 +63366136616364656263323235326637343064316136616132653663393865333136313335343439 +33303636306237376165323462623032643763386435383763653163616532353861613165656462 +39623833653934616437663864333736373766333265613762393035623531666434383064613432 +61373933376330323539663835633665386331343463363966386662636333636639636334633263 +36626563356234303765393335363531363965373264323864636435326265323030373138343635 +62303965623437373263346564323035616530656335613132633230323234353831633463366463 +38336662383065346138313730653564346238386130386431396564333664623466393461313134 +61326338313166303464383534363832636532323438363636333638626536313564373033623935 +61663239666231636139616433396462353736333534373962656564326666373832333639363763 +65643266316262656661376162363636326364633133653634393466616436643333643437643962 +66346265326632323366646233653330343339363764633838336663666439656131633661643664 +34316138653632626164363538303066623330343331336234376561386562616332313965623361 +32333430656664663730326366666132646439643638393861303930633735623464373639373431 +30653961383962373161366430643234353138376536373130653935343762363561326261353066 +31376534666434633362396438616431643636366632313439386632653064343631666165366134 +33623435356664373862393938616632323037663265343633353333663933333765643738366434 +66613332616339303466346534616233393738386661343761396434313836366535626139303732 +64393539333538333064346663333538386265626264616538333366363539626264373864663237 +36626436373033383037643262616431633762613932613366613936626239396136636265363261 +62653038383164623662333964363563336330313264393164313533336262623961393339623934 +61663734366232663137363735303161633933653038313765306137643335663261336539356238 +64623365316237643365663232396561666165623737643364613932303361386263363065316565 +37313330613264633266343965323466353530313033663837333264346264306164353034353761 +62366663653764393065353861306239633935356361336630663362356634613134663461623231 +33303436356632356136383933656638613338626665616461366139363464623764393264333037 +39306637336535393966663635393163633062396633373434393635373838633066366561346663 +65383566313462306262353135333663343964623237356639383561313631313464653664633362 +65316436336161326435383332646538303165623630336162316563363137383366316362636533 +62303537306663666239356437656530633738343033316539326637303938386230386462653961 +64363535366438323736386338393136336661303362336638386530623463353035613739663937 +62393630653633326336666366323564396132366139633464376636316664343532383636356636 +38613131666264333539336237653635373633666164363839613865333937636437373563636533 +35396535366133343736333062646164633734343661373937343133653830643263363462343639 +63623837333136646565636133323132353238336633613830383930323561356562386339343230 +61363461633763336463616433663236323331646134373034633462353064636432333534363965 +37323738623338616464616232613934616263653737373237363165303231623465393666653363 +36346333663863666235613935363165393464306130373830333730396338663464333231316263 +64623238636332386133313566653134383738613135393530613261393366363237643766643433 +36633532643336623038376465303365363734383166313630653039613430333234343438383866 +35386263313065623864623838343661326539613961636531373833346335646665383537383262 +32323439653833393931616666336131633032663732303536326661353866623766373938646530 +34383833663166353535646234363264363666633865663836313939663165356361643437383866 +36633162303964623636346131646630633638306261383235333037393865366235623434313166 +39356335646266386563623034653266393033333365646639386163663632313536363461333030 +30323462336436663432393831613833353261613739643031346363346434356131386238616236 +32303863633865623838356665323037646131306534396465336533646636306465663730366139 +39326139303962313232626539636238316164613663333138393537643862366638616230313562 +37346639353839333430383638393361363465633831666637303233656530393534666132613864 +39656266316238656334373331386463306239333166613331363263633231336136623030616337 +30366162326434653966316332626439346437383566333361396133343765393837653565386663 +33343665376661373337653035396231313531326361653735323036313061353232333462353765 +39383038363634306664386138396131393166393266333232373766326666306330616562333233 +37396666393235313530393861626438346136393436326565363338313534633162613966636561 +66356538326365353432386337366464313230643030343431353163663665663262353133386133 +63336664623531363830303066613762396638313438303432306335613836636363303134393331 +66343563366435663738656338326231373232633131633236396332663637346264656564306534 +63623036663032353864376630333231663637323239653431623262333664653864653736303163 +31343962323732646133393562323532636562643261646139373863663632396663303238343766 +37326132633834313632373639303234643937356535663236313539326231386637366131383336 +32393438373764313637353933383738326561323735376636303264613465333635323462373132 +65643235663566353239333534336464333133313162663162633736393466346466633732393832 +30316537353365323037356466623461306432633064343531373864386465386433396638326432 +38303435333238633736613537343534633331643135376431623561343063343166393365643635 +32616462656666636133643266356665633666623631313438336265303637356132373065363631 +65383665643366393039326361313737313664653634396332373238333933633838373262373930 +38313935653933363061643238303366316661666538303737643035616430316631643861353435 +32643637326333663839636663393438373363626363663437316631613934616239333235356461 +33373331393661633730653932643038383630613233386234643630386237633935393235633966 +37363536343962663662656234326661393764663135663831643865393735643762346265656465 +32663563333437663737646338373963303432393738336132663331386239343635666635366439 +62316635353034306430633433306366613639333364623033343565383639666632303163383438 +61323230663834313235356333356333663736383666633364393137393964363834646361383531 +33313331336430633831336636636335313736303365316161663533303138326266363537343263 +38366461353035393037633638353932346663333633633466623761613037363664393837663039 +38303037636336626664336238643636386566353761643336333939356135626631386630333766 +31653766653136366234383563353032393130356536356530326237323132393964343932306335 +33663934323835393039366330343031333765636436396235313964303538366162623035666637 +37363038356438393964666334373234353063343435373665396130303936633164353666636663 +39353631303636353164636139376136326431303036363232623330306330613363643130353130 +62323363373535393131353063663061346430393766306437353565323635383164623433626333 +31636562356162303034353738653863643862343135663038363039353237393262353666316434 +38393261353632386138346236643163643464623531393530653634636666633530646366363164 +31623830313664376337356466313932303030353834356662303938333238636564623133646131 +65633931393630303834303332326662653466373131356366646465336263316165633337316430 +34323831383864333761626561306530396633306663633865326462653533316435623466343936 +30393464323761383835393033643239396166373564373032373732616165376231343664643034 +62306337653033303461653936626464316436653363353461323337366334353065343031363639 +32336537363837616435343131393738393333373132636265366466373138356639356436656434 +63613334323862656336613635633536623463326437613865626637373263303836636166333361 +30616132303737363963626239356535656564343833343862623862353736383666666364346230 +61383562613166373361376166356166636662323765373263363532653433643737663135353239 +39613833616634636137383132386266643465383463653236306265633532336637663666353061 +65386661616433396436646632656266363362623230316362666263666135376538313339303133 +38616232636538656664383437386266626236376561326334653064653938386133373834643361 +36613661366564636661653365636631376534393566373537643139376162333336636464643232 +66383938633664636139326631363833326364626138363163303536356165376661636162396261 +37613961373463306438616661333039303239346239366537376330646466386161666564333466 +66636230626135323236663830333061313430336431623234323537393964316332323761393036 +35386236373764383861386638336636353436306138306565326433396634386435333636376532 +33303930303365633364356538666236633962616263663463626436643966343234353965623864 +33323935336631333864656230353035356331633032336135363935383665303531663930663062 +61373965633233306239333965386163376665343237363432633334303036303465616264393238 +35363239316333383636613132343661633035306161363166643237336537366337313831613931 +66626230353131653736373661633134326366303038656632613032633163613934366661346661 +37363164363566363634303364336430366461383036353462643137353061316233353563656439 +32353561366434356436633633396135666338626333323237343534363262613038303431316564 +66323438623863643332656264353835633034353162663937343432663935306232343531396233 +33386632636564303063383461386434353264613935613436663433383264353361626331633862 +30636163663563656465643262383065373163616165663161616430363166353938633335643164 +66626532303666643962343965326330373962336431383762613739323438626135316630356265 +34356430633435623530336461343266363834393663313661613565643962346665636235323338 +35663936306538643064636533363261333236313330303331613361643638623530313238346430 +33376565366339316635666431303965633762303836333565346235326430396266346132626562 +61373034313566333665313663636366613737323135653363633963343938646365613061393835 +64663535316431346331626536633232306363383463323361356335663130393137633733653366 +64393638663738356631643739316536646564663131396565363138353764366163353530623565 +35376136623664333064353731373331356162333732366234373534663664323165383937313239 +62646534343563333466646635353032353830363331303634363462326263656339386434623839 +38633061333763643132633331636637356563636338316536313930313362656233623438656530 +34366431336564346234313165653539373061393464643633363238333239383236653963366465 +61393766636262613534343365343538643535396665343039353966326563363436353065376164 +64363835313366656461653637666464393230616330636538343062616135303364376331383063 +31373131393536636434393439316361373434633533316235386435626539613062306538646363 +30326235613938383534376266393334356138626436316166316366373237343830666637313730 +36313362303862623237626566656562326333353836336266313237623965353335643962333034 +61643639323265353437366533323162326363636165613539376635633061373834396162353339 +36373332663839366430343035396538663830303332623335363366656332306361613931356138 +61636231313565396134656139623665303266643662643632653739303231313832643762343939 +61656236643331353034613261616361656664616461666365616232303764643634636264663262 +63386435623530656634343835613938393538653861346562373734386533383939633061353763 +30336266333165666365303764333661326234353565323130326562363334343439303030333836 +61353631616132353166336337316264306362383730646438396162656162383531636530663062 +30643036623136393435353936343231393536373333636539333963353136353539386634396234 +32643935653035336266376638393235316336363031363962633664353038366634663237323863 +31393832306162343365366461396338383263346166633332626163636366613263626631303162 +65363432613332643130373431663263623033623539653664303934306636383039633739626339 +65646137343730663734366231626531313138653630313261623264376234643937633666643562 +66643732363838616433663939626131326362653132363561333261353532386639393238646164 +33333538616531363634366262333938333333313061633132303632333837376535346137383436 +65333130656164376130306261363966353561306130646562363433373066366566316334313964 +36333362373361616535316236653732623139313230373466323338356635376436666637303830 +36303238643031336263623431663236643362333236633034336661643134356664626237656134 +30383634646465346161373732313561386639306165646135646366386139643730353662323761 +32616463353137356365643731663738346262326565646433663837343539393634653361353639 +36396164656336393436646337313661376137653239616461366433323536333564663231636431 +35356439346334653064363964616134393965393730663333383838313339306538373933643765 +30346561656436363261336131383862663835393939656532623134386166313533633733636535 +33646537336235333361383138373033656535366663303332396663383461653036323466613139 +62373433616164643934373434663637396638343733333165386166323734646135396135383136 +32643661653862393364323333356662353132373938353464356566396463373137306335336634 +34633163613262313338353761373763333036323735376162656638326465666532383930356462 +38313639663461666266353635333765333536393632393566373933353266346366646162383761 +31643262386164613361623434663231393639313535663361636132646364336432346665663663 +37626336396633333633313235323434623864636466376438396461303863333263323262646334 +32383636346161643332643530343230323135363038316438336263303730356339306663663431 +35373234303961626330333166666537663030616566353762666333373961656363373661386231 +36323939643438363732393136303762396338613139303466356535323738353834333766343130 +30386532653131656433326362643730313130363834353437343564346337346634616266343638 +64376165633061356362363934633338633361343166303031343566313062383734633962616635 +62666237323637323233373235383532323134343237313436323233666161393736323138343763 +33303464653230663666323533366638373866613334313965353761323930643831353166366237 +39653561336566316339663935373564383066373261666565626163333335653264353930306436 +65323062393632613831366335663864643138383233616331663165313539323866646466306461 +62616466653637656437663836653335353063303838663737656637396265313635373931306464 +66656539353734313033646562646232353336373865343138326134313737373866666563616564 +30616263616531663263633364393031363834373431393262643162363963373264666264663532 +38633337333035633939633461643034373066343938356130626538643134326534613438656565 +39303966376564306365633933656334373138653037643033313639633236393964323566383337 +65303037333237393762323566383333346130363334643565643633633539326564633963333461 +35656565393134396339663432346563386436653566306164346530316530636433306233383639 +62313439363764323033373835343233626433346230633835306564633638316638666265663862 +32393539333739393430373239343663613732303161393234663636613232646563353536353133 +31623262336365613362373931623830313732653039386139616636373065313436666431306438 +64643235353233323433393061303032373438373236626336363531343463333837323464363834 +38383261393165383433396561363761353666663232666239623133646532333963363763636532 +64643736646361663935656138393534383031623261626637376332663766653533623033303061 +66656534656463663630333531333266643365613666643364653533393066313135616237353838 +34303330353634623334623839396265366332323166343136323039356330643239663933366234 +30373638633166323562383833396532346130656530373636326439386533616362623133616336 +37643765383263323762323664623633326631333839376434666135303539333864623436633934 +39353662636637333463353230616266633137313762393237333332326338346535386565666266 +61623639613032343630656535333637336361643937396134313066376365336332393732336231 +64373639666164393761393437373131336362326133393035393735393539656330356637643264 +33346666633932613631366635343266366634616263623937663965303936636437613665316465 +36343433363766396231323333343965393564353032646437373965386635323431663339303661 +32376566393239393433386362396561353536636636353863656361316135366565663639393435 +30363734653639393431346162306665396262663332313431396239663765633166613839343561 +30316164386335636538366436653665333633626464626136383436376663303639366561306665 +63363262643763613865633561333136633163616235376130633762353631363535663333666236 +35663431386636616633336537316632313162383233303030396464636537393538313664373566 +64363762366663373536356539616638313435386436393431323265343863626437373936363638 +35326135666363313762303332363436656437333363653938326132316566383462633961316533 +64626564346230613930353436323531316264613866346133616439646261656365356439346234 +31373636643131326332303638383264346631643537366231333033333535376661613662626531 +35313032363331373835373235316336393533616439323366613464663965313532646330383133 +35656166386165363835313235386635653938313563663663613630663465366235306230633733 +65396631353365366537333735363765626665346566613764643861326536626331303933323365 +61636532323335373463363263363464653065393336363662386632616233323036383063316163 +37313061613265613435326433396161326665366666653733373162363238393530383634353334 +65373964646265633932643665373631316236353039646137663161636131653433613430663232 +38326563663830336138396565393435313536613064323462303164386138393139613131646437 +34643334386333393435366536616664633066663233336165323236643566376236343962383438 +63353965366662613065663335323535663830346634386539393362303339616362623535666334 +62623763363335326165666633343964333536636235393266653163393361303930303664326365 +36373464336266633964356262363866643636303335613237373665646234373463646431326635 +66386234643030356435643263346565303766353138653030626361373134303433376463353162 +31306261343034666231343536643566333537363230623539333438636330336466316263396564 +36343832313637636263623433376132626431653462616362633730393761343739613464366436 +32393739663766376563613162326237353230313066313661613166316664336335363736303538 +38383064376133626436643266383937663436306239353565323638313261306431363465613339 +38323164616239363230363336343664326633396133353562613262666364633166353737373430 +37366635366339376435376236333138646461366366323135363864636439376239613362656436 +35353836313133653934613663343638636133316234623365373438636435363232323935353062 +30623263623062353337346361363365386366353631333565353135333033313063363061353632 +62306465313865623436623231333636663135396565336563306336353537393935313637666437 +36393236343838373536303437646630303937316261646434363865326637643361346565643935 +35363865616663343964346235666261633832353437626430313865636437363461303630323636 +64383839323163373064653965633962363738633666653464323063313137356661626436333338 +63653463396134323063363631313463383563353239323336306233353363666461653437626137 +64666136666231613533363063313762666434623064316137323433613935373734653932326166 +35386237616332313331633866643065643564366264306564333963303633393435323738306537 +36303939313239323665623033333762323935383730623335366665653166396264623237316162 +37356662343539323132653264386264396664633762396635313863306164333831363339343839 +31633964633164346663336265323666616430323165313438663331383365396165313265343337 +34653861316364396137373265356437643337366332666463353439336264376430643265656434 +39663434633661353166356261346239376436626561633534366434653931663838333239343933 +30386337333439623733353233313730306131343866366435383636366133663638306435323362 +30616134316632326238343566313238373134353166643336376264366565613665383964653830 +38353331653061366236336562396631313863656664396465623338656639656339383935373561 +38393862396565333130383265613634613165346239666331376664623830636662336365373130 +37386333363662643434653338383430303733646564633133623732363133633834316535633738 +38336436356631363033396231396164306566313065636238306639653635373534333063393232 +36383033663961623732353036323034376364623134396366643961346536646530316435313938 +31626262303862653863646365396632343936646236633561343861373263363965633937303531 +37343732363938383362616232316439336461336330343566633161356361303466363736363163 +66323830353230636562626366313439636563376566653833326663666431393736353238303732 +34623366626264336630313439336138613866373430396237386561393432376339626465356239 +66353563396530363061356366663363323166336161363336663865643739303636363731613164 +39313833613865643661613164643638356335336534343831373262396265386136613264613763 +64303964393536386464393434643630316634626437353536613134383530363232656264353065 +35333139653465386437316662653732366232326339646164396361346331353562323235656539 +30656465316633653730656462623561663532303236663838636465373534646563643332313730 +62363835373034316433663962363235346438336638326231353264633964373464333738663538 +62626532366162666230363732353839396364646536616234336437653236383264636564343832 +34616262396237656636313534383633616230393838396434343737373432616264323133346564 +62633630343665343663336464353132663766396631623736386164363566343335303639313861 +35336433646531643632353763346466623635306130383239653031373236393530396338303034 +37316338383132643633343438393939353430376465386231383530373165656337656436323833 +30636230663734656639636530666630623338636563373233653565613931386538346338626263 +32373664313239316338646230306161643133326162396162363633326431643036386664386662 +39623762353831396237313930333665326666383462303064316634656139646533643361623263 +38373738336266376538333562666136666238336463666536666266653363313436323861333964 +32346361376339316438343961636463333234336638313533356435323231376133643231366666 +64623031393562363462383965636531636435643130653039626666653137643930333761623634 +61646462303635373630323939313963306564616662333039363639316666306536333964333865 +39353961636163633461373933613432333263313832386264343762643335346239373538396337 +39646564663437363438353738363032613137353763316235306432353733366633376231636536 +64356539656532373661663835326134346535396236346264323036313034653464613139383134 +31353762343863656134386537376235396431663935643039393565623061366631623461623933 +64353565343161393131363434636434333130393836393230653230666466353739386635633439 +66656165623239643362386331346235393032336239386364636264343438313030626237316563 +38343465373335653461383062383737373137363137346136643439643335633531313539373739 +31373062633664666135363034333736666239656336383938323764623761656132623436373739 +31623532323233393431633738323466643463383835353435303166373064343735353030343335 +61383238636339323037303066303838313639383731333635313331356433646236636433323339 +63306632653131393530643936366265666632316561303866383738383062633766383461626463 +36306234623035343135383837653039666136303838383334376233336334663336393565383735 +62323163356137636131366131353162323832306133306530616438663634613537666233326138 +66386235386636616664383635666437653233353435363262663134303562626336336636316363 +34316161613132366430636436336261366434353031313835663135366431346466346430653438 +34373239336665343234633936386465306463323138383136656565366264346562646464636364 +34663835336239336564663238623834663331313631393434353633393334623566333465643138 +62643333353535306439343139666361303265343238376361353764633830646137623938306161 +36393661613064386666653838646430323736326438323730396464613038303938393439353838 +33363439373735663535616166373663343138333438623263386566353662376563373930626637 +37353932383936633861353239633135376630646463383537316639613130383130386538633737 +31306166393434343032636136396438636261633534626563373530643965313531326266393632 +36323539373161616333656661306531653338326331386466326662626266346263626633383033 +31363434323463656662643138316637623065646665653362396662313135623832333661623363 +33313261663030343861656137363163663963343937626632653833363366393038316539613362 +35643265353034333663373134366438663737396563343364663830386364376235653138643830 +39363032633835323165366337343236646666343033363263646134313061363366656465623962 +35386533623538663237303066343062646162626432633231623361356563623037323163633530 +37323739626132613262303539626265376236353563396430373865636636343735363962396462 +63613561316562326234363130643936356235616532383637343533636231353065613038326166 +39313636613966353764326563343930623335386562343061353866663662326661363831636663 +33346234376433333162646338613037623062623361396632393165373266323431666630386330 +36613638613639636238326534343362373234393363626235326133356539656138383936306430 +65646337393537393139326636323231316637623565633062323138316135303432633632333232 +36626638646165313965386563623236353538653132343433643134623435386632613630666562 +65363932363164376263373162633735653831373134663238623166623263306561396531353265 +39323739376336363230386261353664313631396665386665366638383865623735386437643439 +30313431663661663362316330383337663863616438643132663736303136643832393536386631 +33326234653165393735393432336635363337353939643236626631306439353935633166353235 +36313337353438346533383262323565633935376439643064383436373438653634616632326134 +33363635313934643432316466356161366131313538313762396363383531343137663761333365 +35316637373833353330336232316332306262666565303539323064323963313864303930666330 +37623532373566333065366462386234353361313736356133323731393564336637653564346637 +39356265633430323436383831396264646533316535616263313166383535613736616339316530 +62383562326265656166383965613564356536343362613533376235623232373539633338616465 +37383364306361613831343061343830346561636466333261653332663934626130316261643665 +32626265633035616633643763366530366637353663623339663136376238343439656436623961 +39393733646537343939313061363363356633323136306233306432636233643162653637626436 +34326136613139313034666561393461393566386238623163383332653933636232343231336130 +63353433656266346232613335656339616361626461313761643039373463316635376364396533 +36343731323032663565613865613736343532363037383961316437643362633937316432366234 +64356231636264383163666661626334336138303564653065313032363439323737663635363561 +39336235346563383065633765623839653230303436333063353537626630333431336536626462 +61396533353232376661303962633865633531616634363638646661656536663139356632363131 +34663638376435316666323337386530343232373565616266643136386532626361313438636565 +38373162313465343930666330613062646666313663316231343034653030653762613035666263 +32643134616665343338646361636130343835383837346236633032326639363133643036646661 +30396136613630323266376433373565306464616136663237313635666232373730643634363435 +61326135396237643530653561383339646463383939356163663230386237373636383931353836 +65613734616636363565363861643163636437623138346237353236633531393838653966646565 +63303436393037616636633639643337363766303066303065653331336436333763383833336532 +35636531373339343135366162623465393132616138363637366237316434356338353365313434 +62343362663836663335346239346537316661326539363765333363366439653962646437363661 +33376264323865366363316336623336353665363834646538636133646138363131633033323465 +61363565613662336131363432316435373338643033373265303964626264376130303932316433 +36336562353462613766386334326464363637326438663039643064656135306537366265306534 +66323239663036323031303138373262383865316337393063343761343738396531623335633439 +62363334643636316462666664366132353765396263383835643363396233383430656238636162 +34623565616235363131336331666164366438353130313237346664356232623565343234653763 +35323232373737646133323565386438353438306133646233666364383064346331383534376233 +33653761353262636231366263396239313138396538326166316533616465643162663235643334 +36643234363562346331303164653434323934313561303061643862663664646165613766303037 +65643638653637383461353938613665326332646437336135353666363135313935383438393763 +66656436376439306430666133323464643632333964313366646636323533653335346439633232 +35346563366663313030316237383663626130633138643934636133626462616630653731366363 +33313566366538643239653637373364616239353439356433656130343261313338393665396134 +64366338626133616563386461333131663132333533656135346530333936643536626430666564 +39393938366161616635363630303161303161613334343962666531343533643239343061666137 +63663431343962333662303233643933323733623366623535623633333166306362363464393662 +63343533323732323864633365373837303165346433326164353235323466663763623763393864 +36316164313131366262356164303934313333383836393130356331373932666663303134353465 +65623461363737326566633362633739623362366332303863623836343264643730316137366665 +36316238633032303966366433646133353737393239326333393564663334643236613937376561 +31333662303734396562623464623435343037623663323139383630623437613438393836663731 +61646166623832646230633035346337663739373966653863333031363561333235373635656133 +33323861643365396361336632616533333934386532396236363232663761613561633639363731 +31666133643034306233623133366439656261306437303930366635313865656361353636396562 +31626235373933356138303139636162633030653630303266663865653666306364393361353961 +62633663333333306366666637343865313433613965366636373864656364363864663530646163 +33386536323165316361393039663735633166336331303433336538616138363462653936363931 +62636238616530636361353366643266343065396233616134313130393264366334383765643036 +33623861323634303063626137366232663032666466653433626131353138323238353665386535 +37383931663462326635366639313630343937623633306665333238356530363338383462363031 +64663361666161643635363865653563393137643234396230386535366335333437363436636432 +30333233316334373639623236356265386630313465643932653430616566396531356432376235 +36613965666366643764656139643064633636373431333161366463336335656361656338653666 +61386538323732316335393739353239613430313130356534323838353038393139633566363430 +38313264363933316135313462656430356666636133323739343330353964616662383539653638 +64653233346234393433643438316234646139616562393663613532343032343865653536633662 +66333634653731386637666262333762613633346265633731653137383562383662393734313336 +62373463383265333265613234323630386230356536373434363530346463373365613662643661 +38313136393534643833346465336164383566653361623965666363323336643435356439663663 +62646662636635653230393338323862363231626366646665396431623737356133643531393135 +39656637666132646562626134356264663235376335323562326462306666306161323330656432 +36313038633735366662383030386638613264613561643465633163656232636135303134633365 +64303039663761656433363038313639366639313739353531623339326134633138396663383063 +31303434353966633839346366613366376130633661336631323861666161376638343532323635 +37333435656631333933303737323933646464343330303331633061666364616332316530303661 +36643665333937363833336437393039613537613563626335383830336332383731366462316235 +36353232636263626264333530316235323036633566303836326335636439653562356539643463 +62613736656539653136623263613730383632333332656465646138393036326431366562333561 +66626439323761306533623335326439356433366133353538393633633665353034663432633335 +31663837303034346338393033323439383933303936333437323431613538393437653532666631 +32633965653963666634303730643833376539396666326634633933623235303162656466326363 +64623862646432323934333332323166643039353338366665356366326239663563333532386365 +31316562633635663532666165326164643433366366663765643061363865613964643962303431 +30363733313534626363626631616436636239663536303361623166633032656361613239373630 +36383434343962336562616365313833383836333062633334643864356662316466666464613833 +34616466373739353331626632303734376138306435326430353434666233383865373434613739 +63643761666137353238643733636532333565623336376262316636636332636664316261643034 +39353533366434353233643634393036366237373934646433363830326466663663383263303234 +66306162623464343930356538633838373836663737323735623865663463353834613933303062 +65623766633265613466333138643138313763326462313631623466666161656432636435346334 +66353861373636303937343766616138626530343932656630663165666663356531393535626161 +31643532646139353434313333623262386636383636316163343432333532363233376663393531 +63316139613361626535623935313037623636633634346332646237626137383264306330353132 +65666466386462343166383937653364656638383466633461636661356236383761363332366164 +64636432653063306536623330343966346262343334333863653732336163326330326533333636 +63653436336263323862666237633663393166363339343131613930326336353338663765616132 +31653861613634616132633462366166323865336666353737393865616631316332646133623162 +30333666643430653831363932353334663639623965376239343839303832386264373430303264 +34623564313831653233353234613435626137313162663639653039636364613861353164313430 +35363962323062376532376334336534643536346266373764303463353364383039633238333634 +30346365666661663165616139636662366138383164646434646164383966313866343864636263 +64343666316564333431663337363731613163636364633766653537383339396632326363663835 +37656136633166303535363137353364643361376339613265656661316434396630303438383237 +34346632616162336337356461383561366538303363333338613461636234663663396332653338 +62323734623064343264366237333733353838363637613766643566643339363836396364393362 +39386139343266343633633063306431323631633239313064363062353161363339343337346138 +64373666303739323432626236663437653935336663633039303636636165303437393539353262 +35393932316534643338356438373539373036383262616163343166346361323136623362316530 +34616163386664626233393966326663336161393835623639316539396464363564663666313339 +30326438323236333137643137396439396333646438636430316238646264393131366234373563 +65373962313531633165666530333266313462666335396230343031663434646238653530646232 +65353036366237383739306462343639366363663032363462393639343862303533333165343966 +33623264353530653536376534336166663438663965333130316131326238343037663231646462 +31313761346661633061623933353935663132626665663264316261316466396432646130303464 +64633966343837326537353436343637626334363061353333313534303735643264373539313534 +65323663346234623437363963633862613235313633393838623132343064316263326166383438 +63633366353862653934303166643033333837383865616332303365383362366436326439396333 +61333838373932653132353133646634313431346262313931626332346663396536353861643061 +33346438386332633162623931613932373033353235636232363464356635376663346464323332 +32393932666536333735663337643861613236323337626563656261323135396238333961663965 +35653038396432323735646337356631663230363433333164383130336536613937373333383434 +38363762376332316363333237366431303533643439366464323532386162633165663639313430 +66376133376266626661393230653263653932313136343639303761306439636239616437643461 +63333630323633613366363737333561343662333732636566656466646366616338643530346239 +37663531653038643763353037353139333337663866326133316639373665363265346261346634 +34383966613230636433363932366164326633313232353537656631366666323732633230323463 +35653932383431616666386539303663653831333630343130333832656237613762386539336661 +30383836316339663565383964383732333538626663616131396630613632313939646336633666 +62373562386662336237613031386364353264653961333266636333666539316530396336396333 +63643539353331613934626261643065396238353738343466316439663936623830616138386562 +34376236613061633965353361396134363133613133373131376638393462643138326136653761 +64366535306333653162366464343633646537393938636334346533333334623238663732393938 +61313666626138623733643030613333306537336130623863313938323862383632393736323436 +64653661643664376539323933343561613163303630626662353765363832306138623739336634 +36383633353066636332336337666533376534386364313466643261333135623434356331383062 +36383637613666323632313637383665633231356364333334643931376563323364313165323638 +35626638623538656266643462363536376662326330663234633834373965303735373966396136 +63383230643138623838353462623966386232336539663631303133636338623339373965383435 +61353763376537333662373934303431306438303637653834663334663861373632626431353633 +32646462306431623936626433636637373262326562613965333638623261376661633237396230 +37376134383731393235393538316266396463656136643935326663306636306466663163623538 +37666434303930346635333334316661363539393063346131313833316462633261646264356633 +37626533323336353937616439343039376135373765626132326133626465346439613164343663 +37653666306364376231313336336134663832626465626135393465353133353631396534396535 +36373837623434663836636362306130386331653961393766353634623164356662306532663937 +36653230646366363535616431373034336437326137653465666330646239353430386337643366 +64616136393334343033386532326464316561353163633532666634313236626537383333333161 +36343463353733323762643332663736613663303838643334643039343862653764333535623933 +37623265613030363339326530663633616264346163653931336230643234633838316432623239 +66306263376162613366353664326565356664323939303230313366393631356461643739633430 +38393064303136626133613935656332653730366534323366303336376138393831356136633263 +64313666343137393563303764333138363737396663393936386231623965633764353937356164 +32633536333434643461663565313664336639306533646532346238366566613664386335613934 +36643866653461393133366333336266326464356338353932623664636431643763633836336337 +38626437326232363033386631386665333336333663643139343734353934313634313265383864 +62393836623762316339633931396438376165653235643935653565376530353666336666303731 +64373536646461636339646532393535666332346266666438333230333961373463366338376135 +63383161303139633933376162613630343337383035313635666166663137363737616239653663 +30366638343734633161333936663335643563666534623633353938643637666638616334396431 +63363166623464326161316637643964363931363835623634333637383232653630396632633935 +30393061626436626138343865353734353937623863323531336633383730616537633733383839 +31363532643764363038653131393465623434326566383430373132333530656338356530376331 +63343163393032356262323063336239386134353664346132633533343061663861663332353039 +61386437656335323231353565323935646136323133323233303266346162343564653531353234 +39643838306137386437393961313239313562646661333636353036313539323137326137643331 +62376139656535623736613035303430663265356364656364323162346164396338393665393663 +36393430666538336461313335393638303738366333323563376434613465363433393536306233 +61356266613931303530373763323334336337323864383831643737303239626538336433646333 +32326438356637333136633238646336643932343339333130633962326264616331303033633466 +33396234353239366461663465653266633833663762363065313664373563633664333332393834 +39363263306662396432643033613832306239383361616261366437646336666638663231376337 +33636633666535633033336534346130626463326566303439323030616662316234663264336237 +61663861353335383664356333386238626531396366663664633439393764326538656332356438 +64326266663032663863356538333833326635393738386464353830643139653937613730646362 +39643465663636316635303332366366383439353865316432383466376263353764303738623533 +37396162376239343831663962386232616430333130326663663361313534656434653331623265 +30336338313238303135613438313761626633646164383538616533616161616132633333396231 +33643135656134376265336239643839663338303864396236326638613431363163376362396563 +35326439313665663936373134643363653939386262343163646337626335393434303261313764 +34306163366239633030343830626631326438333562356263306539386239386337373434316365 +65616239346665663938303665386664643236613336336134393666633061643930326436663830 +34353735316238383536356537336366336230366262303366343166386335616662363166386530 +31343665373364333636303530623739386534616434396165623934366434346231613136363339 +37393162393261663037353131303134373466666331653165353761333837396230656637343063 +32636433613762313135333965656466306430323237336565323337323761313161666561346637 +61376364313433343732616137643137373936636231326630313634323631663961336433663039 +31386135303762353436313864623135316130666231356264333537623961326236656338353339 +30323634363865333266313263356236613338366364613633376666643737323062306634346430 +31303937326637613533316237653031396565636563316438346538666535396234656564366632 +65633331393131356637643935316264303730623265653962353234326635616137626532343262 +36303932396239396632363465313339393162363435323336326338353732356262316664356339 +61336532323361333066376336356539633665636130323133353463313537336237323035303964 +39636133623235316432363933643139353032646466633363343164613033333830363538623163 +30313939383532613430653530653162613134613637656462376537336534623435646133353664 +35653161616438373836383962666532663632313036613037353833396239663238613135626262 +37316233623832373331666535303734356332653138626339646137346563346536303833376164 +36356533616665333265633964316533613363393331336539313134633161313731383836336132 +66343461343835656432633763346139623764663736336461393434346130656366323939333263 +34326664636437336237306530636366356235616631326364323336626264333931326361626135 +64636432343063616232346436316363626638376437333338363665316334323433643361323563 +32663233643638313532343237636131633232343739656234346165363132633937646437326333 +30356332626663356133623863613431323637376437333331633537656461656539646533353463 +63663637356138633833666435323966646131303064623962343666636463393031346133633666 +63356338363133666663626231663234316432623465336333633032623930373761666438643666 +32646663346666616138353864616133386530623336323030393564343534306134373236393565 +31633839343463313966386332353330393265393466333131313365656262666639383236363139 +37343734616431333438386131393432353835353364303261653838663963326662393431623638 +31353135626162343963326465646138356563376638336634643034636662353438656439383263 +61353863336162366535306631343865356132636461393161623633626465616562323032346636 +34306533313937633239356163616139353066633932623235663262656261396331663430353664 +39663065393665346334613431613463313239376336313331376438646663333637373133303761 +65353265643134363031383338393339363966616336393963636231373162623733353332336666 +35363034343734393038383134396435646361656638326230646531393662653139343631353138 +66623761356238306632336239623638666431646139383466653138316439323164646430383566 +36376132316636366131316136663236643631346534633330336464636361393136653732336531 +63626536643964386362386239623432613963323464633433633630373138356238633035366131 +33346131306263623736323262643863336439663639646531393064323538336339353866343864 +33373131303035343230643134303563623863643338613433336666376431616332383062386635 +64643266383535323665333634343666623163653561313030323566373635333434316263353637 +36623964386664323861306162343065653762333333303136383231626165336565326532363865 +31623361353764636233616230396364653430636338333333303863393837623436633336613361 +37623435663732363635306236346232316237303639373063333766633466363738323337396265 +30333831333931386136396236393266656230386239316434666161636231396235353866326435 +32326637363866303166623131643564303630616434383339613332646535613764323130383064 +31366437303135623930353631363136663463623061316532623032323038376532343836613135 +33616435343431643862313830356330633737333237393235343432356538333433643265653164 +30616330373562613766613862663131646666373437353238386337356263323961323736623636 +63313462613562363832336466386465656237383031346637366430333665396630656635643436 +32343561393639636661383335323963613263646130396563306662353434316666393361383730 +63613362343534633234643061313365333365613132616636306530636538643065666533333335 +64356534653330383337353638663365623434336366386532643532393835636365346331663636 +37393636393231383865613731316162373165373538303962333962626664613663336237663766 +63376433353964353663633837666331303966346139303436643062666462663837633931643461 +31613565323834386431323463613263373932383861303266616339633365386539643163623065 +36643832333566353634343333313461646333343561646566643165363934623239366566393932 +34356539343731613936343064646530626235663864653733653265663963313937313466333633 +35333665643530663562656164393265373535323235623662313832666432393966346665383330 +33393765623131373535306230386430623531656135363866396234386162373064363336313730 +65336262316666383139623639613565306234303730633339313739633763346139623838323436 +37616634643932323166666366303062366563353032633263373432613239303333613537313732 +62636539646363323164333163663338633533613833366135383133336163336431346665616133 +34386533653130393130623330323266616666643430633737333739336133356265323237646534 +31346330646266653464366234336365656532326532333130636136383838333336646430626165 +30363236353563333638343537316137326638646265613434623164666634373338303561616332 +31306534306630336662346231646263363861643262323732633132616131656331623561626165 +37333339353932333963353963383062666461363031633337643063333137393066313464633236 +33646437376335393263303361313563386635303938383036323630646362643330643931326465 +61346566306665376165613934393038363833343132353765363163383063643939356238343330 +61383134386463336435313362313466366535313464613564313361643762303164356663383432 +33663962653930653331666665656232643865316134613530336637343936343738643934316166 +66393365373838616162393261626435336139633963666564393333633833353965336164653030 +34353130353966373538343365343963363933663439353362393132363361363632303936323664 +32356564383963303434313931653462353138313035376236363865346234623165313834636235 +37646162323466643431656632663332636339633163643836343464633964663461313238656537 +33373233356666343865336237313631346634666139653236323930633065373863656465633639 +39663335383239326438623165633463663232653066376234363663616538346536653638303064 +66663430633866656533373639663737323337393835393361646436323133346666363138613237 +34643436353837373737363137373732346635323331653038306465313963393238326663313230 +36613563366136343662363634303835616335633934313239363034383930343764383662313361 +30306334353830306538353863653734376363666137366235323562363238393365623535643163 +63373038663435643262313463613765623531363961313238326637363830653433623836646663 +37323833373035663437653231633737396666373461353934373562386339343138613131633365 +66326435313266393339333638363466333635653261636165343665613533313039646263356234 +33636231373933646430666436373437376266646266656530383339333138653339656161363562 +38343362373432623464653437386333653139373137653236353337613762616234346632373837 +35623531336134623236336339653565396534653338623631303034633334633431356636343133 +33663637623632623833643538306635383566643237656364343662336431396662343063633362 +62616662343933633131353631336131333563373236383038383737616437613532313264353639 +63643130613939366133373565313635623536353139643861643533383131356230656666333734 +34346633633664316138623862313265343830316335333837613961386631353537386433326534 +37313132326166346634326261616339363432323634333065343165643361353334306664663232 +37383333356537343962623636656130303038663534656462613137623932336266396631346165 +35363137653134656666623966653631633937306536633037613739616461386538633631616237 +38343337313564643965353435363063646366626633646334343331333636386366633939643939 +33643134653931343934343833366466316366366134323534313761333230633530626363663437 +34376164386636326535366235363565623761663166323362396430373330373337626537613333 +31316538623163366563353932643538336534336164306232353131316531633835326634393761 +39343039643033363161636538393238303130303966313030613238653930643131373563343032 +61613562313230333565336436633963323264323035306335656336663734336438616461366430 +35633261643637306237643536633330373830393339303633363134393463646264336533363334 +35386363316462326164393139393332336161333765663462653531343634336564396533356165 +30306164306235383637636433663932343063343962373435323632333335313065643563313533 +32623039626233336131613636366136323061303163373563613161386437326439343962333631 +35373630346235623832373737373539316532356536653938663063613935323934646434623963 +39346134333937396134663564363534393139303439623761346233376162303838633234373533 +34323531613432383737616435666630373361303164633734666630663961666131313334356661 +35366365663634333664373164366430346564363562373063316135633662363731643961353531 +30316230633837353864393962326438656164626134633539663565613731616161386536323435 +39653232363366356137613764663637666663353166663063323466396538336263333231353039 +30663032353538396664636662316336343239323066363638313331333564323730626236353739 +64653237363231656561653131383039333763386336393238633533633339396163613630333336 +33613033366163386564393436323962646564343039346636326561373730343137346465636662 +35316530623061633930626663643234633936363234323334626339373966633831656231363635 +33386137376434303035353933656665653938383663653934663936666138623538633032303135 +33376334666633666466346666393961376136663362303337323761656338633638636336663438 +66306462353634326139366330623434633866623738356161373532396165323162646462393632 +63333066356132323234653630373336393538373137346163303334343534343931343864613635 +30613865633766346261643238313036616438646666396636343738363165396663623939343832 +32613436316537363939303063316163643463373962336362636662633364323737666262313536 +32346431366665653831313566333735363535653166373035343333663135363763336236613836 +30613333613436653733393361646231353866623461323565623538303737326265633034653666 +30653231626662333236356531346131646336316461653264623639386638313239613037383536 +62386637343431373531316263643161333565373533366536326432366163323939633134653165 +32313531306464613230323438656639333764336365663039346631303333333137633039333631 +39643562323765326234336435633064653733346433643364626465333662613831376435613037 +36663165333536373637313835623031316331363535323139653432633439663663663035656434 +31396134376561623338376536373037653839303038343866623937316133313039366263633439 +36646662663936646133343132663938306364326662316338376438313664396535326437613236 +39376338643639343837363932333762363539633037356433356333306663346564303535626664 +36346437333034313733346134303339396562383466356663316435303032643065343430613064 +62633230626230353937616134356339613235313030636435613031353766386636383161666131 +37633164623864376366663564353065303839303337356638666361633338623562333630353330 +30366463313730363662363539316439663438363435343335636337623732633132656136346666 +64633366383230343139643831376263383763626665366565633266343834643335633038663533 +32643236653936376539393766386166666133613437393861393362613066363134303438373136 +30373936653663303831366631343133303138353465366133356231376664656639666261646461 +61396237663834316436346236623236333365633831336134323230363332656330326134386638 +33633331333436646134636464646532626234343234613536386237346363306634656665653635 +36643636626364636164393164653961626263313537363732323033623665343837353234396464 +61343937346630396365633961643866323236623731346533323762663163666630366432363539 +61343138346661396235633734643433313132613132376434353534613563353439343139666166 +64653362316132316437613238363636313132383366313064613066643437333566323337336539 +30303065336630613166656439383131616530323632396632613166306365653466626534306636 +35386361626164313161653334326430623433626630636563383064646437616439376630363234 +62616333633461663538383863653166373232373237363331306462653230633165386434363563 +64666231363930656437363631653162613963633636373764663236383537356532663964623836 +63306362326263656631316337343532303061333066353530643036616436633035366137643163 +34373665633830373566633332373238303132306532636564316463626336616239613631626430 +65356334656439396231373566616364643232306535396431343065343765633433336238383564 +61326235623139663664383264383363323735616436646365306462383633333033336434653339 +35623833386361366463383130666666643066336438366530383862383032646530663363396165 +31366266613335653263316164653431353461383335646665666339316135333131616234363434 +38646632396236653537353131303564336261393562373036643636366566663539356334313432 +33366566643132393831653435323962376636346633666666356365643630666562663835373961 +39306635386232393538333838363531333561313761306163333762633230663563353364316639 +34396335646239306135633361636264383961333366343038633238306532373931353861383663 +65383534653962333765343766326437346435306532626231633038396664643531393163663064 +30373730386330363931306461306137303566373436316233633439373463656665656630393033 +31386664313162626466666165333864633565373734623939633030383863623966303431613463 +62393635393065626132336162623433303637373065323465623439653264316638346262363535 +64653065623933666339383731343763376166313338366235386166313537653737336234353433 +30653935663566316635313763306535383961663963623365376462653066663735333630663766 +34613531626531363265646334643337303933313464373865633735636430326363363464313232 +33346130333032616564383035393265663339393937623931383662373233303938313731303761 +66316139303666343035633834316438316665373539333235343936623938356338343437306365 +38633730396538383337643863626436383163633363616433663361313861336663663537646465 +31666637626664636364366134373234643462323938656364383035613963616537343735303838 +34633466343162396336363835323434636139383331343365313834343436316334313063316533 +33356438326361323334373634343334653561373435346531333632306130313932333436343666 +38663235366163306236326130316636376332613562656564313765663237333037313934663636 +64663633663337343263373262316561336135353133333032363666383765356531386463656564 +32316431376661366433633764303162656434643864626464333230636335393664626539386338 +31666530643135333638376131306238666431666532336137323039363665336532623331386566 +30663738306365663934653463383764373234393465343038636539353738386633393930323166 +64353031663337643661643338393937303964303938636463316165313632326130336435386437 +61663935663864393763346234313735343937636662623363316130616631323262613364343335 +62353533666235633536326339363034363661623661663264633861363666383865356438353263 +38656133353333333131336438336238656134303332373865383031383862303037633531386363 +63613932666439303262393738326238386538306432343561663266633430666664373662363465 +31623035306436323632366631643933383265333166326530303366333230353437633830633362 +63363262363230653639336334373461353166386437383266643165666530313536633662366164 +61386333633334663937396135643933643166613166366333303937313363356539366261323639 +37643436323631393266303333633537303363376234663537643832653830343661646534656132 +36666438393765336361653564373433363266346132656431326231666636613265663430363937 +62326232633835653435363030383830313736663839323030623433323964353363393931393766 +66613664303435323438353337313062333331343364633238643463323437393964333864313530 +31316437363237653839313364663763623234346338373237646361343039653266653630306531 +66643838363764346366313638343339666632376639326333343162333362326134373532303163 +63306262323262306563376134376531316437373238356465366538626231383432666138636462 +32323365393038323234363832396265356333613436336264356634366163653634363038373939 +34353339333638353431383233316638353062623063386361396630633932626461353437316564 +64363662633363633437393730616633646634663139663430643239636337396264346464653439 +64333232383038626639346162633263613137306330626162326330323134636239623331663630 +37333337356362666264653630316337626338636330373435643062373962316663646266316265 +36306265623362623865653739323362636565386133353632623565323830386538343134303735 +64366561343561376662663931646466613532386238633161393836313338303137346638353031 +33663835373436623339376335326335633435653031336232393531313339346661373535386566 +37373563623535303639356635353136373130646362376666386539303139613263376561303732 +30663630303231663438663233363330346435336438303561356539663936343264353634373263 +35356562653963323233613333356332313733383037643663303236363930643631336136616632 +34383531316434633039663434353462343634633835383363356563353431363764653230613630 +37336430373530626233386461356332363833623837313232633462316663653265366266363163 +65353661303437646335376364646539623537616563333632366238376437353236363031363536 +63303831316637363239313462636538373837356365333865616362323466643138313363353739 +36336537646232393165633063356530376530646232646436376165666633643864356564653364 +65383037663735386436306135616533373934353363326434653833633534653035636437383161 +35643439643730623136643437383764383835313566396234636636623230633863666336623934 +38623366366161633632343062666564336236623564663265616335393636316162383562376664 +64396632623366623238366631636435346366373839366263646465323632356162643030363433 +32393939313130326135636530343439633236343935343436323064336463333538383730396633 +31373237356262353038326431663734653864386533383063656361636237343063323361616366 +34616238663664353364373630326165323663303136326637303333623830393533613235636538 +65633464303565343331666664366164393830393233613937356663623838646336333164666161 +32343132383632323565356462313464323439373430663835343637373434376237353732376636 +66396438666133663235323639653933623635663738346263326539373562386165383166386364 +39626265636630656430643963653337316630316332333832313365313330383335336239363535 +34386434323562363937363963616437616631636335663031653364323961363039306332356566 +32303563613832393466613932393930373939373934646665343235336539373237646337313935 +39393334373466363935343832343136653831303138313237323339656532333437633836653235 +61346531663137373431363939323132336263393032653637623231393337336337376366303430 +34346331363430393435366131316330653861623866336438303530653236346666343737393635 +64623739306431623239616233663666363364613439346563396234636533613239336439396335 +32333862323037393066366633373132333031626231353237346131316265313162336366373135 +61373162363866633238623866363937633663343436626263393132356233313236303964623063 +31386463366132333230616564646563373866346132653464323735653435303435393531383764 +34333638643666366337646432323630376130646466346165663461313438393166383435656266 +65333136363466376363663736316265643835666564646561653330383434643231313037373664 +32313836333737383435346164613238383530333639333835633165336630656464313065643339 +66303363303731366261333235626366623261643335663537373865333035373939626132306563 +36386566393030333535363832633532363033663038316538343130633531656535616135656365 +34346462303064373364336432616337313463306533623530346137653338313564396333376238 +36303366663964666238663862353934373030366366383136326531376237653836373833646331 +65623839616264616565623635616262653836316534613563313033393861373864363736323939 +64363639366530633435623331613839303962356266626636343563626138316465306337366266 +61666534663936313238346330383031623463323339626338643732363864343463366636313561 +37356466343833633035333737363061643161633539386539633033323362313864356336633863 +36333335313830393338346566326364346635313762666639353334313131366138343938623261 +30303032343932353237343030333533303336333866343161613535643535643534643336306632 +61326663666637396536303435343635333736643436653538643462663830393730626331613664 +64353230306235313631383131366538336232356566633732633462363138316335363539366361 +65393336306166393362633637643962316332353031383839623162303231656335363430613537 +62313236393665633565333764336365396532303634373766643930333364376461363638356636 +61326464633134373364346365363132303561303432366266643935323164343133633835326230 +61623766356531346235353262323065386134363536623436336636396433356135663135666366 +61613036646630643662653930376632646138663139376564633563626633633261613663326266 +33643232366464393331643138353138363834626137616363343432386364636338623365613338 +61323864376632633231376134626164396337666463333236666466303964656536313666373264 +31633138363130356365336132636137643036316363396164653534656636343530363036643130 +32366335653231343763373231326234613036333532343431356435306338333933323430623761 +36333338616637343238376432396464633038323338376461303863336137373462366539633138 +36333334633131316635663864313331303630333661393834626563616134376637383939653062 +36323431363963636166653863323432336336626539643138643237386235306461386535336664 +38323533373137353061343734353935386233313832373137633430373332313636653232626264 +30346262356234656336353035356531366630333765393162316339303035333731336564663233 +30643566313462303031333433623065393666363263383935306137623733393337643364633837 +34313235643063383664653764343133653836623265393665666234316566383630626135613939 +39386666653738343762616334383531626261333266373233663534316334356162343131386435 +64623065656635323239393431323137383035643238346230373838623063633638323361333033 +38323064333139303662346631393433626339623661303435643137316438313362316435633832 +64636131373161306631626137343730643338353565313461306534313733363330363635323131 +64366166393236316137636139633130323933393466363266656132386266313531666462616436 +33396334303934633736653936366635373235623830346337383534376162353266333730636239 +37363731373430623133626334326665646333366636373139623433353931353862333264393232 +65313466353539656136663465613765613738363331353332663436383130653561326132643834 +31653132326136356635323235396264363865643135616561653331376262383339346362663539 +36613538663065636631346665626430386533666465336263643666353636353165376235333338 +32643362653263396664363132616530616637353163646536376262353436663236356630363132 +31633635663032633033633538323162343934373264393936666366323265393131663361353366 +39616161636438323232343131663464653832656432333639343931366364613238393339323037 +66326333663139343762333531613637623437343961346463626664646463346133653762326533 +38396236373262643763666166396164653064383263333166373164356437343766653065636533 +39323963383965326433653563306263366131643436376366633532646235663364663635363663 +61313731306665306631616433303035636138326365643433383830373233633062396364633966 +39303238353363633636636261663763393865636664313063373933303962373839653263316363 +64663964363336636130366266636133343636323865363761303533636363343432343561393564 +38396332353532653038363366616661646236343038636139626338666161303437646563376230 +37613536633364386537336132313930666533663438616163323834306237613766656430643233 +33643030373631323961636138313137663765613531333334373238303761623761636335346637 +35363264396135643761333635333031623261623263393662636531663066356664626632663164 +36613434323937613862633030333534383066313166333262316139643339356464386635656163 +36323139613137396235613138383466646262316236393966393332306630383233363463366537 +31303338646637313166323730653832336631363730333639323766316663326237663261643633 +37343838396332663664366237313339373337323963386231353739323365316666333166306239 +37333433316166363030386337633366376362373031666136613063316232373739376633323337 +37326430353138653937303630363833393463633937633463343132336266626263346136343537 +36346133316130306436303030623533386535656665643765613063353937626335373236303064 +36643035646633646439313665306163363862623734346265313563393336323738353762663165 +38396464393761633264353762396339653465353839343338346262393863656339333637663033 +64356235396533636461313037303666643139323638616633643661373765363035326136356266 +64393230383165653233646638333861326137373533633837326161386230343033306130316665 +64346531303165663665653633343735376332346235643631356662356236303366396334653731 +63653932333035306339366562366163636264313534646463383338393432613237353138333264 +36373533623836303964613466333738303165386332623837383762343139373837643533396464 +61306130336533376534636535663434303862316261663932313832633464366536623139656337 +32306631366434653930663166393063383431313236383863303132646535643832343238323063 +64663332643866643534303934616231326664333062383031306662653734393661653865393234 +63383062653764316437356264333830353165373236663662313264633630653363376339633339 +66666236633334643763613133363734623933303366376335313834366661643266613237353666 +39656432383739646230313334626434623031636330303061366334346630373733376138323665 +63643664613535353966373838306265326235666162376138333762333664613436383433643838 +62353833306635376439336339336436663332666265636338396332363565363762616239316637 +64613361343539303161316439383536626664303366663339336262366538353766323938366265 +34373331646632383666306231323066643637313139396566656234346461303530323566366133 +36326466623736313862626433666231333238353832636434396435343938373538623462306266 +31396231373166353339333234396230336535653361356265656131316236303963356163663932 +61396133343831636532376534646434623864646132316536383031386337303036366139333833 +66383934636461636135316537623435386364393565633565343961633964663765323066353834 +61386134326333666139376532613639386336336166326563343938363037363661623731306362 +30303335353634363439393734636631383136383263303530396265363334663736376131343032 +61656535353337356666653336643663623830373031313639633532666335616238363065326638 +63373737316533343036373762353561306232316661633433646165353539303335653664393530 +65303766363162353337346339663936336631633836616662303136363933613665356161363437 +66366634366161383237363934666632353434383838346462316130336462636132353632633130 +34626233323532623663366431623538323065363964363364633538386438626566383262353834 +33646265346235636139393339323938373131316533366434336432313636333635313330333465 +35626535646636646635323061616336633863336539323064346638326230623234366137363932 +31633463336363383837343533623836626634666538363334633866613763383231383533303365 +61343535356630326337396537303035366264353836313933376339343333313938623636356538 +66326664396234393761306663326236303737346330653964303637336162363138396635376261 +30386133356130636536363165373261313534313231636662663765356332303962363334386465 +64633632613337616236316263323339393661326465613036373333306137613062316533616339 +62326363666166313237306239303230356262656135383165373239633633396231343561343330 +37396539353337393466656439303161643862353163343531613466383534666561303062373234 +65396138306430303464613835323436313830303631663030333365666332383363363235323666 +31386262363235303966373666656634303334333664353538623731623532376636616233383763 +32336566636166623336356161383530646339313733366535663831363337353137326632363562 +66313162633561353763386334386630393934383637613566663530313833363539323962343465 +64343230336237333534356435666433333962323432323736633830363535643438346465303834 +35356461353166656333306638303965663030383961656532343264393832623130366261323734 +33376563393561316164353235613437666236623035363265306234333838633364396266646134 +63383864663831383739336437643833346232363238333932306638336635323238356161313835 +32376432393836323539663433623435393161343166376163393262336637316233343762623236 +66613133336333633937393661663462343661343937323565633036303266383934383266353064 +65316365643666303230303336666162353735303165373938366130653330643837303936653439 +30636532636334353965376664663563636338373633653261643733323536393963303062343233 +39626364653166396161303430626537396435666139306334396466616666373665356531393566 +66393631343063313334656439623430336535663439346363383863336231666233366136323064 +34643734336162323739336662363635646637613063376466343861373236613361353537353937 +30323763636364303035646266393731383138363730323532626438373665303636363632386533 +35383064336165626666633362616136316332313137323137323531643035633833353631636230 +31353265343830353338623031626432663937396566333664373962326236326637303637383931 +35313932643363366661623664646337313561323037303437373838326632656664616564343932 +61313434393939663263313237363466613331653235343565626631623864396334376266376239 +37373435666130356234613766333464393235616261346537353861373464663530323538353266 +65663066303261303437616438303862313263373465373833636265383238396534646435353862 +62396364303934333962613961656163316131626333656661343663376334326333363463356235 +61633539366361653065383636636539323837393038653630636135373364656431386362343639 +34373465376532393562663366393437336432383538343638376539343539633839373163636163 +39386231643263396462326662636338373435663062613537623962373862353865663163376531 +61653430663265663034326633646363366438623236393461613736656433373732326663623931 +36616632376361323632303537623837356233636430623833373162353866633761383563653535 +64303634333338333732376633663738653538626261326437653130633830386235383836373561 +33363435313934386663613766376239643438303633376630666430623630643137373633666264 +63313966653966336531333361303237383665616366376161636236383664353436626330383035 +62666135383430616232623063613232363261393839396437396662356437376532356666396161 +36306262623433333733343562396639346565373636633932613333316364303061633964353333 +34633566393639326566633337643361653636633836626462356433653637333236396538643438 +32643462346162623935346637353136653634306233396136306465323035386464356132626337 +38323135616335636437303266303239396538643331623165343265323530326136383866636133 +34336333373962343566383164336531626530666265346166303265323837623866643062666634 +34313565336262303261643235626562363962613733373065383766336565303564663734623739 +35643664663834393933616564623735336264396239373861313937383139363539373233363031 +38353366643530363131313839306237373263623836626661663566313364306632343563396166 +66346165396336666465646634313339373730363437393031336330666137316365386665326263 +36616433366564643563393835353763373930373464396637666565333235613065613261623538 +31383735636363356663646635346338336635396563643434353337396336666562616264303430 +35643033333934393834666365366365653834363333396136396663376335353431646239636164 +31336465363131653131396336393665396639343438356166636434616531613633643530656636 +30366534346264623864633339663436396633303636366439656565303065356466663831356461 +63333135646537626462343166376539373732316662643134346431393765366134303236316265 +30653838666164383538623735653839643161343763663063353238653239333938613737663365 +62343664373932346636356533623834303635633032333763633837616631323266313236393034 +66343830393038623936666461313938643361303362626630623831326564316432623866313733 +64633662613364653338313537396463366436646561616231383839326462643039626534333362 +33663564636632366639613237653439333665633930363465306435323165353064366365353236 +37356430313236343531306434326431336433343634383631613266316232623361623938306232 +35313664383235643736376637376337613966643962613961333331653137613962386433613939 +61356364343233663263343735333832626533393766633563656233336435666633363763393833 +61303165366431623033346233353233343562643163393037326365396338656165366437633264 +35626139393735626663666237666162316532623036393431306335303766393964393262306432 +33353335326632396439633962373930346230323963323432623166373862613437306234316465 +37636566353666636562393537333064633761363965343764386437633461353439353764386166 +31346132336138363762663933363163396530626537616235653864616331333762656232303965 +63356136616635326532633661346166313939643730346137643461323336636338666531653932 +32393161613935613362643762343538616564316533666333386635616161336232333164336264 +30303035663963306334643236366132353337383166386362393531333964326536623237653735 +63303162386632306163616536376536386139383430303161383834356632653139323663313632 +66646463613561386435656433336164313838366666316236636237636664363163396338366637 +62303932363231376335313561613866326338366330316333666566616335643431383235633038 +30626461363337666438643932336666626363336636626162633564663638366564313534316235 +32633362623462313537643362326133383934383138633438343436373336613532656637653066 +33383234643663393331346536326535343163646231366662363262653637313933323563626663 +37313932656363323837306532626432326234383636383461656139626637666363343139383930 +32396162346436363064653961616435623466313664636338633632653563626637663834613236 +36346131353861383662343636646563353632393361633565356364326561383439323930646636 +32643239613037376434656562643561383438643135373834316132353666303439363965386465 +36383837623032373862333064633435316463323039346631653532376630316634646434656531 +35656664343735653936363963346663383766636632386561393439363162356565343264313934 +64353237343035333238363336383165653063306363646130326137386337303235306235373436 +32643034656564336130396539613961346239653330376338653231623731393534333065663230 +61646430373035613732306132623538393436313531613634656566613537363463303730626161 +65366665663032336536653938326131376664323666363162633938326437376430306634616561 +66626565366162363439613662663031336162363731646136323930623461366530366432623765 +65343632663763666135383438336463346165343832366632383361663135626236623537386632 +32613532313438323334663666346463323366326137646439616330383538313332643565613737 +63663832663661636439343731316165383532376163336136663438356665396336653331326438 +61623834386536346664613830386537373133363161626337343065326265356334636161323530 +61366531353032356139323433343834303337663864353737613463633164336563326261653861 +31353662396263373135396536626562303931336363653666363337653032636632346134653138 +38653861393637386631386435303339336534343966353565653463646365643933346564393564 +37346538313834616261646537303134646665333061366562353765613562306261653635636566 +35346165626233376434633133613830316362356138313537616564633862663031353663623637 +32333163363061346434373237323662393533373736393130333563653734343437316231623131 +63636465396339393362373935653562316531666461316130306261396266313162626337656438 +35383932383163656164323463343535343738343238653430663963666338656562303730323933 +35653033333665393335643733626662323537346431643030346330313164383962656136646434 +36646439356439326130656665666232633536323537613738316331353634383066616138386635 +33646431343161636264666365346133346163376434393339356537616464353532313839326137 +34376230616339363962393032363731336265356239636461343930343665343665363962303230 +61666566643063326433386636343235653261333536326131626162626163356266383032383434 +38643335616164393933663039326466646264366438336134656264356437313738356237613332 +31393261363436323537666139306566313630653461363363656638333931363633636164623135 +31613835303961386563646437333237663733666531393030633132653439633564333631613165 +33396131303831376332633166333763373561313365373464343266306237626134613465333161 +37656136393138303965666438333064363631633866653633616630396435323031643431623261 +63343162383135343231336162316531383563663031643765646565376134373339613861633639 +38336164333333636334613864303233393731623535656137616633396532313766313766383864 +62313237636634303238313264636162643833353861626565353036616636646230356634333062 +38323233656561363065393935386533663462303536316636623664306163343761666663336339 +31653935353235633464303631313530353331666335633862356530356435326537613630316266 +61303639663536383866396434623937376239353265616539326466373163363632313934626332 +63303934343762346162373131616361633038626233306434643938313564326131303663643532 +66666639333531633362343733316239383135646533613132633430343235653065326630613934 +38363033623361653266346331376334656233663030626261626232626665333333393561663032 +33386566643533396633323036633065373366303739326262333166653662636263646330663633 +34303537663230613064323739313431666662363231363934326438656434656638643934663937 +36613537346235333033646661653531323238323437366631666433366265386233623538613566 +30373464306332653866326134303733376139643061396234336465326562663338313561343435 +37346361323036613539316435353031636332303531346531653935613465336432306466383039 +32353039663435633737393766616633373730623461376661653661626237303166323463333931 +35623863303430313063656162383131313664353735656439643934636163633131653435363035 +64653231303366656533386663386266623464383863373032363364383865363265326130346137 +39313338656565323838373563303337613031313338656433366465353930366432616161366639 +32633430343338623732313430316535303038633334643832333533633035346538373836353436 +63656331343734326165616235636430383965613333636231343330653163613539333263386139 +62623430633664393763373334393065333830323431396335663166393462393337626361326635 +61363135623235333565653833613538666531303737653039333466653763323366353630646431 +61383861646465313665323439376337636166376332633432383533323837353938363332383734 +64343565393933353461386362396638386461306561386465613334646266393934643738663737 +34366266306362643931336366306133343361616132663866663738326635663136646362363231 +36396462653862386136343635646166356465373162613363303261366235613963343934376337 +62373266646132376635343234613264393065353635366434636438633035316431333631633333 +61383764646231666166383563383563383130343030663835323633363832316461303564363032 +62633338626439333034353237623365353439346161373665616363383365643464656139326663 +33383961363737303035366466616132633039313131376135656463333333336237343733326461 +63396430353866336266333636396230393133363037343437313364306331613537626463323262 +39323639653532333966336438643633633730663730343931646338336361636139323566636433 +65666236313637633138313431393035393661353835663664646135393765643937346132366262 +64643139626636323736306365613061643134356334343365663362353630623863623132346564 +66653136343633313264323331303761653837353161613037626134393565376434656164333062 +38366234383337663137373966646662653030663036626462326437623261663333363662326432 +63336233333738653930393064313761383139613733363436313239323561383536636164643037 +30653034653665373062653461323234646463313039313161396438313864623430353035393865 +63353631303763373932626136656238626461653263613163653930363861323338623635306433 +33396262306362373238383034343861326138643937333235366535616136313336643231303333 +65663033343130373036303230323736376534656436393634316563653330643035326239313431 +30343734613838396438343863356337346235656637316364356364643834303064643130343530 +31336132373361366665613065323731326639393966303938653638646161356639353661343139 +61393832353838326661396337326536626334313732363666343366373265643536656133376461 +66646466656461396465623039383562333566326561666535393665633963303062633664636163 +37343265323463643138303361393961323839326436353265316664353765306662333364316337 +61323033626663623466363265353266373239656439393339396231306664333638646430306439 +36303532366666353166366664323134343364353266383332343061666239313132303465626538 +32323866396631376563623738343335376461613837656237656631343233616336633831316435 +39653332333663333635373263373364323765626563343934653637383435376532346631346537 +33356166373730666466616539383237396630376365623435353938343335643763663438316237 +35663663363766323537626437343435306530646234646335323032323334323733643139343235 +33613034323730303033303363383930653961393065353234306435353064356636383736636635 +65616235383534353332343431613631613339323062303466613236663731363539316138383265 +30393765376265353436666532613765373337393965323761336431356133343033336634666136 +36313638616265663839616361323232383437646234653565356536356661396637326434636662 +37326438386366643935313339356435396566336662313738386661653832323366303163616630 +30643533303734616463346639326231383834623833373032356635376435313034326232373838 +31316561626135383735306434353638326335313663646263313939383836376337313230376233 +37353338336235396339323134653538376532326337613266373561633336343239386636656634 +33616531346136333535373030366137626630303862353730373163326430316238303766623065 +33386538643132383662376563613066663665373032383139306564356130333532336535323331 +32376131323661663034363463306162343263666236356661333836623866303232663635643266 +30353364633730616535363262313631336333343664636235343433376461306631623837346364 +63313233383732393237353765383463663731353364363131316232323339373135653437346134 +37643638656639613835383566646563323539313937636639626631326234373537373631363830 +65653636363532383333303163363761656464386431386436623331636635323561616664663763 +36376632633664646262353130613436366462633835363332623066646634393231663235643964 +62313636393762343432636663313262363037656332646334323366333861313637323832366439 +38303237303439343937346334306231346134616461393161636439326333626461356561613833 +30333365363861653534353637373939343064393938666233383036383661303036363036623239 +62373939373632666461376337313237336233353639356232383536323662646333653662313761 +34326663653831353032333066613630666266333833366434343438306137333431626330336636 +66333065323138643761323434356232666463333538663337653238613065303962353633326536 +37656164306539646266393033323261653032316366646435313063326563666262653139313961 +66373132313139373737643261383539346630363166653733383339613939363335613163363136 +30666334386261376137356137393134353161353735306334316537343566626536303463336537 +34653436376634626538366131613531616564323034636332346237313632633337653061633463 +32633466363433326235383830616336633234303465383133363038353732343265363632623163 +66343938663966643130636464323737376366643030613134393132396439626531646566366538 +34343932613237623434633930623663383832653234633863326132336234303666363966366339 +38643837303662373630623764616437636632333761366532363434656432636232366631353561 +39636266366132303835313738343134333331633665316537343561323331643736383332633839 +33366265363238323031383737356234356638303333643633653832386338396533383261366361 +37313063633466613233386561666132326662333838346434343136313339383838343839383237 +30366239386361393137646664316331356539316632333532613439663661383930373163386561 +62336366376264636261613833353830313365376637393638346530323430643932353561353032 +33633033303839613536303739616634613937363666626233313836353363646134666131303036 +38373532303030353332636164313637316665643933656137613862313461376330363036386463 +62316232303761633666393033316463333038363235383734646230633839653738633266323363 +38323866633264333834396564343838323738313735653731613866623436346263346634663538 +33656335343834656164663238363962616639326530326537343566633966663738636466353938 +62323232613234643836396163613161356164643133613732666137336137313132353235343066 +31323438383935323938636363393362343237623033353432663835363463653866343866313061 +61356366346235646431353238373665326536363934363366653934343266336535376334383436 +66613832636233626333383233336661623861393865333934376636613735376365383834383364 +36346638636630626366336465333734376339636135616631373931316239376235343064623664 +61383538643236326636363165306563616465616263626336383437356361653432633165393837 +65646163383435376635623638633463336334396131666131313537626335336461353933623339 +31376233366261336264353961376638373164646265393635363632653962666264326331633566 +63303731303163303566316664373939623539313865323563306462613263313639376635396666 +38306461376536616534616130356166656663373430613764353063356231366463613735653132 +66613037653438343964326132666166376137363332646466363038363436666465643164336135 +33353663396433303638643339336631303939393135636264373465306266616438623339363330 +63393964613164373762313536643765646662623765386636653035343335363238303164303139 +64303762353531383061343063353039663839613435343932393131353035343161316463343136 +61623566383931376134663463623635356162663163383465343164346565323033633435333335 +62633630613131323431656462643064356462323434366266643566373064626435616263643666 +37393532343933396533613833666533326537313639363661663333333832326338343333653437 +64613164313262363731336239656661663735623031373238336239643430353331663161303966 +33343063373030653931613635656237363533343835633265346161666338363739393962353137 +31623161323234643236663239633231366563666538653639343130613663343036393532643162 +62353962666430353138343039306535333963646333353135303930386335653464353834636634 +62636339313932653738623732383539313231646263363636616162626431366430613365336633 +36356261376135383338393938623363383363393863613862333636373665616261393662643633 +64626465383265323131623464303466306135336631653133633433636266323839323366366238 +39336533363439353635663730653933636135323964633062366537326363623736633235666436 +62386339306263613562343332323735666162343039646561613331626263383630326139363537 +37643439333238633764303038643632643063303236643736356338306363303934623561316463 +63326234636634383834346637646431656536666530663031386563643434613735353237373635 +62626333646631353134393436303730396566373834336336643532613261303363613336376330 +37353138643532666331326135633235333239333738326636356633366366616532363661346261 +63366532316164626463333635663131326235313939333039383764373530326566653131636231 +35366161336334313339323333303562386133386139336134643830656432663865666535626166 +33663232663130303638306636316564356132616239333835346630373661306438636639376532 +62363539323837626362643737613361336237643934303161323935666437383364326133363063 +62363638303032313865376334613464343630313831313261383835636136303764303737646162 +36343337343066613061363439326433326162633134336236303162633366376138386134386131 +65313138393066333431663134326230636664346133346461363564313064333030636131643038 +39363634306639316231656630623438646536343232353238393536626338333966383965613361 +64303631656264663166373866626331646139326366366631386338373635366638346665613062 +39356235373362643464313938656133373135356232626566363133323931363966633932646639 +33373532633436313364363161363366663835333532373836656536333264613332646434303236 +38616436343135663332386637653538376438393731343739653432643563353930346564383538 +65663061633735633438633032303165386562613834363365663933326261666431376533373032 +38323165633539636137633535656164376133663064653839333737383733663366343132633833 +61393931353565313963366163633238343731343463636430336164626261383733663237663436 +37623365366561626338636661383336623239376433393233306661626232393664636462316262 +39613037323835326539343366613665313436353361643932363833663264363930623833626266 +35633639663865326537303134643664646238383432666530626236313734313633353931623866 +63386237613339363037626638346562666166336335646366366561643663306161623965663366 +65323636663765613465366536336137383739396162613933643738326330343731613733663636 +39663765303961363366376530623336393563653966653366653032356134633033633033303535 +30633731623737343638616661356630633539393865373239623661653830363230386239393261 +32353435373830663934383832643339653037353564653233353837633562333531353832306338 +64326534353434393262333264646662613339333430666336363363646432663263386130323864 +31326265386638663231343065333164326531343337366537326631613162373039643833643564 +30323962653864356539626263316332333266326265306565333163386433623533633938316234 +64633863353130346166663537383938393436353039326466313765663334366332643733643761 +61336138633764626561613566616433376534653435316238313861326432343364353534343432 +63653331616131363835383265373930633933653034343161376230326331636464356433373065 +66336535306532303636376136383137326135323930323139366532663837323561313863666331 +61306635323630346232366438306336353063616161353563343232393661366136643932636537 +30353132646266643961363638373265643162393832333466333436623130616531343935336566 +63383330326335643965363064643962366465343335366438653633323862343831326436373637 +66653136323733336338316561663235636263366264626539363832373562343762303730353763 +35633936646233373361663035393464313932343430636363626135643439386163613835396638 +32396264323562613630393031616664646536623266313534343965316138383538343036393731 +64663837306234363538663663313135663637656563663961316130376362663733643838356330 +32393831363837663630383562623838383231386565626363353961373664306330666465376639 +61323965666236343161316666613734326334343264626566336565633737646634646637636633 +61626337346565613762643438343764323839643630613864363561633766313736363564373165 +34626366323965653263343264653231333661333663313538316630326637326165383030643535 +62653932663361663166326331626463633936656365623538626232373532326562313937646132 +64376562613262316137656632363838636133376261326363326165353561666330396534663966 +65356434353636396165666238303430363336643534646435396231666239373731383134633333 +39346236623961613062656538386561353861363134303565343433363735663634303533313833 +35666539663962303064353261383230386132356462653632303265663439366431633265626532 +64643965623665653339323932616164303638643664373561363862376461616633666133646634 +32356366303965363639303364383737393265666636323864656538313835366462313330643436 +63323838653262646664303237373062373462353232356539636533653638343064363431366436 +37643734653734363839373133643631333231383938393134383661623938636136623834373138 +66616662366264356138653261323734356636333237623962333566373535353465383333346239 +62613733323861333831613833383633346338643262636230323937333066366631373031656434 +35646232623434306562656131663238396661633361393833613135626234623637353033313663 +39303062613039383637303036656432303037376566636231643538623062633338343861363530 +33323934396164313439393765663262653866323732656466303965643262356336623735306132 +31626231343536366337643633393433633434303339656466333333373432663736353164373761 +34323632353833653234323864383534633865336161633738343334363164343162613334306536 +30306462343032623764666532666164333334623463646233303639646362313766363038393933 +61626336323732323633666535326164306139376638366634376665363665393063326631623230 +35353034333964656638626136623965633862653564326162326564306433646333383537343639 +63373233633833303561656166313766313438346265393535643365343332373939353263613065 +39376162336465623635363730346264386436306238326331656562386436356165636138363232 +63613064626161666430333234383362306231343738626630303936373136636136663831306365 +35643965616139343637316463646266363161393639636662313230666264646536306539656630 +64643038303161376139316265386131323432313333653665613666306666313762353766643031 +38653039306130363566303236356530346362613863303630643935316331643630663338376530 +65663063356264336366373133626333613665333863626137303537643966376136633937393530 +31373732333363663030396237626665656135363130336266303034346531333865306664373863 +38376264356630343238333464303132323966633332623034623836653164626562326565353435 +64353530366438643431376336623331303233623565646664643432343738613363363739313564 +62303736313338363863623565316238363031353432323133333037316365323466646531623864 +33666530326563333261633934643737336231356533396464336238623535616231353561353861 +61326364383039643933376239366131336362356366326332633338396630633862363338333231 +66386633356336613564333337366537363964346666363038343633366136353637343334616230 +61336536333338346166323939333039666139363439373862316630303462666162373063663562 +35353839323239313463373335623064333332626466363838363730353133323262323236306530 +30636436623039633461616537666561663533666637363831383561623831326132346130313864 +38306138613430343239646534316634663462353939663737623935393565663662613661363664 +62366539313535613434303636613436623564613961303261613662626633633765653730343232 +30643162616137336138346637373838393231363831326361613831613530346435623435626333 +35633938656564386339373538326632363134343333623939633735386332633665303939636132 +35366166616230646536366162633730616235303631623562356232343664333539383762613635 +66363063396161353263363330646531343633613537363962663863336639303863383366316437 +64623564656630353536366161376664313233643966366365303238373364373761396536336435 +65376536623266363134346131393964353463303331626432326264393136623734623238363964 +31343531373236373663356366373838336333353166653039306162653164353138643131373532 +66323564383963666264343334303866373132633034373339353261626336353537306534363031 +33333864613634303762616337393838643433653162363633393134333064353264356562316333 +36616437363233356632323735373537316564656335353231383662333831333733353662333932 +34653339626635656365646332366333383965336661303039373361383766393863656264636562 +31303361653064303638396365323261623866383335343066313865303136303233313430303666 +66346139356237656565353338386665336533373834336662333937633161323534343238353361 +32643636346239336535336665396164356632656630363461393635663634316466343133636534 +38383632633063303263336536323830356264313163353435623934313735666130376435653538 +34396534383735373532313933353131343132613135616231616238353164623833366466653133 +35613337666261393965393931363237343033636132623131356362363863373633303032306538 +35313365306139326232623562316665373738656332616463393230373735653236316563643362 +63623131353337666533383862373466386134353664363661323237353431303433306335653265 +39343436333333633637393137313861396631383666633265656338386132663835373637313362 +39393330353438346530613862383132623631306430386532626364393166646639343066386133 +63636435323532613463663563353366383534396237656337346133663166303833643764376461 +36353031323837663237663665386639656365363034663436306632643132376439626133653937 +61333162643231383437383566646532383230316339613430636464643038623465383133376365 +38373961353530386261646263616236353262313365396431373932343766383262633966646363 +64616134383538623139303034646632393939353534656236646631353463353431663564353836 +36656362336436636238643133633838626465393864376161646139346435336334326665613666 +39656363653035333937396562626436643561663666346666333735373163303930383062386332 +31666330333230633034613730663962616432376533326339616438303162326434613962383561 +65666432396665366632616266396436356438313765663434613432346663336566366132613863 +35643033653639323364366136313134373932636465613931616664626135626362646562343836 +64613662303436313762303262643532383034666633336235373165633937613331346333323939 +61363962323236366566666437373833646335373764303565646566343261383731363833373465 +35666636636234626364393661323962626333333130656131663331386264363966313431626162 +39653263643234346663643461386437386262643534663163363363303239343132333163303864 +61363930393430663333663138303939653963623334623863643435626633393731366630623262 +33353631356230346131633562366132373336646634623136366231613432366534323161373136 +33373333323264373964626661613763663137366462663362326566353861626537616664663034 +63633162616135643965323932336461343962623032393331626632386263343131336236383162 +33633161346437336362663738346532363033363166616435653930363333316562643936653261 +66656139666338393963366436616261363731306661656161343162616137653432313561666533 +30616566306432656438313335663533323736653630313537363839633737326539633862323738 +34383363643833633163393366643765386132643562383736383031653064333333666666316665 +64363330666238353566366661626630366539643535623435616433613438326361376330386436 +38643639613763653133323761353431343964393639356535363335366133653664326162666630 +63343235636166356332646662306462313961613836633636663336353266346131643134326135 +35633835373137656465313834366439376635393438373835666461326631633733643739363933 +37376530386435396438363962636266306135396165636132306232653333623233666162623265 +32383335303965313233373134376135373233613838333566653732393535326465613932376363 +35633234393662666263613230393734393361653361326330343164633737646165303531323564 +31313834326162646232393531306136323238633930626234616532643634366561396334353862 +64306362623338313638323733663837633938363030323266353363343864353938636632613439 +63656432363138666536633635393766303139383939613465363534316437323831343366346539 +36336130656263363639396534363038663134393764393532333232653237383337613338386237 +39306233373366333365316635646262366233386437656434653035643933636434393539613631 +30653330396138633830346262653630333166363364633763343934323538316135663564343231 +30303333333437323038373561666134333930633536653433633938333562376562373164363662 +62306162376465343037326562643337646630383436653133363638353236336237663234356336 +66656433333439643736633461316638643434313832313039396439643336643162376534636637 +63313635333331623165323861383266616661626336626233336564323965616139626464363937 +34313139636565623237643237343132653865623939623736373662326438653761616363303132 +31326639386438663363313561383738646566323661383036666236633864363136363331303233 +39613261303461653434393565396235366463616362623963666430333563376431393633303432 +61323863333538643536386466393630376431383364336466333032313532663965366436343433 +61376130376238613532633936313733333061393634323065613864373762373661373331396361 +39373462316635313635356265363263653963636366333630646363383536316161363136386563 +62396637653033386230653466303261396433663739303931613661636666383461386635363161 +37363234303535646233633965383630633963356361346264353037623831646631663338323962 +38643135333936366562623162346237353538653430373932383463306565313231336362326433 +38333533643432386366656666666263333738636630323637326430393639666232376137343337 +35376562666236633934653330306634356666303366356561623066663362626664366666383165 +30383665353536353966636662663430366530343030616334343262303437316631626531313936 +30383037366137376566643162313262666361336238613533323336626166356630343636313136 +31313635396438326466633435323237343931663537653066373161656439366336303136313563 +33623730643139663364383563643561353561383330626636316536613230373461313461613332 +62633130663762663632303366333130353665333931393630336533363630666539366536316132 +30356635646632323264386530626432383337346338353833346637626363643935363535353335 +34393234333266336634313466663233373538313663366633316665356662643235346364396464 +37646662643234346630646362313733663066636537366362303639333563666266353831333362 +65636439313037663164386264323261343666666162303964393466393864643839656563656531 +30373035336438613830316339633762386361353663636263336338366131373238643461316531 +63303865353731653739656364336430616135613036363062623233636432383536656434656136 +31353565333736613366333039373863306136626538393163343462303834653932663633653235 +32326466646661643063383263376139623832393333383238613061323935656263313565303863 +65393931656332353734666439393638353830326165343362323463643739323937653165363730 +32623863313031363930646633633435393239383839356332343662633265616461366332326363 +35353763373734656538333633633234623765343934346534303839373734393762666538623533 +64356330626665666166643663323638653739313565326633653235363031616564653431373534 +36616361313964393937633562363834623130636230326166616430646237306337303732653633 +30336535623739376634306432393666616434663335366339623865613366313035616566643232 +35316363626434353837363936396336666266373966306132396139383362333466386164666533 +35636563643738623063623465346637653336333237346662353563623031633864316366373532 +34346164363137306238313235326638366333663039383939373730623137383232313364646662 +64633334616662383135343964666662633232393864383039313630653261306162326566633732 +66666535313938653864303164313464353464646332326461323366616330656566653937306364 +39353339616435376639616432396565306263366561313134613636396233386365633839326563 +37666435616531306439363439386130333137323364323137393638646337633433336332356336 +62383063333561646261366162623237396662386461373362333562333734306635383536323762 +30313064386336383662396331373436343665326637366331643831353834383832636363663835 +31323734383834386633393466666439313164636233303537303135663736613437353036396233 +36663035363935663430303934303962336263613335333630313939336665663661373766383837 +32363130383462613261393037326234643639313036333237656166613434323138613665323865 +31636233666662666464613762316166333264636335303736346238613361636131383261613533 +36336462613332376430646566376163616263646330663939643732323566653031626332336639 +38336162343737663735663235613761326533373662356366353964653265643061343138393666 +66643130643236653634636261623930356434613962633262393034353134393131343761363330 +37616334373336353264363131363166356432376333623964636566633731376164383534616235 +33306361623734373064653336373665623463616463343364616236366233393662636266393237 +31636337373963306230633130333562663865363037373631653036626335323963633861303534 +63636437643964343366386437363531343635306536656366363162316339343737356130333166 +36633234666639373762393066346536616661656563643862653938323463646132656365373132 +37333762336231626537303235303534326564386638663664636261663637663735376464333265 +33373733316231303864346430323131306336663363323731616233643032663837383662643330 +34346231653236313861626337323864313831653732653035326563306433633135663036326539 +66343761623762646139363764386235353039373431363836663762313838616463353261663631 +63336162393736643232363433633662303935646235633537313764646663623661323438316236 +31323966636530656137316165393963343462313130656336643434636466626133383862343438 +62353437353930343135613032313164383735396532653332636663383664376130323635373266 +33313738636233343935646131326165393134623835613766633830326439616130666137343332 +37373562653766333865316463626236396136333730333966353037376231613438643932393134 +64663230653830643362306235356430393563353766343362316535633032356433356633623539 +34383733656234663763326565353666303162366161373133343961653633663431656433666432 +64316661303330343738663738303761653761636534323333653562356231333533623436653438 +33313134333930303834613239356166626336373633333430393565303062646434386464393061 +34313238303261643464613361663636303635653836353064633364356265373863636163616161 +30346331303964613861623962363862366466316633386266616537373661633332353134366361 +64393535333830316235613037356135343432663139396437636236356665663935363034383737 +37653463313864633233303934613036643335653432346437306265373565316263393730613461 +38626437386131646431356662363930346535393262663134386530623562666537386461623736 +64613766376363396439323733333134666133663931313164353033353262306335373864323238 +61663166666466613763663538303236646665383737303736633131653766653566646138373465 +31303961333664663862613433303430323962393065616333336561653136656432623538656437 +30666631616264616164373861323666666432363030663838623637303862383631626333393466 +38323164353561336439383834623035383561616531373332303835383834333731386635346437 +36336162613530376363646439613836303934303834333361653134613632353366333462623139 +30653339363935333335383334393234376461323663656235643566616461656465663463663734 +38383865353537383535333666623035373465633031613037396366633635323733353739376265 +62323230343133373236656566656139306663636531383735363963616664313634666661613361 +62663162633038343830346433346665623965363832366534656439373136623739353665303333 +66646161373438613962376432386137326531616133663632316632343030316261626362396166 +36643530623864313266633332366138613364386461313066343534643037393633626433373464 +65363032653262356634383435643836623062393563303139653835643735653538663234656135 +32303231323531306464373337313435366665333436363462306463386135623862636363623837 +38653939313266613265323963316433633063663738343064616237616364356162663264333631 +30303564376534383333323566343465646533346238326334323332383564373863626161376233 +30393338343363353139353239343765366363326437643638666136353831323836643561613532 +37653363653963313431323533653764373766633833623932653263346161333231626538323238 +39313962613261656663636635393562383261396532386461633831623463376133663033353061 +64303232303565616437396439313930376431656138626132326236303339653332666331343465 +62643735643766643066343531313462646566653832613666313735643766363931306239393835 +39636239613034656138336162376138306338663236363930343738323362366230373934313565 +65396533376263316664643032613239373331643066613466336135623063333635353232623563 +66306232313664326630383937623637316361306338613362306431376263366231333437346336 +66363836316364313562353234313262656435303662333131653662623463393165663336343365 +30663134353362363431616638343835343137363363346163356133653965366361306433623833 +33353666653338613062363337656132353739636137303533376131666136616239303161326539 +66626166383830626635663233623435333465363566333065343632626637666230346361353833 +35636531333630363035343333363134383331306332666639303233373237383633313332373934 +35633764626434396563303064353035303330383638616539663738316438363736393936653031 +66373163623737613966666632396339616330396535323363363037633666633135343632633637 +36396135663665666464303466353836623932656539303334316132393161386563303963323439 +63666363346637313634363661346135613330336435643866616166363232393731366531393933 +38373932353536353137626330313036643531646134323336356437313435333161383464323862 +35353264396530616464363832623663623933343765303936626232643763313435393437666363 +36643331636365326465323966616633643033636462363133636263343332643139386236643131 +64626539303134316565623139303038343832636133323161383832623334653634396232353033 +33366262396338626238336137646138343131386463383335363061643538373366616237336561 +37646530636231323032333263323462666166363261383935363630366666383762316434353737 +33363462316165343862653333616331383438653930373764336661373939303865373039366162 +38373962353934636166313666386633386136636532646631333165303133373034383838666466 +34343637376338376631373038323561623664623330626536616261343833636332633261656166 +35613965643661636534666237393732613135386666396566363361303635303633326137623637 +33616636626339656333633437663730363832366365343363653361663036626364626336323762 +66643064613963316263373939643263393037623765616336653964353036633632353134373230 +36306462616136323336656338373438363264623066643734343863393164383964306432643136 +39313966616232613737313365396336636234313938333963356632613965373331393966633031 +61306435303230306333313261306663653564343438636262386364383665386638396433386432 +31626566363035346163393237323566343661306361356265633030386136373266666666653130 +38396336383766663962653230353037666536383038656361386637323761653431353062353832 +66376465303263623766353533323435396663326666613561623765633331613364633731656236 +36353330396538393462636538343534636236353433353638313461383537333030373838363238 +36623061333066366135376235393833353030636561393262343033343666636339633837643537 +65393262376361646262643039623161353764643037343134306533323833386164353366343833 +65633838303539393737666333643261663533353333633631373062373433313431333963663430 +35643366383035366538366236646439633937663836393730383238303561666539353964656636 +30393034306638636364313865363965313234376534303861663439313734326339653162326533 +63653032393236386131616264323839663266373064313631356535313130663430323337643861 +64633164343131396265383138326463363236343365313236643733383934626263666463326566 +33653033316337663035393637303639366131613764333264343938636433653339643766353862 +66363064636231363133323466303962383463316636346236386532346430633934653661346439 +30316664666635633563623530646530633634636339316137646134313436393534386534346666 +37616634323766623761616165383462633233376335623837613163643333346633366632313864 +64386435323563333034366565653831353230633134623864323732356539336139393137353730 +39623062636665646430373363383664333735633636373963366361346531356536353537653030 +33613438633265643839323266386138643731353330623163316630626363346131363735316338 +62346334323864356336386638383935306334646561633839643230353131363638363564633433 +30373139303463643634366530376165313165666566666238323739343961326666643266393131 +35306234383838366166623839363137616666333663623031373732663863623530663964363136 +30646230623530653436366636363736613866633030643930643337363031326661626665333965 +37396638343561353961376438383662343934343361313361396539313334326230643565373632 +35356633663932303861636164663839633334646437636333616234313664323562383865613831 +32363736323731633837373235653036356330646666313932633035616233623137353436656130 +30356462323065363166376338643739306631633539363963363334663638643166333264623464 +30633865663838323138393164306162336538613238316137383333386561336635366131336363 +65356636363430386463303639633162366131346435633462626632323532383961656131393763 +37633130626436356539623939643339373136633934303339386437306362313464613334303033 +34303739633464333935393365353963313761386333363931383035396563623365323735363439 +31656336643132616235373136323039363032653162316336373133333236353234336533343863 +65306631323432616132303031343038393063666132313136633265366531316165613266373464 +65663135616333623338346437643334393366333531356237333366386261366638303866636564 +64383161373434643365356339323534633337643838336133323336373635386536643830613465 +39626666393163373065626131323863303135303333626466613038663736366432326665323435 +39373365383936626165633262653463373061303439316234326164333534356131376433326265 +64666237373566386638353161353835323733373931303933346530623038333064656132633361 +32646531353037386664343163666262323832383763663833626462326363396131663830663061 +38303064366466306365653461326166356264396434353832313032343435393964643139626234 +61643333623363643037323562363264656265376134656161383330666564363835373137363837 +37316438373339343730333035613661653866636261396361636663653936306662633132613239 +61393566386666383764366537663262623966383735336232373265376466363066386262376435 +35373461316535366236623262333032396561643032393066306336383731613264363235656637 +65626535393837613930333630656336356433313833653533323433333639393565363832313832 +65376531323137323030393232376466373135633736663464376333663630616164386237353133 +33613537323230303633373131656430653538623366333430363837386461376239343639303233 +61613437393535333464333239386164386330353438666630643736363462383137633737333262 +66393639343736343830653161646330303634626133363764656462383732633635313365343365 +34343335653732393639306361356563353939666462323136393166643835633539636334336638 +33376336656632613365383165623862316334366331373638373138646365393535303961613738 +66666264333164616433323264633664303165303137343036663364343862663936326231646335 +64633835386339343534646330376638326331356264633630656534356565656365613362636233 +63376364396661303661336433663539396238366636306630633535386262356562353664666563 +64343730353163623830383463373933343735383964383164316133363232376437653836383462 +35393931663739333163303038636262656261616266626338363937323065336531396663636330 +37386437626365643761393939633862363264313934383433623735363139393532633836396264 +36636633343764363161653830663463323730333733323739353866336234666233626463613166 +37303035623631616465353631393662626135333261646336343939633563393664656232343933 +61323766336263633139646436303966343139313762653137663135366134363962623635363439 +38656563356162336233626561333033303339646361393232653531316137366630303838623865 +64666335333664653262326462313466666235633737653639613562633433663433363135316430 +32306333613266363863393666616639333339316335393336323130346338353132663337336333 +34356366666431666364366263383834656230393166646435623466663363613731353035383362 +36383236316234336365663730323737653462393830643938373764373433376262666631393636 +64373262343161616330636165323538616238326535393930663966343263653362666562306265 +64346135663639623566616663633133336635376165343963363736356534636534653166326635 +33613763616636383835336231623362383233643936356431306266326364313761343034353266 +34653931386162643362326135376164303130636132303062663337666133613766356231386136 +65366230333233353765376431353663353434323937623466303863326439393531323632643465 +33323863346334323237373531303437616135656137323632333366353636383563356166333437 +32313234313865666332643135643637393965613430376233373362616462353638636464616538 +61396232663135616136323166306534323230383137653839393533633733376339666531396364 +61613633653131316233333262383237376236303339396436333962623032303363653137333066 +36663631666536393966326638313163636631656662373831363037613135363466663466393038 +35613032646232343438346134623664323839363435353535346163386264353265353366306336 +61313437623432316630363564363237343965353433653736383237656438646338346439666232 +63613135303436636362616439303438366535326264323565353265303763653562656435323434 +35353033336365633162303533633864336162313838633336343139313263303361363335316630 +32613766373964333732373132613338373330333063646537323631386538663536383935346363 +39656535626233316564666432303661613563386533393934306635316432313936653536303336 +66626137323738353034633534336432353038386662626133653332386661323838383430393031 +63663563633934373030336336366437393661323931366638333735633361383736346463623237 +62333631383035666636306132333766663132653335613261616435613865633337623461663736 +38376265666531643235356132666134646664333265366430656534626162626536623937396262 +38393138643232333065613536653561393465653066313538666536353732613262663933643736 +38353431643463623364383763323539613238313561366264333439633565336336333337666637 +63356136663862383636613331316339373936663237306563366533323937326162363962373532 +65646232356463396366343036356631316235656135323436343731626632613433313066343537 +61383631346237396436376435316666663462666136396631356638373239626530646163663536 +63663839663431646362376339326163313463643365623765313634373330643438343336626135 +39383636356661346539663731343432393537303035336135343131383536396338383266643963 +61313936383838613536636138343936363339373066303266616530636265653534623563656330 +38366435353431376663393138346439363037306230373135656432323863343362623463636435 +64653139386231313965336233306339663336386463636235613130643936363238333637313033 +39373639313631636437333863383365313430623434373535376437333666396462376263646362 +63373832633635353237333935666434343862343265663237303031633737623432353163353961 +66346430663137653034356535313564326162643034633563313364343532623431616530653237 +34636631313039643865363931346433383932356433333462336137623436656361373665663331 +37383137383664656431363133636666353637323361636133373666396533333566653039343031 +34623564636166666133613861376433363134636363313235616563306236353133653030653263 +35353730363664623465393166613537643263323234626362643335383162626533373737613131 +35373461663930303533396139336132303065313736333236373938363939623861316438643665 +62643030613130393739663163316564386339656333653935646136613336313132616365373265 +33636437643465646531343937633831383136336630623066303963383139633637333739353737 +37303462363062333363656361366435613566353436343066373163366634643030376261396465 +63623734353765343335653665656562616138313732386262303136613461656530386264303862 +61336432373835316432393835356136336166393933376663633862366436353239643433366362 +36383664363565613566613565643663323032663632633938363239613435636533396131646264 +39646537383662333261303139356663366437363430333235643539373036303532343565323838 +64633561323538316530333362656464643961353936393932386334323635313136353539633766 +61323939313463313062653566313236623030366462663830623235303036393138333133653832 +64633762656434356462646366333536613830646663333864393937653239313635373231383465 +37323966373532343132623334376634643137623032636665666435313063323232346238383534 +36656130306530333763306436643864326461376336346637623632333266323264646539333336 +64393536663663363933346638643233623534376138333161666463383763623736623765353531 +32623335336632363362616631653638386236626265313033323132646364643164333963383734 +34383036636163643530313535383631636633623135663665666634353936313964386635623564 +66663938343234643961363466353232386166376238343265623730306539653763363433366137 +30353738313138383866323835636134383834653062666332323238333234373662616136396330 +31646232306666633763393366376435363231393632316664383132343662373234616565306663 +31653438313561656136613064386437626430653264653362623566626463653034396635303639 +35666435333339636531326137396562656365653338653064393064643334653761656137396263 +64383135653233663435363061643938346137316538613132376162666438343666343634633265 +34326363373035303437306430656339656337323736666532623131346539623735656133376635 +61313165383432353430373766386465623964326163353161343239646332376430633063633436 +36343166336161613464626330366239306564373838653532383962313361616561346565303535 +66343563623364306432653932303731663037363730623231353430653664666632393232626239 +32633166643833323865313565656633663864653663393063353739623734623865393133343636 +38396566393632313633333834376564613833383836303938653132663730626335353233343464 +64666235366534386666306237393534373530363166333238346262326665323833306639393264 +37303531383530663765393036626536303731656338363137646666613066646134323263316666 +39353834616565313865333536616334646665343462316539313339323631323963656539613039 +62316135666461396536316430623039383839383565633638613565316461666165643835633966 +32353161366638343566616362316631633535333938656533343539306537303062393161383238 +66346361376238383563616136366438623031343038366565343964666638323930323737623534 +34666530623762383163636539316538313430626662393763303562386431636332613737613933 +34623831666231613031666534336237373932316538653933633761623338613966393333353539 +61613662383461386335316366383138353963663932373163663964633865663934373163623038 +64353361383938383665653035653530643739393838393332616166353730623331636636363033 +64323738633833653139656438653136326539613539336562383335343766656235646437343864 +62663238303763643336366630366366383232316431373261356531373265363437376231633962 +65323633326138363131373034643364356632323335333833663836663162636530383137316630 +30306238653832323561633630636165313938663138616431383635343762646230616565636563 +33346638303036383066363534636338386663373763383439333831613165303666663638663232 +61666261623331653462356538643335323733626238656138353061356462323561323530346361 +32396463366137623239303037313962353537646561323936383137613538303061336133333537 +36616135353034386532323162366364623565666131383539303237383734626335633962623737 +62353937336334346431373832623861336239613063633533336439393665623235363431313331 +65666439333665396639653764393732396163343033336133363166616162653436353138303762 +37633664616434333166366265363333653136383561386165363334336533306631366561356431 +64336533323563303566373761666166643461613336323763386239343732333038653731383962 +38353265656139373237643338633939346666323561623730666335373737386264633238323833 +62656664373734353130313431346633343230306533643361396638643265616666633937366636 +31356364643861343063616563363761386166303439383739653233386466626536616230333965 +38373832336339346338363033333064646537383533613633636564346166393131633263393662 +38333064323061313566333133313761663732376566366132393237303363306430666666616439 +32393733383037616630313736313430313531363133626533353966626130636130386434373935 +64396436366336636132626134373237643965353066346630663166383965623734323236306439 +32393533643033363862306664656662336436666231613332616138383539383366366465643764 +62343031353366323561393936666663363736383865366166636462333639313364373334636132 +37653865393931353238373932663239613361333336666162613631333361653562613833663963 +36373837646462346463623264646138613063306565373762303063663734633232353561306134 +66316337633561633962336361656337636663633035653636336562373834643362333938323136 +65336539326561316261373534383033643265396238363263666434636666653764373564316464 +31323661383064343238303138626237626333356130363035633239373834313535396638636336 +36623261353531646661656233373631643764653031303835313662383664363763346630323036 +64343137373933613130633365636162646538623564363166396565643738303136313635303739 +39393935656430646432343134616465323464646231633962653863313565623436313862656362 +32653863633361363664323462636661336233613030646539306165396637343932623634653238 +32656232336362343763346463303536313965333238653632393732366130323764646464326132 +61326635643435343062363834323163633463633138393036303966643936666265663334613636 +64356263363065626437313462323430666562623963623637333366373762353836656135373161 +31623033636130363130373966343366353231626461313633353134653834316261663933333263 +63333933656638363432613063356265643635666133663662666530366630646462663930653738 +66653032383234393262326365623962383662623664386236373531356436643533363633626530 +65336465353536303465663066376534626236306233336561356237623563396264373130343765 +39633762343330656537336431316137373562653433343532353963656132346230663632303865 +64336434633232336262643065366664366361623938303961343437353963636566363234396139 +36366539663462626135343330656362626665333731646264613132623866386437343064613436 +63306639663232316239633662303435646565393939353865353030383738316565353262653733 +33316231653464643433313634386235376337326135353666316431366135303437643633333962 +34353330333262646462363566343036306361653031613765643533396166313765323632616132 +33393365626563613133363466653961663637386264363233626435653261353665353863303536 +33663933313365343833326132363736336638623638346363316361353336666138633339386532 +34623665643930643530653431613463666237663333633635373864396535616261663435383034 +30333937333936386364366434663436346466373465656336616263653665623438343036646632 +63316263643663336138303861396161393161313939393639643666313662333836386230333562 +63346631666639386661613035383132633336336336313831633435383035373266333831343363 +39396230363337316431326262636438386538333964323537313837643831643536353063333462 +37633236363837393439326431643763393261323739643839343261323133353531343039343831 +61393764616261393230343833303035386133633931373364316133653566313930323663616536 +30313839616266343065363837393266393563333762343065316666383233396661643538653739 +30386138656161643933616263393162323738336264633536343063363562383238363439376332 +62316231383865623834353635303839333264613734626137323239383834386133613037393637 +30613963386438666538323036616663303962663139313833653265353633633332633339633634 +34353963643638626164633065306662633333636139333830636539663330323137336165626262 +35653362323837313831386261666564383163346431613036313339376266303264363461626637 +36333331313463663734353862336665373664343238333631393833323232313737346435666637 +37316633373638373330353838393335306530656634316333353162353339653532303862343239 +66633233363936646566633139623364623834666666663065373234333238306137333134616134 +61353735653031383661626532636630623636666331636135633136313435393038366234613864 +38363431333637613435643466626637643761616162353463353637383935333364616535373665 +34643961363937343738626536336166393538383534313631383430613763643239316132353236 +39643530633665653032646361653637306139323636363864303839633730376261373266323033 +31663839303762303362376139633138636139623763353434306432353463333038653038366331 +61396665366130396436353462616638343934626339393238303337643136333038393565343738 +63646131363665363666363133373138613430316135646139663437333262663332333338613830 +35623931616335356264353134386563623230656262393032613334383832353130393661353239 +36633463303661383532366337356139346330326534646266616139626533613666663264633636 +36316537386666393632613362383466373439323364656561336361353638393162346335326664 +66386136653463616137303631343238613031383037626133653632623736393834643939666136 +62616562396233626331613238333635663539653034616165363765303863636565666561643239 +31326165623837333637396566656336636265643162666664623034333633323639663530666531 +35643764336663303037653662313062356533633762393036343930613334666439323739323433 +63613435343265303764623639363765306262343861636436346534393532646631383735383630 +62373362366261346334623437663531633964366633623062666238373533616563323163316331 +65333933393363373263323730396134373662313330633239613430383132363434343935343262 +36393461376235346636366235346433323034396438316137323866653237616336616636643930 +61316336313235323933313063303566386263346535383332336537663533633130393139393263 +63343939383234353638616637633861393465346630306461626137383531363361623937356131 +31393865316566363662373964366331353464623431393262396361356166346134643265316336 +32663637343564316433333730326434613335383666646266636130386465666430366532396637 +65316537386665386136626465343734623362336564383938623239346166343131353663643266 +64653734623162613366653336383866393932656564343435653937656139353839393461373232 +62343830353761333339303061343764623331616235636438623234663662366632633065623463 +64643833356338663335393935653333626630306234386531623963353138623463386438336337 +31356631623639306465313539636565663934663864613230646233316164373439346362343837 +37343763656238336465323435323136373731363139363265633634663230303633383034363932 +38633130343130326134613532643831633234303065633937326461306637663636633865383536 +61353131353632666637316237613261303133633737623731363235343563393965663131636332 +30613066653663333462653030613231653438663136353233363237646435346266613635333231 +66393830646135383964343636303362656338613031653464613034373961313939643339646131 +34336230633766633236336562383431313133633838636237376635636530663638313563666662 +63396462646538666164333863663330393963623263373334626131333661366562646130383134 +35383365333839363265626237383564383265663362646533323732306337653164643262666538 +63383130613765346465306463356533323538383165326132376639366661383236306334303035 +66303336333932393036333239666262343834653839333932663733623638353766663534383139 +34393764313765316633396165316231303337633364373939613263663235323333376163633832 +31663862333866613536393265313437393231656165666639396536353934613936376362333038 +65623635366135316535363461383937346135396230383738323837363962343263323239663462 +65353364306230343433353634653930643562663531363064336433633039393031643831666664 +38646131356131373432356462323036636636393364643863373462653136663736393166613330 +66313762306663663166393765376434396439333835356535303334326232323461336130306534 +39393466313161316137313134623139363861303733363063633038623637383234336362656439 +38306437646164346432666330643830613739313334396233346432326130383335353963333837 +65646564653164373565633563333935316465366465613137653262663931356235653433303439 +39383339326339393431616330363162313135386134626237613431623066373631303233373161 +63356334656335663163623864303836653762306461363466643964636630353563393661306531 +62663361356263373037663633356664633735656139333131333938313865623161396438313334 +30663333336535343936663061323639626233366339636634313266373161653333383037303866 +38313935303234383662653433363162363761356166366534393433343766663662333432396635 +64633062333139386463353334333261313261346563653535613739613835373363313864653635 +6639 diff --git a/etc/kayobe/vault/OS-TLS-INT.crt b/etc/kayobe/vault/OS-TLS-INT.crt new file mode 100644 index 000000000..a4a64f7e2 --- /dev/null +++ b/etc/kayobe/vault/OS-TLS-INT.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFMzCCAxugAwIBAgIUQJBuLmrWDkZ7Hb35ZZYAytYUTjgwDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAxMLT1MtVExTLVJPT1QwHhcNMjUwNTI4MTM0OTMyWhcNMzAw +NTI4MTk1MDAyWjAVMRMwEQYDVQQDEwpPUy1UTFMtSU5UMIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEAvUBHYw4JkOERu2Z8NwPRnDTzduN2tsrv/dxHVU5n +XqN6bPffFF7SAf0m/IOlhPaM9DAcFhIWtCnj4thbnMYA/AZx7liEWeicCBGeGznx +jTGHm85MJ0pt3rwpBJCHA7GY3c+TWoxcqWOS+62Cc7IzGviPj8YgpkUjRl8Agza+ +TwgZ0o2UYz7gpUqcWpj3RsCoGhvBixBxdpinF8SxirPCK9tYMudxW14O5+IxShXz +dyhTtkDvxNTTmc27PRuX5WGJq0BKzGBlGYW9rT2YPG+TCMf+t0G0wFgVx/4P4bU7 +aitNjd5QuEmWz8/VdOTysfFnwgcbB1z8SYpDNwHCx8f6/7UZkyJyxfaiEqq/JuFe +xVLIOzpAxQj1KXj4xVO5p+cHPetLeP3KAvhNAuPlohd0wqzV8Rj5BL0EQequXode +Jx2k9JRlsfXZh4wv+Fh8KsOFFmLO4B2eZqWkaTJ53+kt/LqoJqtB9IkVmvEP6B7p +oNSViPvoA0zE0IVF6LjAKo1DxQX0wFd6hqxM4PwJCsY4WOsrmFdBkguapa7iFBRB +PuTKki2jtti6D5A/JqD2qd5HZHJ4+b0wBRlJ2G4vGzJ+h1dzXnK50qP1IiSW3wxX +7A6Mde7oSJrsv6HTvntqELRJX0a8YFDw0PgZIgvWJucPwUVqdoO7/B93tSzwp85O +hjkCAwEAAaN6MHgwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFCLaSl+L+PxeHrXG0K8FxOcXvEMsMB8GA1UdIwQYMBaAFKlsl4DbjRfZ +PwxXo+b2heDh/HhuMBUGA1UdEQQOMAyCCk9TLVRMUy1JTlQwDQYJKoZIhvcNAQEL +BQADggIBAIWHKuvktZd5kV0C+hyQ+SYVAVeqYNl10nkNy8Msl434FwBp4bByWnJ7 +7PTUROPh3qYhsXmkxg4i+lpGmL7d+ZN6WZ9WzCGKk8uqZ1uzZEGXvenHOSOoM+l5 +x3sDWuokVL8kZmOU5ek53rhWB2RPyoTvYb7gypglf6kEb0ZK69bSbYcKGSlepDZw +iHtOS1ZWn+3N0TZLSVUzkZhAWwrIw8nUXcSfsNf1OvY7tNM6FlLDQ3YY5rr033EM +VeSBgZsi2mi/F3cd6NE1nvkasyny5MK5urVHYWFSDckcDUl0EirUv2XLuM6i6vZM +gJxK+CDxDjO+z8krPLKXZ21AVjHu5hSBwPONwsPFUfG/18YH2/01ejE1UtwgHcsU +ylhtrifK+QtMYhHFOTqFbac1N8rlmD1LS9F9GgoD9DmI1zfiKHLTvuf8W7XS3wxQ +uTKUCpBHuLFxHiZTWMgc0dz7GkaXNKDBe4j+E8VzuXBwuEeTm38BrrmScwU1IJSp +iXoKQHDvw47VJlSW7b/rOnHMYRJsYZG8MdMb3xc4uLPdACd3jP6RC4/cqk5m/90F +0XJtqcQkiryUpAPSOssN0H0lFojOGAzHUR4bzTskiPZ/KXUE6T9Xx3pFh9Tw6U/o +r6JPCWlEBjJOcjHM0sAU498JHW/+WjJjRQrxUPqSReM6WzPYOv0x +-----END CERTIFICATE----- diff --git a/etc/kayobe/vault/OS-TLS-INT.pem b/etc/kayobe/vault/OS-TLS-INT.pem new file mode 100644 index 000000000..21312ca93 --- /dev/null +++ b/etc/kayobe/vault/OS-TLS-INT.pem @@ -0,0 +1,354 @@ +$ANSIBLE_VAULT;1.1;AES256 +64383530346564383334323937363463303231653831306635353931343564386235353563366232 +3761363034333532613166323631316436356661303031300a383032623034373038643932323835 +61303834653461303333396662623937623933613734383532336436363861396264363234373732 +3562303663333238310a356163316233623162623837396232353461316438336364396438396636 +62393261306461356564323863386666356138366337633866386531313435393630316263323535 +33646662333936323138363561633761323838313766623338316465303139353562636536366232 +33303866633933383566386437376331346639383335373566326434313763656335386365336661 +36353763666438376336393462613636323635623366646462353432643134313361663866376362 +37646132626434623861303562336536653333633732633930383433303735323335323230336362 +65363737643732393235383361653738316563613666316561333838386165636232623065333264 +61626530386632343465303134313036663334626263343439623531383730383238386339373634 +32656637323537616164663936666135313164386564653863396635666636326437653338666363 +38356162346431326530303763303032613337653232623333373965363862363537376532393339 +66316131313266643464373331656434393161623961643264303365646433623764333462616334 +30336130623866336532373434303334613837663431323332303764636434323865386663363764 +63303138376465653064313539383937363561333733366137613965396231356161386663643732 +38653132666530653739363532383064643238326131666462656233366334393334303066646132 +30653039656433353863656533303963373862396563663835333335363130636562616237633565 +61373035316566386364646238663436353762326336353536616639356366386461653462613532 +38646365646435303334623436656635346363373766653634336533653438663466623234356466 +33396463633463663233333737643466303337656433386233666130666634323731653532376661 +65303337643661633765636638353635656433353434326361356262646565393031396338303364 +61343136356662653139636562633439623939376330336462623331383638623665643233653033 +36623562383664303739666331646338333861376235386435636137643636323462346333336635 +66313930366632643063336338366264353761323431386664306132346532356430333631376337 +66663161663534356362613161616562393263666465373566343761396539333234336366303239 +64336332663833383036623838643064666634653039376364323430323133623565366131643930 +34363164316638316165653161323833623563626339363661616439346233353236646434613538 +39633737313166366439626161366231396664666364636334373231363939356562656638323063 +62663633356666333833636632333533623362653061636232623234383562326536623930363633 +35376363333439316162316532333663303538313434396436393966383463343833343064663061 +33373564306332353062626165346530343461353337303937323730643638653935653662346137 +32323030343561626337396433646264316464643434633837386362663531633837646634343461 +61656565346632313731346536363463303637356561326363636662626337343761633261613466 +64353464613334393934623332363866376234303134306538616435653637343762303534356135 +61653561306230323662646666666664376534623363643130376136613162656364646538383766 +64383166323863623834373837613532653531373837643164653664613963653762313530303231 +37373866613335623263343638636437343064613931343231383061613862303639386539626330 +37653561653039623731366430386230353730636333623163663031393061353962363362626634 +61363561643966656263643862366264663531353336323735363830363762623663363162633330 +62336165353436643133356164346565656634323961663937373461303630303562333530623034 +37623464353132316137383764616665326334353432343162376133333730316536326630633036 +63356663643835376261353737636666363138633033333637393236626633333138623135313166 +33353531333932613539633434386265336539363065383362386339623865386530373633663862 +35356366373766383739613534636235616165636239663166613961313863613662323136323763 +64613730336133376134636538656636643962303736363338363737653565613630366364303439 +34656262646431333463353334643863333231626562333134336334346637363531666131643634 +62393733376131656234636362353462666363613039386532636536363439623939366163303239 +62623763393563363066356337666539653536643861613661616661613439643131396634646530 +39656164343430643934633965313364396133303233626631313038333966643265343563656166 +64633464306137373633633032623264353266366436356335663730666666336261613866613135 +33303866396630643630343034636161376230643666353264383031333930386431643335353662 +37613137316563396232636437383732336564333932386361343438393862356664396266363030 +63643732383635643362616230656563613664343562663339643963396639303034363838353339 +38323139346362356238363036316331333233363165396430313030356531613337316330383835 +61366364343964323338633365323366343933336463336666616261666364313536653938383166 +64656637326266386433623739303338303739636634336235323333303437393330376561633963 +63343335356132656337386134313465303937636535346565353739646234653362306662383663 +31616161316232643636386630623164393664343062343432323832303735323963336566636338 +32333131613561326637346433616337653439326165633462643136613233316633306266333736 +63653739363235383837333337316565646465643264386237373735323331386638343936343338 +35366263383330303833363134336136326564383463353831316239346539343638636634663464 +34633131323038613135303563343134643233663865376139616566623564643039343738656634 +30326434626430316139626363386132386639366333613437373463363738623730636264306536 +38623035313739646234353032363766303736306534666430626237316132666432396135386633 +31373666356261376463316430643066623336343361636436623339353164396561376665396366 +62366433393938333539636234633832646235393364626666356164336566656131623135613035 +62633862333735623630393439393434653932303731363130353539396564303230306534343434 +62333164623439343364656262626232333537613933313936343764396162613536346230303364 +38616438313537326238343438383539623262326538626565393037383864353833383538663937 +64353334666564313034663564613339343533313330353162353435613539323031613134363834 +38653063313563633634356239303766356462363030653832376432336331663030303762393962 +62383061663963393436643962636565376661303532386137633034303338373536333965333536 +62333534666361336530383763366235313163313865363839343832633062393632313861346433 +61353539613433636462353065623331633438643132346132343861356134363933653065653038 +64666434313331333235383431626336346231623362323934393763653337623938326432303063 +33366338623462343738326662383664303563313137313439323234653861316138326133373931 +36646533316237663466613165393464353032636231656662656433376663383864313539383930 +66343934653538623134353634313734376639323462626463636435336432366437306236396335 +63616637366363306534373736653536343464636430386530386432383331366537613365643131 +39376138663937333036306565346436653036326333613833643437366534623661393461356566 +31336330613163653365383036623939666238613631363366646232386138653338396461666266 +34613730386535653331373031303039386633316435623139396436313239336532316164656538 +63636639323562356630643431663263393363616134353162333630626363656236303265356564 +63663963643335326234343533303330653161363964643133623739633931343365613931633462 +33646134393339376132363335373434353538663534313536616633353837396132386636363466 +33313738383932356138653238626162633864313266356132623331323937396162356630666162 +33343332323939626234386364383334623637323237303532313863313935646461346634643333 +32383137623661333237623563316339343030396464353762383039626338306139363532663733 +30383061626132376530636136353263376433626466316330653732346334663533623131336135 +37336233626662383762303063336634636432643833303066656533313365346136303261353564 +38326535333231353832326639623434303430656466386561616665386238306165373239643063 +62656439346634343466653832636461343731363739663733653464623631633935336636313937 +31643339393738386230643261636435393138396130376338366364636362323637656636393234 +39396562323231383765353536636261613539623963386163623732373963316434663265626338 +66613161613966643433386161336339373636396437346166663830343165393533366432373866 +31336263663736373863663637613933646237666634626461636566626530313735383534313234 +35646332656339393863346539313630373730653732646536396338353263333935386338613039 +61356534346332343835643266313737363063333436346139313339336332653666343137623666 +62323237613438356561323366393236663038313762386130633932386364303435366664303364 +30633462323064666564323832643366363837303630643435643764373163303938623265616366 +62323238633635616133643966303532656335376136316633376365653639613332313038653966 +32343031653966366332303161333934376438613636303139363234333664323636666630363639 +34653935613230313732363562356265616430643938303137616335323431313765656234353261 +33343764636531373630363763373834623638643166326666313135323833303161633639666534 +35343364366337393861313264346338646264396638383034316438343138306433646136656632 +37316536316232366435313433343762316530633637346262623539613035333134626332333830 +62666139313763623338333434383239323235333836386433313434316539323361653939326437 +37366238343566346630656239343365303764646233386461653732633938356633373363623865 +61636561623834613034616635646137343366653330653961613930343362666439353165333332 +38356630363336383661306261653366313637643033386339633332623836343665383961306137 +35323631313039363062326562316630653065656363656633316535393839333462653734363437 +61393236663231353135396562363936643763336333623435656331363039313833386134373932 +39373332376334333436346335623837666666363664336539656564343562346633303233363138 +33303662356533356265343062346162663337373239353365666636316636346336366263323533 +32393566313165343030393261626231653661356639643466376335633832636135636237353831 +33393437376331356239666566616635356163346632663361636431633336396636333762633639 +37326534656134323232643161636134666162646434326431316336383162393965636462336464 +63656130336239653061393463343861313364663134343635626161383831613261353365633838 +34623634353837383631376232646666396666613164383766316233316264303761386336376338 +66613535353264323862373461633939356664643135643336356235326333636262306433646661 +64383065313161616537323031393266356263613637613861303432663962306333353136633036 +38663938306138646236386534326363333031346338333239356262613736643663633762306134 +64326233343431663562623932646461396166653637643235383363323635373062623164653432 +62303537656438363233336332653765663537366565656566623566373037303234343866376431 +65363562333864343334626662666266636530636436646639383439306537633039636363626265 +34333538666664636561376262356232306665343464636639396566656238323862373336333662 +62336666663735303332333332643331353532373062623961373132623433333963356533376331 +61383432663765643261393963353434353065363736313661326363386461646130393939336434 +38633838626335666365663030313765623134653737363066633733336638623431616639633565 +33663534633934363637383464313037656365376534663764313538633135376535383738663931 +63323662643337313932643237656664343632323833636233623433643037356362333433376261 +61393262376437313132623163303262633136666238393565663331623961323864386136626333 +31366437356363643434316266373162323331623439336131333438396632383532656234623534 +37616235616435343930316335613438653363616136336136336264633265313734313933346638 +35383865616332326533393336333234623238336639666665643663326265323730653331346631 +33346564303634313338663331636434366332363830613631366164313130346138363666333434 +37376633653338376132343737643338653362333864313462666338643139333036643436333034 +38316132383563646463653962333264616138353365616634373762393830626462643330653461 +66323862346231313038616333343231346461333930386139373665663462663836313934656263 +38353664396366343435666233363139363331626535613034633031313265623638666563383466 +32356336623634653336663933326661633664323939383031356264623132316463633663363035 +37323130633563623536643562313233303337393937666230323332346564666337306466663533 +30303463653534653061386462353533633831303338656434383531383336376561323435656662 +65303233376633633165643330303338633139633661326239656338643363613331616162353335 +31363934333932656664346136346338633332613634353632323338616438323062643863336565 +34663332656464343936333466386262656539623538353632383637323963663230333634383531 +61376165343132326538363032323231393432336339613433633936636636323636653537373437 +62313530323238646561366433626363666236313932346662333230663538653262313830353138 +31303835373063643966623166313136346635623537613332643130316165393263626163633032 +38646632656666636536646466333966653263393264343831663832343330316537636633323234 +31373138656636303363333637616161373533653239623935386433386464323362346564613434 +35323266613431616137613139323039363434303733303637616263353030306535306664356665 +61653930303563363262366338366230616262353136383561666366646232363931353835363266 +37316231643862643563383436666232326632336339396438613066303565626664616430303633 +32613433666466353937643738343866393934663839326434393633376331636532363438336463 +39323632613536656261653032646465353461643238323862326633383039306335376235653030 +37643663646536303532633939373362636262383762326465353364333633643066376238356461 +64396131646337303334323232643934393731303930306636336264633735373837353530363035 +31356161343430653432316538396236666464323631636538396639393730306466666566666537 +32616530656433616233616663623234373333633937353837383232303832363231653663323662 +61353763613633666263343563663930326437636566383162326339616466616165343833306263 +33326438356461653238376234363233343833333363393662383831396665393835666564383436 +61393936396138646530336366303639636332633237313338363364653534333665356232643238 +38316336653636353632653863373463613437646362623230633962663235623333643734303166 +32656335663836346139383131343162313665303764336532343938313666376664333162346532 +63363134666433373139646366383065373730386133386637373631633932666162373130383338 +64386461633835303331663836623234663739366437323638363339343561633631633462666539 +35323532643738626237653461316337376438633931653737366634323963363031356439323161 +31383564613164396561616336306534333833623035343130653264346563613731323530313032 +30386537643433643761356236346630373861316339663731616135633961353937326337616334 +34356536363733313539346264613236656333633632656636666139316530636466373561373266 +38666234646661396333363130396636356635663666653632616166323336643065666135646631 +30363334333063626138303134613638366235373363326137393061323561623238313730623839 +31623035396463326464363832313530303738323836333231653930623331623233346165363762 +35326665343664623137656364633261316238393464333064366130643331613062376266326564 +36613931313733363434616138613463373430663161306139636662613634323862373633303030 +31393337636332366562343966303832326437323561303564626333316665363934613831333937 +32356237343437363632303130663566316461343332666335663463663666643561363862636363 +66636436663031323739333534646232636437663537313432633861373666383635333239613262 +37636365643464393761323032656535323965393465343436363030613364396634616137643665 +31376638333065353761623366303937386462323931316436376139393333353930356663363065 +33366230653061633265323438343230653237636532623066303365373537376232383533363737 +32313033646563626466303331343262333963643130653664376536376262663135623037623364 +38326262366264373432363136333434343036623630353132303963333338356234363863303037 +64326132376530316637306233656336356164343832623137666133336136326630626631336363 +36333435323864633364643138346438306165323733316435663936626433343965616262643334 +31643237393761363665336534646465333863376662623439663335663363643262333163656564 +33353936636163323933316362633465393763353530316564373733306639656433313038616237 +62326136356539643835333637346335653330666338393638623533623466333466383966366431 +35343733316566383738376435623264623364393136653738383864373039333034366363626661 +62356535656235313866663061393262626161333539323162303734633438383066306262386335 +66316165333232366633316564343963343635623362366565636430383836323932313963373536 +34383562366235313731646438646435643139396261656134653762663966373931376235323733 +38613061326337396165376133386132346166613432396465363161343938313433353064313065 +64633931363430313330636237373261633931313539386634316266396166316535663361363466 +31643865393536636362653964633863333232616161396164323564373139333930333531336630 +31336631633034343830626134333233303363343463336462316330366564633164323636363964 +65306532643538363962336330633661396532356165366562633934303437396463316132633330 +63383531393364633461353239616237393336626564383130313531353732336466616239383937 +35306235643739633238393161363736363063663238363537646631363735386464323666316433 +65643966623433366236616363336437623838643330346439666132323633313831373737626639 +63323435353134643738333439383164353939383030363034306335353736616537623430316133 +35643338653538396433373130323961633530383537313231333432303464313064643334336530 +34306534643231303134383637626362363065616263303234663439623435353031656462333637 +33633436666264346534336461376564396333346536306339313438636261303765663966623361 +34633730376633383662323964373364396664313830373530646363303661656633646335613366 +61373334663462616265316231363765376337343635316164643232393437323466333537303761 +32373966656464346135373530373137333938626237303266316563643664613031633338666431 +35333663656464303532326436623630393137383162653166323733626535653433356130666466 +33616266636666353035333561333233313239356166363639383230363331666130343137396638 +35386466613231623663336638306338346665353830346137636339336137346462326236313434 +65653938353533303131613938393939333162326336613632356134623964383363343566646331 +65393735356436623432393034356261663564373834356634346338353565386132653938353566 +38393363643761336461663561353965636462353934316431326165663766313330363135643238 +61343732633761326230393038363863303266663737643566393233383435343165653861373130 +31386435623138666433383366653132626439626261313564373335373530356131393330393038 +37653930623236376635623238326630333730666461633462663834353031316366343861383661 +61663837373738326464363863356538366231633136333938363235336364396163303232613431 +31393935386664343439393531386664626432393633623666323030313461383230643163336536 +61333336643030393034343763323237646565393461366562656336666233353762323766343536 +61303138316332623866323433353939396261323931623665393537396330323030316666343765 +35353433303133646665643563336364376363366464373335373834616165353264636666613938 +37613863323931356261373161356438623637616132646664313039383465386662363130356563 +32613463323466363463356135613730613264623165653065666537653861316563643862623564 +30613165616431616338306162633465663066363239373432373263363061633632326439333963 +33333965613637393333633535613934656333393930653738353563623930663261353762326432 +61336132653065393264626135613966333539376361393530396432646139313462306233623934 +36636331666432663138613839653938346234386331653965376135386535663635623539366537 +33663538303635623933336331643037366261326431356137613666646533643336383062653937 +35383661646530646636326530356131356233323938626561643535666631383762356233616662 +32363465353533326662396464383537656666373338656335636136336236383430633634306464 +61313533346261376236663536623830306134316462646433666534336430306639306365356262 +65306262393263653363353063393934613262386462373861363361366466646266313563626437 +61336533386436653364666561633332393437323939386630653330386665663731663036383564 +37333964623566383030316334343839303065623039646133346636643061336436393165363139 +38333862356663333938323234393861666137343534323134363964396261393763356264346430 +35356435333936313566666135333466613330373932336637323161663434656332313361613563 +30616435663932376537313737616361363765626531666532306562386537313762633530306338 +66353532643131623064393432616639633464646132623831313638383931663461353536313162 +35616338613330343730343134643562666332303964613132643764653866373838623761303038 +62303666613363396262376235346464653034343737643439656230313562653430313831396166 +66346365326430383462333531323634396532393963643732323239343162646136643665373338 +63333161383632363864366338323835376439623666333163623132356635343265323136346236 +64383565633761396536356637313334346132313839626462366464333133393635343765613365 +37343266313466353061653831386662343834343533306164396431396134393332366162626330 +62316232646237653830316137343063613564326533626337303962653163386365663064313532 +33643862316634356634333739336534363062633162616638643466306138326535633863346632 +33643838303039666261356563323062363631336536323436353635626436363639623039383830 +33613938386137616638386433336666633435366665643164373263666138336534396135653735 +34623362313034613131323338303334663534306331313033363931663862646430396432633263 +66616266303061316337373166616235636135316161643362353235343662636433656263363765 +65636537346333313033646563346532353139303634616364393162393333616638373336613434 +34316663313363636531323930353665646635656430613763633832343262303937663631373331 +63313466306532343064343261376166323931376264303336623434363362656535653563376339 +31366433633630356536343930393637653564343530366131613930356631393561396630373261 +37346562303632336438616331623939656163663536343030346366346338636138313236393364 +36363733306336303738326461306261313763323931316432303637343036633935373937376430 +35313462383732656262643938666566306463616632323435613632653131343863643464646564 +31653861326163623934663361643932333161353830306536353264626261363566333438663736 +31666333653039333963376130326534646366343961326262646264376232313463616534626237 +64343566363331646561616265396233316635393734383932373466393961613264653531366139 +34656439666663636536373734363731383964633261393036336335623937646136393730383837 +31366162666362363437633865303439383764353033383737633133343530386434653134323330 +35386639666463636336333965346236363365323132353165353034663439643634343764646164 +66326538383539616162346333383531316464626363653663626238386334646134303965333331 +30646236613934346234353933333837616161353766633830386637353164316332363433363931 +37643737613638633064366365643632373230303264343830323833643431333836643635303734 +35656264653233326238333163666336353333306364303536323237613733383839366332623862 +37636466383966333162613431636432613430306133656233353733386434323061366135396139 +66306633323938386165383434653832393865346533666637623834346265613735626464336333 +62333133393538313564313562356665326433613132666166336634643465303935623736343338 +37303839343832653065386231303731346238386563383834643163656564626162356431666233 +66616463333930303039326564373462393134373832346166666162336562333439653038653730 +33636430613339616162396536396161653930306434626139373361303031383764396264656363 +34353630613438313531363233396230363838356437326637333666303931316330313961306438 +37363163623161393435656430353337633038316165663230636166616338646238353839663734 +38303737386636363035373530363863383337343534653162626361366335363561343934613965 +34633937656266613834646234336532343866633733613132343735616334323239333662323531 +61363834616134643162303135313166663435653436626163363265313736616561616133373433 +63643934353230626130383838396230626136656236363665326164663862616139333563653739 +65393965623338336438313064393636633565373037623136393432653131353331306261623337 +38356531353432643266333135326136306436383131323431643634633632616235386138363163 +62616132643334323962643736626335303333353362373164656661373963313030663637333039 +37393435383164333933323761333130633262346230333139383831363061313262643630323533 +31343865363563376435643532386232326439613362656266383864316531396464366539363033 +38626637316132333266336130386132393062333465656632656439633832343334366332623033 +38613632353034386339636330623231656439333631363635303066383662336165333634613734 +36653366646566633861656464663664666133633666383534353232373133636230656332313739 +33393161303230313233333666393965653032636235363631363037666137393631393236376234 +34613935383438343965393533353163613966356435386362396435303466643038346437626336 +61393237613333333433623039376432326236643636346430363761633833386539303632636538 +39623334396266383332353731323837356261333134373962633437373832306362316439336435 +31333031613737393861323766303230653535323838386139336133303430393537376637393961 +30666635346561323066363563306437313732383630366639373833303364613439353538393336 +32356637666330386632333039616263353539346230653264616162656332656630633934613662 +38396135646162666232626162363262343934333330363032386364303639346235653365663365 +39643166323262646433386633633630653030613964326533323035333662316436306461626566 +34653365343734653361393062363931323862653362383436306331353132346531643434626437 +33393132326436303339373435653862616339313365613966353030616561386663666563346332 +39363833653939313032376262326535646237353532323133616433323738356662343939623032 +36353862666236376339356464646664376634653262333966323139383533313738643465623365 +34656534653765363162636163666638326630643739626531316234643233643133613932373861 +31353363616531313832383335663564313761633236363761313364313232646239666466373631 +36663462343037396266646662393239373735353266633731323762383366333263666331336431 +35626165303535623464323631633764613161376331393062376564616166316463346331373065 +36356234643038333338396464356263616136623262356663336161666361666137353037346436 +65313034303439663962323366386134353134363033646361623865316533396264663163623931 +32363234386234333439343366383337653532653961363961356130386432323939636637396634 +33623165373438346438343530313534323936393636616333313432636538366562663761343765 +36316433323362623232386237336334393239306663393162633966313834313636626362346562 +64386462373533393435383733633838346461343666633832366538323330373864656339313165 +36306638353337303361313134653465333336396236393935346363653064343363646663663337 +31663235393665313966633664663666323861303534356165366166653863383262626363666537 +66306632363137613139363739626435626437343236386436353831356463613136633236313465 +66663539353666393832666330316532323463633236633361653837306433393431313264663964 +64323166616232306138343233386437666130656230613366326539313437623463646666353031 +65306164616666386233636335373339613062633062306532363063326666313665356331653434 +36666531393138343962616536366331666638396631653466333932626562376465376537643639 +63303538353137613631366665643763306163353163356362656535633837623932366531333566 +64653131313462633430313963663163643066356164663333363637333239346338323334626239 +62666561623761663561656266353366616164626537373634616666373861333239333638646531 +39613931643864613664393632303766363461643237323662663239623932616233383563613233 +31326163343635323434663139346662653330636439353861343365383864356637303431643065 +39326631353664333939333761663130346234323237313834313562356636646463303031316166 +30376535303063623935356235363036316564333064376233333035373034623433363936303433 +66383034663931316636346334646138316536356265323133386137653565343533333563666565 +65323661643035646663363061666662653063323366666433633264643766663732663961336465 +38653230343830616332333431663535306339386466613361313866373131323134353966636666 +63353561343735353862653735663266336238353866393238366537313233353537393062626339 +31653365363764646535636435386339663263663139303538346331373161353161336536636436 +65396165646236323832376263336430633433393430333938303564643264666437663731323638 +37343530383330396466303131333439666136303666326637663933316266626234313139316337 +62393435393131623731646139343930653865336661386435383061663131373961316534306363 +66393739613866313630323263636666306463353535313231383738313637656233336430343336 +39353861316162393363333139326661313966373335373061316131313836333531383761366163 +30376338393537643332393338373738336365386136383436353035626665653231356565353632 +36393765386639346539306431633736646163623863306566653139666161393861396163373466 +34363932396534363835663939396466333131646539373431373131386234386133626231656133 +33323639323432306234343462663165353661623437633533336434313133626135633733623239 +34336365626462633366666138613361303234363938396636356436363338353437346436623335 +38343131646639313863383763626535336364613664623762393238313262336639343833666564 +38643134613230303166663938396137326562663730636434386264623231393364356534356132 +62303964356532626534356464646233326338346231373363373465343737613538333532376538 +63323261653962346366313062316337613563306166316466616333613138353134353930393230 +61313832346665646466626531306430613838633137303337363533333232343861653337633131 +62646465626231386665616366633434326663366334393064323861356430346533653436653032 +37366666313566376236303263613964306465333932306337616230633130613262656233663631 +35356535393936663166353063343363316261643064376538356230613739666436383035306336 +37343661393034356532663261323263613563376564323734313136623536353563633839306232 +39376137643933303031393239643562316165363533333432353236653935373733363464613733 +36303834656632363539616439376163623630653736326165653938363339623738646335626238 +3436 diff --git a/etc/kayobe/vault/OS-TLS-ROOT.pem b/etc/kayobe/vault/OS-TLS-ROOT.pem new file mode 100644 index 000000000..bc9e76c42 --- /dev/null +++ b/etc/kayobe/vault/OS-TLS-ROOT.pem @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFNTCCAx2gAwIBAgIUFolBp+BcUfgTVdoK9/BhU7TGWK4wDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAxMLT1MtVExTLVJPT1QwHhcNMjUwNTI4MTM0OTI4WhcNMzAw +NTI4MTk0OTU3WjAWMRQwEgYDVQQDEwtPUy1UTFMtUk9PVDCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAN06mTtkxUgmfakRoz/OryNxaak6Y2dTGKx2GHH+ +ijL8xYbT5G5fEnEZTef9Z5BjX7BqGcSfCFE3K86s9PHj16zgzUnfAENmCrAmFs8/ +OXd6S5RlRRkA/CC7TKud4tpRKCTGZQ+BrQKAkpRUngRH3PP3rzJggseu+Tho0CRY +FpQF1iQ6/9M4Zdw46sThfErn38Yz6F2NIEr65yfmsqn07R/I0QO2j2DOP0uu8rrk +25z6ZIdl0AckDmuK7umZXrLhLzg0S7y85VklLtL6cvkfI4kJAzdnX6lJ05g9alhZ ++QftHgHwotB+Sg4u/aPeTXrgTXnM4HhMZBAc4uSPBhOTtXho7DUsfb8+0V861S4X +Vy4NvRBo4kk/3pO8WT+XjobXcdkf6cqQLwuUD5L21k0WQ9csBmkGg5iIttao6lea +6OBl0YwTVc42aRuB7CZGbeBtDbRWQ7d6AVyz/OXgcjdLAN8PjK4NOP3wtOXiTwF7 +m0KAHUc+xJrknpHeXB6gSxAjtnFHGj1uwj5If5kCslUYdQmd/vqdISeLLShaJAsy +R8P52NJY0ol/Cot288fGh9qJ7YoR7WYEpRG0xL+10SNeATSF/PzG6Q7ER4ikJf8c +ySSDRoOOMlhvID+rYq6OLYMeP2Z3F76vxB/s/0GypS+OvsvUlxB3p1wtu69iEjfS +50qLAgMBAAGjezB5MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBSpbJeA240X2T8MV6Pm9oXg4fx4bjAfBgNVHSMEGDAWgBSpbJeA240X +2T8MV6Pm9oXg4fx4bjAWBgNVHREEDzANggtPUy1UTFMtUk9PVDANBgkqhkiG9w0B +AQsFAAOCAgEAQBj+I853QAXmfoHC4SFD/EflIN9d0PTsU0zZUUwXlWy+agp3cVmY +LKmtgKDWVJRaxoszSXl/gztZtq1ARNsn/lalfDn1EM90Vf6zYY/mX7ZqHeDwomhe +rtZNmURWHta95jNXeFu6qb1Ku7Psyzhl3p+NNyYwDq0gDkkvwGO+Jzz6GOYWND/Z +0gMzj4efjnTe/Tfh3hZjQGHbWDN1Y8iiJytMWqdKiOdl+ZVdivTdkaYGxnnlmxRM +Lw2o45A+pnQ+UOCCqkrYp9ZHaBC3ZG6w5Kz1+M5PMbEv2tKV3+fPIGEjpTop66MD +rd9IiLaicl+mZb2mOzNJr825EndWZP+bWH4bg8A8gAJjMN4mLRuX4XSNZg6XF5wb +4gjWcVAKO6bl4BaRgQLU9NDfZ+syMXEcZtTScVhyYJrBVVjCr9fjSpqo7Mqth2Hg +OIISANlHPwZGGXybrhFbjTWrmfZKbtYGdDeZjgKNHbjgQkggNO32qn7E0wPFlBh0 +pB8qV8kdKxseuX+QrFm7jKfn+DkJQCoDyyWlTGUp2KsGHmpW7OwH7x1BUzX/aRb/ +gkdXiHM66Nu0r0B92dhd4B6HQXJ9EYl4kmp94vftxOh4PXrqd1GsGfyvkFYFRh9L +kJZQh9KPo2pHu0sXPr7e1gK0d2nmKOhjr0TNUoa4bfATGa7vVWiaxoE= +-----END CERTIFICATE----- diff --git a/etc/kayobe/vault/overcloud.crt b/etc/kayobe/vault/overcloud.crt new file mode 100644 index 000000000..22577904e --- /dev/null +++ b/etc/kayobe/vault/overcloud.crt @@ -0,0 +1,62 @@ +-----BEGIN CERTIFICATE----- +MIIFkjCCA3qgAwIBAgIUMRz4U0GEliffXSFuYnNRwI8jZyYwDQYJKoZIhvcNAQEL +BQAwFTETMBEGA1UEAxMKT1MtVExTLUlOVDAeFw0yNTA1MjgxMzQ5MzZaFw0yNjA1 +MjgxMzUwMDRaMFoxCzAJBgNVBAYTAlVLMRAwDgYDVQQHEwdCcmlzdG9sMREwDwYD +VQQKEwhTdGFja0hQQzESMBAGA1UECxMJT3BlblN0YWNrMRIwEAYDVQQDEwlvdmVy +Y2xvdWQwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQD0jrS1EqKHdc2+ +lLBxC6TQUruGBnx9PAwfXiHrol46PxH8JJthXUOSb7zW0LknzmNKxq3LNyadCsZh +h/y6QY07q37WdZiqkkYX2itGL0VrQhMmNZdZ49CnNF+6ccAH98aWRXgxvp/3zCiY +JHjcGP7LdrRea0I3v2qjoRg5xwshv30y23G/S5+OnGC3BvS3yVlVQi7JdIjhv/M0 +Kp2dmn4oOLZKOsCFoZvMQK5TABoZf6xz9E5Ht2GnAhZjFO+7v2SiYDEdBD+7BaiE +D0VBil9abGUt/s6vjnzoDc7PWOiiSAy8Kcy27L7vmIKhIiLSKZ+ifhln9VtdpNWG +2wUUfIKRmEdBFYL/V9DHbYw2BoqfH3oBSAJ7D4IlFFGkaYWO2ajRkJQAH53hlU8K +2UKigNgPWtC/55wpb7sezlaUkHcURrC/m+iotb4+79evYUlZo+lJC8YHBEtNn68W +XWsNCemi7waTlEjb5m/9nif+9aLQUebynW2niYwN7k9lEBtd5rovgncOxK4Np5Ke +/bmwU1pCkLnPgLGsFHz7vE4V9qm4T69UZg2ItSCd5+EL+VwQKows0Z8wIZWPKrSb +pwC6GULBMVjqT9/Rn0zzUT1H2ohClen88ST9ISM1TR9tTCWP0RJVxArDaiBu4hOq +gWUzO3g16KMgh4TGfIW1JpSjKgnJrQIDAQABo4GUMIGRMA4GA1UdDwEB/wQEAwID +qDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwHQYDVR0OBBYEFAgdYfyX +izOu3qqIXm3EfMl8Q0ZDMB8GA1UdIwQYMBaAFCLaSl+L+PxeHrXG0K8FxOcXvEMs +MCAGA1UdEQQZMBeCCW92ZXJjbG91ZIcEwKhkyYcEwKhkyDANBgkqhkiG9w0BAQsF +AAOCAgEATxe6vFQdPkNfcbW8wJIgQjjjYOI4Ov1PnIAXiVcbQMdYGmrV2IKlfLKi +TDsKNRXf+ix0daA/H5RzpK/xc8Dlc4VSBADNj7tXFZT77KQLVttw9QFYKXWFuz6+ +WG19sWHi/QrFrol962jLN3TwUANievCXDZwEuC0fYPldpx4/BhYdwZn+gPb7vFHS +zP6Bv8wMnv2e+RKuS3YJiBnRMMDvcuqKcLcTeUVh+rHBdPBda1ArVTbzMjlA9F1C +7fvzTsapz1KKRgJk/oM8j7LG17b0GhO+CLKQRhqq9cLmGeLWdk7ZrxkqO0xdD8fR +AQsuzP8Z8EaKV4JlWL+JEU97ialBNFOgTREIu0ujYWUHeD/VTCow6ywuPtqi0RSz +u8ys91XaUboeKKj7t1y338EF5t7vrBRm2licdO4gVQlX3FF9RHHy+6nd+oO0vCDO +nN9jbTc2G3CYmsOh7xnY17WLhydtsr5gi36PyNt5vpC5Mok82itFwTbWz8z4ZrR+ +gMlCM1FTpzwlumL7lkCqiPXY8b/QB+J50iwXubqEbbcXtb/QeC4j01LDKa2Jl2CT +zKaYFIB1o4Dv82ul01J3pAMnDezTNJycVQArag3jEn3on1IlAjdWg2saWAmw0+A/ +O37oBIC/gvd3ZJrcbUet9lylxjjSjumOSkcX59kupYvqlLLtAO0= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFMzCCAxugAwIBAgIUQJBuLmrWDkZ7Hb35ZZYAytYUTjgwDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAxMLT1MtVExTLVJPT1QwHhcNMjUwNTI4MTM0OTMyWhcNMzAw +NTI4MTk1MDAyWjAVMRMwEQYDVQQDEwpPUy1UTFMtSU5UMIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEAvUBHYw4JkOERu2Z8NwPRnDTzduN2tsrv/dxHVU5n +XqN6bPffFF7SAf0m/IOlhPaM9DAcFhIWtCnj4thbnMYA/AZx7liEWeicCBGeGznx +jTGHm85MJ0pt3rwpBJCHA7GY3c+TWoxcqWOS+62Cc7IzGviPj8YgpkUjRl8Agza+ +TwgZ0o2UYz7gpUqcWpj3RsCoGhvBixBxdpinF8SxirPCK9tYMudxW14O5+IxShXz +dyhTtkDvxNTTmc27PRuX5WGJq0BKzGBlGYW9rT2YPG+TCMf+t0G0wFgVx/4P4bU7 +aitNjd5QuEmWz8/VdOTysfFnwgcbB1z8SYpDNwHCx8f6/7UZkyJyxfaiEqq/JuFe +xVLIOzpAxQj1KXj4xVO5p+cHPetLeP3KAvhNAuPlohd0wqzV8Rj5BL0EQequXode +Jx2k9JRlsfXZh4wv+Fh8KsOFFmLO4B2eZqWkaTJ53+kt/LqoJqtB9IkVmvEP6B7p +oNSViPvoA0zE0IVF6LjAKo1DxQX0wFd6hqxM4PwJCsY4WOsrmFdBkguapa7iFBRB +PuTKki2jtti6D5A/JqD2qd5HZHJ4+b0wBRlJ2G4vGzJ+h1dzXnK50qP1IiSW3wxX +7A6Mde7oSJrsv6HTvntqELRJX0a8YFDw0PgZIgvWJucPwUVqdoO7/B93tSzwp85O +hjkCAwEAAaN6MHgwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFCLaSl+L+PxeHrXG0K8FxOcXvEMsMB8GA1UdIwQYMBaAFKlsl4DbjRfZ +PwxXo+b2heDh/HhuMBUGA1UdEQQOMAyCCk9TLVRMUy1JTlQwDQYJKoZIhvcNAQEL +BQADggIBAIWHKuvktZd5kV0C+hyQ+SYVAVeqYNl10nkNy8Msl434FwBp4bByWnJ7 +7PTUROPh3qYhsXmkxg4i+lpGmL7d+ZN6WZ9WzCGKk8uqZ1uzZEGXvenHOSOoM+l5 +x3sDWuokVL8kZmOU5ek53rhWB2RPyoTvYb7gypglf6kEb0ZK69bSbYcKGSlepDZw +iHtOS1ZWn+3N0TZLSVUzkZhAWwrIw8nUXcSfsNf1OvY7tNM6FlLDQ3YY5rr033EM +VeSBgZsi2mi/F3cd6NE1nvkasyny5MK5urVHYWFSDckcDUl0EirUv2XLuM6i6vZM +gJxK+CDxDjO+z8krPLKXZ21AVjHu5hSBwPONwsPFUfG/18YH2/01ejE1UtwgHcsU +ylhtrifK+QtMYhHFOTqFbac1N8rlmD1LS9F9GgoD9DmI1zfiKHLTvuf8W7XS3wxQ +uTKUCpBHuLFxHiZTWMgc0dz7GkaXNKDBe4j+E8VzuXBwuEeTm38BrrmScwU1IJSp +iXoKQHDvw47VJlSW7b/rOnHMYRJsYZG8MdMb3xc4uLPdACd3jP6RC4/cqk5m/90F +0XJtqcQkiryUpAPSOssN0H0lFojOGAzHUR4bzTskiPZ/KXUE6T9Xx3pFh9Tw6U/o +r6JPCWlEBjJOcjHM0sAU498JHW/+WjJjRQrxUPqSReM6WzPYOv0x +-----END CERTIFICATE----- diff --git a/etc/kayobe/vault/overcloud.key b/etc/kayobe/vault/overcloud.key new file mode 100644 index 000000000..0a538e90b --- /dev/null +++ b/etc/kayobe/vault/overcloud.key @@ -0,0 +1,167 @@ +$ANSIBLE_VAULT;1.1;AES256 +62373161613534326234663232353330383563316438623437343034353631646363656264663839 +3036306332303838346234613436386330643639376538660a383232656335343438633837306333 +62663733323538376132373636656334326430313237613832336235656535623838366538653432 +3266646432356663300a396162336237343963396164663633303539333963613261393564383863 +64386635653666303431346366363031333238373962306238393366613830396532353034653561 +38656632326437626239323436303063623739343065666233373931376261626535333736366230 +32356239353763653638383438346231373335396132376363343161313766373266313365636434 +37663339393961393330373533313363633537383165383966316439386561323833653261616633 +33656466663631336565336537653836393034353431326163323133393863346136386337363437 +32636431356362613066636566343863393937373339663439633137343235326335363831616236 +64613833626664613833653936656663303431663331366361376531363132636534643164343162 +61316663643366326533666332366233666166393362326365663636663335633039666663316232 +32333137386235656562363134623233386166306337336562323832333033386438623438383336 +39663934336136363036396261323538366463666239363533366463633765343339316565313364 +66356264366666303536636462613334343565363732326336376130303938373037616265343937 +35303234383064383164343561643836323036343361373735633533343666366231373833636135 +64396664333434336639613866666632353634613639393662326663616665656532303364393862 +61613438386638333939303931376262306365386138663962363161613062626536333837663238 +61316530346362353734356531326533633331313063653033376261623331333836656266313632 +66643235343034366631666162306634383832663137386462646466343439623135616438393564 +37613237393738633265336564386634386636313461363039376437373637336637303433383634 +66343166383562326133316539613162333739313239666534623063373538623364363431613039 +37356339396330376562383937303735336562363639653535623431383962353164363461313338 +39353662396335376164323664633837313065626332623361656462613239633963383633373865 +61353638316338356334643361623263376263333264663533646163363535633339653739383633 +37633139366537303032303161666462386532303434633634623434616333386163323230663930 +62643630396565343362616237396432303036336135323538643864303465313563353839383632 +34323464373630613262653637303161623434666436643132636165633364633330393939366365 +32666665663635666664643933363162373031346365613139353739363834663935313332313735 +35343237623434373166383062643765373534356339356439386336623664313534393731396536 +61313164303533633734666230353161623731626561393135303163623664356562333132363138 +33376139656463356332386631376134323161643530636434376232376138623933343064666163 +61336639363463343930343833636366643965663439626435363730333466303731356231666536 +64313137353932383133316466626566396130396139643936393665353037633334653434316330 +63393939663430306538646230646132656531393766613263653536336266313134333831346161 +35366132643938326438643161626437353130313165303166306532643034626438303338373336 +65326432333365656436653164666530313166333738616532383061376631323165613639666431 +30376338353735326266323562373662376265303662393732376537656631313234383436316130 +36643139323633623636666137653232363530633739346133363266336530653665323137343335 +66656465353532363835366530356262316530663735613030643061663730623736623330633761 +38336135643764643031323366663664373631316231626430383661653135323664353337643265 +65353533636261626364386263343438363661303762313233623839343961323734373930613839 +66666266376465663461343633623732383166636438323332306635316364653834353731373866 +30613737333534323932376366663231316239376530303764623030313837663263643933373936 +65383235396231666261613635326234613231613361353564313532653861346165633233316464 +63653938623232356536323639616162623062663531356637666465383762356664343430613238 +65636565653865313239353166336363616339356664663732613566366162633635353135323962 +35653664633832313533626236383365303661313130353162306363383435366636666339373633 +39373566333138383732646165663038336462663833613563303838653234343932626663363232 +64363563666234663933343339363439333132343362323732346630376465333161363464303631 +63646361653964636231623965666233313161343265323438323234366164616261656633633332 +61396562366137386636323132343163306437373331373232663039353862323731396632373462 +63326363623435383834623566613034623731343732303635643264383839646333306465316639 +30393034613132623264616536623430343039386162656534336533353531316264343135356665 +66666461613236396266613864353266613131353662616430393863393262393262343636353464 +34613164383236353461356236326263633064376438633133383866643332366330356136636231 +35653266316535393435343131643239363662626638376339353033316139386334323733393734 +62356464656333326438393961366237323931353530646466613834363233323833656662323263 +65643166393238333332643364383239666533626335666366313865303835323639306638633137 +38656331303563316134356261303864643138623437313231613333623431343435666561353936 +65323463333337376666303135396331396233373632393133316364313630643233373034616438 +61633663353364643762653438313934323930396433383966383961656366376630643135396462 +31663837306463633234356535636536363235633661306261346465616435353932383066393534 +66613765306438666265353564316537626564393232616362316639323136666533613735383139 +64636266303231393533623435313561303538653962613433663230643362306463396634623539 +33373034376363363065666136303336303435646665306666363730626434343465663563396439 +63363738303463363735313765363034633837306336666338393639363031353564636538336665 +61626339373530643961333864363339623965643563653434326663663031623133626430623939 +37623663663536303439643332313866383966633563363139303030346562333431656231633836 +32326666393231396263316330363234356336343531653763343165326336306661636334336237 +37316463333964356431636532326362323039336566343831653133336631333131663735333466 +37613936343534323530303838616564353439373836613064613365666237653130336339366432 +62383463353665653561666230323534376536336133643061346538666363303436646366613538 +66343436633232353166653735396234663632343934663230303033343236393036376264393536 +31376233653330653765383163333736386539323065313035633063306238313634303966363134 +61353934376562636463626465636261376261613063653835373330396561323263393332313761 +39646537613438636532656265366266363332386136663132336265653230623035336566656231 +66613530613961303236323666303835316330373238383463336666653330663636646433323233 +37323539376662323336623538393634643562613133666535363432636534303931333035656661 +35643238356263363730623836356136373962666133303266656432643239313064643335306331 +65346132653231373763613565646436633032623630306139386634666637323130336639383237 +31666534356562376132343064616132386535333762336133653934333231316632613535393164 +65663136643736383132636361623737306439306335633937643136343563623666383439646434 +62323838666137643166326634636664366230383863353338346439666439396365303665316263 +63303131373536376631396163353362356434386636643138646165323230353836363262643538 +61303462363963373439316635663532393833653139666235353233313533383735363263313238 +64386337626639356164353537356465313464313461616233666566646434306634336566363738 +66633338373834636364613236373337643265353364646630353035313236623264333930373136 +39383439653663373037373632366462323139323063326138323433383130633330623631376638 +38626335353331386432663439303833303632666362316539333966653430383264613839373164 +63623939353537366637306466623130346433366236346237346661326638396265346134363939 +34396331663564333839353861656665393338376233396266636431316239373535333161633134 +30366638373966333262313966363763633466353137373035366165363561643964353261616265 +36656530333161353635313730316239353432373466356665373032363136383766353433353964 +39336164356235613065323262396431616430633330623730396339323033346532363830353835 +30643163373062633561376434663534356633326262643130336630613266356235633135303662 +33643761633361316538356537323365323236393039363032613932616163636130346462336165 +35333434366438353233663231356337633736313864383131316564303336646438353135386565 +37666134623963653362356233393661353432376435346232316432363162386130393063323932 +37613362326239663333393761353037613331336337363330383333623666653863383732356130 +39336162633035653663343337313763356138616632633666633739346135326232613563393263 +65323437313936663033323363636437313465623631363935633431393030643033343164333231 +61383332636539636336336237373333303838663030663230313735323566653034643962363239 +62633964366361396333653134306466646165346562633238393933613235393266303965346164 +36393965373563323764613130346463353465316230656362363334356662643738396232353737 +32663832393465353762616130303533323466386231626336346235353536306333373961663563 +63663431323362326638383663616636363638643066613166613833636634663136336637393337 +63366564383864346661303632376630393466336233376361653435643539663433366236323664 +32633030316337653033633936323061626265626666396239373865656230346261366134623065 +66633236373564373562393862633132393435626665666239333135306236633032323631646539 +63366532626338323537666539326536373562336431643263393761333037303763643163613266 +36366537646361613635376437386262393034643839393032623532323234613337363830326133 +32313031373766313634343638623639346432336562303163643532616339386533373565643533 +35393336653963663738333465636233376139653730373864613664663138643234356462353136 +31396436616232653637336438623631316461346339633331653138363962323561303861393463 +39626264643132643163383137386439633637333735303966373834353033346136353766313665 +38333962366138646139313933613733326530636231383165653562323764666362356161303538 +39323261383735643064373239613065666433383263373361643939363439646537666135646561 +38363239303066653435623463363638333731363233613535353533663532303766366561313564 +36363534323736366563306133646665333535316336313063306265633630396633623630643136 +35316634373733306439656261616134326138343833366330383638336161396639396433333262 +33393066346231383162316231313965326662653135643937623637633736313536393938623730 +37356332633538343934336430356532616164613263303836623738306531633865353037653361 +30663033383663643638633635343165613433386566373265326631653135326561653032663837 +36646535336432363765386439656336366261663662343966653864363230663236643062393530 +39366132326336383333396234373562366632356663373436346263313532646139633138373035 +31613231316135383162313632613165383330386438646534316163613234303034373862393065 +65303131343639363936616536383932653665656433633762306536356161353465353031396133 +34373661666132666564393237306231316161343237626165636165373031653436333263656535 +37623264656232326566373039623636333062663861663161646261623738366133666262356164 +38666634306132313339333866323263323430646238636132366462313862316564396331383362 +64616632316133393136656635366666313033343437346134646431616238626135313032363538 +37373064353662653463626265363061303663383436613962323861376637636632636438323234 +35653333343962636437613366326237613164326262613634386664383866653131626531303363 +65393661373537643232326638396162383638663937393361336433623963663361656539616665 +61393066643832663263316539643535646566393861373037313166383862326365643037303163 +39303534393834303134346136306634613565306338306634306631353163613838393932353032 +62326262313662326131393235376233643765303631623762306561613561303536313233323631 +62643962326335376665363538353163643632386235326536346165346137613665333738353838 +61646361366162653835343938313763363931363932303462346134656161313063353832373837 +66373837376665643836386133616538336664313230336132373432656530656133366437656665 +35383263626665393639626465623463363831313734393465333161303763663137376136656533 +38623966306337613533303036363136303133316564613032646331323966313332393333323237 +61393131383437316265323832316634303665643562643932393561383639626462633733303964 +35333063646434616532643361646439663135313237326663383235303937613836353230626166 +65333134353463373034613962613963346332383833346661613935646234353265646665643636 +31363134393933396362616133643030643562373532313065393634303633643561346339393063 +33386137346137393234643338366433376666353432393639613362346265326666386161613731 +37616436386265666231303034613331626232313764336637666236653439626564376530353730 +35383333306166363439343431396366613837313631376465636539356533626364613131386464 +61306461386162336431623834376461303661633062376666353562363163663938643234633161 +31356633316366656661323137306263393866333732386362346437313333383831366133343463 +35333030316235303935306531653461613836313665356630623666313062616631363264653132 +38666661633961636232323635343734326432346432376638363632643735613464353830363063 +36323730336239313963636638636335323337643937613335643561333531666239383736336430 +37313064633331333538366365393135623431346431323539353866313930383262643032353263 +64386532356566633938643666363066376534636461646332353364373032366366383632383935 +62386435353335303233616337623666633130323266383163613439363835663934666633383863 +31613063366632646462323162383361353462383634653266356533386436626530643532636438 +30376435613738326630313238633061656464353033383833333263653964643066386237646539 +36646461656463653264656635383030666533376238626331646335653162616534386131393562 +31386633386435613064613661376262393563376263626638613733353638323032346439343432 +66323434393062643565366430646663373263393931363964343332633165306665656161613764 +65303862376533366266346564653162323136376162633734316363386237643739663662326333 +30313061313134626565366333653932656437376130623036353064333364646564353730373330 +3366393439633261303161396465663762643030633266643231 diff --git a/etc/kayobe/vault/seed-vault-keys.json b/etc/kayobe/vault/seed-vault-keys.json new file mode 100644 index 000000000..80ee48381 --- /dev/null +++ b/etc/kayobe/vault/seed-vault-keys.json @@ -0,0 +1,46 @@ +$ANSIBLE_VAULT;1.1;AES256 +61663930303733613662326663613361383861393933343264343261373634356533626631373135 +6162633762363464373835613333363461353138366335630a613338656132626136363966616631 +39353332336165623661336131653432306435323133326232333539373966376366643439346162 +3865653234373139370a343664653536623463333634316432373663333162656661363761333633 +37373730306236373034343933343139663730623538333066363532623066343235366234333535 +38633762373562323533333539323965323432323333363664346630643730636134303637303538 +34663766353962333535306463626136663762343931303534626634653964363830303331653832 +31333561376535663631346237363365366635336138653562386439366666313033383633373663 +32386433306130666164646662333463316432633239323732353361346262316531336131613631 +33326138383863626133626438336363343362656331386664633761313563386639313165613031 +36613535363838343563373031393734326630623830346231363662316131393133663764386437 +61636336356234396666643162323532306439646364383061393032383536386130363030356463 +63396238303038633564623264613032373238663431336663633565646130663932373336343131 +66353339336461646337346339643439396464633838656234363162366335393866626331343237 +35396563376236393966663132306239373031613935646333613339326530336632316237386232 +31373739303535613162343636353363643236323165316234323866666634343164633637333839 +64636661343165613062373330353331643438323563306336306538383361396138663833343862 +61323735633034383932313236333235616335643238616533303530383463373137386139623536 +65333333336336633633343966623666643666343061396332363737653662666239343662333833 +62356162643966643532646330616635346564396431393935353831633665353361623965623932 +35643138306430323861626562393366306134393162663031336263316132396362393865376439 +31386565353338306264346633386464366239376535396432643531343562376366656265653366 +37643231333132333766356365613262646332666639323261376632656436643038316336613136 +61616163343232316165396664356133336537343038363166306362663162363537383035336636 +38303631663536336433646164646535613166373730323932393339653037643763666632643333 +65623736393437333038396538366433333033333232643034666266653433313737636433353034 +36663763363533653134343131346636333939353832373463333563613438636630316562306234 +33663931366130646337613731336534633035363032346533616339623137383331306534363731 +38383537306437373539373037316239323938643162366230383366643632366166333037613865 +39353839616331616637643664383662623832303630616664333135353339373631623466663662 +38343939633565333164626466383862323038653565353231613962306231333030326336343662 +37366132396665623434643739353634396262366636383737323539633966386265643761646538 +63373966393935376230383936363764333138343966646363396537633865323864646661633739 +31323364366538613630616662336431366438333634373733396165383866656231343731383532 +64333835336434323062363535346364383731323531326363333263383830383162656536626632 +38663664643637326465366332353432633237323139633536623966326534373036396132393339 +35306366353938383862346135663633336431326237376135646462393763353631663030613061 +30386164393633373965303032396361633964386539623834393035336235383536363033343063 +63366664623130663864313237363262313338663466356239383966653235343430393465663735 +38303861373365646566363461626561616231663033323061336466643763656537333236336333 +33353435326430323966343762666266636339333739343635643737376234333636303162306135 +34626234306437383962373037373436343966353534643037303238323537336663353564633862 +33326631666538326234376361316635343336363563353563303637666266653564636131643064 +66386563363431393761653533386661646530636362633933666134373562653364343138653861 +61313466363837653733386130373762363566323865323834346639323938323365 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..791e87348 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +ansible-modules-hashivault>=5.2.1 From e6dd06a6d656aadb4284249b560f65a162e9a3dd Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 30 May 2025 12:53:17 +0000 Subject: [PATCH 069/103] Ceph cluster fixes --- etc/kayobe/ansible/requirements.yml | 2 +- etc/kayobe/cephadm.yml | 75 +++++++++++-------- etc/kayobe/inventory/hosts | 11 ++- etc/kayobe/kolla/config/cinder/ceph.conf | 12 ++- .../ceph.client.cinder-backup.keyring | 37 +++++---- .../cinder-backup/ceph.client.cinder.keyring | 34 +++++---- .../cinder-volume/ceph.client.cinder.keyring | 34 +++++---- .../config/glance/ceph.client.glance.keyring | 29 +++---- etc/kayobe/kolla/config/glance/ceph.conf | 9 ++- .../config/manila/ceph.client.manila.keyring | 17 +++-- etc/kayobe/kolla/config/manila/ceph.conf | 9 +-- .../config/nova/ceph.client.cinder.keyring | 34 +++++---- etc/kayobe/kolla/config/nova/ceph.conf | 6 +- etc/kayobe/networks.yml | 2 + 14 files changed, 179 insertions(+), 132 deletions(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 793b3816e..09bce9565 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -3,6 +3,6 @@ collections: - name: stackhpc.linux version: 1.2.3 - name: stackhpc.cephadm - version: 1.16.0 + version: 1.19.1 - name: stackhpc.hashicorp version: 2.6.1 diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 5848c3480..a4cee2f55 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -67,29 +67,7 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" # Ceph OSD specification. cephadm_osd_spec: | service_type: osd - service_id: osd_spec_default - placement: - hosts: - - ds1 - - ds2 - - mds1 - - mds2 - spec: - data_devices: - paths: - - /dev/mapper/mpatha - - /dev/mapper/mpathb - - /dev/mapper/mpathc - - /dev/mapper/mpathd - - /dev/mapper/mpathe - - /dev/mapper/mpathf - - /dev/mapper/mpathg - - /dev/mapper/mpathh - - /dev/mapper/mpathi - - /dev/mapper/mpathj - --- - service_type: osd - service_id: osd_spec_default + service_id: osd_spec_r760 placement: hosts: - ds3 @@ -100,7 +78,29 @@ cephadm_osd_spec: | - ds8 spec: data_devices: - size: '>15T' + size: '15TB:' +# service_type: osd +# service_id: osd_spec_me5 +# placement: +# hosts: +# - ds1 +# - ds2 +# - mds1 +# - mds2 +# spec: +# data_devices: +# paths: +# - /dev/mapper/mpatha +# - /dev/mapper/mpathb +# - /dev/mapper/mpathc +# - /dev/mapper/mpathd +# - /dev/mapper/mpathe +# - /dev/mapper/mpathf +# - /dev/mapper/mpathg +# - /dev/mapper/mpathh +# - /dev/mapper/mpathi +# - /dev/mapper/mpathj +# --- # Additional arguments to pass to `cephadm bootstrap`. #cephadm_bootstrap_additional_parameters: @@ -123,24 +123,33 @@ cephadm_ec_profiles: # List of Ceph pools. See stackhpc.cephadm.pools role for format. #cephadm_pools: cephadm_pools: - - name: backups + - name: backups-data application: rbd state: present pool_type: erasure erasure_profile: ec_4_2_hdd allow_ec_overwrites: true - - name: images + - name: backups + application: rbd + state: present + - name: images-data application: rbd state: present pool_type: erasure erasure_profile: ec_4_2_hdd allow_ec_overwrites: true - - name: volumes + - name: images + applicaton: rbd + state: present + - name: volumes-data application: rbd state: present pool_type: erasure erasure_profile: ec_4_2_hdd allow_ec_overwrites: true + - name: volumes + application: rbd + state: present - name: cephfs_data application: cephfs state: present @@ -158,20 +167,20 @@ cephadm_keys: - name: client.cinder caps: mon: "profile rbd" - osd: "profile rbd pool=volumes, profile rbd-read-only pool=images" - mgr: "profile rbd pool=volumes" + osd: "profile rbd pool=volumes, profile rbd-read-only pool=images, profile rbd pool=volumes-data, profile rbd-read-only pool=images-data" + mgr: "profile rbd pool=volumes, profile rbd pool=volumes-data" state: present - name: client.cinder-backup caps: mon: "profile rbd" - osd: "profile rbd pool=volumes, profile rbd pool=backups" - mgr: "profile rbd pool=volumes, profile rbd pool=backups" + osd: "profile rbd pool=volumes, profile rbd pool=backups, profile rbd pool=volumes-data, profile rbd pool=backups-data" + mgr: "profile rbd pool=volumes, profile rbd pool=backups, profile rbd pool=volumes-data, profile rbd pool=backups-data" state: present - name: client.glance caps: mon: "profile rbd" - osd: "profile rbd pool=images" - mgr: "profile rbd pool=images" + osd: "profile rbd pool=images, profile rbd pool=images-data" + mgr: "profile rbd pool=images, profile rbd pool=images-data" state: present - name: client.manila caps: diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index dcce7ed34..e7ca0a8a8 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -3,7 +3,7 @@ localhost ansible_connection=local [seed] -srcseed +# srcseed [controllers] srcseed @@ -50,11 +50,14 @@ ds8 storage_me5 storage_r760 -[mons:children] -storage +[mons] +ds1 +mds1 +mds2 + [mgrs:children] -storage +mons [osds:children] storage diff --git a/etc/kayobe/kolla/config/cinder/ceph.conf b/etc/kayobe/kolla/config/cinder/ceph.conf index 461f51fc9..860fd7355 100644 --- a/etc/kayobe/kolla/config/cinder/ceph.conf +++ b/etc/kayobe/kolla/config/cinder/ceph.conf @@ -1,4 +1,10 @@ -# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +# minimal ceph.conf for 359e2b0a-3c64-11f0-bc95-1423f21d0230 [global] -fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 -mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] +fsid = 359e2b0a-3c64-11f0-bc95-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] + +[client.cinder] +rbd default data pool = volumes-data + +[client.cinder-backup] +rbd default data pool = backups-data diff --git a/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring index 107d93b12..08d15a993 100644 --- a/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring +++ b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder-backup.keyring @@ -1,17 +1,22 @@ $ANSIBLE_VAULT;1.1;AES256 -32343030363536653566636631643630646563623061633133376165653031636361346361386535 -3434636436323663313763653061313464646466636164370a636137346132616664656663353461 -39336335633237616234363766313039366661616133326638643030343330303531623838663633 -3664643337656130390a383530613166393131373735643539346162336365356634393263393934 -66376238656538643334386530386332336336353036303339363235333362623261663336323934 -66323736333032313239656162616131663264316265633133343733333430666430656130643165 -39383061373165356436366434333537396437373931666131663735666365616337636633333732 -39383137643336663133326261666335363036666564333865376566323330373532373631626331 -64643365636162353530383664323665613631363362653665613530353130323764323161366531 -30353231623464653131366566383561663737396466326532643131666330343630356333363032 -33356635376366343932333262373063633233323131373436646235333834663365356566636139 -31316438343035323337316463303562623834366236356463376263353966366364623465363232 -37623537373062333834333431643863613136633466303330636462316139633666346462316230 -30373435313666616366336435313464613537666537383731646466613933363234303238383630 -35663838623761313638623665643439613662383731343563613364363533393165616562613766 -31303733636332363332 +39336263393262646638643239303436366235633462353936623135396234313239333361663235 +6463356332366263636562643937363264616166623433660a633365393038663865323037333362 +62393161646638366535303738623339616333316562366261326636363833373461326433323938 +6534643962383966310a656465363638636532623835383039323436313765366438363832666364 +31613839333039306364646231666531326131373136386533346164386633626230353439333631 +66366666633530666235383636393537636439653332356635313462316133353039643861393265 +31636630356163626439333566646564656632393036353634633930646430316265343663303464 +31393839663261663335613935336235643965376664306335346466653762353235643937383530 +31396363323339616661366532613237666639386534386364653063396134373639616165323462 +36613163656431636230656237636635373365356265363735383633303864363366626564663436 +37306231646565336134303937363237343135663766373461306431313961656465653538343939 +30613834336337336362386237336665643332346562666239363964663336613830323464396165 +32626537336236303736313762633134646232396566333165313861303232376163373233373636 +30326664373761646266366331343234333130656231633234356464353736386265303463383133 +35306536616534383838666366316238336539613339313739663061623839636336626437313732 +34383763313435623365376563643336313234653664393161663865656531323739653633373831 +61393833356165643331373130313231333164616337666536366434383336396438646630343838 +39343234633162326563336331376565663533336165616438333334353633643432353432326138 +65383434613735316630326536343261386232656664643630343433333133376665306435613365 +36643033343837666265636234656663313637636430323066623163396566393231633538613035 +38386461373130663538316266313932393730313736366565393762346266386535 diff --git a/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring index 0755939f2..8b7d7041c 100644 --- a/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring +++ b/etc/kayobe/kolla/config/cinder/cinder-backup/ceph.client.cinder.keyring @@ -1,15 +1,21 @@ $ANSIBLE_VAULT;1.1;AES256 -61333337303062373834616466393336356566363838343931393764666238656265623461323531 -6536633237336366663131613038613161383135643663360a646366383635353462346261636438 -34613937666666333238633065353537333632353664333965383836613661653965306137393434 -3061363366656234380a323534313766636336386635346633373436363738396430636465373230 -66336666333339613430396338383061616362613535306638616162303834373361636535616336 -64386136363363383132336436636630333933373331663037636136363163333234653934633433 -36613337623031376234643461643736336236376664336237636262343735353966666665373364 -38343835653630343437326666646561663332303438393431303235346434306239623233363831 -35383733613766653834313539633532623836613134663132353630346235363661636230373764 -33633866376131633163666365616238643565396461323336666365303336613334343431363738 -63363164616232643764613561326163336636666465663636343334643235663839656266343964 -30343366383939383562366466323163373739323737623737366661643330333862323632323333 -35366531356662303431306231663665663465613163663935613365633134396334323935613466 -3832663233346334653462313131616330306161313464383031 +32636666613239366539303432306634323237386534353131326435373935653935646131623166 +6234366466626538623166363063653365323534653366660a626235353936663831363566613538 +66363031393837306561393439663461386136366234396662396263653235626364393862386431 +3363613731316366380a376533393664633565346361386264633065636362656364396265303532 +38376231306433323664633234383130346330393538623339303039343364326230356666653938 +31353865366634316332386438366435376436373265323933396235386433393339643261363732 +33323533386262363735623535383637383065636331363139656436666134643465396535643538 +64333534613963363666643466356461643862323738333763323638653036313934336633663239 +34663662643532313264346532616364386336663238646562666432333966316436353333376332 +35626563666338396531363765663033343265373234306636353866646138623833393032666162 +32396534613438346131333031336235383539613934313662373838643833313662613039353934 +30316264616631636561633538316338653039376462303762373861333733383434663730613163 +32343532316366346235363665353737313062613266313866343164333035613736643161323861 +35346466396238313030353066393830303164376338356439653038386365643039363161306662 +32316263353662656363346233386130333733343637376334623931663032343432623561623265 +39643133396433643336366137366262353362386339303831333537376236343564623762333961 +38366535613761646362613331336164633334623262313163376165353430386637313834343765 +63316264643430373261393735353830656231313861626433396631363864376365376131323638 +34303337636633656135373963386666356539663066633231303565366638323962336161366636 +33633632326561663030 diff --git a/etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring b/etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring index 0755939f2..f18cf688e 100644 --- a/etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring +++ b/etc/kayobe/kolla/config/cinder/cinder-volume/ceph.client.cinder.keyring @@ -1,15 +1,21 @@ $ANSIBLE_VAULT;1.1;AES256 -61333337303062373834616466393336356566363838343931393764666238656265623461323531 -6536633237336366663131613038613161383135643663360a646366383635353462346261636438 -34613937666666333238633065353537333632353664333965383836613661653965306137393434 -3061363366656234380a323534313766636336386635346633373436363738396430636465373230 -66336666333339613430396338383061616362613535306638616162303834373361636535616336 -64386136363363383132336436636630333933373331663037636136363163333234653934633433 -36613337623031376234643461643736336236376664336237636262343735353966666665373364 -38343835653630343437326666646561663332303438393431303235346434306239623233363831 -35383733613766653834313539633532623836613134663132353630346235363661636230373764 -33633866376131633163666365616238643565396461323336666365303336613334343431363738 -63363164616232643764613561326163336636666465663636343334643235663839656266343964 -30343366383939383562366466323163373739323737623737366661643330333862323632323333 -35366531356662303431306231663665663465613163663935613365633134396334323935613466 -3832663233346334653462313131616330306161313464383031 +66633562386538623338306134376265373061653033393439653364613136313033303266386239 +6362306337353837646635363735353636646665343738380a323263346634376338386333663237 +32633663363636333434633861323936636339386462323832383866623963353563663830356331 +6232333964393963330a303538636338316138336338366363333938353463336632323762393038 +30363835616133323661313966353936626331643033663030333933653165316262396234636139 +33636534653238623133623362323566336134663137333761323966613431393063656462343534 +36633263626134633464653439663530373864386434663134356566613164303036316333363532 +66636132333938623061643937376636326236616132363235373736343733373432656232366132 +34373639613035313334316230613639376132636332373930366630393239343133633764363635 +35366532383032373830616461623465386465646464643834616465633236333636623431353361 +63623362306666313433393939626338396233306263333032636464333263613037343739393362 +32373035663161666531313135643664663463353864663735633034633732343039613663663736 +34616463376339356561393339363837373932333930653833393236373432333238643761303065 +37383938396434386664316133323538323563396532616336323332303931306630383361383162 +65613537303030366334363939616239313561363034393834313938353836323039656234643935 +64663061306135623634643631303532353836373665616630616635343662613866646163663435 +62386262646331383930363036336664656235383663306331636561633966313132356363616566 +62323135323865663966613262653265393936623238366534663761323366343962333833353261 +32393561663832386365333564313138383236353834303735613334336664633361373861313530 +35323833653565316138 diff --git a/etc/kayobe/kolla/config/glance/ceph.client.glance.keyring b/etc/kayobe/kolla/config/glance/ceph.client.glance.keyring index 4e7dfdd06..acd0d46ee 100644 --- a/etc/kayobe/kolla/config/glance/ceph.client.glance.keyring +++ b/etc/kayobe/kolla/config/glance/ceph.client.glance.keyring @@ -1,14 +1,17 @@ $ANSIBLE_VAULT;1.1;AES256 -31326335316637383363383335383035326562323936646166343238323735343137363666333932 -3530313933373765353963643062636235326435633733380a316531316261616564633732306465 -33653631616263616633366461373234353237343632623136663632666665343536636434316461 -3631636532396130370a656664333939376237376637333165643465346437656235396161616538 -36373732343232333563336331366336323936336638613166376465316636663234623565653035 -32366334633062383834663038366437383737326331633462663230616330333361663139373863 -64333665646134353032646134393733303537663039313966396232323031363466323930363036 -66386438653265393035646637373034633663353833613462363735386465376465373566346332 -63356330653061393839613330386635343738383663393664346535663336343061353865313834 -61323461396636326265326432646363373062643065396161643035613937303934303436353038 -31333634663032653064626338353531376239323335353262336161626637313437653961373334 -31663138323862346238346435616134383733323733373462383731353038396630323733343366 -3362 +33623831343334383064653930323863393661643464383431363865316230653333356138633231 +3062306531633365393262643931383535653765663062300a336461376231623136383834613465 +62663864356564376232376236376436663162373064363235356437613531386562613331316634 +3530306263366236370a336232393433333434386336353432366364653134353835643935666265 +32663830326261343334306638303735313934366333353336356664613630333031616462336233 +39336138666263326338366136616631643330333164316662306466653031626665613231303939 +64663165323338326432633165313235633966383531653534376431623862316532643265663865 +66373032643866373736646233353035383434316135363437666134303338363265343131653933 +64303663353130353532323432386533636163376134386233393866323436346436616163383637 +62653235363331383362633236383438383466373136303836333937316439353463363538343533 +38303038323430376632313234346265653133386464383133613236393738643931323833346363 +36643532613063656663326138633239383134653464383838613938306565633064386532323738 +62363435623136383565626331356630336261323834303439313430366464623335386332396361 +64383031656331343335613632306134323762333935366366656533666464313366313430353366 +66383966363561643865663034346439373865626161633439613333393831343335616262623836 +62613933343535333939 diff --git a/etc/kayobe/kolla/config/glance/ceph.conf b/etc/kayobe/kolla/config/glance/ceph.conf index 461f51fc9..120d28e18 100644 --- a/etc/kayobe/kolla/config/glance/ceph.conf +++ b/etc/kayobe/kolla/config/glance/ceph.conf @@ -1,4 +1,7 @@ -# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +# minimal ceph.conf for 359e2b0a-3c64-11f0-bc95-1423f21d0230 [global] -fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 -mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] +fsid = 359e2b0a-3c64-11f0-bc95-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] + +[client.glance] +rbd default data pool = images-data diff --git a/etc/kayobe/kolla/config/manila/ceph.client.manila.keyring b/etc/kayobe/kolla/config/manila/ceph.client.manila.keyring index 6e1d7072d..f9c49b042 100644 --- a/etc/kayobe/kolla/config/manila/ceph.client.manila.keyring +++ b/etc/kayobe/kolla/config/manila/ceph.client.manila.keyring @@ -1,9 +1,10 @@ $ANSIBLE_VAULT;1.1;AES256 -61343063333633393830353031646336643935643536353332386133303562643162353633343438 -3132323162393263623264623236373062343732656139630a353063646335333961303937363533 -31653030306366666461653664626631343833323033663164343266616465326335663939626466 -3536356361396231300a623862646234656338633432303234333134613237643834646361666437 -34356138336235356162653161613930316635623130373037396234643965303733323939376565 -62663138363133336162303731663463353865346635653962383439383039383634336633613661 -35306230356464653639393137353938613462363534623039363766353666656131336265653261 -62386639316661353261 +64303765653364623133333531333264373165393136356662333238393630393066616133343331 +6137636430636663643064636331373261383336333336380a383637323062663565333536613562 +32303463343835386337303065653033653935643237383364333031643235326630643235383361 +3130643436643634360a663437366266653038353836326637376464656562633762373131316135 +62366130313636653961643530373566313165373739383466623431306339353036613837353736 +31336261386338373930313232666263316361636130336361616237613833643433663132666562 +34663565336335643364313338353664373933303663366238316539663566393235396536316634 +34343365333837656632633132383364366265346337363065633861333962633630363664386138 +61303865643532633563366135306462633862313138353031636636393534656262 diff --git a/etc/kayobe/kolla/config/manila/ceph.conf b/etc/kayobe/kolla/config/manila/ceph.conf index aedf6f27e..8c275bec9 100644 --- a/etc/kayobe/kolla/config/manila/ceph.conf +++ b/etc/kayobe/kolla/config/manila/ceph.conf @@ -1,7 +1,4 @@ -# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +# minimal ceph.conf for 359e2b0a-3c64-11f0-bc95-1423f21d0230 [global] -fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 -mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] -auth_cluster_required = cephx -auth_service_required = cephx -auth_client_required = cephx +fsid = 359e2b0a-3c64-11f0-bc95-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] diff --git a/etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring b/etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring index 4226fa4f2..54ac10a39 100644 --- a/etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring +++ b/etc/kayobe/kolla/config/nova/ceph.client.cinder.keyring @@ -1,15 +1,21 @@ $ANSIBLE_VAULT;1.1;AES256 -64306563353463633066363230393232353939303234666166376634363764633535636662323032 -3662376265643233636261613463363764646331333062310a363732616238373461336263336139 -39616137343132336233653232626233386630626264623832613463326461623439373038656230 -3534633163353461320a383366353534633236353732323761333834313332376635383539306463 -63336338303135353938393765313233666230376130376230663631303330643264656261336532 -37353437646262313732373034363266386632383264303838343933656661373135336233663637 -32313239663939343534393137343434346133366264663434376663333566336439343637313632 -64653937636131326165616439393766313635393230326464633135666631336434616332306163 -66306365383964626530316665633730363764303338623034366164663965656664653331346630 -61613364666361653765666235653363623032633732363563613735333961393839623339666266 -66323564616438383234396430383832373231333239626565636433626465343962393438363732 -62653864396464363966663663383233373538656661386631343735313463313830623330643164 -37333531346637613734633834376566326335343834333366316265636134656438613562646234 -3732393862376361316465353863396161323065346130313863 +34613334323263393035613637646664376661323335616461386633336165393665623932336363 +3938623139656439356563336365636235633663343338350a386164383634633535663866336537 +37393635653230646439323332653437623966306333353537666562666366656639313863633935 +3936383030393931320a613466326639346361303934386331396238616537663339353033336634 +34663964366665333738363532386465383235643639616139643561656161303935356430306263 +62613262653838386333303865333932363762623764623765643038356565363631323436363430 +36393564373937373364626434306437373063313535656363633639333866666466623032393538 +33613536373962646664633564383463663834323435303737376537353932303763346662353537 +39643038653439663932636130653834373263363630393234333731376538356363303439393763 +33333530643333323131343233343136323564353030343332636439343832323536636363363936 +38326639316436353034326338323166643661613061656231643038373535306165626139343235 +33383966666339323532653266316338396261363030306133336139303432373932336637386663 +63653433316165633432616565393062343264333662633630663938313834386133336661343261 +64663733656662353334646565326239626538316462653936353664613738353364323061313336 +63653338383365316463616364373338613733343465303761636131306539623033663136396231 +33633266393830373232336631323633343632616636646266633739623231313031393939323136 +34356466646339306530643065366263626135653639393465316431666230396532623761356336 +63633632353964303334356239313833363134613031393935613930396332616164316364356135 +65636137656239323162613836383931333261633033303865373638653737383461303530653265 +35343633363464356230 diff --git a/etc/kayobe/kolla/config/nova/ceph.conf b/etc/kayobe/kolla/config/nova/ceph.conf index 461f51fc9..8c275bec9 100644 --- a/etc/kayobe/kolla/config/nova/ceph.conf +++ b/etc/kayobe/kolla/config/nova/ceph.conf @@ -1,4 +1,4 @@ -# minimal ceph.conf for db775a2e-4e6c-11ef-9a2d-1423f21d0230 +# minimal ceph.conf for 359e2b0a-3c64-11f0-bc95-1423f21d0230 [global] -fsid = db775a2e-4e6c-11ef-9a2d-1423f21d0230 -mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.203:3300/0,v1:192.168.100.203:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] +fsid = 359e2b0a-3c64-11f0-bc95-1423f21d0230 +mon_host = [v2:192.168.100.202:3300/0,v1:192.168.100.202:6789/0] [v2:192.168.100.204:3300/0,v1:192.168.100.204:6789/0] [v2:192.168.100.205:3300/0,v1:192.168.100.205:6789/0] diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 9bbdb682e..2eb21140f 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -97,6 +97,8 @@ jbo_gateway: 130.88.9.250 jbo_cidr: 130.88.9.0/24 jbo_vip_address: 130.88.9.4 jbo_fqdn: api.openstack.man.uksrc.org +jbo_allocation_pool_start: 130.88.9.15 +jbo_allocation_pool_end: 130.88.9.131 data_cidr: 192.168.100.0/24 data_vip_address: 192.168.100.200 From 0c148e0a8c8dbf9fa5303bb51b763c4953951c8d Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 3 Jun 2025 07:13:10 +0000 Subject: [PATCH 070/103] Fixed spelling mistake --- etc/kayobe/cephadm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index a4cee2f55..06fec2004 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -139,7 +139,7 @@ cephadm_pools: erasure_profile: ec_4_2_hdd allow_ec_overwrites: true - name: images - applicaton: rbd + application: rbd state: present - name: volumes-data application: rbd From c3bafd6d20577835e2f4d093381cca8cf54462ec Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 12 Jun 2025 15:08:26 +0000 Subject: [PATCH 071/103] Updated osd_spec with ssd_me5 --- etc/kayobe/cephadm.yml | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 06fec2004..0eff84d1a 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -79,28 +79,18 @@ cephadm_osd_spec: | spec: data_devices: size: '15TB:' -# service_type: osd -# service_id: osd_spec_me5 -# placement: -# hosts: -# - ds1 -# - ds2 -# - mds1 -# - mds2 -# spec: -# data_devices: -# paths: -# - /dev/mapper/mpatha -# - /dev/mapper/mpathb -# - /dev/mapper/mpathc -# - /dev/mapper/mpathd -# - /dev/mapper/mpathe -# - /dev/mapper/mpathf -# - /dev/mapper/mpathg -# - /dev/mapper/mpathh -# - /dev/mapper/mpathi -# - /dev/mapper/mpathj -# --- + --- + service_type: osd + service_id: osd_spec_ssd_me5 + placement: + hosts: + - ds1 + - ds2 + - mds1 + - mds2 + spec: + data_devices: + size: '890GB:900GB' # Additional arguments to pass to `cephadm bootstrap`. #cephadm_bootstrap_additional_parameters: From 297640bee7e689e28169a6ec7b41c6ba333353af Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 24 Jun 2025 09:21:55 +0000 Subject: [PATCH 072/103] Keystone SKA-IAM config --- .../idp/ska-iam.stfc.ac.uk.client | 14 ++ .../jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf | 4 + .../idp/ska-iam.stfc.ac.uk.provider | 196 ++++++++++++++++++ .../keystone/ska_identity-mappings.json | 1 + etc/kayobe/kolla/config/neutron.conf | 3 + etc/kayobe/kolla/globals.yml | 12 ++ 6 files changed, 230 insertions(+) create mode 100644 etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client create mode 100644 etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf create mode 100644 etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider create mode 100644 etc/kayobe/kolla/config/keystone/ska_identity-mappings.json create mode 100644 etc/kayobe/kolla/config/neutron.conf diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client new file mode 100644 index 000000000..11b533b4f --- /dev/null +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client @@ -0,0 +1,14 @@ +$ANSIBLE_VAULT;1.1;AES256 +66326165353134363533313035316536343934303966616232613136633062333639323431303865 +3961626538623962326565636464646163626364363337650a666464333561373665343531656139 +31393466626464376333663036626436303535356230343433333233393766316434653233623464 +6162613666646664370a303264333131316262383934313961643665333531636664303533613934 +37373132393938643431376361343139336231343465373762306538313666653034343962656538 +32306536633930393530613038393235646564623362323333656238623761316535616434323535 +65336161383439653866346563643738316630663637306361343938313239663130663266623766 +33306561373735396364356138313462303532353136323964313364346131643034306133303039 +39363632666137346363376266343662396430386163383638656237356437393937356561313936 +34386264623737396536623135303065363234633236613063653963313537383161313030373463 +38343864303530653164313865323966303633373634363765393431356634626262306364333939 +62366635326533346266633532373564386266363137646565363631343664346231303761333061 +33376164386661353630343436376239363136653766333865376331663164653565 diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf new file mode 100644 index 000000000..617ea9486 --- /dev/null +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf @@ -0,0 +1,4 @@ +{ +"scope" : "openid email profile" +} + diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider new file mode 100644 index 000000000..20042bea5 --- /dev/null +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider @@ -0,0 +1,196 @@ +{ + "request_parameter_supported": true, + "introspection_endpoint": "https://ska-iam.stfc.ac.uk/introspect", + "claims_parameter_supported": true, + "scim_endpoint": "https://ska-iam.stfc.ac.uk/scim", + "scopes_supported": [ + "openid", + "profile", + "email", + "address", + "phone", + "offline_access", + "eduperson_scoped_affiliation", + "eduperson_entitlement", + "eduperson_assurance", + "entitlements", + "aarc", + "wlcg", + "wlcg.groups", + "storage.create:/" + ], + "issuer": "https://ska-iam.stfc.ac.uk/", + "userinfo_encryption_enc_values_supported": [ + "XC20P", + "A256CBC+HS512", + "A256GCM", + "A192GCM", + "A128GCM", + "A128CBC-HS256", + "A192CBC-HS384", + "A256CBC-HS512", + "A128CBC+HS256" + ], + "id_token_encryption_enc_values_supported": [ + "XC20P", + "A256CBC+HS512", + "A256GCM", + "A192GCM", + "A128GCM", + "A128CBC-HS256", + "A192CBC-HS384", + "A256CBC-HS512", + "A128CBC+HS256" + ], + "authorization_endpoint": "https://ska-iam.stfc.ac.uk/authorize", + "request_object_encryption_enc_values_supported": [ + "XC20P", + "A256CBC+HS512", + "A256GCM", + "A192GCM", + "A128GCM", + "A128CBC-HS256", + "A192CBC-HS384", + "A256CBC-HS512", + "A128CBC+HS256" + ], + "device_authorization_endpoint": "https://ska-iam.stfc.ac.uk/devicecode", + "userinfo_signing_alg_values_supported": [ + "HS256", + "HS384", + "HS512", + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512" + ], + "claims_supported": [ + "sub", + "name", + "preferred_username", + "given_name", + "family_name", + "middle_name", + "nickname", + "profile", + "picture", + "zoneinfo", + "locale", + "updated_at", + "email", + "email_verified", + "organisation_name", + "groups", + "wlcg.groups", + "external_authn" + ], + "op_policy_uri": "https://ska-iam.stfc.ac.uk/about", + "claim_types_supported": [ + "normal" + ], + "token_endpoint_auth_methods_supported": [ + "client_secret_basic", + "client_secret_post", + "client_secret_jwt", + "private_key_jwt", + "none" + ], + "token_endpoint": "https://ska-iam.stfc.ac.uk/token", + "response_types_supported": [ + "code", + "token" + ], + "request_uri_parameter_supported": false, + "userinfo_encryption_alg_values_supported": [ + "RSA-OAEP-512", + "RSA-OAEP", + "RSA-OAEP-256", + "RSA1_5", + "RSA-OAEP-384" + ], + "grant_types_supported": [ + "authorization_code", + "implicit", + "refresh_token", + "client_credentials", + "password", + "urn:ietf:params:oauth:grant-type:token-exchange", + "urn:ietf:params:oauth:grant-type:device_code" + ], + "revocation_endpoint": "https://ska-iam.stfc.ac.uk/revoke", + "userinfo_endpoint": "https://ska-iam.stfc.ac.uk/userinfo", + "op_tos_uri": "https://ska-iam.stfc.ac.uk/about", + "token_endpoint_auth_signing_alg_values_supported": [ + "HS256", + "HS384", + "HS512", + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512" + ], + "require_request_uri_registration": false, + "code_challenge_methods_supported": [ + "plain", + "S256" + ], + "id_token_encryption_alg_values_supported": [ + "RSA-OAEP-512", + "RSA-OAEP", + "RSA-OAEP-256", + "RSA1_5", + "RSA-OAEP-384" + ], + "jwks_uri": "https://ska-iam.stfc.ac.uk/jwk", + "subject_types_supported": [ + "public", + "pairwise" + ], + "id_token_signing_alg_values_supported": [ + "HS256", + "HS384", + "HS512", + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512", + "none" + ], + "registration_endpoint": "https://ska-iam.stfc.ac.uk/iam/api/client-registration", + "request_object_signing_alg_values_supported": [ + "HS256", + "HS384", + "HS512", + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512" + ], + "request_object_encryption_alg_values_supported": [ + "RSA-OAEP-512", + "RSA-OAEP", + "RSA-OAEP-256", + "RSA1_5", + "RSA-OAEP-384" + ] +} diff --git a/etc/kayobe/kolla/config/keystone/ska_identity-mappings.json b/etc/kayobe/kolla/config/keystone/ska_identity-mappings.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/etc/kayobe/kolla/config/keystone/ska_identity-mappings.json @@ -0,0 +1 @@ +[] diff --git a/etc/kayobe/kolla/config/neutron.conf b/etc/kayobe/kolla/config/neutron.conf new file mode 100644 index 000000000..b36726cbf --- /dev/null +++ b/etc/kayobe/kolla/config/neutron.conf @@ -0,0 +1,3 @@ +# Global physnet_mtu - all created OpenStack networks without MTU set will get global_physnet_mtu - overhead +[DEFAULT] +global_physnet_mtu = 1490 diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index d8ff0b13f..ac29a3468 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -10,3 +10,15 @@ enable_horizon_octavia: false # Enabling Ceph cinder_backend_ceph: true glance_backend_ceph: true +# Enabling SKA-IAM +keystone_identity_providers: + - name: "ska-iam" + openstack_domain: "ska" + protocol: "openid" + identifier: "https://ska-iam.stfc.ac.uk/" + public_name: "Authenticate via SKA-IAM" + attribute_mapping: "ska_mapping" + metadata_folder: "{{ kayobe_config_path }}/jbo-idp-conf/idp" +keystone_identity_mappings: +- name: "ska_mapping" + file: "{{ kayobe_config_path }}/kolla/config/keystone/ska_identity-mappings.json" From 55eba15748c2bbcfe72db226298cc9e689b4b1fd Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 7 Jul 2025 10:18:49 +0000 Subject: [PATCH 073/103] Add external certificate --- etc/kayobe/kolla/certificates/haproxy.pem | 467 ++++++++++++++++++ .../keystone/ska_identity-mappings.json | 65 ++- etc/kayobe/kolla/globals.yml | 2 + 3 files changed, 533 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/kolla/certificates/haproxy.pem diff --git a/etc/kayobe/kolla/certificates/haproxy.pem b/etc/kayobe/kolla/certificates/haproxy.pem new file mode 100644 index 000000000..96e4b8434 --- /dev/null +++ b/etc/kayobe/kolla/certificates/haproxy.pem @@ -0,0 +1,467 @@ +$ANSIBLE_VAULT;1.1;AES256 +38353032383462393432383163373637623363333230306466643236363634386263376466656438 +3365313364383532393234626633396638643262353438300a373933303765393532326665633136 +30653930303233396364633331316235376662363334343030343731343531373635303031333431 +3564633162613934370a333161643935333462316661616330383531383636393461346439353630 +63323739386361366634336530393635663239643266353430646634666237653132363034326137 +62373164626638616363396531633865303631396532653432323833393335646163613162393339 +66306235323063383434303131633134626239643939653566633466643237653266343861313132 +35613737323663656435366666636163633361326133623964656661306634633664633966303136 +36343563313531623666396364396464613664373432333332363337643766313064643366623434 +33363361356535336365303034666564613261313932613931326139616662616665613935666364 +33376537616366366438333430663630653534386666393861633431613236363164323563336439 +35313763653034643166653634656535666333653432656230333037313536646464343863356135 +66383431346461373230613239323738666163633532353135656435303938346161303435613166 +30656530383637303761336233633531393935346266373562373362383839383561646637643239 +66373539666263613236363362646638346433633938386339633233613230633431363032336566 +32393635623630346565396166393630613462646433656566373530643464666139386437333461 +61326162386162316236613533343536353630373362656138303066353534616534366532656563 +34353438643162646366303264336562366161626635336539336533646161343337306662303161 +63333435643534313831323231303536636338653261373832346533613737656330396462343933 +39326265323666366265666433626337383531633338613731613432303538373566393436663866 +65636363646666303636346632653932343931346134616261623065613762323933643738373835 +30363864346237323231653464623838353732636566353437313764613064313339653332663231 +65653335643864353439653034323231633661613265623439653234326162396635663665643562 +39346266393866623030623733643230663666363234383861643234353363353764346537353836 +62326661633030353961336535323833303666633231316662613833396163333637643030316134 +31346161366536613265343961633338366363303165396232653038323465323130333134653432 +66663330333661383563666334363439613263623538306535616164343663616538313239346132 +36346263383536613564346638616135303935623031653862353563353737383662323436616230 +35343036663366356330363238326564356536396430333363646438336361353933613361313330 +61656262636337633364393332626532323062356230633230343833386266663039373131636263 +39396237386339326263336232313233316536386336343939373634373761383966653334666237 +38353432343138383733306639613331386237353961363532663036316331663662386664656334 +30316130363437363762353732633436396265323462313839343437316563656438656263353065 +30383938636631373963393261373861363737313062363430643236383763656565633562356433 +61656438363561356537373565636338383636313661303734386263373335653832616330346235 +38363138353731333766386330663534656261613437306663636665613238346466313933623561 +34316232313665616434626336396439646265626464353733313363633738643234363736643265 +37343531343130383165336638386161313235633464396138383739383732323736663865303730 +38623530313864626530323462666162326331623331366534643232306663666662346461306331 +64356664353033633738303133386531656335663563386562356639366539303661323635623436 +39343565613266336466323232656365646165366561666163653065353334363161393533616664 +64623134323639343938386365663435666632316666633263393131376536306466383961373735 +32366266633965373932616564346263316632613534376261643633653237386662343638656166 +36636264613566393966356132353733366636373264643830663331396261656562613262313065 +64643233316131333737306264393135373939336139346137343633393939366663663635373464 +37303434306337326238636639656637343837636232346165643832396339316264653662396139 +33333861393537383437613135656136393761633865343563613537316335353839363030653831 +62323939653965323662653261393139343365616363303065346266396230336136623230333731 +35623631313261613137623032366562366662393264633037353962373662316166373230376534 +61323631663665363830376438343664393935343736633065613133356461373033383863333864 +64396437636565646438343235616138613536343033646135366133393836633137373362343761 +39316436306362633066353563353563306663323362396363336663613230663433613136353435 +33393964626531316466623433316434633235363363333333343933343433313534383437383739 +35666163616134353936316466383462346438623530613036386663373737643435323037316566 +66323837383533313361643933386461616234306666373566666134313165663062613438366463 +33663537613934303365383333383666656338393239396635346665383061303830366433383139 +65393738343137346365383336623733346631626335336464396362303063623839613733313937 +62643963313030376235336234346663333430373634636634336532663361653565393463353837 +66313162616166633537626431613965316333313962323664353730323732613264353534393831 +38353333386537366232623863663431353837313965623537653363386366623838343766303138 +34643039303738303930623833316166643837393134653536313334353230666230326532303336 +64633338323465613962636565633032356664383335346339303536656239393234326132383538 +63643133356661656638306333313134303239303439336638633730323538353834653233333030 +64336266353838643333343537666161376532666662666266666339353866643435346266313134 +32383739616264386236313662323462373137326538383131363436313565656263636165656363 +34333235653863336439303564363162373738383538323339666238616466353336346432316531 +34666637663139653663363130363564663335653737373638373633653766343137373133353763 +37623637396132326461636438356132666363666562373334333535343930616230383830383730 +65393063656561386438643434393939643839313336396266633635663234383234643833663638 +64623732333734383934383531363838626437393466656535346464313166393437616461363163 +63393162383430343630363365623062623932333632336665626561303165343364393037663033 +61346535633831636631356363643933316434396664623535373331623332623033373734376662 +66386163613739653564306564303562366534336230303436366130353866336433386439343835 +33383965313561343939373733363931303535346139616539666465393931303634303364353165 +66386234663235323131623936363339633964386630633730663730636663323632633765383739 +66346662356331616430643262356233653339643832626439623663356535626335303631363037 +36643639626638353139343366363336626235396638643362366165323562313663356163633734 +63323463323630333233646138383333306436616334653266666265313461343564323738663035 +38316239316263353631653930623034396537316133323132306665333137353437363534633263 +63366433306133383862373830363436316531353238323865666639636263383730373962393139 +32616264353835633533346437323734303935386465306565633434376139393064323564366135 +65333064646632346531393032396135653164313830633563336439386430366533313862653864 +30396238373364333463656135356462656162316130333066336134383363363763656466356165 +31393335633735393634653161366130333136393231636334666164666363346433646665353039 +30313933626461616663373236663966303430613635646565653264306338366663353131646466 +35663533386434383666353836313361323935323564353733643031633066393831333731326539 +36616633346635616462663735653330333735386335656436353037646332323330306365643362 +61306263633365343330643635373237353231366335636237336232356135366330313133346661 +37376530373831653266303533663633613962376635356366346635373537646239616363613565 +38643631613331363561366464373733626461386366363038636332383164616663323833636266 +31313432393330383335333537373335613935376465366131393838356566333164656465656435 +34373065303736383531636134663362316538333230343964373736623831613730373063323631 +66343530363130363731383538393466386239323664366263376431353035643233623363663635 +62633966383163316465663135653837383731633538356330326563313437333438363565653136 +63303634303063343433663133653334376237636532376138366238636461623231613162346235 +62373662313234636535326139393733626238343436333133366664316535616363333863333635 +64616430653039333665623531353463396233643265366163326163633034393239653732643031 +39326262303732663265363235616163653934643766326330373339356139386331363730343633 +62353138646138616330333561373966613838313633316534316138323531343064643532623965 +62623636376231646138303931666435363131323364386530313538363763613966373634636166 +64613037323638666264333933326536343262343533346335343763316665343131363737343165 +63613137323531646263396438306165383332326331333432633738383861336137626466323363 +36306433393636656638313031383330343833336533626364313430356632323237366239386365 +31343764656361316632346637323130326139623033313564616466656336386631393137633966 +37633239346533313065663863323466653862313335346535656533643466376462356462613337 +61373139366462346362396465646535636464363836373130666633616361386230336332366633 +30643363666337373166323432313338323831616661346531363833386538613765613835376234 +61656433316137363566366430363134613066306235656432313863663661336434653663323436 +37653032643030343232373661656264386639306438643639323334326133383032613536353862 +62313734316338623739356636653532653166353662313463323531623031313662353439353232 +39343862663932616334633562623031326339366463393537386562656230366133326530383265 +64656439663733383234313533363330633636663632633538623265373332306138633262366165 +31356233353561326361326434393561376265633832363235373734623363343835633232646636 +36356135663133343133656266373063376563323032643230653431316535656133643938376137 +36323963626663343463656332636533326361356433623563333964666339633363333331353963 +31316565636337333066613332626631376362336463643632623334643436613933353834616132 +66653930393262663738323238313764663466346236653534393336633335396439613864366364 +62613030323263303161623337313164323836643739326261353461616461366161353861363238 +33633236363535623131306664386565326363366435386531396636343564376436333032623931 +35383734623231653731386361313562336263613934333937346361663438643031663036333139 +64663066633564326638396633363937386464313935653932393235643862373636623938333634 +30373033316561383436636463373037326161306336626262303737383261623739363031373135 +37633662663564656534336234356230656263623432346539343836343232396333396661373032 +64656432363239656435373734373862663336383238656232373765643738333938613634373436 +37366430313132636138396561376166663535306163376662323762386238643562353135356361 +66393530633935326165333130336165313731633061653463626333323964396261396138643030 +30386635633363636661373539613731633163386534366163613065646530643932366631653663 +38353437643161346266326165336133616566613131613935653133656366633933633133313865 +39313237396538343135373834313466303964666234396234383463386466376163666633313538 +31663663656131643833393338656365353863343936326538376264303662613063656666616337 +39353434623133653563373262363235666234656166383765653265383430383064346464613763 +30623838346634343431343261636330336466333361396632316232623931626363366530326266 +66633938633932613135613566363633336139393938636333346136613833383234343234336166 +64666331656137656661333837343662376164383832306139646533383436343862623565383830 +35643630653939363931636432663765353830366438333861333139313063343430376231383663 +32313830643337343235633263363763353034363364386138666136336661323835333631656635 +33393130633461643362316136633232643163383462663964343639356238636435653632356238 +36323135353837313235353934643930303034326339316430323339316239663563353664323964 +63343038313837333037333036313965623133633439346531666231373238646661373436303939 +33303862626163316230636164363866393636646239323164663932346334313832323364633638 +66376561386164346565333036343865303663303662363531363737363736376162356431653464 +35363564303934343532646134393436356439613232343837363030653364366432323564323337 +65386261396565346434646237353136333733306235666138393734333731306564626165366531 +63383837303665343231373136616266303764303061643533656637666561613630643062306139 +31366338626132623637343033633937386535656366613566386335653932303864656530386238 +30336535373164636466313935636366643833326563393636623430313939363035653531353463 +36363039353863626634346531623134346231326532353131333234366237663939623137396434 +31663630336266646663306435323861373038303139653865633165393835323036646637646334 +36323765363739653666383966396533353966323039643030633061303465653433356262373266 +32363233373435343733626135333062653539356632633262346432316362323231616232376664 +63333063663136353135333135396239343333316631643738383636346439663736333236326161 +64623934663364326439613165356466313664653464373837626137386263323563383236336330 +65313361623232323732306431386463336139643933393936316630313232353263336537376631 +61376161303962616533316439303863396465323937363466373635663332383731386661313133 +63336135663832636539653265326432653731373634363732663638303430363937336231626264 +37333335653637613130346565623862656262396233336661626466616362363731666265396233 +33343135383139313163333437323230376535626336396565666566393034663563653530626232 +39323461346430646263323636336439333839396136333666613030333362373261396439393739 +66353333383034643263623363303961396264343834336561306633343966616531333138396461 +64316235366233333832636238636339643139393431333430363163616233633134343865343861 +32303765303434353237343034323566636238383537636365313231343136373331656566613332 +65333336623730353963653464646239633363666565663034393237666563333833653133343136 +35366638656237343737613330386136383833383130656236393338303933343833623363643931 +64396131623262653464313032353266336462346130356466323032383839323736366437383238 +39316664656331396337656633363230363938633530316232633038386235353064656437656363 +63303333656166646364396535386332316163663165623539313639653063616665616330386163 +38396130323663363663316163313038613139333762646236373430623131386563303233303635 +34383936666239626234633237333965646538363166616331396361643037353861393237306638 +35646262396535633339303131396538366436666366666634616465303832633535653064383033 +31376563396364373939626338633732323839316264396439393165383531383932373034343337 +32636164343864663436336335656437326431383365333834373038373561653436663133646431 +65393439393838303966366530303761333534346665336230336637383461383334386365646561 +62333566643135396532346535633165643338353537356166623865323031396537373163353232 +34316461653463613535383465636263356665636632343932393938366530643535613937666663 +32313937613766353538313261393036303537656565636366336464386332356562353139653838 +35363830616233336638633263393838656563636436376139393532333566623334313331383731 +33316632626166386563613630653437653261376438646633333537663030323238623731656336 +62333739326436643138393336376361353965323139343163626439353034306364333837616335 +61646632663235326166326665653232653163386235376534383230323461326231666535316531 +66396535393039303135343734613664396536383231623034643134353539323461363965333036 +31656533346332303566336662663731646135396661353163626236633130643462633863623634 +66363939666532306336366238653235393332303239393031613236303362336633396537376465 +32393635626535666531323363383138353535346662663734323664626639663338663963336562 +31376636613365373233656131626638633237333638386538636234353362613334336136303437 +34313064656339326437393838333430636435353739353031326432653035663639306533303935 +33356434666364313934663063313638353637386665653531396433346434313565643863316433 +30623332636464653861646164633061663361383730333265336465313138633535653665306362 +31643738633732376363366438376165623565646434313161616561346332636430653562613834 +30313331313962653639376238633466316130386365623130356136653238656531613864633036 +32386165386265666536653136306238383331613266343032663631613262656563316331303765 +39653661326361386364383834343832323365396338333633646533656538383861343465346534 +36653031653630346638643836393837616534363465353832366163383165373534303466636561 +65323430386530356631306561343664316438343232636461346564623861303663383761653231 +31653863653261663838373161613633323632396134616462366361393433393935396661643634 +34613164303365313333616134393831663664336235346566333064353465666162626333383761 +35363938343562616366343931643330306439386437326230336530303463316664376631613465 +62663665386137383032363636303835623239656432333434333533393230623833653966616532 +37356564663465613264373531653664666138626536643261376266636362386334643233333965 +66323033636234646532636636636232666264323139633066623166656437663434303536363533 +39353337363839306139666461346532366538323535643365336665336138306237353064616631 +66626233623661363435626466366337303362323531613638376164306662616639643338363437 +65383131363664613030303362393032376366336563396538333730353134616332623162383538 +32366166393435303338613834396261326635626637633637626265666232643734306337343336 +33643664313465613534643532633161383866636437616134306438393932373263643363373335 +32663863653837386133633062303762663163303338393965376232663131376261633336393339 +32393639626563373261376464636335363261396661633863663262613135653531643034616435 +30333130656237363764616136333464623164663636393136376362343931333239346333633936 +64613866376164376438393235383430653231353132613165383735663762303534653461383763 +39313131356263643835656632343530633939363764343434623762396533393366653434636637 +31663862626436666366303833323536386238376437323132323538353939333964386430323131 +66313630363639373037303535646237663766343237643761346164623839626364646462663633 +33623661616334396433323133313865343238653865663163396334376430666337333061313662 +37633231386365333465616566333966616137646631373733663863383666343930376338623433 +64376235663535336161333263396561643963353165336434623735653165343263393563363330 +66626335663130633835663964333732386438346235623130633836643366656433656239623436 +36373332383933363231363831623661646466356262613138326133373636306532316135623133 +38343737343363346438313934633632323034616236346136383430623734663066343232393634 +61396332353365363433353932393637633234383239346638666131323062646664346236666362 +34363965343339626433383131333066333531383239653034663662666265626137323433623233 +30303030653834626134643435383131646436626662336464653266383363613062343837333234 +66633663623866623031633666643430643736363435646566366431653064393831313061323730 +35333633643439346431363330663632383365303032666132613761626336666131313763643534 +34646461613331396662346234326439653662643232396536383766623662646133363366366365 +64663639633831303739343130653932623538303239336337396365396638326261613839306566 +66643566623664653436366264383634383063373631366638646531623332393537313239376539 +65626234353136356337336363633031313535303838643438636266623466373262386337623331 +62333661386137633535373866346331323932636631643639623161363236343965653233623836 +64333538376432326166373563303134323462613931663865356339373962306437343232373731 +34623636396462393665646333323263633634383738353631636635646239383530333435626462 +33363431653834383238623865643034623564623533633130363034393463373333633733666136 +64396362353333373861323839623336313031626234303030353466333465633831613766633334 +66326539323336373962653233633932323661313036313735653632326635356631623761653030 +39363335656139316133636361353366396235343933666139396533396631343538656635356438 +62376231353661343862376361376163376535323231383531393037646164626633393066653632 +36336565326666353230333263656334613938666666313665636137623331333666396564303835 +34306561343532343166386330626262343761373231636336653663363062393866386232616137 +39333332663833326632393631333234333630346632643861313639316665626637656466666231 +30363663666261323033363161363734623462633130306265663939633532363062386637373939 +62303736393431666663346131613736323036313731626464306636623031356362646236333031 +33356433323136613332363431613036656133323736353162656539363034343535303235363831 +63613935326362646666333463356137653230323062393035633462633438616462366231383563 +34373131326565306436653139343631656261323231386132626632316434353435366264386338 +65633932616330616430336430653535313235636138636462343164326661366165323766336264 +39306136386264356364623734663263326263366332653061653938616337386439396264616636 +39383462363235396232623637353865656663343465386135383234616137303436396262376133 +31343936343134633134333933346264653934363536346166333137666261656261356364616231 +35666338383336366531623065623264326131613237393239326364303461383565623638623538 +30613837613139306466333632366461373864393837303731363636643238336337623333316132 +35663661336166666637376633666362626335396339323430376339616435653363356432346663 +64633137333662323038323633373164313666353937373434363235623437653934366562623961 +38353534623766333762343361663666396237363365663138663061336631613261373263623763 +31653231663464346665336337396264393564393932346131643630373633626337313361653065 +34653664323233643961613833643664323733353232306265343133366564376332346631373133 +30366637643565643233396237386566623939396164363763656131663331366666633335366133 +39653134346331613166373262323731336263396331653732323931646336386166373933623736 +66666337323335306530633232333932353233373430363939323633616438336465623765373934 +66633832363631616435353535613730643738326362306334636231646237333839623538653433 +35313332316163383930316532653035613735323066643636626363343965306333376339633537 +34646634663638393761373965623737356433353865356335373163623264346166376236636632 +35383637316635363837383339663038376162313163396436353931343135343864656131356362 +33363337383336656136616363303166636236343061303539303034303661366639613464646364 +66353235373564636561366133353931663165376637666238316332346336383531376532643562 +65326331303233653138386639323961616232343235373938383662313961323537353732373161 +64626165643637636463636566333164373137313735663534653136356233356261316630653539 +35633135363565356636376663376136336662393930613462633336633363376439353363353563 +61363466316164633038653030396166396633383731363936626131383038373030303564373265 +65326632343931356639653036653862383864333561653766313030303262613661313263376233 +39656437393063383761656261613035333965393534366533393765303162356565616430366165 +35346231663833656138613863663631303236616636623361663839643637316564643936383639 +63656631303834626235626330343962613365626465353636613563653862323833623235626230 +30353637333162323738306133303031613263303238626431336664366635356431336664303864 +61386639333865356362623361363365306337303636353734656361636632313134313135616364 +65313936393966356435666562643430313933386437343364616235376463323833303533363132 +31643364306331323331313135356436343866396636633737613564396464623633626332653031 +35333734616639613638373966343162633862363931636536346136643162353633633639373732 +62616438356336363931383365333362633164313235326631316166653965336338363833663835 +32366233303262333038306164623766343565366566306532343435343139393131383664623062 +62656264613137336463386532663532663131303233313261356165316564336561373330323866 +31626365633734656138613436613234323638393039316231633866646238393638316162323231 +32343935393038353030653535633635643765306636626366653834653061313161626661633938 +66616563626330383462636531316362356536663334353335313236633337653633356665346532 +65353638663633376330363865336635393934623366636239363536306538323733656631623464 +63346463343263366539663535333937316664356561303839623338626266633039643164633737 +31363761353539626363373765336433396136333461346136306330316364656236633165393666 +31376565336230646664643831383632386535303164633235346131306663326636373635363430 +31653765326235663364303430333935386537653832363661623735316437323565396366303839 +36306564323238313232326565313934353362323539303765373537343166316530366330363032 +30396239336266336133303931343030343937323931613234326533323839306238646631336165 +33333631646438306666316265356435333030653461663330366237613332313232343030656133 +34366361323732356338336466363633323536323137303861663364323762313637653034623065 +36653538356337383533326562333535343639623132383362346537636565666662643465383031 +65396261326261373534333735663433653063373838643565303439376433383461373463363537 +39306234356535646138663431656639643164353734623436396432643639333762646161623430 +33616261363631626134653937623863663163353866326337346530653232636434666537666431 +34613164323165376162396633646566633263363239353663636339386338336366326636383932 +30643365336538343235393538393366343633616166666464363133393934663834356566303335 +61396539306161633962356466393461643665316261656463373134366636613866373630323937 +63646463633631653235366561303934333433346235383266626239303037383830663233383335 +31303533363834316239313561343061373930383264333564306639343664633063396264393364 +34373639383936363862333039663431613538333337326139646634663137383139666332646237 +30383031363031633839353163356363653864333137313137353064303937613632633436346162 +31613962336430333537306632336266343463386139663562626161643661626365333837626633 +39313161303062343835336636656465316431616338333832386238643564643630303564356534 +33346162396333343166663337343230623639333761373433333630636165393635643036323239 +31666164386166393633326466383832396163376130356366386362343636326137393261396531 +64623965623139313832633366313133613065646634303133663638616361343930383263333132 +66303739353936646134363461626164333365646134383233343737363163376364386435633563 +61633330313634383635303564663362633832323237376664623035373261373364316638306332 +31613333333164336466396136356562663361366463613563383135316537633235666263306139 +36326536646432333761336332646266643333313334396235323862633064363666376330656231 +31633833613439343661366531393832313336336663393364313437336364303431383361643030 +63373130363561353763636362666232323734393534306238626630316664656166393031316237 +39393230333631653635393231646636633738373734363631343938366337636566386231383430 +34376661383039633365363863613861366438646532346439646166333534666436643135666234 +35393964373665616534643564643465353737636532393761653962396661373362636132633061 +66613465396131346263663139373637336563316339346262663632353739626232363039363465 +30346565313039623831623231666630656331323064323362653030663061303132393331653630 +63313936663063663265333162356330363363313934386537623833646565386538336361636262 +30623030643230373161623030373731363337393161396336333235323038396638366531333664 +35623834633764383666613434336333346461623137666266373531663637613266303661383463 +32313038376236366461646437663934646564356530623931613930336530356236386235633437 +37336538323837343237663138323134653163333231623931326232396132636338353265323432 +62303263326536323865386163613566383434356131363233356563313963646661303730343862 +64306237623161653133653963333836663337376632333737313562393362336130616361326666 +62643638336237663339303230663035376664393263643332663366373030326438336239333162 +62306538653633373831376539633135636564663233613763646562666364316262313364316333 +38306333373065306264363338373365373539383266623439376436636363656636646139613834 +31386466633031363365343763373032623539366164666231383839343639653039326232383431 +37353633306634313165386536303239313238303566346233363537366539383834323364346365 +34626439346562356537383033333732333637366362313638636566353365653063353037663034 +36306438353665373864353164613066376163336663643232336563333730353737656636653564 +35643930303262333935643861323265643735616161396363326132323239346331326334326265 +37383034353461366633623166646133643234636335626666346239626164643333333564323662 +33346338363533373035666133346464656236643533313162616233613135643662393732316539 +30363330666134323337626463363139333839653566336432663832326333643433353466313864 +37313339663366373564376431303332366538323639353838643234633636303562336533333434 +61396632616562353837613732653263333231613939366232643737616234356664393231386630 +37643635353064623238623339326331613664333065393564653234323033323465386264303365 +31653639663933643032653661323638386534626337396364323234643134633462623431363838 +61326336656666393565623933663831623034343430343736636235396230376537323933376361 +62633533316263306564353633373132333335343361633764373938626238316564353739646331 +34623132323866666431393734383063366337356331383163646639613965623737306532663163 +39333264613239666537343837643265653530313461653566353962656462343434663862343634 +64323436363437653661373761616536313436643566356666613235623166393631656561643032 +65643534326338626537623234666335383965326666343138363635326365366163303565653231 +34656238633632616665656630313963396666346235613063323131323164313231653239393962 +61313237313237633263633763616430323035386465383064343233383938376466626136623139 +36643934303964616535303530336137666238393230626562616639633265316164326231303165 +65646164363636393339343664636330343565343938363032646335663630646566363833646533 +65383533356135306561353237646131333233653562363361343438313236623036333939666166 +62396238633361653433643564313532646332653130343561623434313832653462666337663135 +30613238653435653561326635663135323763633435656365643064653732663264376562363838 +39623834343064383234393063356535636164633231653538633436303135656434663133326430 +31343639393964386666366635316462326463333235313534613665333036353339373938623861 +32613034643962626564353739663034663861363661333365663531623765383638353737626132 +61363562363165643031336563633666396235373535376163663635356163646333383763613834 +30623134386332653166333238386163393730663662383537643366363230346266346133356665 +34663166666230346561323232633837613635636230636630626362326664613236613866383864 +33373162353864346231633163333236373839363932343337396636333337643661363064386134 +65663539623661653931653139396665663363663966323835623930366566366232663138653562 +34313035313664383961316563363038353432656330653766356666326138613062313732623963 +33326634356365616335626530306137363663663562356639666563356631353236343835313035 +37373432623439376638386662366533613231363539626634393035323264643233616234373035 +34356337363865323461656131303332666435646662383636643131616434346164373936633066 +61323364373533323130313634396231386463313336383265373464653763653837326635313564 +63333636353663646463633832316536363732393462303865653265613232306230626538663939 +35333835313063623534326138346434323230363034346339396339346538323937323739333962 +61343137383235323433353964643637373362316566653538666432373237663236663730346161 +61356662616133363263366138343030626263346331373439313338616535643035336635343337 +63333934303239653761353233383561643835643265366230623662643234643538356533643730 +32343537663534323535326263313430366135336434356637303832653535363030656333376166 +63306666333337326563643064396661663536613561636133623061353463666335623030383538 +35356561663531653366386631323761626632376135643438323537316134333937333466343932 +34323466316166366534323365616332363635633762356231666232363238326330386232326335 +37343433353562653638653466393030653537653631323532663265393264623964316264396161 +33646136376538316434333036623564613939633937323939393761383133666237346239643362 +38373637353435313334663331356465343162383764626131306132633261353162323232376432 +64636165313864383730333739373964623766383861373533386330643630633536323032323934 +33363030626432646430316336323663393131373439313365323536336531373361643434363638 +30393236643362613662303362636639396236633337653462633532653164333430306633613438 +33653262323363386334323265396639643538393463363439353164393738366237326639363161 +39323330393033303537323333366163383265653131646664306431366536336661316231356464 +30663064333534323339363331323466396331373030343730613537613862383431326236366463 +63313761313836636665373635313766326234313435376333356365396638383736316465336134 +31663164353364633433653364336463616366396135663637356134633133653133663939636239 +65393431393138633138666330626365656336346630383831653663643763623034616138643333 +31656230323836643763643031383637303539336236303135323935383335623433396439333732 +62373939653035343739323833656366396330353738366165306333303566376634326432613435 +35646665663239376262626330656364646664366466633838643065613136396635343762396630 +64663539653533386230303530313134663233306562303230303635366433643636353263646432 +31626132313261333761306430356637373130343639636461616431623939656365353636323863 +65336431366433383666636633616232613537383133343332616236356533303733636564666332 +61353962346331313632643632306635633864643935326131626232656439663831346661366165 +64613032353239626131623537316564306665636363353234343734323335656237323262383364 +63616164613238663362623733353838633162363663383438386462623739373866333437306263 +61653265343065626635653762383961633165613835346363643666613136373435306334613831 +61356436316633306563653636373936636536323231313933326432383237666239373761376365 +61316561313232373134313836353235346338633031373536313439393435663331376366646431 +36306363393830393237393636333561363964633738636463643939393932633037313165643833 +34306562346238663465353939376434303062333862366662396239633430633366383861303332 +33613930373465373435656337396662393165373863306532313562653564303736393462643864 +35626339623336366139613631663531356666323438316536396665353630373837373038613833 +35633538326461303337353264366635353739633664383161613939633935313433613563393161 +38663732313534313966663164343636343666346465323264346166383465643130386631316139 +66656365633033363839333734336165323963663165656437363836376630356432633566656666 +63376338333331333533633133653966323832653335393232343739613130333237643565393366 +37373738653666386339303238333761663130303833336136623637633033323964653665306237 +35376130376636626162613736643063626233316430383736653931373837323235393565343932 +65353066663839643130383737613535656532373630393237316431343130623266653661363066 +39366430623337383135643539386434646631633666613232656431363265383438613435323830 +64636564633064643936616433323833313863396433363631343430313834333038633061623336 +64303632666665323062306637616137643735663235643536616536393730366630393964643037 +32343333336135663037636138326530396663303865343130376665333864373234356231363666 +35313163623763323331363838323666636334386263396633373138663731366564396563666433 +38346436643234306230353534333932353662353035646438376536653539613732383563383666 +39343862386536303861306532363662333135363037376261656331356132643233386330643364 +32323565336438656235613161353534386662303332303162393431643531656134633436313038 +39666465336630666134333539643131633530313638306563346336393234616162613136336631 +65663637643930343333623665646262613039383736353931356663393934383338633534626335 +65636433356636363831613036653061306236323435303864613564373736616435353162316530 +34313631353733626630323836623663383961326533343562613739333830386166656565396366 +31313665393638326263343335313436313931633965353630303539393036643564326563653862 +33306666666433653837363133323062336361613835643062363232633637353134346365643864 +63636330643362623439326261363939376664356264346230306232323836653964343335313534 +36346466396633653932346433373661336564646532343434633465333630663136386536636462 +37643237353666633533613666343837363561653433326166616238373463323063306463393462 +30643939613161396166616466303237646663633333396632306133383631643438646136363433 +63623633346561383936366533363962303765633866653936396639353731316239626337333665 +38633830313439623865323032313339646633303030626632303562333966613664666462313437 +38366233633561323464333036626339316130613435373131646164343635653963386461623339 +33623031626634396364663362386539626330326437636432653131303762343139663534343963 +31616165353033306538623665306631653432383733363231313339636261366131383134373134 +34643239303235396364366234363031316432666162613764616233623736656662306664623062 +37366133653239623362333264393761366438633266326531326230343438346138653736616363 +61376166613562646338643138623133303638343330623233633662303535323437393863356539 +35313066383736643935666165386163353437356136356562643666383366636238313739316136 +66393534396661323763666130653831393366316231323238306130393335316436373238363562 +39343664323566313661636630393832383065316161303532323865643664626436623463613539 +62323065656633376230646465623766353462613239393632316530623162653432326431376434 +65373337636430356464633663393164316366306336633562313936623561333631393239323336 +38373837613562303230303535613465326664626563356665643566613036356132313737343034 +39626162346163343834636264393430323632643065643331396333366466333333323161333164 +31366233646561663038356134386236386536393032363839316331613336363531306333396235 +62376665396564343131633334303331666431333861383631386233393663663031323530326262 +61383034383133323263326466643038633636303538656266363930373231613039346331633330 +34326562656366656530623264646463396366643437323333653366353537316632363362373335 +61386533623934343835666662326166653731373530623562613439323234333132353730656262 +34373361303632376435336562636237393863636664303435663933383730346439306139313839 +34666138623732303061386561343565653137343135623663323830653335646336323362646165 +66353664626361653234376661636364656162643866346636613331393266613030326137653031 +38393834333535346565646365636162613461356462613030666135653234303161303938333334 +34613065383136306435396662626331353732313936343566326330386361623861663336613631 +64313237393466326565626136393564306234633135623462623764646664666338616639643562 +32373232356265313938646232666135616637356338356562636334356138316132366132633538 +64313662643534616139313534336138643533323162356431636539333166306334623130636562 +32376263653161313662656332393162316238346133663630393237393134653961633436663963 +37656463353763653235363561383731383362303761353763303032616639396664616635616163 +34613430363064343236306337333464336634386236666634313437633335636235653537656261 +37626336343133646432643535323639346361383437343964656536333834373433623131356165 +33303961326566353334393764653666366336343666383630383936333732626233356531626336 +36666461383062333136373931303865656665336539653264333331383036623635373533663462 +36326365653330306261323764366530623765306261633636623830366135363437646535626231 +65383838663830343736663838373164396637646166363630373462313864323632313464626664 +37396134643564313766303436376365623865366631326233373330316231396265653834323232 +62393661393365616338366566353165383663656531386362656531363664623266643938613234 +3833316565633363663338383564613537353037303233646466 diff --git a/etc/kayobe/kolla/config/keystone/ska_identity-mappings.json b/etc/kayobe/kolla/config/keystone/ska_identity-mappings.json index fe51488c7..d340605d2 100644 --- a/etc/kayobe/kolla/config/keystone/ska_identity-mappings.json +++ b/etc/kayobe/kolla/config/keystone/ska_identity-mappings.json @@ -1 +1,64 @@ -[] +[ + { + "local": [ + { + "user": { + "name": "{0}", + "email": "{1}" + }, + "group": { + "domain": { + "name": "ska" + }, + "name": "ska-prototyping-teal-users" + } + } + ], + "remote": [ + { + "type": "OIDC-preferred_username" + }, + { + "type": "OIDC-email" + }, + { + "regex": true, + "type": "OIDC-groups", + "any_one_of": [ + "prototyping-groups/teal" + ] + } + ] + }, + { + "local": [ + { + "user": { + "name": "{0}", + "email": "{1}" + }, + "group": { + "domain": { + "name": "ska" + }, + "name": "ska-teal-lofarint-users" + } + } + ], + "remote": [ + { + "type": "OIDC-preferred_username" + }, + { + "type": "OIDC-email" + }, + { + "regex": true, + "type": "OIDC-groups", + "any_one_of": [ + "prototyping-groups/teal/Lofarint" + ] + } + ] + } +] diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index ac29a3468..a7d7d5784 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -22,3 +22,5 @@ keystone_identity_providers: keystone_identity_mappings: - name: "ska_mapping" file: "{{ kayobe_config_path }}/kolla/config/keystone/ska_identity-mappings.json" + +kolla_enable_tls_external: false \ No newline at end of file From b67f29e0550307387e16e4b7f81cf2229f4f6f52 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 10 Jul 2025 10:35:23 +0000 Subject: [PATCH 074/103] Enabled TLS and encrypted haproxy.pem --- etc/kayobe/kolla/certificates/haproxy.pem | 1017 +++++++++++---------- 1 file changed, 551 insertions(+), 466 deletions(-) diff --git a/etc/kayobe/kolla/certificates/haproxy.pem b/etc/kayobe/kolla/certificates/haproxy.pem index 96e4b8434..a8d7b3877 100644 --- a/etc/kayobe/kolla/certificates/haproxy.pem +++ b/etc/kayobe/kolla/certificates/haproxy.pem @@ -1,467 +1,552 @@ $ANSIBLE_VAULT;1.1;AES256 -38353032383462393432383163373637623363333230306466643236363634386263376466656438 -3365313364383532393234626633396638643262353438300a373933303765393532326665633136 -30653930303233396364633331316235376662363334343030343731343531373635303031333431 -3564633162613934370a333161643935333462316661616330383531383636393461346439353630 -63323739386361366634336530393635663239643266353430646634666237653132363034326137 -62373164626638616363396531633865303631396532653432323833393335646163613162393339 -66306235323063383434303131633134626239643939653566633466643237653266343861313132 -35613737323663656435366666636163633361326133623964656661306634633664633966303136 -36343563313531623666396364396464613664373432333332363337643766313064643366623434 -33363361356535336365303034666564613261313932613931326139616662616665613935666364 -33376537616366366438333430663630653534386666393861633431613236363164323563336439 -35313763653034643166653634656535666333653432656230333037313536646464343863356135 -66383431346461373230613239323738666163633532353135656435303938346161303435613166 -30656530383637303761336233633531393935346266373562373362383839383561646637643239 -66373539666263613236363362646638346433633938386339633233613230633431363032336566 -32393635623630346565396166393630613462646433656566373530643464666139386437333461 -61326162386162316236613533343536353630373362656138303066353534616534366532656563 -34353438643162646366303264336562366161626635336539336533646161343337306662303161 -63333435643534313831323231303536636338653261373832346533613737656330396462343933 -39326265323666366265666433626337383531633338613731613432303538373566393436663866 -65636363646666303636346632653932343931346134616261623065613762323933643738373835 -30363864346237323231653464623838353732636566353437313764613064313339653332663231 -65653335643864353439653034323231633661613265623439653234326162396635663665643562 -39346266393866623030623733643230663666363234383861643234353363353764346537353836 -62326661633030353961336535323833303666633231316662613833396163333637643030316134 -31346161366536613265343961633338366363303165396232653038323465323130333134653432 -66663330333661383563666334363439613263623538306535616164343663616538313239346132 -36346263383536613564346638616135303935623031653862353563353737383662323436616230 -35343036663366356330363238326564356536396430333363646438336361353933613361313330 -61656262636337633364393332626532323062356230633230343833386266663039373131636263 -39396237386339326263336232313233316536386336343939373634373761383966653334666237 -38353432343138383733306639613331386237353961363532663036316331663662386664656334 -30316130363437363762353732633436396265323462313839343437316563656438656263353065 -30383938636631373963393261373861363737313062363430643236383763656565633562356433 -61656438363561356537373565636338383636313661303734386263373335653832616330346235 -38363138353731333766386330663534656261613437306663636665613238346466313933623561 -34316232313665616434626336396439646265626464353733313363633738643234363736643265 -37343531343130383165336638386161313235633464396138383739383732323736663865303730 -38623530313864626530323462666162326331623331366534643232306663666662346461306331 -64356664353033633738303133386531656335663563386562356639366539303661323635623436 -39343565613266336466323232656365646165366561666163653065353334363161393533616664 -64623134323639343938386365663435666632316666633263393131376536306466383961373735 -32366266633965373932616564346263316632613534376261643633653237386662343638656166 -36636264613566393966356132353733366636373264643830663331396261656562613262313065 -64643233316131333737306264393135373939336139346137343633393939366663663635373464 -37303434306337326238636639656637343837636232346165643832396339316264653662396139 -33333861393537383437613135656136393761633865343563613537316335353839363030653831 -62323939653965323662653261393139343365616363303065346266396230336136623230333731 -35623631313261613137623032366562366662393264633037353962373662316166373230376534 -61323631663665363830376438343664393935343736633065613133356461373033383863333864 -64396437636565646438343235616138613536343033646135366133393836633137373362343761 -39316436306362633066353563353563306663323362396363336663613230663433613136353435 -33393964626531316466623433316434633235363363333333343933343433313534383437383739 -35666163616134353936316466383462346438623530613036386663373737643435323037316566 -66323837383533313361643933386461616234306666373566666134313165663062613438366463 -33663537613934303365383333383666656338393239396635346665383061303830366433383139 -65393738343137346365383336623733346631626335336464396362303063623839613733313937 -62643963313030376235336234346663333430373634636634336532663361653565393463353837 -66313162616166633537626431613965316333313962323664353730323732613264353534393831 -38353333386537366232623863663431353837313965623537653363386366623838343766303138 -34643039303738303930623833316166643837393134653536313334353230666230326532303336 -64633338323465613962636565633032356664383335346339303536656239393234326132383538 -63643133356661656638306333313134303239303439336638633730323538353834653233333030 -64336266353838643333343537666161376532666662666266666339353866643435346266313134 -32383739616264386236313662323462373137326538383131363436313565656263636165656363 -34333235653863336439303564363162373738383538323339666238616466353336346432316531 -34666637663139653663363130363564663335653737373638373633653766343137373133353763 -37623637396132326461636438356132666363666562373334333535343930616230383830383730 -65393063656561386438643434393939643839313336396266633635663234383234643833663638 -64623732333734383934383531363838626437393466656535346464313166393437616461363163 -63393162383430343630363365623062623932333632336665626561303165343364393037663033 -61346535633831636631356363643933316434396664623535373331623332623033373734376662 -66386163613739653564306564303562366534336230303436366130353866336433386439343835 -33383965313561343939373733363931303535346139616539666465393931303634303364353165 -66386234663235323131623936363339633964386630633730663730636663323632633765383739 -66346662356331616430643262356233653339643832626439623663356535626335303631363037 -36643639626638353139343366363336626235396638643362366165323562313663356163633734 -63323463323630333233646138383333306436616334653266666265313461343564323738663035 -38316239316263353631653930623034396537316133323132306665333137353437363534633263 -63366433306133383862373830363436316531353238323865666639636263383730373962393139 -32616264353835633533346437323734303935386465306565633434376139393064323564366135 -65333064646632346531393032396135653164313830633563336439386430366533313862653864 -30396238373364333463656135356462656162316130333066336134383363363763656466356165 -31393335633735393634653161366130333136393231636334666164666363346433646665353039 -30313933626461616663373236663966303430613635646565653264306338366663353131646466 -35663533386434383666353836313361323935323564353733643031633066393831333731326539 -36616633346635616462663735653330333735386335656436353037646332323330306365643362 -61306263633365343330643635373237353231366335636237336232356135366330313133346661 -37376530373831653266303533663633613962376635356366346635373537646239616363613565 -38643631613331363561366464373733626461386366363038636332383164616663323833636266 -31313432393330383335333537373335613935376465366131393838356566333164656465656435 -34373065303736383531636134663362316538333230343964373736623831613730373063323631 -66343530363130363731383538393466386239323664366263376431353035643233623363663635 -62633966383163316465663135653837383731633538356330326563313437333438363565653136 -63303634303063343433663133653334376237636532376138366238636461623231613162346235 -62373662313234636535326139393733626238343436333133366664316535616363333863333635 -64616430653039333665623531353463396233643265366163326163633034393239653732643031 -39326262303732663265363235616163653934643766326330373339356139386331363730343633 -62353138646138616330333561373966613838313633316534316138323531343064643532623965 -62623636376231646138303931666435363131323364386530313538363763613966373634636166 -64613037323638666264333933326536343262343533346335343763316665343131363737343165 -63613137323531646263396438306165383332326331333432633738383861336137626466323363 -36306433393636656638313031383330343833336533626364313430356632323237366239386365 -31343764656361316632346637323130326139623033313564616466656336386631393137633966 -37633239346533313065663863323466653862313335346535656533643466376462356462613337 -61373139366462346362396465646535636464363836373130666633616361386230336332366633 -30643363666337373166323432313338323831616661346531363833386538613765613835376234 -61656433316137363566366430363134613066306235656432313863663661336434653663323436 -37653032643030343232373661656264386639306438643639323334326133383032613536353862 -62313734316338623739356636653532653166353662313463323531623031313662353439353232 -39343862663932616334633562623031326339366463393537386562656230366133326530383265 -64656439663733383234313533363330633636663632633538623265373332306138633262366165 -31356233353561326361326434393561376265633832363235373734623363343835633232646636 -36356135663133343133656266373063376563323032643230653431316535656133643938376137 -36323963626663343463656332636533326361356433623563333964666339633363333331353963 -31316565636337333066613332626631376362336463643632623334643436613933353834616132 -66653930393262663738323238313764663466346236653534393336633335396439613864366364 -62613030323263303161623337313164323836643739326261353461616461366161353861363238 -33633236363535623131306664386565326363366435386531396636343564376436333032623931 -35383734623231653731386361313562336263613934333937346361663438643031663036333139 -64663066633564326638396633363937386464313935653932393235643862373636623938333634 -30373033316561383436636463373037326161306336626262303737383261623739363031373135 -37633662663564656534336234356230656263623432346539343836343232396333396661373032 -64656432363239656435373734373862663336383238656232373765643738333938613634373436 -37366430313132636138396561376166663535306163376662323762386238643562353135356361 -66393530633935326165333130336165313731633061653463626333323964396261396138643030 -30386635633363636661373539613731633163386534366163613065646530643932366631653663 -38353437643161346266326165336133616566613131613935653133656366633933633133313865 -39313237396538343135373834313466303964666234396234383463386466376163666633313538 -31663663656131643833393338656365353863343936326538376264303662613063656666616337 -39353434623133653563373262363235666234656166383765653265383430383064346464613763 -30623838346634343431343261636330336466333361396632316232623931626363366530326266 -66633938633932613135613566363633336139393938636333346136613833383234343234336166 -64666331656137656661333837343662376164383832306139646533383436343862623565383830 -35643630653939363931636432663765353830366438333861333139313063343430376231383663 -32313830643337343235633263363763353034363364386138666136336661323835333631656635 -33393130633461643362316136633232643163383462663964343639356238636435653632356238 -36323135353837313235353934643930303034326339316430323339316239663563353664323964 -63343038313837333037333036313965623133633439346531666231373238646661373436303939 -33303862626163316230636164363866393636646239323164663932346334313832323364633638 -66376561386164346565333036343865303663303662363531363737363736376162356431653464 -35363564303934343532646134393436356439613232343837363030653364366432323564323337 -65386261396565346434646237353136333733306235666138393734333731306564626165366531 -63383837303665343231373136616266303764303061643533656637666561613630643062306139 -31366338626132623637343033633937386535656366613566386335653932303864656530386238 -30336535373164636466313935636366643833326563393636623430313939363035653531353463 -36363039353863626634346531623134346231326532353131333234366237663939623137396434 -31663630336266646663306435323861373038303139653865633165393835323036646637646334 -36323765363739653666383966396533353966323039643030633061303465653433356262373266 -32363233373435343733626135333062653539356632633262346432316362323231616232376664 -63333063663136353135333135396239343333316631643738383636346439663736333236326161 -64623934663364326439613165356466313664653464373837626137386263323563383236336330 -65313361623232323732306431386463336139643933393936316630313232353263336537376631 -61376161303962616533316439303863396465323937363466373635663332383731386661313133 -63336135663832636539653265326432653731373634363732663638303430363937336231626264 -37333335653637613130346565623862656262396233336661626466616362363731666265396233 -33343135383139313163333437323230376535626336396565666566393034663563653530626232 -39323461346430646263323636336439333839396136333666613030333362373261396439393739 -66353333383034643263623363303961396264343834336561306633343966616531333138396461 -64316235366233333832636238636339643139393431333430363163616233633134343865343861 -32303765303434353237343034323566636238383537636365313231343136373331656566613332 -65333336623730353963653464646239633363666565663034393237666563333833653133343136 -35366638656237343737613330386136383833383130656236393338303933343833623363643931 -64396131623262653464313032353266336462346130356466323032383839323736366437383238 -39316664656331396337656633363230363938633530316232633038386235353064656437656363 -63303333656166646364396535386332316163663165623539313639653063616665616330386163 -38396130323663363663316163313038613139333762646236373430623131386563303233303635 -34383936666239626234633237333965646538363166616331396361643037353861393237306638 -35646262396535633339303131396538366436666366666634616465303832633535653064383033 -31376563396364373939626338633732323839316264396439393165383531383932373034343337 -32636164343864663436336335656437326431383365333834373038373561653436663133646431 -65393439393838303966366530303761333534346665336230336637383461383334386365646561 -62333566643135396532346535633165643338353537356166623865323031396537373163353232 -34316461653463613535383465636263356665636632343932393938366530643535613937666663 -32313937613766353538313261393036303537656565636366336464386332356562353139653838 -35363830616233336638633263393838656563636436376139393532333566623334313331383731 -33316632626166386563613630653437653261376438646633333537663030323238623731656336 -62333739326436643138393336376361353965323139343163626439353034306364333837616335 -61646632663235326166326665653232653163386235376534383230323461326231666535316531 -66396535393039303135343734613664396536383231623034643134353539323461363965333036 -31656533346332303566336662663731646135396661353163626236633130643462633863623634 -66363939666532306336366238653235393332303239393031613236303362336633396537376465 -32393635626535666531323363383138353535346662663734323664626639663338663963336562 -31376636613365373233656131626638633237333638386538636234353362613334336136303437 -34313064656339326437393838333430636435353739353031326432653035663639306533303935 -33356434666364313934663063313638353637386665653531396433346434313565643863316433 -30623332636464653861646164633061663361383730333265336465313138633535653665306362 -31643738633732376363366438376165623565646434313161616561346332636430653562613834 -30313331313962653639376238633466316130386365623130356136653238656531613864633036 -32386165386265666536653136306238383331613266343032663631613262656563316331303765 -39653661326361386364383834343832323365396338333633646533656538383861343465346534 -36653031653630346638643836393837616534363465353832366163383165373534303466636561 -65323430386530356631306561343664316438343232636461346564623861303663383761653231 -31653863653261663838373161613633323632396134616462366361393433393935396661643634 -34613164303365313333616134393831663664336235346566333064353465666162626333383761 -35363938343562616366343931643330306439386437326230336530303463316664376631613465 -62663665386137383032363636303835623239656432333434333533393230623833653966616532 -37356564663465613264373531653664666138626536643261376266636362386334643233333965 -66323033636234646532636636636232666264323139633066623166656437663434303536363533 -39353337363839306139666461346532366538323535643365336665336138306237353064616631 -66626233623661363435626466366337303362323531613638376164306662616639643338363437 -65383131363664613030303362393032376366336563396538333730353134616332623162383538 -32366166393435303338613834396261326635626637633637626265666232643734306337343336 -33643664313465613534643532633161383866636437616134306438393932373263643363373335 -32663863653837386133633062303762663163303338393965376232663131376261633336393339 -32393639626563373261376464636335363261396661633863663262613135653531643034616435 -30333130656237363764616136333464623164663636393136376362343931333239346333633936 -64613866376164376438393235383430653231353132613165383735663762303534653461383763 -39313131356263643835656632343530633939363764343434623762396533393366653434636637 -31663862626436666366303833323536386238376437323132323538353939333964386430323131 -66313630363639373037303535646237663766343237643761346164623839626364646462663633 -33623661616334396433323133313865343238653865663163396334376430666337333061313662 -37633231386365333465616566333966616137646631373733663863383666343930376338623433 -64376235663535336161333263396561643963353165336434623735653165343263393563363330 -66626335663130633835663964333732386438346235623130633836643366656433656239623436 -36373332383933363231363831623661646466356262613138326133373636306532316135623133 -38343737343363346438313934633632323034616236346136383430623734663066343232393634 -61396332353365363433353932393637633234383239346638666131323062646664346236666362 -34363965343339626433383131333066333531383239653034663662666265626137323433623233 -30303030653834626134643435383131646436626662336464653266383363613062343837333234 -66633663623866623031633666643430643736363435646566366431653064393831313061323730 -35333633643439346431363330663632383365303032666132613761626336666131313763643534 -34646461613331396662346234326439653662643232396536383766623662646133363366366365 -64663639633831303739343130653932623538303239336337396365396638326261613839306566 -66643566623664653436366264383634383063373631366638646531623332393537313239376539 -65626234353136356337336363633031313535303838643438636266623466373262386337623331 -62333661386137633535373866346331323932636631643639623161363236343965653233623836 -64333538376432326166373563303134323462613931663865356339373962306437343232373731 -34623636396462393665646333323263633634383738353631636635646239383530333435626462 -33363431653834383238623865643034623564623533633130363034393463373333633733666136 -64396362353333373861323839623336313031626234303030353466333465633831613766633334 -66326539323336373962653233633932323661313036313735653632326635356631623761653030 -39363335656139316133636361353366396235343933666139396533396631343538656635356438 -62376231353661343862376361376163376535323231383531393037646164626633393066653632 -36336565326666353230333263656334613938666666313665636137623331333666396564303835 -34306561343532343166386330626262343761373231636336653663363062393866386232616137 -39333332663833326632393631333234333630346632643861313639316665626637656466666231 -30363663666261323033363161363734623462633130306265663939633532363062386637373939 -62303736393431666663346131613736323036313731626464306636623031356362646236333031 -33356433323136613332363431613036656133323736353162656539363034343535303235363831 -63613935326362646666333463356137653230323062393035633462633438616462366231383563 -34373131326565306436653139343631656261323231386132626632316434353435366264386338 -65633932616330616430336430653535313235636138636462343164326661366165323766336264 -39306136386264356364623734663263326263366332653061653938616337386439396264616636 -39383462363235396232623637353865656663343465386135383234616137303436396262376133 -31343936343134633134333933346264653934363536346166333137666261656261356364616231 -35666338383336366531623065623264326131613237393239326364303461383565623638623538 -30613837613139306466333632366461373864393837303731363636643238336337623333316132 -35663661336166666637376633666362626335396339323430376339616435653363356432346663 -64633137333662323038323633373164313666353937373434363235623437653934366562623961 -38353534623766333762343361663666396237363365663138663061336631613261373263623763 -31653231663464346665336337396264393564393932346131643630373633626337313361653065 -34653664323233643961613833643664323733353232306265343133366564376332346631373133 -30366637643565643233396237386566623939396164363763656131663331366666633335366133 -39653134346331613166373262323731336263396331653732323931646336386166373933623736 -66666337323335306530633232333932353233373430363939323633616438336465623765373934 -66633832363631616435353535613730643738326362306334636231646237333839623538653433 -35313332316163383930316532653035613735323066643636626363343965306333376339633537 -34646634663638393761373965623737356433353865356335373163623264346166376236636632 -35383637316635363837383339663038376162313163396436353931343135343864656131356362 -33363337383336656136616363303166636236343061303539303034303661366639613464646364 -66353235373564636561366133353931663165376637666238316332346336383531376532643562 -65326331303233653138386639323961616232343235373938383662313961323537353732373161 -64626165643637636463636566333164373137313735663534653136356233356261316630653539 -35633135363565356636376663376136336662393930613462633336633363376439353363353563 -61363466316164633038653030396166396633383731363936626131383038373030303564373265 -65326632343931356639653036653862383864333561653766313030303262613661313263376233 -39656437393063383761656261613035333965393534366533393765303162356565616430366165 -35346231663833656138613863663631303236616636623361663839643637316564643936383639 -63656631303834626235626330343962613365626465353636613563653862323833623235626230 -30353637333162323738306133303031613263303238626431336664366635356431336664303864 -61386639333865356362623361363365306337303636353734656361636632313134313135616364 -65313936393966356435666562643430313933386437343364616235376463323833303533363132 -31643364306331323331313135356436343866396636633737613564396464623633626332653031 -35333734616639613638373966343162633862363931636536346136643162353633633639373732 -62616438356336363931383365333362633164313235326631316166653965336338363833663835 -32366233303262333038306164623766343565366566306532343435343139393131383664623062 -62656264613137336463386532663532663131303233313261356165316564336561373330323866 -31626365633734656138613436613234323638393039316231633866646238393638316162323231 -32343935393038353030653535633635643765306636626366653834653061313161626661633938 -66616563626330383462636531316362356536663334353335313236633337653633356665346532 -65353638663633376330363865336635393934623366636239363536306538323733656631623464 -63346463343263366539663535333937316664356561303839623338626266633039643164633737 -31363761353539626363373765336433396136333461346136306330316364656236633165393666 -31376565336230646664643831383632386535303164633235346131306663326636373635363430 -31653765326235663364303430333935386537653832363661623735316437323565396366303839 -36306564323238313232326565313934353362323539303765373537343166316530366330363032 -30396239336266336133303931343030343937323931613234326533323839306238646631336165 -33333631646438306666316265356435333030653461663330366237613332313232343030656133 -34366361323732356338336466363633323536323137303861663364323762313637653034623065 -36653538356337383533326562333535343639623132383362346537636565666662643465383031 -65396261326261373534333735663433653063373838643565303439376433383461373463363537 -39306234356535646138663431656639643164353734623436396432643639333762646161623430 -33616261363631626134653937623863663163353866326337346530653232636434666537666431 -34613164323165376162396633646566633263363239353663636339386338336366326636383932 -30643365336538343235393538393366343633616166666464363133393934663834356566303335 -61396539306161633962356466393461643665316261656463373134366636613866373630323937 -63646463633631653235366561303934333433346235383266626239303037383830663233383335 -31303533363834316239313561343061373930383264333564306639343664633063396264393364 -34373639383936363862333039663431613538333337326139646634663137383139666332646237 -30383031363031633839353163356363653864333137313137353064303937613632633436346162 -31613962336430333537306632336266343463386139663562626161643661626365333837626633 -39313161303062343835336636656465316431616338333832386238643564643630303564356534 -33346162396333343166663337343230623639333761373433333630636165393635643036323239 -31666164386166393633326466383832396163376130356366386362343636326137393261396531 -64623965623139313832633366313133613065646634303133663638616361343930383263333132 -66303739353936646134363461626164333365646134383233343737363163376364386435633563 -61633330313634383635303564663362633832323237376664623035373261373364316638306332 -31613333333164336466396136356562663361366463613563383135316537633235666263306139 -36326536646432333761336332646266643333313334396235323862633064363666376330656231 -31633833613439343661366531393832313336336663393364313437336364303431383361643030 -63373130363561353763636362666232323734393534306238626630316664656166393031316237 -39393230333631653635393231646636633738373734363631343938366337636566386231383430 -34376661383039633365363863613861366438646532346439646166333534666436643135666234 -35393964373665616534643564643465353737636532393761653962396661373362636132633061 -66613465396131346263663139373637336563316339346262663632353739626232363039363465 -30346565313039623831623231666630656331323064323362653030663061303132393331653630 -63313936663063663265333162356330363363313934386537623833646565386538336361636262 -30623030643230373161623030373731363337393161396336333235323038396638366531333664 -35623834633764383666613434336333346461623137666266373531663637613266303661383463 -32313038376236366461646437663934646564356530623931613930336530356236386235633437 -37336538323837343237663138323134653163333231623931326232396132636338353265323432 -62303263326536323865386163613566383434356131363233356563313963646661303730343862 -64306237623161653133653963333836663337376632333737313562393362336130616361326666 -62643638336237663339303230663035376664393263643332663366373030326438336239333162 -62306538653633373831376539633135636564663233613763646562666364316262313364316333 -38306333373065306264363338373365373539383266623439376436636363656636646139613834 -31386466633031363365343763373032623539366164666231383839343639653039326232383431 -37353633306634313165386536303239313238303566346233363537366539383834323364346365 -34626439346562356537383033333732333637366362313638636566353365653063353037663034 -36306438353665373864353164613066376163336663643232336563333730353737656636653564 -35643930303262333935643861323265643735616161396363326132323239346331326334326265 -37383034353461366633623166646133643234636335626666346239626164643333333564323662 -33346338363533373035666133346464656236643533313162616233613135643662393732316539 -30363330666134323337626463363139333839653566336432663832326333643433353466313864 -37313339663366373564376431303332366538323639353838643234633636303562336533333434 -61396632616562353837613732653263333231613939366232643737616234356664393231386630 -37643635353064623238623339326331613664333065393564653234323033323465386264303365 -31653639663933643032653661323638386534626337396364323234643134633462623431363838 -61326336656666393565623933663831623034343430343736636235396230376537323933376361 -62633533316263306564353633373132333335343361633764373938626238316564353739646331 -34623132323866666431393734383063366337356331383163646639613965623737306532663163 -39333264613239666537343837643265653530313461653566353962656462343434663862343634 -64323436363437653661373761616536313436643566356666613235623166393631656561643032 -65643534326338626537623234666335383965326666343138363635326365366163303565653231 -34656238633632616665656630313963396666346235613063323131323164313231653239393962 -61313237313237633263633763616430323035386465383064343233383938376466626136623139 -36643934303964616535303530336137666238393230626562616639633265316164326231303165 -65646164363636393339343664636330343565343938363032646335663630646566363833646533 -65383533356135306561353237646131333233653562363361343438313236623036333939666166 -62396238633361653433643564313532646332653130343561623434313832653462666337663135 -30613238653435653561326635663135323763633435656365643064653732663264376562363838 -39623834343064383234393063356535636164633231653538633436303135656434663133326430 -31343639393964386666366635316462326463333235313534613665333036353339373938623861 -32613034643962626564353739663034663861363661333365663531623765383638353737626132 -61363562363165643031336563633666396235373535376163663635356163646333383763613834 -30623134386332653166333238386163393730663662383537643366363230346266346133356665 -34663166666230346561323232633837613635636230636630626362326664613236613866383864 -33373162353864346231633163333236373839363932343337396636333337643661363064386134 -65663539623661653931653139396665663363663966323835623930366566366232663138653562 -34313035313664383961316563363038353432656330653766356666326138613062313732623963 -33326634356365616335626530306137363663663562356639666563356631353236343835313035 -37373432623439376638386662366533613231363539626634393035323264643233616234373035 -34356337363865323461656131303332666435646662383636643131616434346164373936633066 -61323364373533323130313634396231386463313336383265373464653763653837326635313564 -63333636353663646463633832316536363732393462303865653265613232306230626538663939 -35333835313063623534326138346434323230363034346339396339346538323937323739333962 -61343137383235323433353964643637373362316566653538666432373237663236663730346161 -61356662616133363263366138343030626263346331373439313338616535643035336635343337 -63333934303239653761353233383561643835643265366230623662643234643538356533643730 -32343537663534323535326263313430366135336434356637303832653535363030656333376166 -63306666333337326563643064396661663536613561636133623061353463666335623030383538 -35356561663531653366386631323761626632376135643438323537316134333937333466343932 -34323466316166366534323365616332363635633762356231666232363238326330386232326335 -37343433353562653638653466393030653537653631323532663265393264623964316264396161 -33646136376538316434333036623564613939633937323939393761383133666237346239643362 -38373637353435313334663331356465343162383764626131306132633261353162323232376432 -64636165313864383730333739373964623766383861373533386330643630633536323032323934 -33363030626432646430316336323663393131373439313365323536336531373361643434363638 -30393236643362613662303362636639396236633337653462633532653164333430306633613438 -33653262323363386334323265396639643538393463363439353164393738366237326639363161 -39323330393033303537323333366163383265653131646664306431366536336661316231356464 -30663064333534323339363331323466396331373030343730613537613862383431326236366463 -63313761313836636665373635313766326234313435376333356365396638383736316465336134 -31663164353364633433653364336463616366396135663637356134633133653133663939636239 -65393431393138633138666330626365656336346630383831653663643763623034616138643333 -31656230323836643763643031383637303539336236303135323935383335623433396439333732 -62373939653035343739323833656366396330353738366165306333303566376634326432613435 -35646665663239376262626330656364646664366466633838643065613136396635343762396630 -64663539653533386230303530313134663233306562303230303635366433643636353263646432 -31626132313261333761306430356637373130343639636461616431623939656365353636323863 -65336431366433383666636633616232613537383133343332616236356533303733636564666332 -61353962346331313632643632306635633864643935326131626232656439663831346661366165 -64613032353239626131623537316564306665636363353234343734323335656237323262383364 -63616164613238663362623733353838633162363663383438386462623739373866333437306263 -61653265343065626635653762383961633165613835346363643666613136373435306334613831 -61356436316633306563653636373936636536323231313933326432383237666239373761376365 -61316561313232373134313836353235346338633031373536313439393435663331376366646431 -36306363393830393237393636333561363964633738636463643939393932633037313165643833 -34306562346238663465353939376434303062333862366662396239633430633366383861303332 -33613930373465373435656337396662393165373863306532313562653564303736393462643864 -35626339623336366139613631663531356666323438316536396665353630373837373038613833 -35633538326461303337353264366635353739633664383161613939633935313433613563393161 -38663732313534313966663164343636343666346465323264346166383465643130386631316139 -66656365633033363839333734336165323963663165656437363836376630356432633566656666 -63376338333331333533633133653966323832653335393232343739613130333237643565393366 -37373738653666386339303238333761663130303833336136623637633033323964653665306237 -35376130376636626162613736643063626233316430383736653931373837323235393565343932 -65353066663839643130383737613535656532373630393237316431343130623266653661363066 -39366430623337383135643539386434646631633666613232656431363265383438613435323830 -64636564633064643936616433323833313863396433363631343430313834333038633061623336 -64303632666665323062306637616137643735663235643536616536393730366630393964643037 -32343333336135663037636138326530396663303865343130376665333864373234356231363666 -35313163623763323331363838323666636334386263396633373138663731366564396563666433 -38346436643234306230353534333932353662353035646438376536653539613732383563383666 -39343862386536303861306532363662333135363037376261656331356132643233386330643364 -32323565336438656235613161353534386662303332303162393431643531656134633436313038 -39666465336630666134333539643131633530313638306563346336393234616162613136336631 -65663637643930343333623665646262613039383736353931356663393934383338633534626335 -65636433356636363831613036653061306236323435303864613564373736616435353162316530 -34313631353733626630323836623663383961326533343562613739333830386166656565396366 -31313665393638326263343335313436313931633965353630303539393036643564326563653862 -33306666666433653837363133323062336361613835643062363232633637353134346365643864 -63636330643362623439326261363939376664356264346230306232323836653964343335313534 -36346466396633653932346433373661336564646532343434633465333630663136386536636462 -37643237353666633533613666343837363561653433326166616238373463323063306463393462 -30643939613161396166616466303237646663633333396632306133383631643438646136363433 -63623633346561383936366533363962303765633866653936396639353731316239626337333665 -38633830313439623865323032313339646633303030626632303562333966613664666462313437 -38366233633561323464333036626339316130613435373131646164343635653963386461623339 -33623031626634396364663362386539626330326437636432653131303762343139663534343963 -31616165353033306538623665306631653432383733363231313339636261366131383134373134 -34643239303235396364366234363031316432666162613764616233623736656662306664623062 -37366133653239623362333264393761366438633266326531326230343438346138653736616363 -61376166613562646338643138623133303638343330623233633662303535323437393863356539 -35313066383736643935666165386163353437356136356562643666383366636238313739316136 -66393534396661323763666130653831393366316231323238306130393335316436373238363562 -39343664323566313661636630393832383065316161303532323865643664626436623463613539 -62323065656633376230646465623766353462613239393632316530623162653432326431376434 -65373337636430356464633663393164316366306336633562313936623561333631393239323336 -38373837613562303230303535613465326664626563356665643566613036356132313737343034 -39626162346163343834636264393430323632643065643331396333366466333333323161333164 -31366233646561663038356134386236386536393032363839316331613336363531306333396235 -62376665396564343131633334303331666431333861383631386233393663663031323530326262 -61383034383133323263326466643038633636303538656266363930373231613039346331633330 -34326562656366656530623264646463396366643437323333653366353537316632363362373335 -61386533623934343835666662326166653731373530623562613439323234333132353730656262 -34373361303632376435336562636237393863636664303435663933383730346439306139313839 -34666138623732303061386561343565653137343135623663323830653335646336323362646165 -66353664626361653234376661636364656162643866346636613331393266613030326137653031 -38393834333535346565646365636162613461356462613030666135653234303161303938333334 -34613065383136306435396662626331353732313936343566326330386361623861663336613631 -64313237393466326565626136393564306234633135623462623764646664666338616639643562 -32373232356265313938646232666135616637356338356562636334356138316132366132633538 -64313662643534616139313534336138643533323162356431636539333166306334623130636562 -32376263653161313662656332393162316238346133663630393237393134653961633436663963 -37656463353763653235363561383731383362303761353763303032616639396664616635616163 -34613430363064343236306337333464336634386236666634313437633335636235653537656261 -37626336343133646432643535323639346361383437343964656536333834373433623131356165 -33303961326566353334393764653666366336343666383630383936333732626233356531626336 -36666461383062333136373931303865656665336539653264333331383036623635373533663462 -36326365653330306261323764366530623765306261633636623830366135363437646535626231 -65383838663830343736663838373164396637646166363630373462313864323632313464626664 -37396134643564313766303436376365623865366631326233373330316231396265653834323232 -62393661393365616338366566353165383663656531386362656531363664623266643938613234 -3833316565633363663338383564613537353037303233646466 +61326164646564636461623364633765663830656336353839653037643930313032343536343631 +6563383161316464633163613331396564616636653333610a633434393433346239613036323339 +64323165633263626365326231306536613036666134373039303037663039653734656339346462 +3533613266346531300a353661636266383338633466626338626362353832326165643632366633 +66393263663438623664363031663137373133333464373333633936323230643462626131663131 +63363464333139313464346130663732303431373739363438633361383733356435363736366233 +66656131333737376162616531643635306137373031373966633038643739303032363562383832 +65396233333235663933366432376239336466306261666130316636633136623162653962326133 +66343132303337323234353266353366393137383261363736616234326266323431353935633434 +37653833633036383836316561653165646530363831646532356365373031653737666339306161 +64303335366132363735393634336236306132393465366463323161326462623130343138653936 +37643964393264386138366538363132636561356236396662326231396666393965626166373231 +38633033373964636437386238303632643065633964643862396461316332383266316164363936 +61313866336263623265396432393362336566323864336363333535323639326134663535383134 +31366333656135376165316265643638653763373533616466663733616365613730623136363062 +66613662333639383265303736333432323062306633383737343931336130303831663037353334 +63333934616661343235643138323564623430313438656365343662646532396334613737613832 +32353335616636646138626662336530663462656631666237613233663066386337653630343262 +31393031323439653336646635616132376638396439613430653964363230383330386431386637 +31663338666662383762636233613865323434646363636639623164633337653462623562656239 +62303330316131643533393434313961343431353766393735356535356538623837363161623538 +31346538343766653133633637633466373034326639613333353231316665333036613930666638 +30616438333835356537613933333562313837393130656465373134346165383366326564323637 +32386465336230633733396164376239386532306663646338333563333334663462303739393666 +37356666636232663531333661303763656635653266656462306234333332323035643065376235 +32666662613261346661316532333937386161333765306436393839396237396630666630636533 +36353466616531316665383061363764663466626139303063356335323039643762376437316132 +36316633306163386532376332343737396663636661353865666130343162376238326164323334 +62613536633930643262336434313135396231393431313564393164343362666531313666336635 +33323532363336383530373936663132366230343664386366343836643562366633663135366233 +32396637333630353239326263326663353933313862633665386535323232353862373733366665 +31343034396239313734376438633361386437303961303539643435313064646566656635386365 +33333562363864633733663233633261356434636438303334366135616134613161383839306166 +38363538616338653662666561363464373332613639303833373862626330333937363732386438 +66316663376532386232376637623030333461653330316638616535663164633865333961303861 +35343265643036623435373661373631643436643531383066386466646262663833646433656165 +37633137323365653534383636366464643264303761313635343931663362653737313230636137 +66323737633433323066623964663064613862343863626461326465316531373465623162663532 +64363433386138653437633439333462353264393165356235396531643233336165323262303938 +65393363386430383063653035633137373562353762666165616366336532373031613566656633 +62643166313465653530623361643633383963663963613262643263353033666536393532643835 +37376633306438303866313332656133386564393033333865333766613135616462663033373861 +36353830643263663364343164383364613430393834396336343365303234306533623237383937 +64383839613533633861633036343663356239613539316432396431306462333830616134383036 +32366639323634373833323761353538636165633463343135353435636163356266633933366638 +38393434363063323439326562623334303362346139646562393034613863643163653864663937 +35316563303561613338616234663131353338383066653932383737313435343837376433323466 +62633264393431333635326139653666373532336131326435656232383261613037363861656139 +39303530626131386664363238376134633961306433636464383539313437303638653134646662 +31316162326366376539316134613462353235313763646439353732353462623331386539353130 +63303865303539626237303330386666346230363363303361656137633338646666653235363238 +37633033656162346635663161393563323636316431666132643266616235393966343162663064 +62333365323936323136386336636266393631343234653063303835663733366563613533393338 +63343436653766383865643064623965303566386663386232363135626431393039323233373562 +35386163613433613465353036623236316238343965336632303963393634306138616661613161 +63633139636366396532666339333361383362663062633562643131396337343463303136356533 +37346339353163636639666664386234363239313439623064326133633230623831343033613234 +32313062373034633331643961313031636339393838633436613063373864643131373561353164 +35653434653166383034653038666362666466636139323037636136616338336663623034643236 +30613834353066393831626561396439343463366136393234383632666131353337653464336263 +31666265626534626364333466663765663437306230356265333431363435333131323466333563 +62323633663861663636643531656434306564313739396362303466343636623562353731636537 +63396538653366643730373063353266333661336338383932613937306134626364323131353065 +61383834666630336535373632333236323063363165313661346237333765663136663630613764 +37623335636134393132323732363837633761653064353338386564613765393665633336356362 +33313231386362346131313134323034373933626438303939313462393439643337313266636262 +36666162393363326665383563396662393537343539336337656137636632623062313663613438 +33323033393435366133663533663366303462343334613238313230373032383730646236336364 +30333036633064633034373835313366626531363039346632653865373432386563396631366635 +66303866393263613139663161393766653765323938343863323933663966333530656165396432 +35366432383062306564666639613666393932393837383864626163306232636330653138373263 +34353936333032313164306632376536393532653165336661336234383765656536366462613436 +64346630613635653439663231353566333761353666303231396631613533323461646233376564 +30303937366362303861373763306131373230336464386162336334363230663735636337343637 +30373265306639626330343235376461333266313665363635336563323532353766343763343038 +33323132393534653065663335646138643165386538393334343131643663636339656333633534 +63393632613331353864623832663839636637343264633862353634326336353639313237393835 +34333633613266353037376337396661663031666136663139343832616231393136323966376532 +61633831626236636565643536666164343130646266333530366139663534653866356262626236 +39653033373134326635666339373665363863316533643833346330343364336364356533613431 +32363366623834316266383039636562666135626538383733336136653630643966646134313665 +63323263303964326633633065616462343964306265333831343762376133666163333837326437 +66363162393165323565656434623838376666663062373566663133396338646331343263386131 +63613033613862643361353564303333643162363532623364373435353735346165323963366336 +62373539313864623537626266636437633137396531663166383864323939646533393334393035 +39356130306662646137636463643233633532326563646434366363643130656565666235656262 +31656338386533393930623063386139393566373764653638633036386531636166396163313163 +64316166636135626137326565663863633731393830353731363666313265366131663737373061 +66363463323664343162386537303635653962613039393363333730663833666632613434376137 +37616630316634306365386531626231353164373835323632666663313139636439343837303337 +37636639343061643962396437626134353739303664313932336130623663306533333731393135 +33346263633063356366366635343037666363333665623065306138656537363364666131326330 +35376637373836313036333132316631363332643932373564323035363631353730666639616232 +38313239393233393866353930303262343766616563363961393566363138373136616232633662 +30643932303134323633653563366466346430633862643039383364656161383939616665333030 +32396339623435383562323836323433393835313964663334373562653034663864646134623338 +31663161323438323162393966623432393934646237363638363362663262373135326439636233 +31343566353662623732663334313732373466633939303439353630386231666161356431383832 +61356661303039666433353864626537393238356532396263343964656661353964366530383833 +62613038393262346137306530623933336266656266353138376136666163663563323030396435 +34353865663939393739653262393039353062333162623933623135333733623936353036346633 +64356662343633323039303364333362383338353239376235643139353737616637376466663431 +30373163356662376166623731356462636535373431356234306262323462343330316563636331 +63623537383731623634646637353062666463343538656138633436353434373636663563663965 +63373938653934383966386562393634386535653136313338643136663138663164386531393066 +38306134353439393438656137316339646532646333383339316239666238626438363038316565 +64663263356636633934636533316338303663323037353031376565313934396635356639616537 +31366233623237366262636564613538663635653033613162303036646530666364373730633366 +61643964303732643833633763383537303433333762656439386431396366396638633864376162 +61363435326134303763313864643566656665313362306432313738353935633964333838346264 +63363236376439373162336439336434633962313833353766316666353239343231646164303231 +37663631373838356362333531366533306666626536623531616562613633643863326564623737 +37643862666533306438616635363134313137376361633264333035656339643132396466323466 +62663930653130376138336636313862663765333835656131313535366236303636323835376439 +34666230373837353538333439666138613965316564396433303265313835383431653435396437 +39306233633830396664663165613132653664363061316130373130643732383634383564363163 +38356234343066643135613432393036333465643665333739386134626130396238633732626463 +38633861346532666232326134326435376533346463666331353033306537636634623830663238 +66323464393536386562306234326237663435383533646337396466313432363465623830363765 +36646566636535646534383537393732336664613230386162636531393335623235313436326365 +34346631303539613862326534353661356365383962646131353533376231333135356562653634 +38306635363139333461636562666235356131623533653032643962396263366666616566336130 +38616532616339383665393536366132646238333732356232666163383964343061636463623039 +38336234323436363435376536653065646662353863316432333334326233333465326165613362 +64326162616563643735303038376462656239333461393237666561323461643661623430343536 +35396266353337363263626636373030633233363734396333323365643336666632353336303230 +36633665633031643766366434303965336662656539373461373362303631636163313064383036 +62353231346439663135646663383833663534346133326239623537303561626464383261396131 +66393035313734633234383064613635623166346333393963316563653166393634326161333265 +65623562643133306265643231303464333831323666353765623830653038373462353263303665 +61386331366338336365393235396537343539306161663665613532306466666432336363646334 +65343362656630633963396364353039396339613266373831333162363461623533376331646434 +34323631383662336330396264626338636338343038643835383333366534353433393338386464 +36343866626431636263626533356133613261643265626538386339303136333639326565303333 +35326235613937383362663233343466626635306465353536623233626537313962623836643562 +32353031663164663865623232396663616333613462366233663766373933623834653164396637 +65656638656334396639636333663333323964316166333437373337383635663130393737346166 +65633133643631333966623161663064313830316535646235633939393937653265366231376431 +66323630343462663239626465623836336532663963323730613631303330663730623637373062 +35366338313437343339366537656461363037346161386365353966663631343037666162363436 +64386637376566666538333663613132326664343936353638373930346630353934386663636536 +62356363643833353630663366633439313962623334663362336663376463333361333134613830 +38303662393430346533313030373030623432393662313134656339316661343234356433386634 +64396462303339363937343330656466663738633139636365656639623531316264346135626566 +33313336633535313438613937386639373666373661636537643339323033616532336231323338 +36616363626234633563653632643531343833306332366430613564623234333162306633326438 +31386437366539323137393966306232316439366166643863363964303964646431616462643964 +31376462356235313033346665303964363339316339303839663036393166646635613630343533 +64366530346238326264643331356432376335373962323163323039383535663130396563306330 +63633066363937383336626663616166633663353231356463353637646163313430333862616238 +36366163303238346464383966336362393838303837396135626535653236396339376265313030 +32346331613535356365336165376465313664616264333732633166626631316661303664653564 +65623133623537626637633237643434396361626666643431316361623331653562393436373931 +63356536636538623132316366643863323761633964666634663437343635363462623164366564 +34316638663465623937653065356262373536346334373538643335383836656134383363303961 +38346266373161626634646432343733656632656561343539306530653635336333306233396266 +65383436383761643130613837333732393438383063643564393337666236626666356238643931 +61366531336334393462383531316464616331383466663064303532663262306464353464356366 +35396261613932393134313063373532646237663965303866386262333031303236633561643235 +37316464313566333136386433323839333333653565376364373637623236653338346133353332 +64323732343230363837393437313036303563373339366236623033653333373862636362366162 +31393036343764346537363762343330626265303963656637633738656665303864383561646336 +37353264343631653961303834356164333462333264376330393038646365383565396266613135 +36313061633432393961343964343732356164343236373831393563663764393037353164303131 +39383565323137353265636564343961633566616436343933366465333964373562643163326132 +34333061396536303638636531373864366237643362386466353330663033663738363365373465 +30353162633533666136393635626363623238656535633565343537383530376333303262346263 +33313535353430336139633735326135346466636339386639613066663030323537313230613966 +37396462626630383837326137393135616136306363336366343036373762303565373334396133 +34313935326665313565373837613933636465643464343561316666373862646138333034383831 +36343163303239303733656637363833383265373539623030353165306363626664373138633634 +38353565376665346364386131323932366433323063636338653531366131393134393238386364 +62333533396364383631383930363466363038323337636263393334393634363833323066613037 +36376535353538333034303463343835623236343639656662323764666434336665363338666438 +63646336323133396637383534336433323639383362656664343131663830366438313766353766 +35393562643332323530306334306262636365653363303936643338366636393234303131313263 +61396335303639353265346561353531303235626263616432353862303937366662656634396337 +36343338366162306535386533396439393232396166643039336361303166633264323934313431 +61623330336330373166396334363462343631383163393161326338653337353065343261343937 +33613438366330373936656130383066333839373835386162316431303334666632663533323731 +65636566366230653039666131663065373632633339653537623463366238653833393762663235 +30336132366562336365613633636535346331333364333863306137623565366261353937646464 +64383666333038363535313733363632316232316366646662613934303938653534333535653662 +65323161613835643535306464633436653666616631373364623864336630656237656461323631 +66333032313935626433353138623936353363373663333165333661623638333765323430373934 +32666136623464363262616433303236353166623137343864396537636135383933633834616161 +63333663633630633939333266643566393133376166373730306230396364396430616630656663 +66376234653761356635316236623663623634636364633735326436353866303062306134643163 +63376335316638653039393831363237376439346538363161356238373534343532366537623366 +35653430346561393235613330623463336433653264643739663533303132323733663633383062 +34336536613139376638396138316338663237656466363231376531363634633838653638653437 +65326539613261383030333538333563633138633033363165636336396565653261633135636336 +32633733653330663835313935333261353161313134346133303964353139306163363363383664 +61623764383261646132386639356638656462386339656363336535333864353732323939363738 +66313333626135383734633561626230613334656237666537306265343839653733623838633137 +31373362306636663263323061336636353937636532316266303738393138613732313930383363 +37363036633464656137623862613566633436386639393464653565333263386663326433336338 +37623030653231383931613833313538336435383532346238643030633734353037613964393637 +64656262663935383839346431306564376630653438383630386262633061626261386433313961 +35353532336666663639323730386466616366343438663466633233386433623435356136353634 +38326132643365306235663631373566393263343038373935333230663230303761313662343133 +63663564623361613634353738306263363339376638366231313337613766643964383861663566 +64663231323336376136666134393939303637663863356561613235316337363337363134643162 +33356232393931356335616566636561346266386138633930656336323961353366313039643364 +62386331373037396230333030623431333863326336616235643365623237326136363066343930 +61376636333036633762353933616432656535383934633833643064343365656163306436643433 +38613631613138636330363562386435663462653039656430613662656465326166336331316638 +36396138393032666663393530303534333662343961333461313461303539646530343064383331 +34613435373561326161333235386535366564336333383061616537633263383439616531303932 +30306137616135396562306236316632333634396638633338653333316334343163643435626231 +39373364333235386538356365623861636161363963643730373966386135363166333637313562 +62373332333035643434313162313538626639656461333563646565626439386464353537633064 +61343939363033343435656430663836313166633463356132356436303639353932656562643264 +32383337666438666638333433633038396430623936613162656262353165376435613035613131 +62666635346634343239343735313031336534306161646637613665626362313331343265393763 +64396161663563376434306237653634646363323737636439636533353130303734633932313763 +65643638366362313364383265393265326434636463343732656638316261306161323661303363 +63663230643838633666396164343163656633363236313562356663626438663162616565633063 +63653364343030643832323866373337333432353436643736306331316461326136633733616163 +31383665383630313763396462326335396665616338313462333763316330626533623964343438 +35393330353262313662333634623961393166303935643531623732346538613833353761353262 +31646630386563633164386132343634346534306133663962623765323133353235336163646633 +34393338356537636230646563366536623035333339303433656262366131363165646232383038 +64633533623532616131393833336137303338633436386633626338656235663937356238666234 +35386166626135346331363664303532323630666135393737303238633966626465356333353630 +37306335393132323461393931396237386462316430343261643037623666613033653563663438 +30643032663837333437633639363836633035636134333437626533313633663565663663353433 +62376635323534396136323038363633613565656165346565306531656332343464333066393737 +37353037356332363530313238653264643734326562643234326132333230393938636666666161 +33393965626330616263633361383234323862376263653366643132396436343236306134336264 +36623832386437363362636634653438633866303634636136343462373262663434653165646632 +39353438643431623832636332336661386233613263393035643539313632306339346631643336 +34336139613362613764373265653963353461383734326533643864326333373236386266643033 +33613261666663393839363836646339373065386539376535366239666463653035656439376164 +63666434366464636231333535363061313335303737316362643633626137653837366630346535 +31653765313861306366373930333931623765663561326432343835353339373734383238303863 +39353363376136326632643239343339353732373664663539363063643630613238353934376232 +64303836306166323433313732313534616664653832343861323035663264646163663166303934 +66663535306361346264616464633761656338653765663766313863666166613666373862383239 +35326636326566313536316161653466613066633334653364663033616466626132336365616138 +65333764373363616134303131353934323064613737346630396536623534666438633361666531 +63333938363462396630383062383861613464373631643763646536663835363031643463326337 +63666439316531303636353435613439366331353466306632316536643633656238626331343963 +38663131393366383564333037663365383338306535643334353531653861386639373839383637 +64306532336638383331396234663462333138363933633064353932613961343862343031636266 +33623331316639393337316432323863316338316632336262616261613936356337346431333538 +65303961393633306530383632313331393063333739653966623839623963636638666136313266 +32386436313636343233633631373364616433343364366133656539316462336661343333623631 +64386435373261663463366638623566383164623335363564633334383230363239376430333461 +39366133373230346133303539363834633665383338356136646366386630383939613562646339 +34316464663537323435386137396235363261613162323633396265663962326563316230326662 +35393461363738393663393932323162363765633839353734656665303832616264623932353664 +37346133656437396630636438666164303835393134653334393938396335343335666465326635 +31643666633032653634626262346533653935383136356366343230333962363464356565386362 +31313366613032363166643933326266346562623061643236653337383363303237303466616665 +37333364306237306630323432643531663431393766633865663562316437626637316539373361 +62333261316335363364653732616338336133323632336338613130336634663339633738333866 +63633165346138353763303639326537306433613233366533346438343134393962643038313136 +64383163306661383761353433356261326535363134633665383163643337386365313131636535 +64636139356638663838396138313132373264323864613533323738373437383632666265633335 +32613130623164616564333664303138623035663265306330666661386532343536326536323935 +65663538643361616463313066333137373161353433373030633138656563346537333566656134 +35393237396563623463303035663639306135333265326164393939373566393433633138383735 +64376139613337653662343133396266613038323863313139306461343839333937623963383064 +61373965393061616439643361323262643135646366383936663966633065326533323830613332 +37653832656165373866623833613534313466353636663565646465373566323666313539376338 +61633733383761646630343238393836653863343262383831386633626463343666316139616463 +61383465376364643630366433383865363565616331613330646232366363393035343637663839 +64306562336461663837303332313365643931313535646263623264613531366462326165616162 +39656564333837386337313264656361623561386331616665656437373930393264626433633632 +61393735373061346231363137303261316435643937613538653630313738663932363639623966 +64396466366466633130626135363765396233323966366134366564303134656161353337353363 +39353839313731363065653061393761653565643364366136376664613438346432633864393864 +32363331333636383262353335303337646664326534346135663164323434666337643964363539 +61646434643562383636616261613361616634616539383134316466313666306563666662663364 +38376633343634623163653937386434393361656136356432373138353136336162343964356661 +39386134303363343731383661306531613364643139303432313266393066653835626536333563 +31306532363136353666396465356138636434616664623131333632333335353563323831333036 +64313235653337316539646337633237306439633332646566313835653330356462353731376232 +36363863343131643762386363356535336434306335633639316232636134323964346363366330 +65653761393634636236383935313239633930653037613234353865366263633239613137633031 +61353134666537356163356436346337343735303133643739616231356236616230613961356636 +32353934313962333330333636663737663833653665396534656630353632396637336337303165 +61626235633562373231373930336137363039346366616265346239633836303037336130366263 +30316431323663623066623063333239303864313537306363336539346339653838383663373330 +33303464383865313061623265626430633831396139643334353263333633653237373130663439 +36393161643237626235366465316132346534626334326338303666366233373566613238363335 +36613661356234333232306136643865363833633065396439393762653165343630303564616666 +62346162333930666262333861303235323662346232393530663631303037363230303133353330 +31326565386638623733343735393662316632316436663336323864393334346233643566646464 +62313738343335373236373134366431386234336363306139363535363133393564643562363661 +32343865616466346631656134656537383534333634303964373966313262356335313938306237 +63633038663538366164616637303535653030613838646430336562336637383935613636623739 +34383435333534633034376663383261663861616162626565613666323037376231633033333961 +37316266633464626335353939316365663265383634393738343737356439663935636165386462 +64643233663031353065353338343837616130333236363931323030626338643738313838313437 +37353963663635626233613939386437363032303134646461646161376139393730323038303137 +66346436636531363766316236353565353661623665613733363333326330333437633633363034 +30313739383232613331666666333636303036363935373435313132613164393830333536356136 +65343536643661666464653231366431636563303133653332636666306437316665646437383462 +64386163636439366234303364666133636234376634343364363534396236666432313462393963 +64626638663739383665396536666262396631343062616237383739323338623136346432366665 +38303564626630373961386636353263326366383334616466343333346337396562336163373462 +31336130316464313064353161393861396639306466306130323935633333626236613437643364 +30643632376165343136323434643363383830663765363664333337386639323661336237653265 +38653837373535356461633463363237643137313834303931316230353531663239386639663565 +63656336623435663532663037643863616564616161383638656537383163623962363665353065 +31313333336562333437626439653937346530616234663963626466656434663537323964363765 +30393864393462306239316465656536643436333634363263643639333933396636346234623731 +39383432396538376563393338363132643466366431396664396238333833643837316333356266 +31316434306434623837343031336332323061383937666639313036633139633632363166623264 +31386639396361343938666238353261643635343162653932626131643763623235336532666634 +66373166336461616439373838323461626466326438343966613464373064626136623136613562 +62303338346265363232636434366663626635323830666535376561313638646330393332363235 +35393839393765376438306261363930376430666463623136653731333266306139663364613061 +34346231643365643832373731343236306138643832346337343362396438613766343563323933 +39623932663938336565386138383866643264643137323136313762663464303739333439656262 +37623738393537383333306235396536656534353937653234333238613335396337376236313461 +64356564326132393666393434323339653532383564373734306661653165626234663430663230 +33616262666266613333303665613863346237336630323264353230373339666339323739373765 +65613235363337616263353166633861313435613133666435616664643632383136363538666336 +33343137633933353666386432326336323663663562383935326237303737333830336538363831 +63383138323032643535386562633864653362393435356432356330623530663466323264663732 +62346338633734353036373239643566616662393063366637663431373662616330623839333438 +62646464336462626438396135656535643262366238336632613132616631373530376336336236 +66306564333638346333633261323035623864343338633365663663313864626562363938383530 +31346361393463396166306437666630383231303435376633633230366562663431333166643930 +32356133663839666661363966656137386530306231616438653338376633386166313432363435 +33396363393338373337396232373762303563636631663161663038383830363365616135373134 +66303165306363633862636534336534353565303566326163626635356439656464616631393766 +34373733643234613035376263313932316262346538316361323533303834383533356337313438 +32646136616162396334323131343863333538613238316562323837313536383064383062376164 +66303566396563333634373462666334363464313138613665306635363138326263653935373735 +36666430336639633438343537656464323835383931643630336331373766393762303235613535 +65306239663366336137643437656261303766363066346234343235376537396537333866656362 +66376538313439613530356465333663303331623933353635346162313833373336326433336464 +38336664666362303237376130323731346436313533306332373330306234666130336462303735 +36306237643864343965393235333462356661663833333230636264643235313234313136633134 +66633665313361343465613932666364656362363962323532366434333133356130633664306638 +37633737396262663032623430373666353366316364393739336439353236646366363465613535 +61303863336362616134376365313238616237303866646337386462326461306238633735643431 +36656364303533616433313462623834613239643334336632346430323261373232633033303363 +61623835343661383230306261653164306233663133343634663534616630356666383937663666 +31303262613434346630626661633165333639323633643137643062373139303066333636663034 +33653164396162333066613335303562663031316430393235363963363934336232306262383332 +32363532363331616332356335366163383735613236643834383961313835373539653131306338 +61363931613037306638386639636663303634346134643035326564336535653434363632373163 +63393836363466383364366264663063363037343630353937336466666136396163323235656138 +37346531323061623965356563663361373938336262653134366332613264656366323736313565 +38613231623731316532626535653261323531343737306338366338653431386464633834663632 +33613337653335313262626130626632353434636131643931303334636439646163333930376233 +63643331613563383934643436343833383439316431313535643839643937356133383630653430 +30376136323834323164343837656132656165383263346562393033643237646138323364353830 +32383462383033666531633632633361333837366632326263373031653465646138663732393365 +65363434396236646133303065316436303431366461356435316565326335643437333861663838 +32346333363264663339316535386134383432373061633436323266646637373366623730656331 +37636464643037656336306234323763643633333930633130613564386636353535653733376335 +66613330636533373433393230633631656634373634376161616333633065653963316432616438 +32396264626536333161353630343730623861393066363239396465386636633436376661623633 +66653238613931346433326338653638613161386635663236303162626333373933383731656132 +61366437616437613761656230656235326362663336636132366436313935653634346434313066 +37376364353563306661636564303732393761633034643139383537343063653939663534633132 +65646337396336653863636537313764373565386662653561646136646431623665643065643835 +61643363616161393638616561336131626533333233366662383434656566396463376437643965 +30303561353231326165333938376263363263356431363065323530373135386563333964306666 +33303264386236303432303262353232626535326665666530616565636166356461363066333765 +38313163623361656436653536396665613039643832393130383738326235636363653633316666 +31356339393861383337656266316661343063626232643936373661373838623735643236303832 +65386133613835353165343037666232303464386632343237633833363434373263386433383064 +39623763343236646664356233323566316266653134343337613961396564653638613438376665 +62346565383934616230333463363537323838633564373131333965393266373961373037303961 +63646130356135376336383966333433396535316664306638656635653462303138656561336666 +32323666393231663231663462383661393835383234383361623034653263393432336134633634 +62333561393662303733333165643662383236323162303634626365306330346631643564653239 +30333139393063376363313264326566643962353132616132376534653765663430386236326364 +63306462383039633230613833623432343265623135313439643437323935303063613031353266 +39653233356432333830303433373961343836333630643536363335626465636131316239366165 +36623335656234353062643930633161623663383763323564356166316439616162366463643236 +34636139613233666464366338616465306337306138643233336232613531613162303437636161 +61643438643331303234303661343361316636623237633165666338303333633235323832373834 +63633761623431633164376633636464346137653238396336613233343832303333653636353236 +32666138313639306263323030363037313164656465353239316635306565303235363862373064 +39613639386636303733656233393339353234333432663961366665303735356362353266316261 +32393333383230623230366166663632343365353835353233396636343238363837613962633064 +39306661653438393436383762323765343266336232373661663731316436653434376332393133 +30613930323331396139393139383961363763636136626634663663326164303637356565366563 +31343566333538616663353530623635373262613030383365663630386531656335333562663931 +64326365306463326166366439393665363862366564383037346437326336306435613739383038 +66306531343437613034663738333734363661333334646237636465356661313966353563363937 +63633338336365373962306531326434663435643831323731656363346339633134336331393264 +37646238323132613139313664383062623930643664353937643438396661613335303236336233 +61353762303662356365306232663662393761663861373230396165613336356532633133346432 +63613861616438313436313764343835373032663364653039343662323331343539323764643536 +33643466623837616134353065623832346564653261323566663363653335333634366436383538 +35323263333833343437383565636232343161613533353130353463343162373365316235363538 +30336636383866316666306535346233376663383839656265656531396235313434666439313030 +36313934343330656633313337346230363335313539333762343461316161303636336133616538 +66633636393462613864336134626665656561393636373466643934616530316465373236626334 +30343032386337393233333032383735363965646432366662666664333535613135343764656333 +62346139653533303165396133353130323132613866373162303463366565323262636532396638 +32303835393632333130366463643834313161623131333364393637383636646465303038323337 +34623032663336666536343266343664373961313639666534643236623662626333393434323537 +34663865666365623764353062336532666261343038636362373431373335353635653934383034 +33313164623437623137326238623139313634653637636161663933643032393965613438373066 +31326362306564653963633537633938626264323436646366356365623632393264613162316161 +63363531316439333066643333653465353262653534663638326436383366313965363339373334 +32626339396166633864633166346336333932623032393535343235313337636532353364643763 +35623431353434633536383064363436313330666230383965633633373236336536363565663961 +30663235333538623233616164346233336134393032386235643935363233653837393163613866 +33393464303566336263616536356233616133653335663734353162383065356630396462363362 +33633162663130343766386461363164663061363936623031656134643635393632333133396364 +66666236343938623866633230393233366535363361356539666262383963636133646536363866 +31303266326430316136393066623333356130313039613633633034643365663361303662626365 +31313831646366633835666238633932303637366562316232303634343234633664373766356262 +32396463326634386464386636306336343337633665326338366466303133653635323036646564 +35353465666232633665383530653263346637613335366362663865646564373163323339376230 +39343763663432373763336131613338633561343663313238343063303864313963383435353264 +30633632666537363838653035363862336466373935366331653566613439383062663165363262 +39316437613464353763626132396166373034336532366333393439626233613238353531633266 +33663834323432343532356334613563333534356266626662643638356330356435323237636237 +33633066333965393865353466393339393366643135313661646330353961346164643064656532 +62646435313165343132393966343566663765643039616633666265393537346361333365303566 +34323263396339636266356666623561613231363731326364383935653736386562623964643065 +35613962333265313566306531306162393034386330373334363363623132303938346531336666 +65613466633065646366316664313862613963646561643732363663653066316265356533633734 +33653264633065653166313338653065396561363534376663653631393233326430616366383765 +36656633383764383835386263623839656637376139653763386330633233343062303133663136 +36316261666635313137623939336165663730633065656437316262626136646334343430363965 +62333434633662326435306662373934613365643738356233373339643831336132366433633035 +33646566373833393033316335393265663934353066623139343061643836336133636537373739 +62663262613934613439333238313035316331373232636639616261373737353664636639346533 +31666366643339663930373864623637623632316637303634333030663134396466393638313464 +61303835653539326465643436303866623766643834376365346136653032323935313131333166 +34613362363731363734636135343732343062643431303837323862373865343031393432346439 +36353362373630373934336435316431386532313461356465376664313232346262656435306332 +30653933363938373635633533656261626530336264353365313031643539396633343062303965 +39346633306230363435353237613430363930343662656330373039613130643535633033303630 +65663965396264633230386664613862653035383863323431633530386537373437373135356461 +34303265393133646235646635336166363965353132366135356330316164623362393566626433 +63646232373061623936373338613435373337653635323364383931643030613463613234616237 +34366339373161656533363231313530316133373066626135313438663763366364303534346130 +36346563356137396232643866636166616465613536376261326433633965666534663761636331 +32353639303736363734636462633935356236653833343835626664373235626538303437663861 +35353534386566383164343431326132386365616465626565646638343164613265376133663931 +33613332626436306638323830613666643536653131626535613065346433663661643730386435 +36386134623639346361333037393038326639646536333938623839333838633134653334646330 +39333836643965643637626662393338303336313661623734343665656237653939383036353238 +61376630396563313561643163393365633564633266383933393335646563656636323538313235 +65383330363533343964383936373562383863666334343136373536333539636264356234386338 +65663833346335386237373533373032663034316537653863666365313837306666363633383166 +38383730373665346534383538346662326534366537346236666432636233623935656461626634 +37333466383336303962373636346538623034343861623461663263626432633732353661383033 +33366336383365303631303132356434333739383466643165353133386266386331363265356637 +34346534306634326231393133623630356432653436396631633734326430393933313830636332 +63656163396536373861343332623634653732346162323036303661643930386533393738333939 +34336130356463363865396331643934643866663539656464343261663461333738393363636139 +32303339653466303365643637636562303335363130646366613732343162643334306531623162 +39663437313333356338386239396365383861393038313862313761326164316537623761643536 +61633433646437653462633039633730386639646130383330373930386464353761343639363934 +34313166363963633866343165373039326132396532373135373838633636393732323732386230 +39333662633330363366306634643731656432363535323333656163393639623039316135643464 +34663263366332633934306531643134313730343735393839326234363438333038646630366635 +63636335353435396530643436353239316537383135396266303562366364623662633831333736 +37363536343562396630663137393130343934643834623439393739376435383334326263613061 +64306436396337373062343361646466323138316138643434363736336339616632333364313865 +62653037363039376665653961366336646665323536336537343830393636333437623166356464 +62386165346663323236613662376430633832343733376430633037323532623232656339656162 +31303237626233363664613434653239326466333630383437366661353465636333626466626632 +65643933376135396635326161653035366637303230626463386234643031393631653633613133 +61653565346462323830333363636239313337383335386638646637616137333734663839386363 +33356631346364396661313134393833373863613931363532303239646266333332616664356632 +63333963623961643362643231653636613930306365653662663938303262643361663634363439 +31313431656334303663386565623462613361356537346362633334373864653961636130363430 +37346665636333656263626663396634646433303663326336623132623265333835666238626262 +37363361633839306133343333303266363832656565303665636663383830313436383634366433 +38303639343637613033623432333838653362633733316438316135363662663931383832323036 +38303563643130656631646261316130643039633165353738393565393938666464333732326632 +64336530313234323338343037366263303830643936393361303831343734383035663561373862 +38363330343762333164333062616634613866633534386264636462303335653834336361626333 +31623539653463663264306436636662316635346331326162623761376331666632303465623632 +64333939653562616234343263643430393234313264333835353062663663336263396434643162 +38313961333636376362333639633861373230313539306463623733303936633039663861376665 +65336666323833336532393335663536306138363835656236373563343266316638656236363731 +66376262346132366430336637306239633061343430376261643565363832313836636162393832 +62356530313464633361396166373237366633303834633930656166623164623332616266333231 +30333064623036623937323030356136613764343138363437346232313863343432386665346330 +63663339653266643161613632383662383133343236666539623338343435336463656138356331 +62643461376262343435666465363265373164623732313435353534643731666663626431393165 +61396231313638666338636135613235373633653536366631376537383231373938386438643634 +34653334666235633030376165366463383861306637376534373561343530346332393562636234 +39323038373361333030333037326332353538623536336234383937323034303432623134343438 +62633161343639373635663331373031666236623839623564663662333630393732393535653637 +65666536373061306162363737313734626262633466333936373432313335343131396134613437 +38396237383666353566623738326362346635613363396535633463393463376336383966333764 +36616466666538666233336531366539306564393831306238333234303533313566663131363834 +34333833373431343133613232383061313533653762656661336132366361616662613131343038 +38393735396435623762316163613432333039656362353665303538373966356233333664396561 +66313166313462343135376163613839653337393466616563303335623830626637316431383336 +65313364613332393237626433643433616534646131633565653136613238346661346136626166 +36373363353133366237636231323937306135353034616333393662356534396335333533316263 +37656136353066383334613430626462363330616661316537353562636236633263356566613539 +30363432393339363538616338316139393334366666643763306131666131653566643664656430 +63666664326134636566313862353961646637376665646561303761393835616638636433363939 +39656237646164663230303038393831623362393435643833656438626264653335336130313562 +63666236666231396639613935326166373830303237333037323133633561663330306666663562 +64383036613665313165633239343064653739393563336331393935313033666532343235623439 +32393131313935303665633036313135383036316334313763623263313734636131333533316262 +66326332633031336364323561383461323733633635653162613731323634383738336139326630 +64626536613830393963376461323035376338313135373131373135313037613366633136383430 +33633666656466613737646333653337636438383830666637306437643837373966373231336162 +36393564353839626266653166316134356533363163646430383832643434313135316264383966 +34373234666339333235303661313938353963336363373863343962616265323735333266366665 +39363834306537313962636161376161613434303039376664656631613333623261636530396661 +35333036616166313761373539646164306566656362386463626666373464653332383035643862 +31366430363161653838313261343464326162653437373636633438653838306333653962383461 +65383932653134616163633636633438366364646231336532623235363635613937363862653866 +66343964663163363438616663623865653533636462663634396534343433623134366434353638 +38323536623165316134383134343032663861656566646237306439656239326231313536636661 +30373566333232346631663531323161666335613165383165326335386366613361336165616661 +34373036633531646335353433623136613139646230663635653037326261663661393162323336 +66336562303062353663613034616632336663613631396465643065373361646464316233643666 +35393238323536383933346232643934386639653936313230343466366532626535643936653933 +65386630346438663466636462343066313734363862306566616638626362333439623936663339 +35386332326163323630343061343335393835393264353865303736376534333939323030653531 +30633830656130313633626563363039643132386662653061343565326361346464333939633635 +62613561326532383762653864326633663462353763616633623239373930626164656362333966 +38343731656536323839383535616230353435353331366535633866643634303066393062336438 +35336365613962306635373431663939393263353461336330393232633337396434396364653630 +35313637313638366633313431346635306365326166383037633536373333373663316461656435 +37303639613132376236646635376634393332653237366139316263323563643134386338356264 +64653934366465316166643762333963313033383761653861393539653335613635613737663761 +38343561633636373432323838656435323333333639383235343536333939326463306131393661 +39356432626337313133303537623436356334353661643737393131353064636534306336346135 +66633963633232376235313131383262326238336533353666633531646661633439373464653635 +32643337353136393931346462353734393537643236383236623130656232313736653935666331 +39333332323630346161316431663135613037376335396434613139383166386263306232303434 +66333030303830653036663064383862336231323535336235303734323966333463363763383665 +66353363636561353465383236343136623339653834663262376564353264316364343334623961 +31613732613466336462636261633937616434656262343362336532333632336637633166363932 +38373166386334623263363633303833353939363535323465633232363965646535656436613333 +65363738613262336135646639666238306439613365343632626164393132303335383739643566 +32376363326332343934303038663030323832663663313166636264643731323934326536316339 +65373063366162613633363837643536323331336333303734383361323561353765643538393434 +37336539653262383636353434313664313238663465326565356666383330663963306639383832 +35333662346633356431613933663434353462313665366662663761383431326336363539306337 +39316335383433393736616634303234373965656234373539376534346539346235306338353138 +63346664366163653864613162656164623364646562393261623663663233353264613439333035 +34373537663930646264663666323637633632613861363366653036616337333832633536393034 +36656137646661333537323130656363396661393539363164353836613266663764623761336439 +63613862633830643238326132383339353733616165336161613466373836326232323562353934 +346364393137303864666537373361363137 From 7ab4ea2600a6aef576031586d5459e3b3d70405b Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 25 Jul 2025 10:11:09 +0000 Subject: [PATCH 075/103] Attempt to fix ska-iam auth --- .../idp/ska-iam.stfc.ac.uk.client | 26 +++++++++---------- .../jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf | 3 +-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client index 11b533b4f..bd195a8d2 100644 --- a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client @@ -1,14 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -66326165353134363533313035316536343934303966616232613136633062333639323431303865 -3961626538623962326565636464646163626364363337650a666464333561373665343531656139 -31393466626464376333663036626436303535356230343433333233393766316434653233623464 -6162613666646664370a303264333131316262383934313961643665333531636664303533613934 -37373132393938643431376361343139336231343465373762306538313666653034343962656538 -32306536633930393530613038393235646564623362323333656238623761316535616434323535 -65336161383439653866346563643738316630663637306361343938313239663130663266623766 -33306561373735396364356138313462303532353136323964313364346131643034306133303039 -39363632666137346363376266343662396430386163383638656237356437393937356561313936 -34386264623737396536623135303065363234633236613063653963313537383161313030373463 -38343864303530653164313865323966303633373634363765393431356634626262306364333939 -62366635326533346266633532373564386266363137646565363631343664346231303761333061 -33376164386661353630343436376239363136653766333865376331663164653565 +61653238343631383635633631643635386534626565306632613734326261313862393863663834 +6661306534663732396231646436366439653432646163330a316664366136376166653637633465 +34383461363065383534383835323530353133623164666132663435653263303138323436306466 +3733623634626262630a393834626135346361323731356166326530616634663934613264313862 +64656464623238333830386430663930303961303034323266346664323238373366386364303761 +36633639356632303132386361343532363364353165663561333035336661636334336138633039 +64346133373338303137653337343133666365363531363131636530373538396130633966323436 +30616463376132623130626230336337616161363138353039396534393435306461366231643264 +66343136666638323438333530383734323538386437633864366139363164346663623161303533 +31313530626263306363663632373562303532633661643364373234363131346535643330636137 +35643331613539353838383034363133663233316333303039633236303566643437383633616161 +62633963353231376666643265656365386330666337363839363237333039303832643230613862 +34333664656266306134393663356665366263333665333230333639663136633762 diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf index 617ea9486..1c3904207 100644 --- a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf @@ -1,4 +1,3 @@ { -"scope" : "openid email profile" +"scope" : "openid email profile offline_access" } - From a42c8fc716337af3b4facf2c9802026bce4cba0c Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 25 Jul 2025 10:19:34 +0000 Subject: [PATCH 076/103] Attempt to fix ska-iam auth --- .../idp/ska-iam.stfc.ac.uk.client | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client index bd195a8d2..cd42bc4d8 100644 --- a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client @@ -1,14 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -61653238343631383635633631643635386534626565306632613734326261313862393863663834 -6661306534663732396231646436366439653432646163330a316664366136376166653637633465 -34383461363065383534383835323530353133623164666132663435653263303138323436306466 -3733623634626262630a393834626135346361323731356166326530616634663934613264313862 -64656464623238333830386430663930303961303034323266346664323238373366386364303761 -36633639356632303132386361343532363364353165663561333035336661636334336138633039 -64346133373338303137653337343133666365363531363131636530373538396130633966323436 -30616463376132623130626230336337616161363138353039396534393435306461366231643264 -66343136666638323438333530383734323538386437633864366139363164346663623161303533 -31313530626263306363663632373562303532633661643364373234363131346535643330636137 -35643331613539353838383034363133663233316333303039633236303566643437383633616161 -62633963353231376666643265656365386330666337363839363237333039303832643230613862 -34333664656266306134393663356665366263333665333230333639663136633762 +34393634643762333662616339396265373361623763393863323633393730386330376330653336 +6634613537353330626234623036323963366132646533370a393438306336376336626165326538 +61313265633131633132623936336537313030643364636264613065346361396432373963396261 +3037613761653130620a643137636265343534646561626562653234633634366531356137643262 +64633037303235303265663631373037653338386464653439633132613839313732373364353530 +30303132633633643335366131316234316461626134343465366538336361343037393939656366 +65643932656238323363356138326432376632356434313664656135376562646537663362353930 +36343239346234636432663364336165306662616535346363333236626337363835646531396438 +33313465663564376639313938343862333963373962396232636364313966303635323762666562 +35653662363838383865343435316537313631323631363734396461343263633032383066396336 +38656361303339636432313837316664663232363838626231383137393733636135353236656266 +30303533396362616666623164663262303031353062343164333964376438343632353738623662 +64376231613163353664353031636165363939366565316265383637643636393832 From cc16455b98bfd844fd82ca27b72eeef8b98ec580 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 25 Jul 2025 11:14:11 +0000 Subject: [PATCH 077/103] Attempt to fix SKA-IAM auth --- etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf | 2 +- etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider | 4 ++-- etc/kayobe/kolla/globals.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf index 1c3904207..9b8029003 100644 --- a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.conf @@ -1,3 +1,3 @@ { "scope" : "openid email profile offline_access" -} +} \ No newline at end of file diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider index 20042bea5..677952264 100644 --- a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.provider @@ -1,5 +1,5 @@ { - "request_parameter_supported": true, +"request_parameter_supported": true, "introspection_endpoint": "https://ska-iam.stfc.ac.uk/introspect", "claims_parameter_supported": true, "scim_endpoint": "https://ska-iam.stfc.ac.uk/scim", @@ -193,4 +193,4 @@ "RSA1_5", "RSA-OAEP-384" ] -} +} \ No newline at end of file diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index a7d7d5784..bd99a11a5 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -23,4 +23,4 @@ keystone_identity_mappings: - name: "ska_mapping" file: "{{ kayobe_config_path }}/kolla/config/keystone/ska_identity-mappings.json" -kolla_enable_tls_external: false \ No newline at end of file +kolla_enable_tls_external: true \ No newline at end of file From c72d7dbb6fbefbf1db8a916ea37d048ec14d7e02 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 28 Jul 2025 08:12:35 +0000 Subject: [PATCH 078/103] Attempt to fix ska-iam auth --- .../idp/ska-iam.stfc.ac.uk.client | 26 +++++++++---------- etc/kayobe/kolla/globals.yml | 1 - 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client index cd42bc4d8..efe6dacea 100644 --- a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client @@ -1,14 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -34393634643762333662616339396265373361623763393863323633393730386330376330653336 -6634613537353330626234623036323963366132646533370a393438306336376336626165326538 -61313265633131633132623936336537313030643364636264613065346361396432373963396261 -3037613761653130620a643137636265343534646561626562653234633634366531356137643262 -64633037303235303265663631373037653338386464653439633132613839313732373364353530 -30303132633633643335366131316234316461626134343465366538336361343037393939656366 -65643932656238323363356138326432376632356434313664656135376562646537663362353930 -36343239346234636432663364336165306662616535346363333236626337363835646531396438 -33313465663564376639313938343862333963373962396232636364313966303635323762666562 -35653662363838383865343435316537313631323631363734396461343263633032383066396336 -38656361303339636432313837316664663232363838626231383137393733636135353236656266 -30303533396362616666623164663262303031353062343164333964376438343632353738623662 -64376231613163353664353031636165363939366565316265383637643636393832 +36303831306362303538663366323134633439633637346535336461326238366239336633313864 +3233613237303735376338376361633531653162316263640a353066333831383962323632313263 +65376433313035326139643961643735346439393963623437346435636631636264346536653463 +3539313737306262360a313738363564353739336135636237336565303137306461653533326466 +39396265373164643032333332636165373234313235356637623763633333363330313139633661 +66636631393433643830633439326138333339303432613230396361353134613039623138613466 +65666534613263333466373165373333353963313461336237346536323137366664663139333235 +64613166336236373838336631316537333963363663303638326636326336326636303764353263 +33353661343038363961393934316439313638363134363334353065336239663265323537316636 +38616261306462383936396564616564333238656538326633336163363061383562316434363937 +37653331396138623837336631366130633938393435323039373634623835346561633733643961 +63656235383233616231633865313362636163663362363536663664613835333733646432393435 +66393035373162323839666339393830663039366238613331653330363966346137 diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index bd99a11a5..32b56bb50 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -22,5 +22,4 @@ keystone_identity_providers: keystone_identity_mappings: - name: "ska_mapping" file: "{{ kayobe_config_path }}/kolla/config/keystone/ska_identity-mappings.json" - kolla_enable_tls_external: true \ No newline at end of file From 85371d7cfa65c54645c88bfee4d4bb73242dd937 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 28 Jul 2025 10:34:15 +0000 Subject: [PATCH 079/103] Fixed ska-iam auth --- .../idp/ska-iam.stfc.ac.uk.client | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client index efe6dacea..43b3714b8 100644 --- a/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client +++ b/etc/kayobe/jbo-idp-conf/idp/ska-iam.stfc.ac.uk.client @@ -1,14 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -36303831306362303538663366323134633439633637346535336461326238366239336633313864 -3233613237303735376338376361633531653162316263640a353066333831383962323632313263 -65376433313035326139643961643735346439393963623437346435636631636264346536653463 -3539313737306262360a313738363564353739336135636237336565303137306461653533326466 -39396265373164643032333332636165373234313235356637623763633333363330313139633661 -66636631393433643830633439326138333339303432613230396361353134613039623138613466 -65666534613263333466373165373333353963313461336237346536323137366664663139333235 -64613166336236373838336631316537333963363663303638326636326336326636303764353263 -33353661343038363961393934316439313638363134363334353065336239663265323537316636 -38616261306462383936396564616564333238656538326633336163363061383562316434363937 -37653331396138623837336631366130633938393435323039373634623835346561633733643961 -63656235383233616231633865313362636163663362363536663664613835333733646432393435 -66393035373162323839666339393830663039366238613331653330363966346137 +39396132343837376464323839356338346465633035666332303365613661666361383661326238 +6637326531366166346364336230333431366234306631330a313232663030643434353636623166 +33313961373730333738313135633832333134333161323536353961353662623138386630363065 +6237313839663339340a353838633463346332393636333633633838646435393239616133336262 +61323361646166356531346438366462376335613439393134313238666432616632356230363864 +63326239393762323434666630373031353632663966376166363734306661663663306665656263 +30633530613834316165636635666335663834633065343536353866336633306336346263313936 +36323336306333636462383834306265633362316131346431313932393836353366653032646135 +33613934393161336463326437363930643965346565396362613437333832613339396232316332 +64623766643764666237336236653866356362306664346463353739303336333435613838376166 +34303962613039346334626362366161346461393333613637396134363463623735346635386235 +61316637336166333739363136656532666131353635313466343261656332626263613333316131 +38646366303665363838366530383261343030666566316433333764333265663934 From 1b502efedc062a424942e3e36fa46c931ff6f0a0 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 1 Dec 2025 13:14:37 +0000 Subject: [PATCH 080/103] Config to allow pods to route to ceph (data) network --- .../inventory/group_vars/controllers/network-interfaces | 9 +++++---- etc/kayobe/kolla/config/keystone.conf | 3 +++ etc/kayobe/kolla/config/neutron.conf | 2 +- etc/kayobe/kolla/globals.yml | 3 ++- etc/kayobe/networks.yml | 1 + 5 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 etc/kayobe/kolla/config/keystone.conf diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 3ad1f9a4b..59fd35a19 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -15,10 +15,11 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. -prod_interface: breno12399 -data_interface: ens2f0np0 -prod_bridge_ports: - - eno12399 +prod_interface: eno12399 + +data_interface: brens2f0np0 +data_bridge_ports: + - ens2f0np0 jbo_interface: breno8303 jbo_bridge_ports: diff --git a/etc/kayobe/kolla/config/keystone.conf b/etc/kayobe/kolla/config/keystone.conf new file mode 100644 index 000000000..4aad4f113 --- /dev/null +++ b/etc/kayobe/kolla/config/keystone.conf @@ -0,0 +1,3 @@ +[federation] +trusted_dashboard = https://portal.azimuth.man.uksrc.org/auth/federated/complete/ +trusted_dashboard = https://api.openstack.man.uksrc.org/auth/websso/ \ No newline at end of file diff --git a/etc/kayobe/kolla/config/neutron.conf b/etc/kayobe/kolla/config/neutron.conf index b36726cbf..3b78066e6 100644 --- a/etc/kayobe/kolla/config/neutron.conf +++ b/etc/kayobe/kolla/config/neutron.conf @@ -1,3 +1,3 @@ # Global physnet_mtu - all created OpenStack networks without MTU set will get global_physnet_mtu - overhead [DEFAULT] -global_physnet_mtu = 1490 +global_physnet_mtu = 1500 diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 32b56bb50..358a22f89 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -22,4 +22,5 @@ keystone_identity_providers: keystone_identity_mappings: - name: "ska_mapping" file: "{{ kayobe_config_path }}/kolla/config/keystone/ska_identity-mappings.json" -kolla_enable_tls_external: true \ No newline at end of file +kolla_enable_tls_external: true +keystone_federation_oidc_response_type: code \ No newline at end of file diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 2eb21140f..be9b065ba 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -36,6 +36,7 @@ internal_net_name: data #external_net_names: external_net_names: - jbo +- data # Name of the network used to expose the public OpenStack API endpoints. #public_net_name: From bc17bc8f67995d4e2d2af5fb08350183097a83e1 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 6 Jan 2026 12:10:49 +0000 Subject: [PATCH 081/103] Remove hugepages from c-1 to stop VM failures --- .../compute_flavor_general/nova} | 0 .../hugepages | 0 .../{compute_dell => compute_flavor_r660}/nova | 0 .../hugepages | 0 .../nova | 0 etc/kayobe/inventory/hosts | 16 ++++++++++++++++ 6 files changed, 16 insertions(+) rename etc/kayobe/inventory/{host_vars/c-1 => group_vars/compute_flavor_general/nova} (100%) rename etc/kayobe/inventory/group_vars/{compute_dell => compute_flavor_r660}/hugepages (100%) rename etc/kayobe/inventory/group_vars/{compute_dell => compute_flavor_r660}/nova (100%) rename etc/kayobe/inventory/group_vars/{compute_lenovo => compute_flavor_sr650}/hugepages (100%) rename etc/kayobe/inventory/group_vars/{compute_lenovo => compute_flavor_sr650}/nova (100%) diff --git a/etc/kayobe/inventory/host_vars/c-1 b/etc/kayobe/inventory/group_vars/compute_flavor_general/nova similarity index 100% rename from etc/kayobe/inventory/host_vars/c-1 rename to etc/kayobe/inventory/group_vars/compute_flavor_general/nova diff --git a/etc/kayobe/inventory/group_vars/compute_dell/hugepages b/etc/kayobe/inventory/group_vars/compute_flavor_r660/hugepages similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_dell/hugepages rename to etc/kayobe/inventory/group_vars/compute_flavor_r660/hugepages diff --git a/etc/kayobe/inventory/group_vars/compute_dell/nova b/etc/kayobe/inventory/group_vars/compute_flavor_r660/nova similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_dell/nova rename to etc/kayobe/inventory/group_vars/compute_flavor_r660/nova diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/hugepages b/etc/kayobe/inventory/group_vars/compute_flavor_sr650/hugepages similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_lenovo/hugepages rename to etc/kayobe/inventory/group_vars/compute_flavor_sr650/hugepages diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/nova b/etc/kayobe/inventory/group_vars/compute_flavor_sr650/nova similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_lenovo/nova rename to etc/kayobe/inventory/group_vars/compute_flavor_sr650/nova diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index e7ca0a8a8..d094cb5af 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -27,6 +27,22 @@ c-10 c-12 c-13 +[compute_flavor_general] +c-1 + +[compute_flavor_sr650] +c-2 +c-5 +c-6 +c-7 + +[compute_flavor_r660] +c-8 +c-9 +c-10 +c-12 +c-13 + [compute:children] compute_lenovo compute_dell From a55a8c15d82e767e94ea33f033bef97eaa12ed33 Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Thu, 15 Jan 2026 09:19:18 +0000 Subject: [PATCH 082/103] Add script to remove hugepages --- etc/kayobe/ansible/hugepages.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/hugepages.yml b/etc/kayobe/ansible/hugepages.yml index f0c2013e2..7d5d9b02d 100644 --- a/etc/kayobe/ansible/hugepages.yml +++ b/etc/kayobe/ansible/hugepages.yml @@ -8,7 +8,14 @@ - default_hugepagesz={{ perf_tuning_default_hugepagesz }} - hugepagesz={{ perf_tuning_hugepagesz }} - hugepages={{ perf_tuning_hugepages }} + when: perf_tuning_hugepages is defined + - name: stackhpc.linux.grubcmdline + kernel_cmdline_remove: + - default_hugepagesz + - hugepagesz + - hugepages + when: perf_tuning_hugepages is not defined handlers: - name: reboot debug: - msg: "please reboot" + msg: "Please reboot {{ inventory_hostname }}" From c6c6f1c37a35359ac467edce82ec072f5e1cad08 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Wed, 15 Apr 2026 12:55:59 +0100 Subject: [PATCH 083/103] sbd issue fixed on c-3 & c-4 --- etc/kayobe/inventory/hosts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index d094cb5af..d94e4b3ca 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -12,8 +12,8 @@ srcseed [compute_lenovo] c-1 c-2 -# c-3 sdb needs fixing -# c-4 sdb needs fixing +c-3 +c-4 c-5 c-6 c-7 @@ -32,6 +32,8 @@ c-1 [compute_flavor_sr650] c-2 +c-3 +c-4 c-5 c-6 c-7 From a1b460b03823f47f084596f379ea3f885657cc40 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 16 Apr 2026 09:11:05 +0100 Subject: [PATCH 084/103] 100gbps network issue fixed on c-14 --- etc/kayobe/inventory/hosts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index d94e4b3ca..cbf6f04da 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -17,7 +17,7 @@ c-4 c-5 c-6 c-7 -# c-14 networking broken +c-14 [compute_dell] c-8 @@ -37,6 +37,7 @@ c-4 c-5 c-6 c-7 +c-14 [compute_flavor_r660] c-8 From 5ea08b29c0359fea4ebbce405e6b5828b4676e96 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 16 Apr 2026 12:44:05 +0100 Subject: [PATCH 085/103] Added stack as kayobe_ansible_user in globals.yml, commented out c-3 since missing sdb device --- etc/kayobe/globals.yml | 2 +- etc/kayobe/inventory/hosts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index 71b57206a..b71640251 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -39,7 +39,7 @@ # User with which to access remote hosts. This user will be created if it does # not exist. -#kayobe_ansible_user: +kayobe_ansible_user: "stack" ############################################################################### # OS distribution. diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index cbf6f04da..aa7073a64 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -12,7 +12,7 @@ srcseed [compute_lenovo] c-1 c-2 -c-3 +# c-3 no sdb c-4 c-5 c-6 @@ -23,7 +23,7 @@ c-14 c-8 c-9 c-10 -#c-11 storage broken +# c-11 storage broken c-12 c-13 @@ -32,7 +32,7 @@ c-1 [compute_flavor_sr650] c-2 -c-3 +# c-3 no sdb c-4 c-5 c-6 @@ -43,6 +43,7 @@ c-14 c-8 c-9 c-10 +# c-11 storage broken c-12 c-13 From f149b38dee0b52f63b2e4aa8d9785e76a8e6c4b6 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 16 Apr 2026 13:43:46 +0100 Subject: [PATCH 086/103] Commented out c-14 since wrong 100gbps interface name --- etc/kayobe/inventory/hosts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index aa7073a64..ecf81238b 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -17,7 +17,7 @@ c-4 c-5 c-6 c-7 -c-14 +# c-14 wrong 100gbs NIC [compute_dell] c-8 @@ -37,7 +37,7 @@ c-4 c-5 c-6 c-7 -c-14 +# c-14 wrong 100gbs NIC [compute_flavor_r660] c-8 From d9fdf6513d5bd910f8d5e7398b3b8cd753be8929 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 21 Apr 2026 12:52:54 +0100 Subject: [PATCH 087/103] sdb issue fixed on c-3 --- etc/kayobe/inventory/hosts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index ecf81238b..71fc8a10b 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -12,7 +12,7 @@ srcseed [compute_lenovo] c-1 c-2 -# c-3 no sdb +c-3 c-4 c-5 c-6 @@ -32,7 +32,7 @@ c-1 [compute_flavor_sr650] c-2 -# c-3 no sdb +c-3 c-4 c-5 c-6 From efd6b17dafa5a8b9d7479f9cb18d5064a44535de Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Wed, 22 Apr 2026 08:27:38 +0100 Subject: [PATCH 088/103] nvme issue fixed on c-11 --- etc/kayobe/inventory/hosts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 71fc8a10b..2162dc822 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -23,7 +23,7 @@ c-7 c-8 c-9 c-10 -# c-11 storage broken +c-11 c-12 c-13 @@ -43,7 +43,7 @@ c-7 c-8 c-9 c-10 -# c-11 storage broken +c-11 c-12 c-13 From 1af7f1abf30bf519d7de004abfcae35a6e05a9cc Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 9 Jun 2026 10:13:49 +0100 Subject: [PATCH 089/103] Renewed SSL cert for api-openstack.man.uksrc.org --- etc/kayobe/inventory/Failed | 0 etc/kayobe/kolla/certificates/haproxy.pem | 1102 ++++++++++----------- 2 files changed, 551 insertions(+), 551 deletions(-) create mode 100644 etc/kayobe/inventory/Failed diff --git a/etc/kayobe/inventory/Failed b/etc/kayobe/inventory/Failed new file mode 100644 index 000000000..e69de29bb diff --git a/etc/kayobe/kolla/certificates/haproxy.pem b/etc/kayobe/kolla/certificates/haproxy.pem index a8d7b3877..ec1801864 100644 --- a/etc/kayobe/kolla/certificates/haproxy.pem +++ b/etc/kayobe/kolla/certificates/haproxy.pem @@ -1,552 +1,552 @@ $ANSIBLE_VAULT;1.1;AES256 -61326164646564636461623364633765663830656336353839653037643930313032343536343631 -6563383161316464633163613331396564616636653333610a633434393433346239613036323339 -64323165633263626365326231306536613036666134373039303037663039653734656339346462 -3533613266346531300a353661636266383338633466626338626362353832326165643632366633 -66393263663438623664363031663137373133333464373333633936323230643462626131663131 -63363464333139313464346130663732303431373739363438633361383733356435363736366233 -66656131333737376162616531643635306137373031373966633038643739303032363562383832 -65396233333235663933366432376239336466306261666130316636633136623162653962326133 -66343132303337323234353266353366393137383261363736616234326266323431353935633434 -37653833633036383836316561653165646530363831646532356365373031653737666339306161 -64303335366132363735393634336236306132393465366463323161326462623130343138653936 -37643964393264386138366538363132636561356236396662326231396666393965626166373231 -38633033373964636437386238303632643065633964643862396461316332383266316164363936 -61313866336263623265396432393362336566323864336363333535323639326134663535383134 -31366333656135376165316265643638653763373533616466663733616365613730623136363062 -66613662333639383265303736333432323062306633383737343931336130303831663037353334 -63333934616661343235643138323564623430313438656365343662646532396334613737613832 -32353335616636646138626662336530663462656631666237613233663066386337653630343262 -31393031323439653336646635616132376638396439613430653964363230383330386431386637 -31663338666662383762636233613865323434646363636639623164633337653462623562656239 -62303330316131643533393434313961343431353766393735356535356538623837363161623538 -31346538343766653133633637633466373034326639613333353231316665333036613930666638 -30616438333835356537613933333562313837393130656465373134346165383366326564323637 -32386465336230633733396164376239386532306663646338333563333334663462303739393666 -37356666636232663531333661303763656635653266656462306234333332323035643065376235 -32666662613261346661316532333937386161333765306436393839396237396630666630636533 -36353466616531316665383061363764663466626139303063356335323039643762376437316132 -36316633306163386532376332343737396663636661353865666130343162376238326164323334 -62613536633930643262336434313135396231393431313564393164343362666531313666336635 -33323532363336383530373936663132366230343664386366343836643562366633663135366233 -32396637333630353239326263326663353933313862633665386535323232353862373733366665 -31343034396239313734376438633361386437303961303539643435313064646566656635386365 -33333562363864633733663233633261356434636438303334366135616134613161383839306166 -38363538616338653662666561363464373332613639303833373862626330333937363732386438 -66316663376532386232376637623030333461653330316638616535663164633865333961303861 -35343265643036623435373661373631643436643531383066386466646262663833646433656165 -37633137323365653534383636366464643264303761313635343931663362653737313230636137 -66323737633433323066623964663064613862343863626461326465316531373465623162663532 -64363433386138653437633439333462353264393165356235396531643233336165323262303938 -65393363386430383063653035633137373562353762666165616366336532373031613566656633 -62643166313465653530623361643633383963663963613262643263353033666536393532643835 -37376633306438303866313332656133386564393033333865333766613135616462663033373861 -36353830643263663364343164383364613430393834396336343365303234306533623237383937 -64383839613533633861633036343663356239613539316432396431306462333830616134383036 -32366639323634373833323761353538636165633463343135353435636163356266633933366638 -38393434363063323439326562623334303362346139646562393034613863643163653864663937 -35316563303561613338616234663131353338383066653932383737313435343837376433323466 -62633264393431333635326139653666373532336131326435656232383261613037363861656139 -39303530626131386664363238376134633961306433636464383539313437303638653134646662 -31316162326366376539316134613462353235313763646439353732353462623331386539353130 -63303865303539626237303330386666346230363363303361656137633338646666653235363238 -37633033656162346635663161393563323636316431666132643266616235393966343162663064 -62333365323936323136386336636266393631343234653063303835663733366563613533393338 -63343436653766383865643064623965303566386663386232363135626431393039323233373562 -35386163613433613465353036623236316238343965336632303963393634306138616661613161 -63633139636366396532666339333361383362663062633562643131396337343463303136356533 -37346339353163636639666664386234363239313439623064326133633230623831343033613234 -32313062373034633331643961313031636339393838633436613063373864643131373561353164 -35653434653166383034653038666362666466636139323037636136616338336663623034643236 -30613834353066393831626561396439343463366136393234383632666131353337653464336263 -31666265626534626364333466663765663437306230356265333431363435333131323466333563 -62323633663861663636643531656434306564313739396362303466343636623562353731636537 -63396538653366643730373063353266333661336338383932613937306134626364323131353065 -61383834666630336535373632333236323063363165313661346237333765663136663630613764 -37623335636134393132323732363837633761653064353338386564613765393665633336356362 -33313231386362346131313134323034373933626438303939313462393439643337313266636262 -36666162393363326665383563396662393537343539336337656137636632623062313663613438 -33323033393435366133663533663366303462343334613238313230373032383730646236336364 -30333036633064633034373835313366626531363039346632653865373432386563396631366635 -66303866393263613139663161393766653765323938343863323933663966333530656165396432 -35366432383062306564666639613666393932393837383864626163306232636330653138373263 -34353936333032313164306632376536393532653165336661336234383765656536366462613436 -64346630613635653439663231353566333761353666303231396631613533323461646233376564 -30303937366362303861373763306131373230336464386162336334363230663735636337343637 -30373265306639626330343235376461333266313665363635336563323532353766343763343038 -33323132393534653065663335646138643165386538393334343131643663636339656333633534 -63393632613331353864623832663839636637343264633862353634326336353639313237393835 -34333633613266353037376337396661663031666136663139343832616231393136323966376532 -61633831626236636565643536666164343130646266333530366139663534653866356262626236 -39653033373134326635666339373665363863316533643833346330343364336364356533613431 -32363366623834316266383039636562666135626538383733336136653630643966646134313665 -63323263303964326633633065616462343964306265333831343762376133666163333837326437 -66363162393165323565656434623838376666663062373566663133396338646331343263386131 -63613033613862643361353564303333643162363532623364373435353735346165323963366336 -62373539313864623537626266636437633137396531663166383864323939646533393334393035 -39356130306662646137636463643233633532326563646434366363643130656565666235656262 -31656338386533393930623063386139393566373764653638633036386531636166396163313163 -64316166636135626137326565663863633731393830353731363666313265366131663737373061 -66363463323664343162386537303635653962613039393363333730663833666632613434376137 -37616630316634306365386531626231353164373835323632666663313139636439343837303337 -37636639343061643962396437626134353739303664313932336130623663306533333731393135 -33346263633063356366366635343037666363333665623065306138656537363364666131326330 -35376637373836313036333132316631363332643932373564323035363631353730666639616232 -38313239393233393866353930303262343766616563363961393566363138373136616232633662 -30643932303134323633653563366466346430633862643039383364656161383939616665333030 -32396339623435383562323836323433393835313964663334373562653034663864646134623338 -31663161323438323162393966623432393934646237363638363362663262373135326439636233 -31343566353662623732663334313732373466633939303439353630386231666161356431383832 -61356661303039666433353864626537393238356532396263343964656661353964366530383833 -62613038393262346137306530623933336266656266353138376136666163663563323030396435 -34353865663939393739653262393039353062333162623933623135333733623936353036346633 -64356662343633323039303364333362383338353239376235643139353737616637376466663431 -30373163356662376166623731356462636535373431356234306262323462343330316563636331 -63623537383731623634646637353062666463343538656138633436353434373636663563663965 -63373938653934383966386562393634386535653136313338643136663138663164386531393066 -38306134353439393438656137316339646532646333383339316239666238626438363038316565 -64663263356636633934636533316338303663323037353031376565313934396635356639616537 -31366233623237366262636564613538663635653033613162303036646530666364373730633366 -61643964303732643833633763383537303433333762656439386431396366396638633864376162 -61363435326134303763313864643566656665313362306432313738353935633964333838346264 -63363236376439373162336439336434633962313833353766316666353239343231646164303231 -37663631373838356362333531366533306666626536623531616562613633643863326564623737 -37643862666533306438616635363134313137376361633264333035656339643132396466323466 -62663930653130376138336636313862663765333835656131313535366236303636323835376439 -34666230373837353538333439666138613965316564396433303265313835383431653435396437 -39306233633830396664663165613132653664363061316130373130643732383634383564363163 -38356234343066643135613432393036333465643665333739386134626130396238633732626463 -38633861346532666232326134326435376533346463666331353033306537636634623830663238 -66323464393536386562306234326237663435383533646337396466313432363465623830363765 -36646566636535646534383537393732336664613230386162636531393335623235313436326365 -34346631303539613862326534353661356365383962646131353533376231333135356562653634 -38306635363139333461636562666235356131623533653032643962396263366666616566336130 -38616532616339383665393536366132646238333732356232666163383964343061636463623039 -38336234323436363435376536653065646662353863316432333334326233333465326165613362 -64326162616563643735303038376462656239333461393237666561323461643661623430343536 -35396266353337363263626636373030633233363734396333323365643336666632353336303230 -36633665633031643766366434303965336662656539373461373362303631636163313064383036 -62353231346439663135646663383833663534346133326239623537303561626464383261396131 -66393035313734633234383064613635623166346333393963316563653166393634326161333265 -65623562643133306265643231303464333831323666353765623830653038373462353263303665 -61386331366338336365393235396537343539306161663665613532306466666432336363646334 -65343362656630633963396364353039396339613266373831333162363461623533376331646434 -34323631383662336330396264626338636338343038643835383333366534353433393338386464 -36343866626431636263626533356133613261643265626538386339303136333639326565303333 -35326235613937383362663233343466626635306465353536623233626537313962623836643562 -32353031663164663865623232396663616333613462366233663766373933623834653164396637 -65656638656334396639636333663333323964316166333437373337383635663130393737346166 -65633133643631333966623161663064313830316535646235633939393937653265366231376431 -66323630343462663239626465623836336532663963323730613631303330663730623637373062 -35366338313437343339366537656461363037346161386365353966663631343037666162363436 -64386637376566666538333663613132326664343936353638373930346630353934386663636536 -62356363643833353630663366633439313962623334663362336663376463333361333134613830 -38303662393430346533313030373030623432393662313134656339316661343234356433386634 -64396462303339363937343330656466663738633139636365656639623531316264346135626566 -33313336633535313438613937386639373666373661636537643339323033616532336231323338 -36616363626234633563653632643531343833306332366430613564623234333162306633326438 -31386437366539323137393966306232316439366166643863363964303964646431616462643964 -31376462356235313033346665303964363339316339303839663036393166646635613630343533 -64366530346238326264643331356432376335373962323163323039383535663130396563306330 -63633066363937383336626663616166633663353231356463353637646163313430333862616238 -36366163303238346464383966336362393838303837396135626535653236396339376265313030 -32346331613535356365336165376465313664616264333732633166626631316661303664653564 -65623133623537626637633237643434396361626666643431316361623331653562393436373931 -63356536636538623132316366643863323761633964666634663437343635363462623164366564 -34316638663465623937653065356262373536346334373538643335383836656134383363303961 -38346266373161626634646432343733656632656561343539306530653635336333306233396266 -65383436383761643130613837333732393438383063643564393337666236626666356238643931 -61366531336334393462383531316464616331383466663064303532663262306464353464356366 -35396261613932393134313063373532646237663965303866386262333031303236633561643235 -37316464313566333136386433323839333333653565376364373637623236653338346133353332 -64323732343230363837393437313036303563373339366236623033653333373862636362366162 -31393036343764346537363762343330626265303963656637633738656665303864383561646336 -37353264343631653961303834356164333462333264376330393038646365383565396266613135 -36313061633432393961343964343732356164343236373831393563663764393037353164303131 -39383565323137353265636564343961633566616436343933366465333964373562643163326132 -34333061396536303638636531373864366237643362386466353330663033663738363365373465 -30353162633533666136393635626363623238656535633565343537383530376333303262346263 -33313535353430336139633735326135346466636339386639613066663030323537313230613966 -37396462626630383837326137393135616136306363336366343036373762303565373334396133 -34313935326665313565373837613933636465643464343561316666373862646138333034383831 -36343163303239303733656637363833383265373539623030353165306363626664373138633634 -38353565376665346364386131323932366433323063636338653531366131393134393238386364 -62333533396364383631383930363466363038323337636263393334393634363833323066613037 -36376535353538333034303463343835623236343639656662323764666434336665363338666438 -63646336323133396637383534336433323639383362656664343131663830366438313766353766 -35393562643332323530306334306262636365653363303936643338366636393234303131313263 -61396335303639353265346561353531303235626263616432353862303937366662656634396337 -36343338366162306535386533396439393232396166643039336361303166633264323934313431 -61623330336330373166396334363462343631383163393161326338653337353065343261343937 -33613438366330373936656130383066333839373835386162316431303334666632663533323731 -65636566366230653039666131663065373632633339653537623463366238653833393762663235 -30336132366562336365613633636535346331333364333863306137623565366261353937646464 -64383666333038363535313733363632316232316366646662613934303938653534333535653662 -65323161613835643535306464633436653666616631373364623864336630656237656461323631 -66333032313935626433353138623936353363373663333165333661623638333765323430373934 -32666136623464363262616433303236353166623137343864396537636135383933633834616161 -63333663633630633939333266643566393133376166373730306230396364396430616630656663 -66376234653761356635316236623663623634636364633735326436353866303062306134643163 -63376335316638653039393831363237376439346538363161356238373534343532366537623366 -35653430346561393235613330623463336433653264643739663533303132323733663633383062 -34336536613139376638396138316338663237656466363231376531363634633838653638653437 -65326539613261383030333538333563633138633033363165636336396565653261633135636336 -32633733653330663835313935333261353161313134346133303964353139306163363363383664 -61623764383261646132386639356638656462386339656363336535333864353732323939363738 -66313333626135383734633561626230613334656237666537306265343839653733623838633137 -31373362306636663263323061336636353937636532316266303738393138613732313930383363 -37363036633464656137623862613566633436386639393464653565333263386663326433336338 -37623030653231383931613833313538336435383532346238643030633734353037613964393637 -64656262663935383839346431306564376630653438383630386262633061626261386433313961 -35353532336666663639323730386466616366343438663466633233386433623435356136353634 -38326132643365306235663631373566393263343038373935333230663230303761313662343133 -63663564623361613634353738306263363339376638366231313337613766643964383861663566 -64663231323336376136666134393939303637663863356561613235316337363337363134643162 -33356232393931356335616566636561346266386138633930656336323961353366313039643364 -62386331373037396230333030623431333863326336616235643365623237326136363066343930 -61376636333036633762353933616432656535383934633833643064343365656163306436643433 -38613631613138636330363562386435663462653039656430613662656465326166336331316638 -36396138393032666663393530303534333662343961333461313461303539646530343064383331 -34613435373561326161333235386535366564336333383061616537633263383439616531303932 -30306137616135396562306236316632333634396638633338653333316334343163643435626231 -39373364333235386538356365623861636161363963643730373966386135363166333637313562 -62373332333035643434313162313538626639656461333563646565626439386464353537633064 -61343939363033343435656430663836313166633463356132356436303639353932656562643264 -32383337666438666638333433633038396430623936613162656262353165376435613035613131 -62666635346634343239343735313031336534306161646637613665626362313331343265393763 -64396161663563376434306237653634646363323737636439636533353130303734633932313763 -65643638366362313364383265393265326434636463343732656638316261306161323661303363 -63663230643838633666396164343163656633363236313562356663626438663162616565633063 -63653364343030643832323866373337333432353436643736306331316461326136633733616163 -31383665383630313763396462326335396665616338313462333763316330626533623964343438 -35393330353262313662333634623961393166303935643531623732346538613833353761353262 -31646630386563633164386132343634346534306133663962623765323133353235336163646633 -34393338356537636230646563366536623035333339303433656262366131363165646232383038 -64633533623532616131393833336137303338633436386633626338656235663937356238666234 -35386166626135346331363664303532323630666135393737303238633966626465356333353630 -37306335393132323461393931396237386462316430343261643037623666613033653563663438 -30643032663837333437633639363836633035636134333437626533313633663565663663353433 -62376635323534396136323038363633613565656165346565306531656332343464333066393737 -37353037356332363530313238653264643734326562643234326132333230393938636666666161 -33393965626330616263633361383234323862376263653366643132396436343236306134336264 -36623832386437363362636634653438633866303634636136343462373262663434653165646632 -39353438643431623832636332336661386233613263393035643539313632306339346631643336 -34336139613362613764373265653963353461383734326533643864326333373236386266643033 -33613261666663393839363836646339373065386539376535366239666463653035656439376164 -63666434366464636231333535363061313335303737316362643633626137653837366630346535 -31653765313861306366373930333931623765663561326432343835353339373734383238303863 -39353363376136326632643239343339353732373664663539363063643630613238353934376232 -64303836306166323433313732313534616664653832343861323035663264646163663166303934 -66663535306361346264616464633761656338653765663766313863666166613666373862383239 -35326636326566313536316161653466613066633334653364663033616466626132336365616138 -65333764373363616134303131353934323064613737346630396536623534666438633361666531 -63333938363462396630383062383861613464373631643763646536663835363031643463326337 -63666439316531303636353435613439366331353466306632316536643633656238626331343963 -38663131393366383564333037663365383338306535643334353531653861386639373839383637 -64306532336638383331396234663462333138363933633064353932613961343862343031636266 -33623331316639393337316432323863316338316632336262616261613936356337346431333538 -65303961393633306530383632313331393063333739653966623839623963636638666136313266 -32386436313636343233633631373364616433343364366133656539316462336661343333623631 -64386435373261663463366638623566383164623335363564633334383230363239376430333461 -39366133373230346133303539363834633665383338356136646366386630383939613562646339 -34316464663537323435386137396235363261613162323633396265663962326563316230326662 -35393461363738393663393932323162363765633839353734656665303832616264623932353664 -37346133656437396630636438666164303835393134653334393938396335343335666465326635 -31643666633032653634626262346533653935383136356366343230333962363464356565386362 -31313366613032363166643933326266346562623061643236653337383363303237303466616665 -37333364306237306630323432643531663431393766633865663562316437626637316539373361 -62333261316335363364653732616338336133323632336338613130336634663339633738333866 -63633165346138353763303639326537306433613233366533346438343134393962643038313136 -64383163306661383761353433356261326535363134633665383163643337386365313131636535 -64636139356638663838396138313132373264323864613533323738373437383632666265633335 -32613130623164616564333664303138623035663265306330666661386532343536326536323935 -65663538643361616463313066333137373161353433373030633138656563346537333566656134 -35393237396563623463303035663639306135333265326164393939373566393433633138383735 -64376139613337653662343133396266613038323863313139306461343839333937623963383064 -61373965393061616439643361323262643135646366383936663966633065326533323830613332 -37653832656165373866623833613534313466353636663565646465373566323666313539376338 -61633733383761646630343238393836653863343262383831386633626463343666316139616463 -61383465376364643630366433383865363565616331613330646232366363393035343637663839 -64306562336461663837303332313365643931313535646263623264613531366462326165616162 -39656564333837386337313264656361623561386331616665656437373930393264626433633632 -61393735373061346231363137303261316435643937613538653630313738663932363639623966 -64396466366466633130626135363765396233323966366134366564303134656161353337353363 -39353839313731363065653061393761653565643364366136376664613438346432633864393864 -32363331333636383262353335303337646664326534346135663164323434666337643964363539 -61646434643562383636616261613361616634616539383134316466313666306563666662663364 -38376633343634623163653937386434393361656136356432373138353136336162343964356661 -39386134303363343731383661306531613364643139303432313266393066653835626536333563 -31306532363136353666396465356138636434616664623131333632333335353563323831333036 -64313235653337316539646337633237306439633332646566313835653330356462353731376232 -36363863343131643762386363356535336434306335633639316232636134323964346363366330 -65653761393634636236383935313239633930653037613234353865366263633239613137633031 -61353134666537356163356436346337343735303133643739616231356236616230613961356636 -32353934313962333330333636663737663833653665396534656630353632396637336337303165 -61626235633562373231373930336137363039346366616265346239633836303037336130366263 -30316431323663623066623063333239303864313537306363336539346339653838383663373330 -33303464383865313061623265626430633831396139643334353263333633653237373130663439 -36393161643237626235366465316132346534626334326338303666366233373566613238363335 -36613661356234333232306136643865363833633065396439393762653165343630303564616666 -62346162333930666262333861303235323662346232393530663631303037363230303133353330 -31326565386638623733343735393662316632316436663336323864393334346233643566646464 -62313738343335373236373134366431386234336363306139363535363133393564643562363661 -32343865616466346631656134656537383534333634303964373966313262356335313938306237 -63633038663538366164616637303535653030613838646430336562336637383935613636623739 -34383435333534633034376663383261663861616162626565613666323037376231633033333961 -37316266633464626335353939316365663265383634393738343737356439663935636165386462 -64643233663031353065353338343837616130333236363931323030626338643738313838313437 -37353963663635626233613939386437363032303134646461646161376139393730323038303137 -66346436636531363766316236353565353661623665613733363333326330333437633633363034 -30313739383232613331666666333636303036363935373435313132613164393830333536356136 -65343536643661666464653231366431636563303133653332636666306437316665646437383462 -64386163636439366234303364666133636234376634343364363534396236666432313462393963 -64626638663739383665396536666262396631343062616237383739323338623136346432366665 -38303564626630373961386636353263326366383334616466343333346337396562336163373462 -31336130316464313064353161393861396639306466306130323935633333626236613437643364 -30643632376165343136323434643363383830663765363664333337386639323661336237653265 -38653837373535356461633463363237643137313834303931316230353531663239386639663565 -63656336623435663532663037643863616564616161383638656537383163623962363665353065 -31313333336562333437626439653937346530616234663963626466656434663537323964363765 -30393864393462306239316465656536643436333634363263643639333933396636346234623731 -39383432396538376563393338363132643466366431396664396238333833643837316333356266 -31316434306434623837343031336332323061383937666639313036633139633632363166623264 -31386639396361343938666238353261643635343162653932626131643763623235336532666634 -66373166336461616439373838323461626466326438343966613464373064626136623136613562 -62303338346265363232636434366663626635323830666535376561313638646330393332363235 -35393839393765376438306261363930376430666463623136653731333266306139663364613061 -34346231643365643832373731343236306138643832346337343362396438613766343563323933 -39623932663938336565386138383866643264643137323136313762663464303739333439656262 -37623738393537383333306235396536656534353937653234333238613335396337376236313461 -64356564326132393666393434323339653532383564373734306661653165626234663430663230 -33616262666266613333303665613863346237336630323264353230373339666339323739373765 -65613235363337616263353166633861313435613133666435616664643632383136363538666336 -33343137633933353666386432326336323663663562383935326237303737333830336538363831 -63383138323032643535386562633864653362393435356432356330623530663466323264663732 -62346338633734353036373239643566616662393063366637663431373662616330623839333438 -62646464336462626438396135656535643262366238336632613132616631373530376336336236 -66306564333638346333633261323035623864343338633365663663313864626562363938383530 -31346361393463396166306437666630383231303435376633633230366562663431333166643930 -32356133663839666661363966656137386530306231616438653338376633386166313432363435 -33396363393338373337396232373762303563636631663161663038383830363365616135373134 -66303165306363633862636534336534353565303566326163626635356439656464616631393766 -34373733643234613035376263313932316262346538316361323533303834383533356337313438 -32646136616162396334323131343863333538613238316562323837313536383064383062376164 -66303566396563333634373462666334363464313138613665306635363138326263653935373735 -36666430336639633438343537656464323835383931643630336331373766393762303235613535 -65306239663366336137643437656261303766363066346234343235376537396537333866656362 -66376538313439613530356465333663303331623933353635346162313833373336326433336464 -38336664666362303237376130323731346436313533306332373330306234666130336462303735 -36306237643864343965393235333462356661663833333230636264643235313234313136633134 -66633665313361343465613932666364656362363962323532366434333133356130633664306638 -37633737396262663032623430373666353366316364393739336439353236646366363465613535 -61303863336362616134376365313238616237303866646337386462326461306238633735643431 -36656364303533616433313462623834613239643334336632346430323261373232633033303363 -61623835343661383230306261653164306233663133343634663534616630356666383937663666 -31303262613434346630626661633165333639323633643137643062373139303066333636663034 -33653164396162333066613335303562663031316430393235363963363934336232306262383332 -32363532363331616332356335366163383735613236643834383961313835373539653131306338 -61363931613037306638386639636663303634346134643035326564336535653434363632373163 -63393836363466383364366264663063363037343630353937336466666136396163323235656138 -37346531323061623965356563663361373938336262653134366332613264656366323736313565 -38613231623731316532626535653261323531343737306338366338653431386464633834663632 -33613337653335313262626130626632353434636131643931303334636439646163333930376233 -63643331613563383934643436343833383439316431313535643839643937356133383630653430 -30376136323834323164343837656132656165383263346562393033643237646138323364353830 -32383462383033666531633632633361333837366632326263373031653465646138663732393365 -65363434396236646133303065316436303431366461356435316565326335643437333861663838 -32346333363264663339316535386134383432373061633436323266646637373366623730656331 -37636464643037656336306234323763643633333930633130613564386636353535653733376335 -66613330636533373433393230633631656634373634376161616333633065653963316432616438 -32396264626536333161353630343730623861393066363239396465386636633436376661623633 -66653238613931346433326338653638613161386635663236303162626333373933383731656132 -61366437616437613761656230656235326362663336636132366436313935653634346434313066 -37376364353563306661636564303732393761633034643139383537343063653939663534633132 -65646337396336653863636537313764373565386662653561646136646431623665643065643835 -61643363616161393638616561336131626533333233366662383434656566396463376437643965 -30303561353231326165333938376263363263356431363065323530373135386563333964306666 -33303264386236303432303262353232626535326665666530616565636166356461363066333765 -38313163623361656436653536396665613039643832393130383738326235636363653633316666 -31356339393861383337656266316661343063626232643936373661373838623735643236303832 -65386133613835353165343037666232303464386632343237633833363434373263386433383064 -39623763343236646664356233323566316266653134343337613961396564653638613438376665 -62346565383934616230333463363537323838633564373131333965393266373961373037303961 -63646130356135376336383966333433396535316664306638656635653462303138656561336666 -32323666393231663231663462383661393835383234383361623034653263393432336134633634 -62333561393662303733333165643662383236323162303634626365306330346631643564653239 -30333139393063376363313264326566643962353132616132376534653765663430386236326364 -63306462383039633230613833623432343265623135313439643437323935303063613031353266 -39653233356432333830303433373961343836333630643536363335626465636131316239366165 -36623335656234353062643930633161623663383763323564356166316439616162366463643236 -34636139613233666464366338616465306337306138643233336232613531613162303437636161 -61643438643331303234303661343361316636623237633165666338303333633235323832373834 -63633761623431633164376633636464346137653238396336613233343832303333653636353236 -32666138313639306263323030363037313164656465353239316635306565303235363862373064 -39613639386636303733656233393339353234333432663961366665303735356362353266316261 -32393333383230623230366166663632343365353835353233396636343238363837613962633064 -39306661653438393436383762323765343266336232373661663731316436653434376332393133 -30613930323331396139393139383961363763636136626634663663326164303637356565366563 -31343566333538616663353530623635373262613030383365663630386531656335333562663931 -64326365306463326166366439393665363862366564383037346437326336306435613739383038 -66306531343437613034663738333734363661333334646237636465356661313966353563363937 -63633338336365373962306531326434663435643831323731656363346339633134336331393264 -37646238323132613139313664383062623930643664353937643438396661613335303236336233 -61353762303662356365306232663662393761663861373230396165613336356532633133346432 -63613861616438313436313764343835373032663364653039343662323331343539323764643536 -33643466623837616134353065623832346564653261323566663363653335333634366436383538 -35323263333833343437383565636232343161613533353130353463343162373365316235363538 -30336636383866316666306535346233376663383839656265656531396235313434666439313030 -36313934343330656633313337346230363335313539333762343461316161303636336133616538 -66633636393462613864336134626665656561393636373466643934616530316465373236626334 -30343032386337393233333032383735363965646432366662666664333535613135343764656333 -62346139653533303165396133353130323132613866373162303463366565323262636532396638 -32303835393632333130366463643834313161623131333364393637383636646465303038323337 -34623032663336666536343266343664373961313639666534643236623662626333393434323537 -34663865666365623764353062336532666261343038636362373431373335353635653934383034 -33313164623437623137326238623139313634653637636161663933643032393965613438373066 -31326362306564653963633537633938626264323436646366356365623632393264613162316161 -63363531316439333066643333653465353262653534663638326436383366313965363339373334 -32626339396166633864633166346336333932623032393535343235313337636532353364643763 -35623431353434633536383064363436313330666230383965633633373236336536363565663961 -30663235333538623233616164346233336134393032386235643935363233653837393163613866 -33393464303566336263616536356233616133653335663734353162383065356630396462363362 -33633162663130343766386461363164663061363936623031656134643635393632333133396364 -66666236343938623866633230393233366535363361356539666262383963636133646536363866 -31303266326430316136393066623333356130313039613633633034643365663361303662626365 -31313831646366633835666238633932303637366562316232303634343234633664373766356262 -32396463326634386464386636306336343337633665326338366466303133653635323036646564 -35353465666232633665383530653263346637613335366362663865646564373163323339376230 -39343763663432373763336131613338633561343663313238343063303864313963383435353264 -30633632666537363838653035363862336466373935366331653566613439383062663165363262 -39316437613464353763626132396166373034336532366333393439626233613238353531633266 -33663834323432343532356334613563333534356266626662643638356330356435323237636237 -33633066333965393865353466393339393366643135313661646330353961346164643064656532 -62646435313165343132393966343566663765643039616633666265393537346361333365303566 -34323263396339636266356666623561613231363731326364383935653736386562623964643065 -35613962333265313566306531306162393034386330373334363363623132303938346531336666 -65613466633065646366316664313862613963646561643732363663653066316265356533633734 -33653264633065653166313338653065396561363534376663653631393233326430616366383765 -36656633383764383835386263623839656637376139653763386330633233343062303133663136 -36316261666635313137623939336165663730633065656437316262626136646334343430363965 -62333434633662326435306662373934613365643738356233373339643831336132366433633035 -33646566373833393033316335393265663934353066623139343061643836336133636537373739 -62663262613934613439333238313035316331373232636639616261373737353664636639346533 -31666366643339663930373864623637623632316637303634333030663134396466393638313464 -61303835653539326465643436303866623766643834376365346136653032323935313131333166 -34613362363731363734636135343732343062643431303837323862373865343031393432346439 -36353362373630373934336435316431386532313461356465376664313232346262656435306332 -30653933363938373635633533656261626530336264353365313031643539396633343062303965 -39346633306230363435353237613430363930343662656330373039613130643535633033303630 -65663965396264633230386664613862653035383863323431633530386537373437373135356461 -34303265393133646235646635336166363965353132366135356330316164623362393566626433 -63646232373061623936373338613435373337653635323364383931643030613463613234616237 -34366339373161656533363231313530316133373066626135313438663763366364303534346130 -36346563356137396232643866636166616465613536376261326433633965666534663761636331 -32353639303736363734636462633935356236653833343835626664373235626538303437663861 -35353534386566383164343431326132386365616465626565646638343164613265376133663931 -33613332626436306638323830613666643536653131626535613065346433663661643730386435 -36386134623639346361333037393038326639646536333938623839333838633134653334646330 -39333836643965643637626662393338303336313661623734343665656237653939383036353238 -61376630396563313561643163393365633564633266383933393335646563656636323538313235 -65383330363533343964383936373562383863666334343136373536333539636264356234386338 -65663833346335386237373533373032663034316537653863666365313837306666363633383166 -38383730373665346534383538346662326534366537346236666432636233623935656461626634 -37333466383336303962373636346538623034343861623461663263626432633732353661383033 -33366336383365303631303132356434333739383466643165353133386266386331363265356637 -34346534306634326231393133623630356432653436396631633734326430393933313830636332 -63656163396536373861343332623634653732346162323036303661643930386533393738333939 -34336130356463363865396331643934643866663539656464343261663461333738393363636139 -32303339653466303365643637636562303335363130646366613732343162643334306531623162 -39663437313333356338386239396365383861393038313862313761326164316537623761643536 -61633433646437653462633039633730386639646130383330373930386464353761343639363934 -34313166363963633866343165373039326132396532373135373838633636393732323732386230 -39333662633330363366306634643731656432363535323333656163393639623039316135643464 -34663263366332633934306531643134313730343735393839326234363438333038646630366635 -63636335353435396530643436353239316537383135396266303562366364623662633831333736 -37363536343562396630663137393130343934643834623439393739376435383334326263613061 -64306436396337373062343361646466323138316138643434363736336339616632333364313865 -62653037363039376665653961366336646665323536336537343830393636333437623166356464 -62386165346663323236613662376430633832343733376430633037323532623232656339656162 -31303237626233363664613434653239326466333630383437366661353465636333626466626632 -65643933376135396635326161653035366637303230626463386234643031393631653633613133 -61653565346462323830333363636239313337383335386638646637616137333734663839386363 -33356631346364396661313134393833373863613931363532303239646266333332616664356632 -63333963623961643362643231653636613930306365653662663938303262643361663634363439 -31313431656334303663386565623462613361356537346362633334373864653961636130363430 -37346665636333656263626663396634646433303663326336623132623265333835666238626262 -37363361633839306133343333303266363832656565303665636663383830313436383634366433 -38303639343637613033623432333838653362633733316438316135363662663931383832323036 -38303563643130656631646261316130643039633165353738393565393938666464333732326632 -64336530313234323338343037366263303830643936393361303831343734383035663561373862 -38363330343762333164333062616634613866633534386264636462303335653834336361626333 -31623539653463663264306436636662316635346331326162623761376331666632303465623632 -64333939653562616234343263643430393234313264333835353062663663336263396434643162 -38313961333636376362333639633861373230313539306463623733303936633039663861376665 -65336666323833336532393335663536306138363835656236373563343266316638656236363731 -66376262346132366430336637306239633061343430376261643565363832313836636162393832 -62356530313464633361396166373237366633303834633930656166623164623332616266333231 -30333064623036623937323030356136613764343138363437346232313863343432386665346330 -63663339653266643161613632383662383133343236666539623338343435336463656138356331 -62643461376262343435666465363265373164623732313435353534643731666663626431393165 -61396231313638666338636135613235373633653536366631376537383231373938386438643634 -34653334666235633030376165366463383861306637376534373561343530346332393562636234 -39323038373361333030333037326332353538623536336234383937323034303432623134343438 -62633161343639373635663331373031666236623839623564663662333630393732393535653637 -65666536373061306162363737313734626262633466333936373432313335343131396134613437 -38396237383666353566623738326362346635613363396535633463393463376336383966333764 -36616466666538666233336531366539306564393831306238333234303533313566663131363834 -34333833373431343133613232383061313533653762656661336132366361616662613131343038 -38393735396435623762316163613432333039656362353665303538373966356233333664396561 -66313166313462343135376163613839653337393466616563303335623830626637316431383336 -65313364613332393237626433643433616534646131633565653136613238346661346136626166 -36373363353133366237636231323937306135353034616333393662356534396335333533316263 -37656136353066383334613430626462363330616661316537353562636236633263356566613539 -30363432393339363538616338316139393334366666643763306131666131653566643664656430 -63666664326134636566313862353961646637376665646561303761393835616638636433363939 -39656237646164663230303038393831623362393435643833656438626264653335336130313562 -63666236666231396639613935326166373830303237333037323133633561663330306666663562 -64383036613665313165633239343064653739393563336331393935313033666532343235623439 -32393131313935303665633036313135383036316334313763623263313734636131333533316262 -66326332633031336364323561383461323733633635653162613731323634383738336139326630 -64626536613830393963376461323035376338313135373131373135313037613366633136383430 -33633666656466613737646333653337636438383830666637306437643837373966373231336162 -36393564353839626266653166316134356533363163646430383832643434313135316264383966 -34373234666339333235303661313938353963336363373863343962616265323735333266366665 -39363834306537313962636161376161613434303039376664656631613333623261636530396661 -35333036616166313761373539646164306566656362386463626666373464653332383035643862 -31366430363161653838313261343464326162653437373636633438653838306333653962383461 -65383932653134616163633636633438366364646231336532623235363635613937363862653866 -66343964663163363438616663623865653533636462663634396534343433623134366434353638 -38323536623165316134383134343032663861656566646237306439656239326231313536636661 -30373566333232346631663531323161666335613165383165326335386366613361336165616661 -34373036633531646335353433623136613139646230663635653037326261663661393162323336 -66336562303062353663613034616632336663613631396465643065373361646464316233643666 -35393238323536383933346232643934386639653936313230343466366532626535643936653933 -65386630346438663466636462343066313734363862306566616638626362333439623936663339 -35386332326163323630343061343335393835393264353865303736376534333939323030653531 -30633830656130313633626563363039643132386662653061343565326361346464333939633635 -62613561326532383762653864326633663462353763616633623239373930626164656362333966 -38343731656536323839383535616230353435353331366535633866643634303066393062336438 -35336365613962306635373431663939393263353461336330393232633337396434396364653630 -35313637313638366633313431346635306365326166383037633536373333373663316461656435 -37303639613132376236646635376634393332653237366139316263323563643134386338356264 -64653934366465316166643762333963313033383761653861393539653335613635613737663761 -38343561633636373432323838656435323333333639383235343536333939326463306131393661 -39356432626337313133303537623436356334353661643737393131353064636534306336346135 -66633963633232376235313131383262326238336533353666633531646661633439373464653635 -32643337353136393931346462353734393537643236383236623130656232313736653935666331 -39333332323630346161316431663135613037376335396434613139383166386263306232303434 -66333030303830653036663064383862336231323535336235303734323966333463363763383665 -66353363636561353465383236343136623339653834663262376564353264316364343334623961 -31613732613466336462636261633937616434656262343362336532333632336637633166363932 -38373166386334623263363633303833353939363535323465633232363965646535656436613333 -65363738613262336135646639666238306439613365343632626164393132303335383739643566 -32376363326332343934303038663030323832663663313166636264643731323934326536316339 -65373063366162613633363837643536323331336333303734383361323561353765643538393434 -37336539653262383636353434313664313238663465326565356666383330663963306639383832 -35333662346633356431613933663434353462313665366662663761383431326336363539306337 -39316335383433393736616634303234373965656234373539376534346539346235306338353138 -63346664366163653864613162656164623364646562393261623663663233353264613439333035 -34373537663930646264663666323637633632613861363366653036616337333832633536393034 -36656137646661333537323130656363396661393539363164353836613266663764623761336439 -63613862633830643238326132383339353733616165336161613466373836326232323562353934 -346364393137303864666537373361363137 +32323664353662306262396536323861373033393331333735633663383836623634323330356361 +6163306262373465653761393266653734333835353030650a626538616564623039633564353033 +65343834313539633864313162356334356439633037373439646462316632313039393136343232 +6331633666393238380a666361616637356234386537396635323337336233643731343365316335 +61643061623733646161323866353035643837313537353131343563363061656334306561313739 +38376662656330656333666238306633646262646533393965633333313431653037396434323932 +39653637353131396562653534343062303834303366323263643261313431346166393962663634 +62303064613335306438333765633932303665613737383536616135646638633161316239326465 +33616137313763656639343834333663633866393830653431653931643536376634313534343731 +65366266626263616261633461616265353436633932393561653635326235393662326431303663 +35336533666537636461633937623839613066306131383734666162336465323034306166613663 +38336334323935363932313038383934663431626364363064663336653762623832663832333236 +64653039623862396431396337356332643561393436616161383036373534396661643533303731 +65626130636464396363323835306563363639666639383133316534383564346338376262343765 +64666538663563376537376265383663393732323536366437653762396233616163313636393165 +38353065303462613766306436396539633566623134366433323132396661303466343438373365 +61336461656461306337636664643466663232623238653236326132623366376262343934313030 +66316566623539306335643962666666646531396565626665663436383233633734346430313363 +33393531636662663738643864386130613861306464613964613963353639646638356535303063 +30366139303165666265356263383336646461386434366262626233643439396330383938626431 +33653931613466646432383032333438643561313639323363613036333564346164326338373932 +62366266363531303166346265366632343335343133653031616633353133396336343766363662 +64333837626634643263303334363433333266346139643336633239366263613763373633616631 +30326464636635326231396439323966666431636263373737666233613165396634373363633265 +61303763383336393837623937383931343030343536653032313637346261323764666564613965 +64356236376639383933663936336639666431336136643434643666333632356230613061313263 +31643531636264623662653962363364383735363131333937313363303030303138366537383333 +32346537303737313136613635653738376565346563336336626565623431353430316430303538 +61343131653036396136356139663239373930386338626537383332626561373135396562663639 +31623834653131653833376265393730303062666161303361366530656338393130656363366666 +32333166313261373839323732616466363036356263616164613164633330663336653761363061 +66373566623463306261306233346132313861373563646430383930656331383333623530383266 +66656561333763313737313438613866663934373538313030303735306566356439643835623538 +33663535646537383333353265643837303630373234313661343333666435306630353861323730 +65623564363738303065343366393036383561353831323738373635613233373334346138343566 +39343861323463643037316563336265373834343138306138373133396663653231333232396536 +36326665343062323331386337323331303065353365633934643930663835636433616163336430 +32323864356333613863616636643139346461333838623637333139663263636364656235663365 +63633636313663326363653966383634633865373736653661303535303133623730626231333437 +62636237613861353032383432393538646366643366323264636431386236396435616633626361 +31303036613264656566323435386165323263383730303263313532343764333732343936613063 +62353432303030633937373835393230333735316466653064376330383233633664306461326530 +61323833663232663663643436623535313162386365653732623662363332616533653563633461 +33356339303331366263323064326262626438633165376230646534613133323166663136653736 +39353631363738323761653566356664663161626464373462303530353965313237343332623465 +65303333313930393037363065613134386363613530396233646562613138663565663137363931 +31653361303135386332333662356165313131313539373435393064646132366638323838303862 +33636361356566626362323661636231346238396361643363343963373735366363306264383934 +65313939633766383738636366613061383433396465366136643565626639613962376532323363 +66306432326439333666363237636631386462316463383861316534313933656164653464313637 +35643231626364336334373564316433313633323134336361336132356663646238613532623961 +64303065623439313033383662343737346461373236343363346461353263316338343337366132 +63633136313365666336626261623265656661303962346437623466313764333866343732613734 +37313230343430636537653961653662363731306530373134326566396538336139316461613234 +64666430336463353063356533353736346437393163666338653930646164326631373834373735 +34306362373361333338643133626430613838386131303533343165663466623963366433623764 +37306163316637623536386437643534353231313830643533333133363438346131626636323666 +63613363353339383237333332336632653635386561653638306337636366353863663966636631 +63346536326665373033663763326331633839313931653161306461323163623631383531306462 +65333161616264613061373564623836336237373537303437363238386332383438656336363836 +62396431343462633663393531386230623132373461656434386538376533363838326537653239 +66343733353534396462373638633332393931303132383463323661366266356632663130313766 +37326432316334626435383033613733303131663434666534623736306134353532343264666163 +39623564653263343566636338313730373863633265626636343038636434383734383131623232 +30356139613464643936303037396166313337626337343230393038616432626632303634303238 +62653962376432663533346265383634373530333066633363626264663034633336323162653333 +65373363336163656335396263343138313166313334396439653464303335383266336537373639 +36396536623936346537626461316361366536323465303931376431366662616364343533343430 +66643164613130363161346533316337316530376665383636653233323136623365616432653437 +61356134643962366534373931376535343931376237373531626430386361353937306332396335 +63383038666231393432343130393531326131326463613662303738346430313230363337623636 +64613037343566326662623932633562326435323932623939613332393133623661636135326166 +61623933393532393163623730323839343361646532326338343735323934303264636339393339 +62343765336365623638353439653263313065616137616133623161323831613431313963376339 +65343535356366623434396138643266623537396537656131346638633933663264386332313331 +33396335333836656533653431336366386462653465623130663036666639323632396661323361 +32633163623738626461323462366434316530393039393230666630613764646535346639663433 +39333931383236343035646463383565653466313233646536343663623138303537346530663738 +35323663383865643261373938353530333436393834393561386239353837303663636165643461 +39353935343561363131346135353638393238616431373530626662333437346230313036373332 +30623864653538623833373338643963396161376362623733336663383461393939663262363066 +39616264306234643738616562396461353333663832313161363761306262653465393136346132 +61346634343833363562656130343030363533366166333530623031383964666537363732313762 +66336630636432383134313863613934333764336437623862303633653662313237306337623033 +32326331343035633136613233303533386164663236643733303163323965303434306662383163 +32626662353262663464343164356461366566613833636338393762643633326138616263656562 +61613934643166616434383461396438343765313062393433613936363335383562613861353864 +35636338636135366536663266616364306130383134373639376563356662326361643937303330 +63373662313664613837393435303031313361373666323039396530613236383130623138646330 +39333735393665383261613039366138623736333632353335343466633032333064383937313361 +30313531373032646463643532663539323263633363656461386539323065396265653232386233 +39383433396461613534303134633134333431326461343562626234643833343930656530353539 +66613136336639626539373932373464393935323338323162363432336132323162636462386238 +33656266653336613163663334373836363839623663616638313337353739313939663036663130 +62366563643562316434303766656637326564356234356663343265386163356335303066653664 +35343165333130653865376233396138333532336463303463363839313263373965633162353064 +30316439643031646532616462313461366238343832303832323939313234316136386539306666 +64323037633063343266323263343234356462376563396139303063323736326665656332326566 +64383036656137633365343335636132353362323934393039363338363634313339346232636436 +33313163393934653766343266626562383434663932373963366636643365383935333731303533 +63656130663431376661303462313264336465623963346235393764656263333435636132383461 +30636366346331353932623365373134343736313662393336363837353833313163316330616330 +37643662656333326461303661656161363363376635383835353737643063613933613032626630 +31303465333563646131353038343935666262653563313132383662343765316562643538396330 +33363138323936613964666439316236643964343666366561656563653234646531636336346466 +30363935333964636231633866353664356563353037646132616339343938373063323366353565 +63373863616637306635393931616232346263623535633036643764306231326664643934353263 +37633862616635373635646333643132383732386433336534616130323761366536633436333734 +66623765336334363835346261643234393333323633666535316166616132313132303037646430 +62326634363164333937316338383963623037383761393131636265653336626266386364666364 +33663333353338333365386333336165613839616438376130663034643439323533616238396239 +38613534383065326362353761363161303564316431376239666433653531616633616538613364 +66303836626565656131343166376165386130663438396431376134303265323432383336363065 +37386136623365626630313163303338373461396630356165346439313037373439323361363865 +37656666373932393336333762623362643837323366623766366136353437353965373631326331 +34386262393962393866313262306362336631303564373864343032373832366230333730313364 +32373536656261336164653838363133343337343663363064353739306430333164303533313763 +37333338636366656265316461353236613666316366646565323136343436326363353462643330 +61383439383535393930356439313764376637306331336166306261633930326231353430626333 +35303566653630396230303235353032373738366631393439303363646237333738626166666635 +39643632656266333831333861383561613036666133626564623237616563333136376432353964 +62643761373864383734636438663664306362613766376363663161343266356637353635386665 +62653362363737363761313035303662333438326139653432346137373337653836333265613163 +30656232303537656461356232393434366331353134626565383833656530396434386336343338 +35636332336335633735393737633933636539343639366465343732343764383536393833626339 +31623331343665613361343633373631313637386239313531386638326263323139346266326134 +61373836396366346137656162353864653263663031306333643762383633383432393937326264 +33666566306332376664363735326533613639626531623762376166393231333964353963333137 +61353133373138626132366361353566363530333065616235323634353237306565343564363130 +30396265386538343236356434343438376530386136656534323561306435376430393032383563 +65373739306535316462663662303339393434393762346235323030663532336239386335353433 +37353732313861326464326163616533663935613238653337636565383930323339376539636164 +36666336316134646264303365613665313032323633393236626565396261616436323066393532 +61313136626561666630643930303462336561656365343639323631363937383037653331376436 +61323630373765333464626230393930626664356465333662616330383866663034333862383036 +30633261653833656430366634626235393337313766316562623762393439663539373766623165 +38376436323439373935306338346163653938656330306465333636313838626663393532336239 +62396463333465623031663562626535613433623731373230663165633764303964663463316331 +39343665326638656138616463323865646362656530346534353736653436326635356433313862 +38316164333536343536316137663262353065383038643039633334313530653666663364316636 +65336639316165363134653562343161306234633134363061356330313937373965326130376364 +61323936646534323635626131626339396362646166643832633234666639323136616135396230 +34373833333130323165343130616661353232636463356633386364303635663234636638303233 +39643137393932613935373962343536613935653863383433356166383262336536633635346561 +63346465613866393038636134313437323132616438373461346533326237623365376665323637 +31393832316239636137646339636433666131386638303735313437643837386134346337306435 +62643639613131393062356430663637306139306138663030633335616363613838336462616635 +64396563366466313466613561346165363634663062353835376565353631303736323064623331 +35363561346633616534356530393833613539356566363337623437363435326462326530613830 +38643766663864626431633666633235623363363732376439633265386430383363636430363966 +36613231373639376137626236343966346364653130353766633163663633656564396537323132 +36616638353131653537386134663033636433656633613935666130633962353463656138343838 +31333031313264373836323839363663386366613236666230383837643932376363336530333961 +39623131656535653466623062643065303738653335633464343465666235393930616138356434 +31616632376462623437356538643166633364656363633636373863373962326337643935333835 +34313634303832633132636661653962313333393462323736326630393236373131306638656462 +66363365323131323839313163313534623137393463333538636437616634623965383337643330 +62313836313261326137356134383730663264343939626361306631636436656139663231323734 +32336432323530336537393130393039363264393336643632326230366534386363396133643938 +62633736363738616564373239363364633532393036353636666435653333333031303038653734 +30386439666534633632386131643364376631393434343865353536616133633835366136653764 +34613636393231333265396437306436363062666466306431656635363234383963303061383433 +31346237343330623261613364633263343037346565323637386336343238323161303364613363 +34383031396336656338376135336333613164373762363965393531323733366363653331653464 +34333932356663383733613333386331346131666631356462366464386535326162363165653533 +64626366393164633862333165623662346231613362623839313734353234343466383039326436 +33396362376433643636383665623539636163653639333238616339363135396231383936623165 +65353765623330616535323264653931643736373232393866363366663334656364373937353334 +33373138396466653862356164313862356239663762373333383864316565633734313633353039 +37393230313639363462306233653531346134343964343262366139313938373966356138353062 +36383065373766346662666238323436353538663538393033323364653161616532633766323365 +35336238336231626637393264613461643131393265393362623434306663656130326565366234 +33656638633834303161353030353430303764623530306466393766623134383238646239653332 +61313738343564316131323961306364393833333865323763393630333931613530313961343739 +61643933393333383861346431383234383732343538376635353532346361353637383666646239 +36373566383134383466346131663037333732313435386437613431646539636437343531323562 +31353463323833613161663035363630353733343639386430323630643466653664343165303965 +62333735323139653565303162646532393762386633346465323962386561366464323938303332 +37353764306561653634303735383564636363613965376238366630313766343563393839313834 +38373065383861393862366563643736383238326231323466376566353539663333653038303734 +35306632653563373536663539633335303331323063666137656462336465343362633964653139 +65313762663537363561346634386265313237316664303034353338623937666637373862663762 +31623931316239313666306233353330613466646465323534366338306439323766333136656465 +36363238333834373036353763373136663964623066353732353130653865623164643236326539 +33363263356539643463383661663439656365646261336437613263616434376266363036363335 +32393762613265383032636239346135393664646264623539323564383535663965643365613364 +64343531373935613737393966306238313332633532653163366265393366393664303065383632 +34323738393765613031393731323631323163623165313336653839376536646638633237333864 +32333536653536303233346633353466313439356430363063623566656138383730373066353833 +30386333353032613066363137313165353763343230366164326538383163613437343536666236 +36343534613166316166343261353365323461323036343235323263393966636336376434646666 +61666461656334373034623731623661366231313764633066653231343734333234313361343539 +65336338346363383630316136333632386537633136316533303462326162343338653131653833 +30653665313838316638623163643233333037316233323636376165636631663437396437353730 +36323830353761616661633264666166316635326661363435613236326165363063643539303061 +62313632366435346339306137343830383932613834636231643430363033666432653962346463 +34363630623731363361656233396365326665363634623861313238316465363135643164623433 +33363162653136623363326639646364353933363631666665313230353534646435623631653737 +36656238303933346365343331363466626665366535336537383132343638646335303463623631 +66393634383339313636313237643963646535653364626334353463346137373035343462383933 +32666462386133356530636266346233393335653230313364393766363165633732393864336337 +32653530626237616336376362346539383962326537386137316435333838626363623465663266 +61333231396561643734663336613034333032393935663339376432346437653234353661353666 +31346536333662303630363266633564376163616634373866356163626137373737666230316363 +33646564396163313935343462313265623361386536633835376437363264663637373932376666 +63373064373333323562306231303365346230633334663430633532393734303234326435623336 +64353438303061356262656562656532616239313761666434656433333262346264323030653831 +63623937366335653038626238623161623532393636653938323664336331323066653938663039 +38653636626537626261613135643137373834373166613930393638633231616666616266306634 +64376266313661353062363635353962616235633235623635633065376364376264626261343465 +61386364626631386235383838666366666632616565353061613933386437663838356563356433 +66653038363264376464303836386163396631663232636334383835663733336139316661323233 +64396237633030653664373863653239623136643766366430393963666638626239613837656533 +37373930653665326436623063666331376363633966623138396434356230333832396339383764 +64336539313732346266343466613136393636396330636130633336366130653261303630643161 +39336133623161386630363566316535306364313465656664346635656639656333393234326261 +61616165393031383637366331306230303666663962333430363030623539316464353835346562 +36353934656562666231343664616236653431376536323762393966376638666532393139356261 +34373534393963333866663937306266376331383262333666373030633135633731336364663761 +31393235366232643030313832333238356333386139646534353964356462333264323039643362 +36663638343263323234313362663032343634656431643962363161643861356532313634623932 +32616465666163646361313332376163373339313333643064396134633431633862323438393466 +65376461383733326236356565376536393435643533323865363931616130613364633464383339 +62333535326465313132666364636232643938313635653232336166313666663862643430366339 +30376161663462366239663139646437326263396263303861333539356131303539613830636664 +62326462333265663035313131626364663935376465356534646133313232653331356137653862 +62313163366436626434353233616437346263366363366435343365306531333539356664643137 +38616137313535646564323533383939373137323433643135353635333863326232303665656364 +30636363376261346334376536663636643934636331303339306562653339666539366664636363 +64316535666532616663393766333161373535316163636235363232346138663137303066306535 +61616439353739623962626634313339313633623930353234303935313232373064613065613834 +63626364393838626632333838326238313235366666323737666264343838613162663564373433 +39346663316565383162363064373161336539393539356337353064663063626266373763326261 +61353634346335633566373035323432616637613062646136386361323937316563363939613064 +39313332616138633664633530643964333463633639333732633466333964613563306464346562 +32353032663439303766363138363736613439343738323462616437343763626262373963656136 +39353764356162323836303861656566326239363837373038343239316536663037356333663165 +32323361333164323631343239626363346535303534396662666566313938613830623865333239 +36326364366231396435383332323035313961653464316662323235326231613865613663323739 +34353939633263636235306563323864633634656438663237613234373566383863346538346663 +34646630626664376434393537623561396235383734623464353636633536363430333462383333 +37343931613036383130636366633563656563306437643033303135653165353763616333626334 +66666339373739653535666433633932373565356266366566656561313164363865663030643264 +33323163303931383838336238383537316265336238636364663662353463356331663462366463 +38323739303634336464383662343833336264616164376561643062323461666464306366653937 +64613633386233373830343066363437303939353763666336613565396138633730373566623261 +66363962373937376665323033623532386130333735646662356333393762303666643132376333 +66643362386135343537613833663165393461643131663661656636653635333830623861393462 +35343137666166323835323638363732633330646664626236663731646561333865646535646264 +65383237333531616236303033343531393161393532653861303032643861366635303761326266 +63623138333530373063623735613834393631366530633266323537333964373361353436333865 +38656462633862653961623166386262373966353032373334633934343138316632373231366334 +37373232323366396438636530383833343836376234386531343466633230613432356265316265 +36316138356261636332666561626336383534333764646561306362653961333165656665333661 +64336366646366373963393133626630303065383162353032383033353737326663346564393466 +32333663633465343966366234663434396266376162303564613362623631323765613231376534 +32623162373764653466343936656634333933653739363565663533323237383535376562316435 +63346637616634343561393762373063336364636664613536376466643962343532363633353338 +33656532653464666433623563643338333534646633383931303938316134353664383664626435 +65333132613136316263363434646336343439333230363639636533633237616131356532356236 +66663034336661303934613832393866336431626331623332336564633133313130616131656630 +37333731343861383038663366373933663034616334656464343930616337323836313439626437 +66363732663564633030393934376537373138323035623064356631613538306664333335653966 +38333261346233393233623639346435663238653833383234303335633362366334643263346536 +33646538356466646466353766643734306265386235313038346233336339343932343935666630 +39363735343132623964623430613733383737386366623162663765613731353736376332393139 +66646661663465303539626534376638396238303032653263646466323938626465646165653537 +61616531633939366366366165353866656637316538646666636436316534386337343964316538 +61653362616333366336356463346661363665333637643862346131366366663265393735373634 +30373363396238346330353962346539316439303531616530636531626537646365666238623435 +33333135633930653737373930623832306566363632376365626165616266653339336539623731 +31393061303062613638613632663634383738356633336234353531353166616463363434393037 +30343761313866306662373631636339366139333432303134363863353536626332616335313631 +34333064666661343834376237376634343666656631643036313461346361386432613036393039 +61386236623465353431373735323062623539323131663030623961646137343432306134323766 +61323261393966393438643439356433303534343132313034626366373837613736663661623233 +65313134636533623530373964343165623161306637356631393663653132663439666431633539 +34643566343635306562653839373862396336323534353737373339336638633632626331623231 +63323161363135333234373335613131386166663365386139353664346464663133366131393830 +61633037366366343830626565336539626531366266623330313332626165306635656165323832 +39626330393662626339626431383862383633333130306539373234663738366261623234346137 +62666630663465643637353566343030336162303765626236663237303066343066663061626262 +34636561653261363432306632393639663864386432303438626339626635643732643436666337 +64623863663666336430633563336533336530653264313861306534373432643634376333653961 +63366136363764333065626265373538346134373938663962643465633130343363363131376366 +38663738393434663933333630323562326435376331653436656530323064343437313230646430 +36623965373535306266356463393932313439336261306331666465333934623335363438633337 +35303738316538313531326230643832323466663232343165646164643838313436333961366334 +33303061623438336262396230336430373138643030616539636563343164356536616266376661 +35623265616231643164393131396430376532663839383363653533313866633464366137363437 +30306636643966366362343766613835336138333637636164393461333363393466333037613232 +34336161633036323664323636653432616561643034323532303364626666323462616439653761 +37623631373036373530386639633439303832666239383432306231316166363038656439643333 +63333037386637366562303134326135326661666262393662343964376435656637636161613565 +66366261623137366430656466346461633134343563643932636663356134303966383634363064 +34656538313836666639313263613033643566383237633237666666643336666361633734393632 +31383964633833323035646630633230333862343665353962663464386463636539623838626364 +30626333383830373839393138313637353636303539643561376561383061306132323664656561 +34383166383434386130366166303831353831646536656339666538363337346665653637336466 +61316336616135323764316330306664313962383563373361623566626434306666313438616637 +36636339326332373764353330663136666538616530353964313433653337323035356436353663 +39373564626231376162613465306533356336383231376533343630633761613034366363303166 +32303839363933346133396430656139356234343937326665303338633038636430303337306134 +66346237303466326265343836326163643965356363353665323338313761386465636362353632 +37366630383331393133346365383539376632613566633762373136666134616638393833323034 +37663231386263633461633934373434333534646466643132663739623437343262333739353064 +35636261656363646436616266666531323935656431653933623139323065343138616363663131 +38663964643361623230643934366637666238326134363330633337346232356530663034373363 +65353530623237613737386335386237326139343166316330313662306334653831626462363036 +66326134313732323831306131326536386630373639653139306132633135363432363239336135 +36383134373862613432376366636633366435613331613561366130333764316462643133623164 +65306438303233643566393061383037343936623462636132393130336139643934386137316133 +63373137326664613864666336306436386338343235343663333738663264613134306636326330 +64316263363961376632343962366463653563396261333965313265386562613264616537643631 +34626237353839383836353461666665313333623363626462393838633430396661386563316533 +63303236306433643037643762356336613037623434386262636231316431323062626233343635 +30313338353236646133396366373462616361326638373964373337626435323432386531323237 +65363464626362663135303266393266376335653538613738303439396136663536383531336135 +66613035613438623539393639323136613332343663663938366662623363656664356438623561 +32643535333234623363343564323930656163643865653132366565323537663236333361333964 +36363762366536326563336232313663663662383631623035303038343664653966346536323730 +37356234663933303430633038666432313961356361326231333531616639393833353638363865 +36383030656438323863356237353562393135303735346466343630343735373534356534636537 +35313639383231353835366236646163643765633439616332636263316434313539326165383531 +32343331613639613735643333363665373034666462316135643037323361376366326538343364 +30346335636235386566646631313534613566636363643264306235303661633038326633363230 +65653336396236363164663236366133656135313962623332306636323961366139303163386236 +30323566623562303766653531616230356537366665323764633532663237333261336434623262 +38326162376261633431633030373937666264623435646163633336636636646539333236616363 +39306365373336623865316163616230343435626531326463313439396230616430313830626439 +61393834386439363138303263313930383439343634323235366133373432616361306664643134 +39396266363931633264663939386566653063633066616664343533393366366335366535613966 +36363163643464363633343663376164323636326462363766366131653333363462376432333030 +61663730326162613938623161383366353166383164386165313731636639623736323864383666 +30616537393566633961396463373135653534383365393066626561313065353262636364386438 +36623536356533393634386336376430393239373930646231663236623038656536356662346166 +62316561646531616331373932656538313530643638393361666439333235343931656237353039 +39333536306663653430373663333135626437373331393731643363663137663437666639663437 +61376531386434373130653363363664656466643934373130616430373965396534323664373334 +36326263386630626536613931626134333465623966613866326566343934323435353262623134 +62383830653333333932326633376235616463363234633662353464643964646631303134646531 +63333733393634623231323937643739666434643462323166613638363063303031356465653063 +65383034376331396262643462663932346433313566616433316339326336343935393332333963 +39393631306238613636333336623732303866353630616135653039613832333961623234633733 +63623133373333346562383466326235393330666464353563646331373063636631343232376434 +62656538373637353231653231643730356433313533393363623863653637303831333932323438 +65646239333061346238393066323861393032303032373930623431333238653864393033353431 +36666235623438663830383435383535646635373661383365356362623538356231326332373637 +31346465323165313334323230633234656464363837666439376239333365386533653666616165 +30633538613731336561386466336365396631303030343962653739636234643531646532323562 +36366537386265383432393061363934386561386131306564366233386362326533633461633033 +39316462636533643638313933643966643633393036326236383762356536343365333430316663 +37666434323933626630656638666264656438333436313739646433663966663431336338316430 +61313034333263313230343035383436646562383834666166393563306364316638663434633338 +39663531383931616265383532666162373330366639646264636362633165663435653463333332 +61323436353165313633316366373932663231653262373765353761623666306435336333373063 +62336635643332363335326264343732396164336264333230366332663739316166326163653339 +65353736366663363264303637363331613663316630633735343235343666353837333331393239 +37643130623335643932363066306664333736366333363162396633653638306366613436323735 +34646536333039353230303535306165393464646633353965323931346436643564396531333436 +33363662656331653131326137653963366437333164626365616532353237633365313433333130 +36306166376164306234646430356235333035653236353036653733636363346432306461633362 +31343531333036363434626230656366353132633631656665316264366364356566366431393238 +39386339656130386431613237326463373666663266613162643762336165656662373536666366 +61313232353936663262613264366466653765353834646435626234656263633631316331373837 +31383263323463333165343830366164323663366636316439303130633734616262613861313538 +38626563613335333462333464366630313566316131626639626663643935646630643862346664 +33366336306565653631363336313864366361343763383735636334376431323337346461643031 +38316635386663353337323764663336626539616561626234386666653762333231333462663463 +65343437336361613630633566323066633632383435623136656136346161656532666433343961 +35346364333733356663383163316235366131313764613165313033333663646237393132616261 +65653666366363366238393263353834316538656561663638633131653535323036363234616437 +62366631396337353532323238663664633436323661626162373765643365333764323766303638 +62633735643730393334373631316637376630306335363362633866313832386337666566383635 +63653934623962663930373862326464613439623132623035633834626430383636643137393734 +39653738666666316437623731366433373164353362613532353639626631333136316131333037 +61316330333833653766383738343339623962303432376639303363653137633238313135316432 +32666663653136666334656632373466383034663937663732316430373139386163653961346335 +39393537666134396536653461323339353064393035303233343034633339373366326633353139 +63316539353463386530343563363635373231343830303830663362323939316466666434346233 +38613265363632383763366363373765343063366565613264646465623164313039663139326332 +33303937663036366632373436393364353437393863656131376532643864316566383335626438 +36386462646435313566333434643732393964666664393739353632373236396564353666346436 +38326534643365616261383631353962356266303966663430646564646466646162333935393162 +34323765646562326539636164353561333862636531333366393363326366643361613033666435 +33666132653332636465316533666435643662346565336633306239333734326531396339383035 +65633932356463356665336635616433363230633830376663666366336438323364333939393065 +64613430353039636437666430346437623130633031313539343430613232643736383033663030 +37313632663431643566646165656134663734636237373833366232616364386466616162636234 +66336533653832356434353936666535316630363033313366396361633366303332313736633031 +61393063333361636538613765363131663465653634316134393736633165653766616364663937 +30633966313361646131346366383130616339343436303234333035336237613866616633356437 +65383964396531393434636162303765633039626331366161613065333637333561326366616637 +31636138613436623036386461393762376634376261393336336232643136346466653561306136 +30303361376536383939383030323336323266366561363138343137363136383335643735356535 +35386565383961623934633364656565353864313031306537333838383338336264376464663365 +38653266653962623932343039666462353930326435306166313364373431323434626139616466 +38636337383637393132373739666139326664663938353964623931393038613232623238636563 +36623364333338653262646232336336653630383436643763323630313237653063376539376363 +64303861306266376238666439376262333639613964663230373366316638373435326166616638 +63643566633531623837316535643637386461313063373138313637343535393362393466636337 +35383461633862356166336631303234646363646565323434383265383333303939346561343637 +30333632336432653239396566316538636661616336376434613665386465383530386662383639 +38333032333361393935383335633231313266663834343336643337623762333833346430306163 +65323666633964626264383162643064613864646134343131303563343532623565623635306163 +61346139306565633763376635613061363134316563303161376663656134333534303962376433 +66653333666537643235356461323037393262313339343363323562363734633230626436306138 +33356262313565353762353463363638323632363638363638393638623835363837323962666261 +39383863386635366565636337626366663936616665343736643761356336383039613966613931 +32636534333232383362643165616439323264333961386334383266633436316636616530616538 +35363561373932666139346130386238633562613838656132633366376464346563373431303136 +30336534646436613432663566383633303061376237356366366239353364313334346333346535 +35396630303935666532373339373332333564393666326334636336363233353431306539343039 +30653030333031323035393035363063313561663037643861633761613339313936666332343661 +61303964396139646330613962633536613861323832323263643162323135323861656530323966 +39616665313236376530326338386561356332376537363963383432356233616531373531316138 +64666332373738386336386464333666616462383334316165366531326431636433396138326539 +37366133393034326633636331343161613531376438396633356561316465366663366361386137 +61623436656339393334363739636639346465353933383739323232366535306238303464356466 +39346631336233313033376430333533356131393861356631303336663635396238383763666562 +61376666393562383563663430656663656534626263633263343833313337376136303866303264 +66363635353466383434656663303436643765343361373536363262333935623431636630343139 +66643464656466376564323034663862623033323533333166643965313032656238363764313230 +37633566366535646263616663363462613334616333343164626464323661356631323865316638 +38376363636338393133646337326434653139643438343938356362323930643535323562633431 +36303863386637653037363333313635663261363238363862613062663465343835616262363133 +66363839366464393331316235303538373866396564646237646466643134663730393966313232 +36643439636662353062303763393761363137323165333065333130376339303235656663313733 +31396262626561656664313466333938396334303036346539383833623363656632353265386665 +64363031386463643465383430393266316638383334383337343962333231386563363064613363 +63636132386663353464323539663130316633616232656135653635323236623265646365356631 +32653739646663656134316238333431306438366435653564656561623931643765363938373966 +66623139396630363362633365363236613865333832323363656535336439343134373763653830 +63616236316532363864646338653238353835323862356364633262633464623536313539366439 +31386231306662326636623535393032353966353936313165373730373863363934336131343864 +34386330353330353933316436666561323661666562386639396431613833386335393931303433 +34393165316366646365646435343362663665323464623632316365623239326336633564656639 +65633533326434303533376436326431646362623265343466386261656262653064343065333361 +31666331353230666664656331383666613337393732383632333536363335646233643533376435 +31666333376137633963306566643162376365653163333333333535393931613764336261303031 +35343739333535653138346231353265633032363533313534303130306538356666336634623334 +61313461323464616364653336346166613931363432383565623031363764323132616632353034 +36613564386661343534313930363431613835623065353862646535363939613233646233323165 +36353430646662643566643664653034363234376561346637353237363464363063323264366133 +39653831666335313565346263613464383764303231623239626635626264623561313461626138 +61383261656433363866326633343732353739343662376137383034616633356363383136363461 +39326637306231343437383538343938666632363530613130636361373965313530393639323665 +66373537646261356365663230633634653436343664623435636536373138353937633561646434 +39613236653332306165396239346562313236343534663866353565333532343263653938346235 +30626337323631316339353965383935383332396636363332373831353430313565316461623338 +61653139383763373438333266346430383934656663376362306436633865633266663038373831 +37353034376464396530316364323638653865623232336638626664326131306432386332303031 +35633235656335396436393764353736616430333166653232333136343036623763346164396534 +63666462366162383139383531346432303432303365303933656636303663363864613863353336 +63663836343838366331356463616362313337316333663866613933666333313636386664643735 +66366330303531613064653634383634383733633164353736313136316336306232633830656465 +62333466303139633436353930623166386364393966383836356665663439343832383632333938 +32653363666366346137653338623936616137363333666262323762646365616466643462616434 +38316663656165613961666332336538373832653164393030333763373963633338633361616463 +64646530326236393633633538636232623665323734386239626563323738323936373130363933 +36393531346464623765393361633835353730646362306638373366653938623036356438363165 +63656265623662356530393664636262613138616362376637373938313238643162356331623636 +34663931343537343933613635353731633830373230666565386137356165653263663737346463 +32323764626430366162643831366362356137333362346262656235313238353965323736333966 +63393132306366663432363865323236396332316533663036323133346233393039363930393763 +35343935333266343430393932666435653139383630613961346531383665343064323337363461 +61336638333135313430396533663961643336306366303439643034346136666562333038656164 +33636534306330663333373266643066646164303364656532653730626234316534656263376630 +34346363363166643934626434653263386434373030316666656238646332303365343230313061 +31663335366663616538356530353737343861376464376239366537643961653433656236313537 +30313031353438653235383261353265663439623134343263656139363432313361313066373635 +33343162383164303265623634363232613330353532366438346563646464663330316164366661 +37346661386331626363646632393937323866663066643365656162653762306263613763396464 +62306433313231313665633530616266666633356334353264396633623938383037353234356663 +34333136366238623336616262333336333531653935343135653734316165616661633137633837 +64643964313730336335313565366536646664616336316535356633336265616536306137663932 +38656238653932613837313436633338333530353634633837383737366438346164376633386261 +35383332306265363335623534363036363861343934653161303730386330323330326238636161 +33643461636262323363636637383338643863386661306464376233316664626339396261613035 +33643136623838353936353836646439323666373132316164306139323835336430336234323239 +31323838653834383366303137343661336263316166336334343766623038373433356362656334 +63373439366264393863373132666439313166653234643266653562323638306430306539643332 +65663833326565653030303431636531643262646433326432376163613335306133353363363934 +31303036383734363663633961363838656135393437643865386161323364313138313362373036 +62636631343739626432313564626264636536633733326164373662356130663265363831323236 +32313030363965323266616437616436656562633664313366613263323634353461346235333861 +38333034363937616530323831383633633438616431626533323066316566326466656465363639 +34323337653732373433303831323232653765343830306235316137616538626266333231313430 +30333136663564373735383030623230663336353865633234363239346138633538316637313865 +35653236393539323431333463353237336332383266343432656330386561633834626134666666 +64386635396434343534323536303861333865326532303866666535363835633562643762653832 +65376634353637353439626162393130396434306237633665656566303431636165366631326263 +38316136336562353737636666393365336465663831646337666131353739393033623831356439 +66316135376533626232346537613730383536393037646639633766356136306631336666343738 +63666634643365363333663065646137623239363532393231653861366139613962326633656364 +65346232623737353438303363613431623337613761346234356665636137303161306563663732 +31386535306537613561643964303938333766366133393332626438396463613964666163633963 +33393430333166393731663066353638656665643766393934313030613664356332366261313061 +61626231653931366462633632646530663839653064383437363264613164383732656438373361 +31633139383336623731633435346665666563613265383938633731656538623836613463633139 +34376261346238343563383631626238656531396434663831366438313830383130626461386330 +38313231313230653930326238343634386161306161313637363665353631303039396337343738 +64366434333133636533323364353835666537656461313961663533623839666331623439633533 +30393265353262623230343235363963616339646436666131363861613432356165386665343961 +64623263656638396237653036376336313936323361343236366466386633666634666664363634 +35643438396461313538346565343236613335623435613361643638653839313766663830393964 +33613162333531306434373235346336323933383032663535366365386566303632653839616466 +35353637623533336536623035306432323235343735656161393237636565376439303862623666 +65303232333566383464663832613838306538336135643831323339626330636337623561666536 +38646535303431336132636461316634633662656633653931616231623263333937643635363034 +36653635643066353539666530383130353234323130666262306531626166376431643239623337 +33383639386432366631663263613562383430626531346337393236346632356262323430346536 +63336638363837383630346233323461373735386332653239393461643130353262623237306433 +35643331666464643334353635613832356435356239343833383634363564356261306463356365 +38353332656531646334313531633430666135623865663734636133626636323735346166643334 +32643965353737663964336565626532363736303361623766343233386539316234333365303331 +65363562316464353237653161663966383635393932383532313738393139306130613539303934 +66313632633431323364666539643139653432373366396536636361623335386438323330326461 +32386236666139646262633963663034656266363834363465313962643764313634303431346564 +61643930633038326630316463646666633836666430353264643731346239303835383238633632 +30356463313038396332316334643733333435323864323138616534663539386331393337623536 +64313566636632643161373166643434636465306337353266653832613738383762393430316638 +30323737666565666238393166363565376164623531326166353130626664336535363237353632 +30313462643636303365383966323061653062316363373134343037333966643663623833343261 +37633130346363636365373533306539336463343734663437343134373737356637643336343161 +30343935643736386262303537666639623830393033353530393338613864636237323934373364 +36363133333766313464616531613030623137316239393736613039373033363062653234393637 +39346233363431613965353363356335303362633831613334313837616539376133313538346164 +34336561323131306238376265323165663266386666646335393831386663353230373039386263 +36616662663561616634383336303961666263346365613731353938653062326331373236636364 +30646535323333393562366437353866366462326638383839396235313631623666343462613938 +35356562646138663739306263623236313330343331386431396334373531393036653831643963 +62636430323561626365336539393639623935613362306138316337336132363162633437306263 +37333661663032306235383562643366363035633063633566393562396264376138623231366263 +64313034396533383662383565383435613764623030363331623161323935393237396535393365 +61353530376635616236306131393761646333663431363861396364383332353565373866326338 +34386232643737656363663739373961323830356237653462326635626462623432323636616332 +34613534303465343832626530333337666666383234316265383739643339323937623831653964 +38393332636331336337626465363766306438393137353031663234636633306663346533646336 +31376536353333613965333333623237393136663164303932353464633438373036633732336332 +35613966613039386135656366386136323830343631303761326639313930326237363739313734 +36303566346462383334646439363462323966336333626234343038613533333265643236373362 +63616436323831373837646361633734343863666339363936613133343165336666653766633932 +66633961323965333039373435316336303334396163643135333139626464653339313937333738 +32396265396639633739623666313630336531666132386137333431393762613866326462623336 +36613433623961356163343339306330666636303933636663636566383733383464356430656535 +33626630666536336365353432613535343830616436653233666636623331346564386637633233 +34326333376132363230643164363436366138333062353331386239633239623034633734643162 +30613033666165356539353830326539383961346133306465303636653864626265316230373036 +35396534313262396561313566356135623837646465653535336261393936356236663561323038 +393134303937616638376561383233653339 From 5ab175139997432608a7d1a52ac6c909497cbe28 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 26 Jun 2026 10:54:02 +0100 Subject: [PATCH 090/103] POG-150 Remove ds1 and ds2 from osd_spec_ssd_me5 in cephadm.yml & storage_me5 in kayobe Inventory hosts --- etc/kayobe/cephadm.yml | 2 -- etc/kayobe/inventory/hosts | 2 -- 2 files changed, 4 deletions(-) diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index 0eff84d1a..faf9809ce 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -84,8 +84,6 @@ cephadm_osd_spec: | service_id: osd_spec_ssd_me5 placement: hosts: - - ds1 - - ds2 - mds1 - mds2 spec: diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 2162dc822..b57128c29 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -52,8 +52,6 @@ compute_lenovo compute_dell [storage_me5] -ds1 -ds2 mds1 mds2 From 8527cd9a5897c218ac26cfe1a188cb4538013220 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 6 Jul 2026 10:16:39 +0100 Subject: [PATCH 091/103] Added temp OpenStack controllers and extra Ceph nodes into config --- etc/kayobe/cephadm.yml | 22 +++++--- .../controllers_r650/network-interfaces | 30 ++++++++++ .../controllers_t620/network-interfaces | 24 ++++++++ .../network-interfaces | 0 .../storage_r760_2/network-interfaces | 24 ++++++++ etc/kayobe/inventory/hosts | 55 ++++++++++++------- 6 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces create mode 100644 etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces rename etc/kayobe/inventory/group_vars/{storage_r760 => storage_r760_1}/network-interfaces (100%) create mode 100644 etc/kayobe/inventory/group_vars/storage_r760_2/network-interfaces diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index faf9809ce..ad83c83dc 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -70,12 +70,18 @@ cephadm_osd_spec: | service_id: osd_spec_r760 placement: hosts: - - ds3 - - ds4 - - ds5 - - ds6 - - ds7 - - ds8 + - ceph-hdd-01 + - ceph-hdd-02 + - ceph-hdd-03 + - ceph-hdd-04 + - ceph-hdd-05 + - ceph-hdd-06 + - ceph-hdd-07 + - ceph-hdd-08 + - ceph-hdd-09 + - ceph-hdd-10 + - ceph-hdd-11 + - ceph-hdd-12 spec: data_devices: size: '15TB:' @@ -84,8 +90,8 @@ cephadm_osd_spec: | service_id: osd_spec_ssd_me5 placement: hosts: - - mds1 - - mds2 + - ceph-mds-01 + - ceph-mds-02 spec: data_devices: size: '890GB:900GB' diff --git a/etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces b/etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces new file mode 100644 index 000000000..59fd35a19 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces @@ -0,0 +1,30 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +# NOTE: The content of this section is very deployment-specific, since it +# depends on the names and types of networks in the deployment. It should +# define the group-specific attributes of networks. The following example shows +# a basic configuration for a network called "example": +# +# example_interface: eth0 +# +# Global network attributes such as subnet CIDRs are typically configured in +# etc/kayobe/networks.yml. +# +# Further information on the available network attributes is provided in the +# network configuration reference in the Kayobe documentation. + +prod_interface: eno12399 + +data_interface: brens2f0np0 +data_bridge_ports: + - ens2f0np0 + +jbo_interface: breno8303 +jbo_bridge_ports: + - eno8303 + +############################################################################## +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces b/etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces new file mode 100644 index 000000000..94c1b7663 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces @@ -0,0 +1,24 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +# NOTE: The content of this section is very deployment-specific, since it +# depends on the names and types of networks in the deployment. It should +# define the group-specific attributes of networks. The following example shows +# a basic configuration for a network called "example": +# +# example_interface: eth0 +# +# Global network attributes such as subnet CIDRs are typically configured in +# etc/kayobe/networks.yml. +# +# Further information on the available network attributes is provided in the +# network configuration reference in the Kayobe documentation. + +prod_interface: enp65s0 + +jbo_interface: eno2 + +############################################################################## +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/storage_r760/network-interfaces b/etc/kayobe/inventory/group_vars/storage_r760_1/network-interfaces similarity index 100% rename from etc/kayobe/inventory/group_vars/storage_r760/network-interfaces rename to etc/kayobe/inventory/group_vars/storage_r760_1/network-interfaces diff --git a/etc/kayobe/inventory/group_vars/storage_r760_2/network-interfaces b/etc/kayobe/inventory/group_vars/storage_r760_2/network-interfaces new file mode 100644 index 000000000..2fe3fd822 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/storage_r760_2/network-interfaces @@ -0,0 +1,24 @@ +--- +############################################################################### +# Network interface definitions for the storage group. + +# NOTE: The content of this section is very deployment-specific, since it +# depends on the names and types of networks in the deployment. It should +# define the group-specific attributes of networks. The following example shows +# a basic configuration for a network called "example": +# +# example_interface: eth0 +# +# Global network attributes such as subnet CIDRs are typically configured in +# etc/kayobe/networks.yml. +# +# Further information on the available network attributes is provided in the +# network configuration reference in the Kayobe documentation. + +prod_interface: eno12399 +data_interface: ens4f0np0 +prod_gateway: 192.168.10.201 + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index b57128c29..db6193d02 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -6,8 +6,17 @@ localhost ansible_connection=local # srcseed [controllers] -srcseed -#c-1 +[controllers:children] +controllers_t620 +controllers_r650 + +[controllers_t620] +ctl-t620-01 +ctl-t620-02 +ctl-t620-03 + +[controllers_r650] +ctl-r650-01 [compute_lenovo] c-1 @@ -17,7 +26,7 @@ c-4 c-5 c-6 c-7 -# c-14 wrong 100gbs NIC +c-14 [compute_dell] c-8 @@ -37,7 +46,7 @@ c-4 c-5 c-6 c-7 -# c-14 wrong 100gbs NIC +c-14 [compute_flavor_r660] c-8 @@ -52,27 +61,35 @@ compute_lenovo compute_dell [storage_me5] -mds1 -mds2 - -[storage_r760] -ds3 -ds4 -ds5 -ds6 -ds7 -ds8 +ceph-mds-01 +ceph-mds-02 + +[storage_r760_1] +ceph-hdd-03 +ceph-hdd-04 +ceph-hdd-05 +ceph-hdd-06 +ceph-hdd-07 +ceph-hdd-08 + +[storage_r760_2] +ceph-hdd-01 +ceph-hdd-02 +ceph-hdd-09 +ceph-hdd-10 +ceph-hdd-11 +ceph-hdd-12 [storage] [storage:children] storage_me5 -storage_r760 +storage_r760_1 +storage_r760_2 [mons] -ds1 -mds1 -mds2 - +ceph-adm-01 +ceph-adm-02 +ceph-mds-01 [mgrs:children] mons From d65a5dfd6ae233d27a23f26233772c6e052561bd Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Mon, 6 Jul 2026 10:53:22 +0100 Subject: [PATCH 092/103] Updated network-allocation with temp controllers and extra ceph nodes --- etc/kayobe/network-allocation.yml | 52 +++++++++++++++++++------------ 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index 91152ac8b..5bc7264eb 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -13,16 +13,22 @@ data_ips: c-7: 192.168.100.7 c-8: 192.168.100.8 c-9: 192.168.100.9 - ds1: 192.168.100.202 - ds2: 192.168.100.203 - ds3: 192.168.100.206 - ds4: 192.168.100.207 - ds5: 192.168.100.208 - ds6: 192.168.100.209 - ds7: 192.168.100.210 - ds8: 192.168.100.211 - mds1: 192.168.100.204 - mds2: 192.168.100.205 + ceph-adm-01: 192.168.100.202 + ceph-adm-02: 192.168.100.203 + ceph-mds-01: 192.168.100.205 + ceph-mds-02: 192.168.100.206 + ceph-hdd-01: 192.168.100.211 + ceph-hdd-02: 192.168.100.212 + ceph-hdd-03: 192.168.100.213 + ceph-hdd-04: 192.168.100.214 + ceph-hdd-05: 192.168.100.215 + ceph-hdd-06: 192.168.100.216 + ceph-hdd-07: 192.168.100.217 + ceph-hdd-08: 192.168.100.218 + ceph-hdd-09: 192.168.100.219 + ceph-hdd-10: 192.168.100.220 + ceph-hdd-11: 192.168.100.221 + ceph-hdd-12: 192.168.100.222 srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 @@ -43,14 +49,20 @@ prod_ips: c-7: 192.168.10.7 c-8: 192.168.10.8 c-9: 192.168.10.9 - ds1: 192.168.10.202 - ds2: 192.168.10.203 - ds3: 192.168.10.206 - ds4: 192.168.10.207 - ds5: 192.168.10.208 - ds6: 192.168.10.209 - ds7: 192.168.10.210 - ds8: 192.168.10.211 - mds1: 192.168.10.204 - mds2: 192.168.10.205 + ceph-adm-01: 192.168.10.202 + ceph-adm-02: 192.168.10.203 + ceph-mds-01: 192.168.10.205 + ceph-mds-02: 192.168.10.206 + ceph-hdd-01: 192.168.10.211 + ceph-hdd-02: 192.168.10.212 + ceph-hdd-03: 192.168.10.213 + ceph-hdd-04: 192.168.10.214 + ceph-hdd-05: 192.168.10.215 + ceph-hdd-06: 192.168.10.216 + ceph-hdd-07: 192.168.10.217 + ceph-hdd-08: 192.168.10.218 + ceph-hdd-09: 192.168.10.219 + ceph-hdd-10: 192.168.10.220 + ceph-hdd-11: 192.168.10.221 + ceph-hdd-12: 192.168.10.222 srcseed: 192.168.10.201 From b1da511d15ed38b79b42cfee6e4465ed2b053721 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Mon, 6 Jul 2026 14:45:21 +0100 Subject: [PATCH 093/103] Merge in latest SKC config for Caracal --- .ansible-lint | 10 + .ansible-lint-ignore | 9 + .automation | 1 + .automation.conf/config.sh | 45 + .../run-books/pulp-sync-content.sh | 29 + .automation.conf/tempest/load-lists/barbican | 335 + .automation.conf/tempest/load-lists/cinder | 160 + .../tempest/load-lists/cloudkitty | 34 + .automation.conf/tempest/load-lists/default | 1 + .automation.conf/tempest/load-lists/designate | 370 + .automation.conf/tempest/load-lists/glance | 114 + .automation.conf/tempest/load-lists/ironic | 477 + .automation.conf/tempest/load-lists/keystone | 1626 ++ .automation.conf/tempest/load-lists/magnum | 60 + .automation.conf/tempest/load-lists/manila | 1956 ++ .automation.conf/tempest/load-lists/neutron | 1005 + .automation.conf/tempest/load-lists/octavia | 67 + .../tempest/load-lists/octavia-full | 820 + .../load-lists/platform.2022.11-test-list.txt | 432 + .../tempest/load-lists/tempest-full | 1601 ++ .../skip-lists/ci-multinode-tempest-full | 1 + .../tempest/tempest-ci-aio.overrides.conf | 2 + .../tempest-ci-multinode.overrides.conf | 36 + .github/CODEOWNERS | 1 + .github/auto-label.yaml | 55 + .github/cherry-pick-bot.yml | 3 + .github/path-filters.yml | 31 + .github/workflows/amphora-image-build.yml | 271 + .github/workflows/amphora-image-promote.yml | 69 + .github/workflows/ipa-image-build.yml | 424 + .github/workflows/ipa-image-promote.yml | 118 + .github/workflows/multinode-inputs.py | 83 + .../workflows/overcloud-host-image-build.yml | 466 + .../overcloud-host-image-promote.yml | 118 + .../workflows/overcloud-host-image-upload.yml | 236 + .github/workflows/package-build-ofed.yml | 248 + .github/workflows/runner-selector.yml | 48 + .github/workflows/stackhpc-all-in-one.yml | 534 + .../workflows/stackhpc-build-kayobe-image.yml | 123 + .github/workflows/stackhpc-check-tags.yml | 65 + .github/workflows/stackhpc-ci-cleanup.yml | 123 + .../stackhpc-container-image-build.yml | 424 + .../workflows/stackhpc-multinode-periodic.yml | 50 + .github/workflows/stackhpc-multinode.yml | 76 + .github/workflows/stackhpc-promote.yml | 68 + .github/workflows/stackhpc-pull-request.yml | 266 + .github/workflows/stackhpc-update-kolla.yml | 29 + .github/workflows/tag-and-release.yml | 14 + .github/workflows/update-dependencies.yml | 189 + .github/workflows/upstream-sync.yml | 38 + .gitignore | 15 + .gitmodules | 3 + .gitreview | 2 +- .pre-commit-config.yaml | 11 + .readthedocs.yaml | 27 + .yamllint | 10 + README.rst | 44 +- doc/requirements.txt | 8 + .../images/capi-architecture-diagram.png | Bin 0 -> 267742 bytes doc/source/_static/images/release-train.svg | 1 + doc/source/conf.py | 159 + doc/source/configuration/cephadm.rst | 554 + doc/source/configuration/ci-cd.rst | 394 + doc/source/configuration/cloudkitty.rst | 131 + doc/source/configuration/firewall.rst | 462 + doc/source/configuration/host-images.rst | 102 + doc/source/configuration/index.rst | 25 + doc/source/configuration/ipa.rst | 35 + doc/source/configuration/lvm.rst | 98 + doc/source/configuration/magnum-capi.rst | 117 + doc/source/configuration/monitoring.rst | 302 + doc/source/configuration/openbao.rst | 440 + doc/source/configuration/release-train.rst | 354 + .../configuration/security-hardening.rst | 80 + doc/source/configuration/vault.rst | 447 + doc/source/configuration/walled-garden.rst | 120 + doc/source/configuration/wazuh.rst | 431 + doc/source/contributor/documentation.rst | 20 + .../contributor/environments/aufn-ceph.rst | 190 + .../contributor/environments/ci-aio.rst | 266 + .../contributor/environments/ci-builder.rst | 207 + .../contributor/environments/ci-multinode.rst | 376 + doc/source/contributor/environments/index.rst | 12 + doc/source/contributor/index.rst | 16 + doc/source/contributor/ofed.rst | 104 + doc/source/contributor/package-updates.rst | 267 + doc/source/contributor/pre-commit.rst | 47 + doc/source/contributor/release-notes.rst | 38 + .../contributor/testing-ci-automation.rst | 422 + doc/source/index.rst | 35 + .../bifrost-hardware-inventory-management.rst | 283 + doc/source/operations/ceph-management.rst | 294 + .../operations/control-plane-operation.rst | 386 + doc/source/operations/customising-horizon.rst | 118 + doc/source/operations/gpu-in-openstack.rst | 678 + doc/source/operations/hotfix-playbook.rst | 70 + doc/source/operations/index.rst | 25 + doc/source/operations/migrating-vm.rst | 22 + doc/source/operations/nova-compute-ironic.rst | 307 + doc/source/operations/octavia.rst | 119 + ...penstack-projects-and-users-management.rst | 38 + .../operations/openstack-reconfiguration.rst | 150 + doc/source/operations/secret-rotation.rst | 516 + doc/source/operations/tempest.rst | 348 + doc/source/operations/ubuntu-noble.rst | 556 + doc/source/operations/upgrading-ceph.rst | 175 + doc/source/operations/upgrading-openstack.rst | 1188 + doc/source/release-notes.rst | 6 + doc/source/usage.rst | 107 + etc/kayobe/ansible.cfg | 7 +- etc/kayobe/ansible/advise-run.yml | 47 + etc/kayobe/ansible/build-ofed-rocky.yml | 45 + etc/kayobe/ansible/ceph-enter-maintenance.yml | 13 + etc/kayobe/ansible/ceph-exit-maintenance.yml | 12 + etc/kayobe/ansible/cephadm-commands-post.yml | 3 +- etc/kayobe/ansible/cephadm-commands-pre.yml | 3 +- etc/kayobe/ansible/cephadm-crush-rules.yml | 3 +- etc/kayobe/ansible/cephadm-deploy.yml | 3 +- etc/kayobe/ansible/cephadm-ec-profiles.yml | 3 +- etc/kayobe/ansible/cephadm-gather-keys.yml | 11 +- etc/kayobe/ansible/cephadm-keys.yml | 3 +- etc/kayobe/ansible/cephadm-pools.yml | 3 +- etc/kayobe/ansible/cephadm.yml | 27 +- etc/kayobe/ansible/check-kayobe-version.yml | 72 + .../ansible/check-kolla-ansible-version.yml | 28 + etc/kayobe/ansible/check-kolla-images-py.yml | 35 + etc/kayobe/ansible/check-tags.yml | 38 + etc/kayobe/ansible/cis.yml | 42 + .../ansible/configure-aio-resources.yml | 61 + etc/kayobe/ansible/configure-vxlan.yml | 13 + etc/kayobe/ansible/deploy-github-runner.yml | 32 + etc/kayobe/ansible/deploy-gitlab-runner.yml | 24 + .../deploy-openbao-kayobe-automation.yml | 78 + .../ansible/deploy-os-capacity-exporter.yml | 76 + .../ansible/deploy-radosgw-usage-exporter.yml | 152 + etc/kayobe/ansible/diagnostics.yml | 71 + etc/kayobe/ansible/docker-registry-login.yml | 11 + etc/kayobe/ansible/firewalld-watchdog.yml | 70 + etc/kayobe/ansible/fix-grub-rl9.yml | 22 + etc/kayobe/ansible/fix-hostname.yml | 30 + etc/kayobe/ansible/fix-houston.yml | 44 + etc/kayobe/ansible/fix-networking.yml | 23 + etc/kayobe/ansible/get-cloud-facts.yml | 87 + etc/kayobe/ansible/get-nvme-drives.yml | 96 + etc/kayobe/ansible/growroot.yml | 97 + etc/kayobe/ansible/hotfix-containers.yml | 69 + etc/kayobe/ansible/install-doca.yml | 52 + .../ansible/install-pre-commit-hooks.yml | 21 + etc/kayobe/ansible/nova-compute-disable.yml | 48 + etc/kayobe/ansible/nova-compute-drain.yml | 75 + etc/kayobe/ansible/nova-compute-enable.yml | 51 + .../ansible/octavia-amphora-image-build.yml | 108 + .../octavia-amphora-image-register.yml | 122 + .../ansible/openbao-deploy-barbican.yml | 107 + .../ansible/openbao-deploy-overcloud.yml | 145 + etc/kayobe/ansible/openbao-deploy-seed.yml | 78 + .../ansible/openbao-generate-backend-tls.yml | 83 + .../ansible/openbao-generate-internal-tls.yml | 56 + .../openbao-generate-test-external-tls.yml | 57 + .../ansible/openbao-unseal-overcloud.yml | 37 + etc/kayobe/ansible/openbao-unseal-seed.yml | 34 + .../ansible/openstack-host-image-upload.yml | 55 + .../ansible/ovn-fix-chassis-priorities.yml | 164 + etc/kayobe/ansible/pci-passthrough.yml | 142 + .../ansible/prometheus-network-names.yml | 40 + etc/kayobe/ansible/prometheus.yml.j2 | 287 + .../ansible/pulp-amphora-image-download.yml | 35 + etc/kayobe/ansible/pulp-artifact-promote.yml | 46 + etc/kayobe/ansible/pulp-artifact-upload.yml | 181 + etc/kayobe/ansible/pulp-auth-proxy.yml | 15 + etc/kayobe/ansible/pulp-container-publish.yml | 11 + etc/kayobe/ansible/pulp-container-sync.yml | 11 + .../ansible/pulp-host-image-download.yml | 34 + .../ansible/pulp-repo-promote-production.yml | 11 + etc/kayobe/ansible/pulp-repo-publish.yml | 19 + etc/kayobe/ansible/pulp-repo-sync.yml | 12 + .../ansible/pulp-sync-publish-promote.yml | 16 + .../ansible/purge-command-not-found.yml | 33 + etc/kayobe/ansible/push-ofed.yml | 81 + etc/kayobe/ansible/rabbitmq-reset.yml | 77 + etc/kayobe/ansible/reboot.yml | 46 + etc/kayobe/ansible/rekey-hosts.yml | 118 + etc/kayobe/ansible/requirements.yml | 35 +- etc/kayobe/ansible/reset-bls-entries.yml | 39 + etc/kayobe/ansible/rocky-97-ofed-upgrade.yml | 106 + .../ansible/roles/pulp_auth_proxy/README.md | 26 + .../roles/pulp_auth_proxy/defaults/main.yml | 8 + .../roles/pulp_auth_proxy/tasks/main.yml | 56 + .../templates/pulp_proxy.conf.j2 | 17 + etc/kayobe/ansible/rsyslog.yml | 36 + etc/kayobe/ansible/run-container-hotfix.yml | 25 + etc/kayobe/ansible/scripts/aio-init.sh | 145 + .../ansible/scripts/generate_fixtures.py | 118 + etc/kayobe/ansible/scripts/nvmemon.sh | 150 + etc/kayobe/ansible/scripts/smartmon.py | 261 + etc/kayobe/ansible/scripts/test_smartmon.py | 278 + .../scripts/tests/Dell_ENT_NVMe_CM6.json | 26 + etc/kayobe/ansible/scripts/tests/nvme.json | 24 + .../ansible/secret-store-deploy-barbican.yml | 107 + .../ansible/secret-store-deploy-overcloud.yml | 172 + .../ansible/secret-store-deploy-seed.yml | 102 + .../secret-store-generate-backend-tls.yml | 83 + .../secret-store-generate-internal-tls.yml | 56 + ...ecret-store-generate-test-external-tls.yml | 57 + .../ansible/secret-store-unseal-overcloud.yml | 37 + .../ansible/secret-store-unseal-seed.yml | 34 + etc/kayobe/ansible/smartmon-tools.yml | 144 + etc/kayobe/ansible/stackhpc-cloud-tests.yml | 197 + .../ansible/stop-openstack-services.yml | 32 + .../fix-houston-interface.service.j2 | 20 + .../templates/os_capacity-clouds.yml.j2 | 18 + .../ansible/templates/wazuh-secrets.yml.j2 | 14 + etc/kayobe/ansible/ubuntu-upgrade.yml | 267 + etc/kayobe/ansible/vault-deploy-overcloud.yml | 2 + etc/kayobe/ansible/vault-deploy-seed.yml | 2 + etc/kayobe/ansible/wazuh-agent.yml | 42 + etc/kayobe/ansible/wazuh-manager.yml | 147 + etc/kayobe/ansible/wazuh-secrets.yml | 31 + etc/kayobe/ansible/write-github-workflows.yml | 7 + etc/kayobe/ansible/write-gitlab-pipelines.yml | 7 + etc/kayobe/apt.yml | 67 +- etc/kayobe/bifrost.yml | 6 +- etc/kayobe/cephadm.yml | 25 +- etc/kayobe/compute.yml | 12 +- etc/kayobe/containers/pulp/post.yml | 48 + etc/kayobe/containers/pulp/pre.yml | 43 + etc/kayobe/containers/pulp/settings.py | 9 + .../containers/redfish_exporter/post.yml | 6 + .../containers/redfish_exporter/pre.yml | 16 + .../redfish_exporter/redfish_exporter.yml | 5 + etc/kayobe/containers/squid_proxy/pre.yml | 14 + etc/kayobe/containers/squid_proxy/squid.conf | 83 + etc/kayobe/controllers.yml | 10 +- etc/kayobe/dnf.yml | 138 +- etc/kayobe/docker-registry.yml | 2 +- etc/kayobe/docker.yml | 6 +- .../aufn-ceph/a-universe-from-nothing.sh | 141 + etc/kayobe/environments/aufn-ceph/cephadm.yml | 63 + .../aufn-ceph/configure-local-networking.sh | 81 + .../aufn-ceph/configure-openstack.sh | 28 + etc/kayobe/environments/aufn-ceph/globals.yml | 21 + .../group_vars/compute/network-interfaces | 19 + .../group_vars/controllers/network-interfaces | 27 + .../seed-hypervisor/network-interfaces | 18 + .../group_vars/seed/network-interfaces | 13 + .../group_vars/storage/network-interfaces | 17 + .../environments/aufn-ceph/inventory/groups | 29 + .../environments/aufn-ceph/inventory/hosts | 27 + etc/kayobe/environments/aufn-ceph/ipa.yml | 13 + etc/kayobe/environments/aufn-ceph/kolla.yml | 16 + .../kolla/config/bifrost/bifrost.yml | 15 + .../aufn-ceph/kolla/config/neutron.conf | 2 + .../kolla/config/neutron/ml2_conf.ini | 2 + .../environments/aufn-ceph/kolla/globals.yml | 17 + .../kolla/inventory/overcloud-services.j2 | 466 + .../aufn-ceph/kolla/kolla-build.conf | 4 + .../aufn-ceph/network-allocation.yml | 13 + .../environments/aufn-ceph/networks.yml | 117 + etc/kayobe/environments/aufn-ceph/neutron.yml | 13 + .../environments/aufn-ceph/overcloud.yml | 21 + .../aufn-ceph/seed-hypervisor.yml | 14 + etc/kayobe/environments/aufn-ceph/seed-vm.yml | 9 + .../environments/aufn-ceph/stackhpc.yml | 15 + etc/kayobe/environments/aufn-ceph/storage.yml | 10 + etc/kayobe/environments/aufn-ceph/tenks.yml | 95 + .../environments/ci-aio/automated-setup.sh | 109 + etc/kayobe/environments/ci-aio/compute.yml | 3 + .../environments/ci-aio/controllers.yml | 23 + etc/kayobe/environments/ci-aio/globals.yml | 66 + .../post.d/10-debug-firewall.yml | 9 + .../inventory/group_vars/cis-hardening/cis | 28 + .../inventory/group_vars/controllers/lvm.yml | 34 + .../group_vars/controllers/network-interfaces | 17 + .../environments/ci-aio/inventory/groups | 8 + .../environments/ci-aio/inventory/hosts | 9 + .../ci-aio/inventory/kayobe-automation | 2 + .../ci-aio/kolla/config/ironic.conf | 2 + .../environments/ci-aio/kolla/globals.yml | 19 + .../ci-aio/network-allocation.yml | 3 + etc/kayobe/environments/ci-aio/networks.yml | 126 + .../environments/ci-aio/stackhpc-ci.yml | 76 + etc/kayobe/environments/ci-aio/stackhpc.yml | 3 + etc/kayobe/environments/ci-aio/tempest.yml | 3 + etc/kayobe/environments/ci-aio/time.yml | 3 + .../environments/ci-builder/globals.yml | 15 + .../pre.d/10-pulp-auth-proxy.yml | 1 + .../pre.d/10-pulp-auth-proxy.yml | 1 + .../pre.d/10-pulp-auth-proxy.yml | 1 + .../group_vars/seed/network-interfaces | 13 + .../environments/ci-builder/inventory/groups | 14 + .../environments/ci-builder/inventory/hosts | 8 + .../environments/ci-builder/kolla/.keep | 0 .../ci-builder/network-allocation.yml | 3 + .../environments/ci-builder/networks.yml | 114 + .../environments/ci-builder/overcloud-dib.yml | 20 + etc/kayobe/environments/ci-builder/seed.yml | 13 + .../environments/ci-builder/stackhpc-ci.yml | 118 + .../ci-doca-builder/.kayobe-environment | 4 + .../ci-doca-builder/controllers.yml | 15 + .../environments/ci-doca-builder/globals.yml | 6 + .../inventory/group_vars/ofed-builder/lvm.yml | 35 + .../ci-doca-builder/inventory/hosts | 3 + .../environments/ci-doca-builder/seed.yml | 15 + .../ci-doca-builder/stackhpc-ci.yml | 12 + .../environments/ci-multinode/cephadm.yml | 77 + .../environments/ci-multinode/compute.yml | 29 + .../environments/ci-multinode/controllers.yml | 30 + .../environments/ci-multinode/globals.yml | 74 + .../environments/ci-multinode/growroot.yml | 2 + .../pre.d/10-fix-hostname.yml | 1 + .../environments/ci-multinode/infra-vms.yml | 29 + .../inventory/group_vars/all/firewall.yml | 7 + .../inventory/group_vars/all/main.yml | 2 + .../inventory/group_vars/all/vxlan.yml | 18 + .../inventory/group_vars/cis-hardening/cis | 28 + .../inventory/group_vars/compute/lvm.yml | 34 + .../group_vars/compute/network-interfaces | 15 + .../inventory/group_vars/controllers/lvm.yml | 34 + .../group_vars/controllers/network-interfaces | 21 + .../inventory/group_vars/seed/lvm.yml | 34 + .../group_vars/seed/network-interfaces | 19 + .../inventory/group_vars/storage/lvm.yml | 34 + .../group_vars/storage/network-interfaces | 15 + .../group_vars/wazuh-manager/lvm.yml | 34 + .../wazuh-manager/network-interfaces | 9 + .../group_vars/wazuh-manager/wazuh-manager | 4 + .../ci-multinode/inventory/groups | 8 + .../ci-multinode/inventory/kayobe-automation | 2 + .../environments/ci-multinode/kolla.yml | 25 + .../ci-multinode/kolla/config/barbican.conf | 15 + .../kolla/config/haproxy/services.d/vault.cfg | 21 + .../ci-multinode/kolla/config/ironic.conf | 2 + .../kolla/config/keepalived/keepalived.conf | 48 + .../ci-multinode/kolla/config/magnum.conf | 2 + .../ci-multinode/kolla/config/neutron.conf | 3 + .../kolla/config/neutron/ml2_conf.ini | 3 + .../ci-multinode/kolla/globals.yml | 69 + .../environments/ci-multinode/monitoring.yml | 23 + .../environments/ci-multinode/networks.yml | 160 + .../environments/ci-multinode/neutron.yml | 12 + .../environments/ci-multinode/secrets.yml | 10 + .../ci-multinode/seed-hypervisor.yml | 23 + etc/kayobe/environments/ci-multinode/seed.yml | 52 + .../environments/ci-multinode/stackhpc-ci.yml | 67 + .../ci-multinode/stackhpc-monitoring.yml | 3 + .../environments/ci-multinode/stackhpc.yml | 3 + .../environments/ci-multinode/storage.yml | 29 + .../environments/ci-multinode/tempest.yml | 6 + etc/kayobe/globals.yml | 4 +- .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../post.d/99-cis.yml | 4 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../pre.d/pci-passthrough.yml | 1 + .../post.d/10-fix-grub-rl9.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../overcloud-service-deploy/post.d/.gitkeep | 0 .../post.d/deploy-os-capacity-exporter.yml | 1 + .../post.d/deploy-radosgw-usage-exporter.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../post.d/deploy-os-capacity-exporter.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + .../pre.d/check-kayobe-version.yml | 1 + .../pre.d/check-kolla-ansible-version.yml | 1 + etc/kayobe/infra-vms.yml | 15 +- etc/kayobe/inventory/group_vars/all/growroot | 3 + .../inventory/group_vars/all/openbao.yml | 82 + etc/kayobe/inventory/group_vars/all/selinux | 5 + etc/kayobe/inventory/group_vars/all/stackhpc | 6 + .../inventory/group_vars/all/stackhpc-lvm | 150 + .../inventory/group_vars/all/stackhpc-repos | 6 + etc/kayobe/inventory/group_vars/all/vault | 6 + etc/kayobe/inventory/group_vars/all/wazuh | 3 + .../inventory/group_vars/cis-hardening/cis | 313 + .../inventory/group_vars/controllers/cis | 4 + .../group_vars/github-runners/runners.yml | 38 + .../group_vars/github-writer/writer.yml | 8 + .../group_vars/gitlab-runners/runners.yml | 9 + .../group_vars/gitlab-writer/writer.yml | 14 + .../group_vars/overcloud/stackhpc-repos | 7 + .../group_vars/wazuh-agent/wazuh-agent | 26 + .../group_vars/wazuh-manager/wazuh-manager | 147 + etc/kayobe/inventory/groups | 37 + etc/kayobe/ipa.yml | 23 +- etc/kayobe/kolla-image-tags.yml | 89 + etc/kayobe/kolla.yml | 429 +- etc/kayobe/kolla/config/bifrost/bifrost.yml | 19 + .../kolla/config/fluentd/output/00-local.conf | 85 + .../config/fluentd/output/03-opensearch.conf | 51 + etc/kayobe/kolla/config/grafana.ini | 2 + .../grafana/dashboards/ceph/ceph_mds.json | 354 + .../grafana/dashboards/ceph/ceph_osds.json | 1471 + .../dashboards/ceph/ceph_overview.json | 3341 +++ .../grafana/dashboards/ceph/ceph_pools.json | 962 + .../dashboards/openstack/blackbox.json | 1162 + .../dashboards/openstack/cadvisor.json | 1506 + .../dashboards/openstack/database.json | 1128 + .../grafana/dashboards/openstack/fluentd.json | 3422 +++ .../openstack/grafana_cloud_dashboard.json | 905 + .../openstack/grafana_project_dashboard.json | 1165 + .../grafana/dashboards/openstack/haproxy.json | 5445 ++++ .../openstack/hardware_overview.json | 1549 ++ .../grafana/dashboards/openstack/libvirt.json | 2237 ++ .../dashboards/openstack/loadbalancer.json | 1003 + .../dashboards/openstack/memcached.json | 935 + .../openstack/node_exporter_full.json | 22607 ++++++++++++++++ .../openstack/node_exporter_side_by_side.json | 2043 ++ .../grafana/dashboards/openstack/nvme.json | 1217 + .../dashboards/openstack/opensearch.json | 5801 ++++ .../dashboards/openstack/prometheus.json | 4035 +++ .../openstack/prometheus_alertmanager.json | 4313 +++ .../openstack/prometheus_benchmark.json | 5356 ++++ .../dashboards/openstack/rabbitmq.json | 5895 ++++ .../grafana/dashboards/openstack/redfish.json | 3588 +++ .../grafana/grafana_home_dashboard.json | 3016 +++ .../kolla/config/grafana/provisioning.yaml | 15 + .../config/haproxy/services.d/os_capacity.cfg | 25 + .../services.d/radosgw_usage_exporter.cfg | 25 + etc/kayobe/kolla/config/ironic/policy.yaml | 7 + etc/kayobe/kolla/config/nova.conf | 2 + etc/kayobe/kolla/config/nova/nova-api.conf | 4 + .../config/nova/nova-compute-ironic.conf | 4 + .../kolla/config/nova/nova-compute.conf | 13 + .../kolla/config/nova/nova-scheduler.conf | 7 + .../kolla/config/prometheus/blackbox.rules | 79 + .../kolla/config/prometheus/cadvisor.rules | 45 + etc/kayobe/kolla/config/prometheus/ceph.rules | 207 + .../config/prometheus/elasticsearch.rules | 99 + .../kolla/config/prometheus/fluentd.rules | 13 + .../kolla/config/prometheus/haproxy.rules | 27 + .../config/prometheus/loadbalancer.rules | 35 + .../kolla/config/prometheus/mysql.rules | 81 + .../config/prometheus/notifications.tmpl | 32 + .../kolla/config/prometheus/openstack.rules | 24 + ...metheus-alertmanager.msteamsv2.yml.example | 29 + .../prometheus-alertmanager.yml.example | 47 + .../prometheus-blackbox-exporter.yml | 56 + .../kolla/config/prometheus/prometheus.rules | 234 + .../prometheus.yml.d/60-redfish.yml | 27 + .../prometheus.yml.d/70-oscapacity.yml | 18 + .../prometheus.yml.d/80-radosgw-exporter.yml | 22 + .../kolla/config/prometheus/rabbitmq.rules | 80 + .../kolla/config/prometheus/smart.rules | 32 + .../kolla/config/prometheus/system.rules | 169 + etc/kayobe/kolla/globals.yml | 69 +- .../kolla/inventory/group_vars/prometheus | 210 + etc/kayobe/kolla/kolla-build.conf | 35 + etc/kayobe/monitoring.yml | 10 +- etc/kayobe/networks.yml | 6 + etc/kayobe/ofed.yml | 35 + etc/kayobe/openbao.yml | 47 + etc/kayobe/openstack.yml | 6 +- etc/kayobe/overcloud-dib.yml | 11 +- etc/kayobe/pip.yml | 2 +- etc/kayobe/proxy.yml | 5 +- etc/kayobe/pulp-host-image-versions.yml | 6 + etc/kayobe/pulp-ipa-image-versions.yml | 5 + etc/kayobe/pulp-repo-versions.yml | 104 + etc/kayobe/pulp.yml | 789 + etc/kayobe/seed-hypervisor.yml | 10 +- etc/kayobe/seed-vm.yml | 6 +- etc/kayobe/seed.yml | 80 +- etc/kayobe/stackhpc-compute.yml | 103 + etc/kayobe/stackhpc-ipa-images.yml | 21 + etc/kayobe/stackhpc-monitoring.yml | 89 + etc/kayobe/stackhpc-overcloud-dib.yml | 236 + etc/kayobe/stackhpc-overcloud-host-images.yml | 19 + etc/kayobe/stackhpc.yml | 219 + etc/kayobe/storage.yml | 13 +- etc/kayobe/time.yml | 14 + etc/kayobe/trivy/allowed-vulnerabilities.yml | 98 + etc/kayobe/vault.yml | 59 + releasenotes/config.yaml | 5 + ...11.1-opensearch-bump-11fc566cd57a5f39.yaml | 4 + ...d-containers-q2-2024-02ed375b9c73ac75.yaml | 45 + ...anager-config-server-5423b1ba9e3bc165.yaml | 7 + .../notes/OSSA-2024-005-74fc3c536cde645f.yaml | 7 + ...or-swap-availability-75e28ed7f913d1ec.yaml | 13 + .../notes/add-blazar-fa6ecce8b21c73b4.yaml | 7 + .../notes/add-caso-f36b98453be10169.yaml | 8 + ...r-exporter-endpoints-f7d3c8b1131f6982.yaml | 8 + ...prompt-to-reboot.yml-4fd1ae8e8d360e57.yaml | 6 + .../add-disk-utils-58b871d9afc9cbe8.yaml | 4 + ...ndly-interface-names-d95bba2f9af28302.yaml | 6 + ...e-automation-support-33dc85bbc8746d82.yaml | 9 + .../add-grafana-plugins-f4856a30529ac686.yaml | 6 + ...-containers-playbook-5a6d3d48067cf0b6.yaml | 6 + .../add-iscsid-support-e05d5dfe76de235d.yaml | 4 + .../add-jammy-repo-yoga-f58bfb9792a3c6a7.yaml | 5 + ...sencrypt-eab-support-807bdc4078d3114d.yaml | 4 + ...itoring-to-multinode-958603f026edbc7c.yaml | 5 + ...d-more-tempest-tests-4c162a50378f9c5a.yaml | 8 + ...-ironic-failover-doc-a0c4f45b1fb48c4a.yaml | 12 + .../add-openbao-for-tls-698ae3834ed5c67f.yaml | 12 + .../add-openbao-raft-ha-e8d78ffe68913512.yaml | 5 + .../add-opensearch-bdc4a9d230ffef40.yaml | 4 + ...dd-post-reboot-delay-2ac3fa99badf4601.yaml | 6 + ...add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml | 8 + ...-4-1-support-caracal-ddb7271c83b75897.yaml | 8 + ...age-exporter-support-93d55c544418b05a.yaml | 9 + .../add-rekey-playbook-0065c5057b1639f8.yaml | 5 + ...add-rocky-9-packages-43074cb3367171a5.yaml | 7 + .../add-rocky-9.2-repos-eff5e5b0b07111f5.yaml | 5 + .../add-rocky-9.3-repos-454c2f825e088417.yaml | 9 + .../add-rocky-94-repos-27caa21c77252866.yaml | 9 + ...-stack-user-for-pulp-c96041e82c13aa10.yaml | 6 + ...oble-support-caracal-1b9e64d4aa2e1ff7.yaml | 5 + ...vm-overcloud-support-92baa7c2ce87ac1d.yaml | 7 + .../notes/add_wazuh-be2999448db5cb7c.yaml | 5 + ...t-for-active-routers-91281912213692c9.yaml | 5 + ...ra-utility-playbooks-21f7c523f4960f89.yaml | 6 + ...ing-for-ubuntu-jammy-d9bf23a34c08f5be.yaml | 5 + .../notes/adds-cis-hook-8cec8d42103d075e.yaml | 7 + ...nfig-merging-options-6f75d63b4b784d7f.yaml | 19 + .../adds-mdraid-alerts-309fb79e61389325.yaml | 4 + ...tch-batching-support-adffe038ea2441d0.yaml | 9 + ...adds-networking-mlnx-05fdc10ea6132145.yaml | 6 + ...port-for-trunk-ports-7301a258dca0c469.yaml | 13 + .../aio-lvm-script-7de0c919f312040b.yaml | 6 + .../notes/aio-script-8221b9dcbcc40c55.yaml | 5 + ...w-admins-to-list-bms-ff0bc40d605d4ca2.yaml | 7 + ...nsible-profile-tasks-3d341727a39dadcb.yaml | 5 + .../ansible-tuning-142bbcf4f646e6e0.yaml | 6 + .../automated-firewalld-a95e7322fd457259.yaml | 7 + .../backend-tls-network-aa7b09008a2e1914.yaml | 6 + ...disable-ironic-debug-5427c6f1160779aa.yaml | 5 + ...bifrost-fix-ipa-auth-9d0e0c3b948b5850.yaml | 5 + ...update-discovery-fix-220d745e22d6fbe0.yaml | 7 + .../blackbox-backend-4415919e10a1aa4e.yaml | 14 + .../notes/bug-2019977-456bbf34fed3f090.yaml | 6 + .../bugfix-smart-rules-53ff5400e7f681c8.yaml | 4 + .../bump-alertmanager-958f90fa2bc9b562.yaml | 5 + .../notes/bump-all-tags-58e916ff669d87e2.yaml | 5 + ...e-collection-cephadm-2a6c988a34b192a6.yaml | 4 + .../bump-bifrost-xena-c0c65bd823ed15fa.yaml | 7 + ...snapshots-2023-09-04-a473edfd3f3b2298.yaml | 18 + .../bump-ceph-images-f9d4721aa1d37729.yaml | 4 + .../notes/bump-focal-7d99ewj84r7h70sq1.yaml | 4 + ...e-to-fix-bug-2086675-303542d3208103b9.yaml | 5 + ...ump-hashicorp-images-2c23873ad5eeb719.yaml | 4 + ...ump-hashicorp-images-7ef66e1328983bbe.yaml | 4 + .../notes/bump-horizon-694d426decbf7df3.yaml | 5 + ...loudkitty-dashboards-e94f706d5a993cc9.yaml | 6 + .../notes/bump-horizon-dae0cd4cfffe7d7e.yaml | 5 + ...stone-image-ec2token-98caa61ccf86c944.yaml | 6 + .../notes/bump-magnum-51e03a61ae8aa5a4.yaml | 3 + ...ump-magnum-capi-helm-6723d89456e6a590.yaml | 4 + ...ump-magnum-capi-helm-6febfe840e81cea5.yaml | 4 + ...ump-magnum-capi-helm-7e4ad37d3d9eecce.yaml | 4 + ...ump-magnum-capi-helm-d766b5956de65d31.yaml | 4 + .../notes/bump-nova-ede0cd4cfffe7d4e.yaml | 6 + .../notes/bump-octavia-273fb50e05836f21.yaml | 7 + ...vs-ovn-rocky9-2024.1-eb5a42d9f890d3cd.yaml | 13 + .../bump-qemu-kvm-pdcm-e890d516f1efc690.yaml | 6 + .../notes/bump-rocky-9-90412343a29bc45b.yaml | 4 + ...snapshots-2023-09-29-e115427edd3334c7.yaml | 7 + ...snapshots-2023-09-29-c736c3d37afd7e5c.yaml | 7 + ...snapshots-2023-09-15-22ca5250d40bd5b6.yaml | 8 + ...dfish-exporter-to-v2-11032fb9dde36283.yaml | 15 + .../cap-pulp-workers-e0f12c0b67d3d0bf.yaml | 4 + .../notes/capi-bump-78cd6f89a0eff0d7.yaml | 7 + .../notes/capi-changes-3275e667680af8e1.yaml | 9 + ...ump-container-images-083b691e83816b3b.yaml | 7 + .../ceph-config-append-1cc6146d3241b63e.yaml | 6 + .../ceph-ingress-images-e756bde5087460ad.yaml | 6 + .../ceph-maintenance-4c4eb0a4f7149665.yaml | 15 + ...ceph-pacific-16.2.11-a675ccb4e4d70de7.yaml | 4 + .../notes/ceph-quincy-e110f3ba270c3a52.yaml | 4 + .../ceph-reef-18.2.7-9d8d3ad3a1ea517b.yaml | 4 + .../notes/ceph-reef-f447dde8c397f91b.yaml | 9 + .../ceph-squid-19.2.3-35c1a6b914d31388.yaml | 4 + .../notes/cephadm-bump-294148e93121da48.yaml | 4 + ...ds-now-have-defaults-b42ce9b13fa6d63b.yaml | 8 + .../cinder-bug-1823445-5d07997488bc2416.yaml | 6 + ...-do-not-disable-ipv6-98bd79ad86555f51.yaml | 12 + ...-permissions-on-logs-81a48ab8ed2d6b5f.yaml | 8 + ...x-es-to-os-migration-b0efd2626f59d977.yaml | 5 + ...ainer-image-scanning-e5adf2c6b540b502.yaml | 6 + .../cve-2024-32498-2cbd205129351766.yaml | 7 + ...deduplicate-pulp-yml-a65b3e949662a284.yaml | 3 + .../dell-sonic-support-a1dd2edd958bbe9d.yaml | 4 + .../notes/diagnostics-378a6693a64d0b3c.yaml | 14 + .../notes/disable-heat-03f3ac52ac610223.yaml | 10 + ...-prometheus-exporter-894d98022a1e926d.yaml | 7 + ...-prometheus-exporter-daf2dda149466a2e.yaml | 9 + ...overcloud-apt-config-5b79b5b63c78779c.yaml | 6 + ...able-password-expiry-3c296c72d36cd9da.yaml | 7 + .../notes/doca-2-9-3-238838fb78e0c7d9.yaml | 5 + .../notes/doca-workflow-017931dbfd580b41.yaml | 16 + .../notes/docker-bump-dc4e85668ee69a21.yaml | 11 + .../docker-live-restore-6b6c55455202422c.yaml | 5 + ...drop-skyline-support-31d683f58f125335.yaml | 4 + releasenotes/notes/dwpd-6b9fb0c8d6d3a570.yaml | 10 + .../notes/elrepo-9-9d88af1dea541155.yaml | 6 + ...d-of-neutron-dragent-14897d9744c7de1f.yaml | 4 + .../enable-mariabackup-97834ea8d87e14a9.yaml | 4 + ...ate-rabbitmq-version-8eaa68b2a9486320.yaml | 11 + .../notes/enable_ovn-b84974f398242dcb.yaml | 10 + ...-aarch64-repo-change-a77c49c408413857.yaml | 7 + releasenotes/notes/etcd-e4927551bf5d020c.yaml | 7 + ...l-unparsed-inventory-c3b4e2ffcb620a6b.yaml | 7 + .../feature-smartmon-65cacfe893f0eb47.yaml | 8 + ....1-kolla-image-build-f78a5524381fa4da.yaml | 9 + ...ot-config-templating-0488dd14d15d65e4.yaml | 7 + ...-no-external-grafana-de1db02c540af6d8.yaml | 6 + ...opensearch-dashboard-d973b43c557c8103.yaml | 5 + .../fix-broken-links-dcd128c8e211b2b8.yaml | 4 + ...-capacity-used-panel-26d495c45f2678c8.yaml | 6 + ...gw-blackbox-endpoint-7af02679b3fd093d.yaml | 6 + .../fix-cephadm-facts-2ee6dc9a1c617944.yaml | 6 + ...y-opensearch-backend-743501558e69e712.yaml | 5 + .../fix-cve-2024-40767-24b9b3c35f61a0c8.yaml | 6 + .../fix-cve-2024-44082-122ef225f674d864.yaml | 12 + .../fix-cve-2026-33551-e37bc84ad753d740.yaml | 5 + ...x-fix-houston-notify-165a7871bda48f68.yaml | 5 + ...-opensearch-template-14bde9b6d5a723f8.yaml | 5 + ...ix-growroot-playbook-6a8ee02d7c0fbcb3.yaml | 6 + ...rview-dashboard-temp-ecc047ba46305668.yaml | 7 + ...-horizon-bug-2055784-3bb729919d345e77.yaml | 7 + ...ix-houston-tc-mirred-bfb16c89f63b472a.yaml | 12 + .../fix-ipa-collectors-f7afc852d5005282.yaml | 5 + ...xporter-scrape-group-b10eaac6ee1e6af3.yaml | 6 + ...ayobe-version-checks-d1fb3e09391e4a3e.yaml | 5 + ...ystone-multiple-cves-cdb096a2fbb94bba.yaml | 5 + ...hen-neutron-restarts-6aafd03d115a1ee4.yaml | 6 + ...liblasso-netiq-issue-afec5b2ee7de2a1e.yaml | 5 + ...ix-libvirt-dashboard-b33327f804bea5f9.yaml | 9 + ...ix-local-image-build-59967df0a2bceddf.yaml | 26 + ...nternet-connectivity-49ab64cd75833f09.yaml | 5 + ...on-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml | 6 + ...epalived-pid-cleanup-5faae4cf78b07f89.yaml | 6 + ...ls-terminated-pkcs12-4f7e32a6f5ca0143.yaml | 5 + ...-openbao-include-tag-dfef2a0e731674f0.yaml | 4 + ...opensearch-heap-size-0cd31054927d4bbd.yaml | 4 + ...zimuth-cloud-metrics-39cf81eebe16b879.yaml | 6 + .../fix-ossa-2024-004-f732e58c12e26785.yaml | 6 + .../fix-ossa-2026-002-65bd525d6aee93cc.yaml | 6 + ...ram-vm-amd-iommu-rl9-6ddb3d7c030a1674.yaml | 5 + ...-ovn-log-permissions-4b067cc1b1a7c3b8.yaml | 6 + ...pci-default-template-8660ab2a7a106376.yaml | 4 + ...er-duplicate-metrics-ca96b3e4b9d6afe9.yaml | 6 + ...g-802-3ad-port-group-cd48d2e6f43b363f.yaml | 9 + ...-metrics-permissions-8b1183de434fafcc.yaml | 5 + .../fix-smartmon-script-db2a38df1b245e58.yaml | 9 + .../fix-squid-issue-11b5f03719ab8b45.yaml | 5 + ...overcloud-host-image-b395c8afb6d5b820.yaml | 7 + ...-vault-standby-alert-501b743a40971926.yaml | 8 + ...-custom-sca-policies-f16c7f0eae8265a5.yaml | 5 + ...s-InstanceDown-Alert-570a295e3d5006f7.yaml | 7 + ...ith-custom-placement-e98f1621dc5c873d.yaml | 5 + ...file-descriptor-leak-3d5c9340e5983e9c.yaml | 8 + ...ot-for-software-raid-3852bdea5415a0be.yaml | 5 + ...ild-api-version-2.93-4fb297328d55bff8.yaml | 7 + ...es-mdev-uuid-parsing-4c9e972c3c424760.yaml | 6 + ...xes-osd-size-summary-9924ef4aac61d2b6.yaml | 4 + ...ffer-too-large-alert-682893410855387f.yaml | 5 + .../generic-blackbox-588859073c7441ca.yaml | 4 + ...ultipart-bug-2043740-e7a122a6453c45e4.yaml | 5 + ...earch-cluster-health-154275e8d39dd89f.yaml | 5 + ...-openstack-dashboard-e99a6a44e927363f.yaml | 3 + ...move-angular-plugins-1c10e83ddb6556f4.yaml | 6 + .../grub_timeout_values-c39b8b1a2c34a602.yaml | 8 + ...board-instance-label-836b93921e964680.yaml | 5 + ...ashicorp-sync-images-96dc726e85323104.yaml | 5 + .../notes/hcp-2.5.0-8e30c7b1910f2bd2.yaml | 5 + ...fix-multiple-domains-51086e9ba3d197b7.yaml | 6 + ...age-build-sms-upload-4829088864b15c5b.yaml | 6 + .../host-image-rebuild-957b0a69de1044fc.yaml | 5 + ...ph-pgs-unclean-alert-98306d397344b572.yaml | 6 + ...ncrease-pulp-retries-79cc258da9aabb4f.yaml | 5 + .../increase-tls-ttl-c1eba5cca7767d0f.yaml | 6 + .../infra-vms-ntp-group-45ade04fe885c3f8.yaml | 5 + .../ipa-build-workflow-cde24f3b9e3e0667.yaml | 8 + ...-inspection-settings-133fe91b1d855fa0.yaml | 5 + ..._cipher_suite_errors-c8915956be17a28c.yaml | 6 + ...-fix-online-upgrades-36e86b3ab3ac081c.yaml | 6 + ...tor-blackbox-backend-ff1bd68297435a36.yaml | 5 + ...patch-cve-2024-44082-87c70b333bc28ea0.yaml | 5 + ...-prometheus-exporter-f9301ef233d47291.yaml | 6 + .../notes/kayobe-image-f1227adacb28f40b.yaml | 8 + ...-bump-cve-2024-36039-07f18e18b5c86980.yaml | 7 + ...-dependency-workflow-6ff5520ee0ab8e15.yaml | 5 + ...kayobe-version-check-f44d43c9c34d1b89.yaml | 13 + .../kolla-local-builds-eca0277a969803fd.yaml | 7 + ...olla-rocky9.3-images-a3ff235485093191.yaml | 4 + .../kolla-update-fixes-5ff55225ce85441f.yaml | 4 + .../local-pulp-tls-a8e7464d8cb0d114.yaml | 6 + ...-openstack-dashboard-6e345ff7f16c0658.yaml | 5 + ...w-memory-alert-units-a6fde380ff9b7839.yaml | 4 + releasenotes/notes/lvm-3c785770722f768a.yaml | 5 + .../magnum-bump-tags-42u298jfj93r3ok32b.yaml | 4 + .../magnum-bump-tags-7841edcbd3da4b9c.yaml | 10 + .../magnum-capi-1-1-0-68f14759413316c4.yaml | 6 + .../magnum-capi-driver-2dd8cf6e68a1919c.yaml | 6 + ...num-container-footer-dcf6dca0667bcb94.yaml | 4 + .../notes/magnum-pin-958b39182e565697.yaml | 6 + ...m-remove-cert-append-8797b640f25644ea.yaml | 5 + ...manila-for-multinode-df2186276c6b434a.yaml | 8 + ...nbao-vault-playbooks-0c78364f4a86f6e0.yaml | 8 + .../moved-wazuh-certs-b5173d264f543b14.yaml | 15 + ...h-manifests-pipefail-9b860f83d0f93f1e.yaml | 6 + ...ultinode-magnum-docs-00607f139dc2b53e.yaml | 5 + .../multinode-wazuh-c9edd65ab5cdbcb8.yaml | 4 + ...-bond-degraded-alert-d2a0b05002609ac1.yaml | 5 + ...ork-bond-single-link-766adf41a3c2fd4e.yaml | 8 + ...ners-built-from-fork-e9f3c287b694fa8b.yaml | 4 + ...dadfailed-workaround-9cd82d29ee1fba59.yaml | 5 + .../node-exporter-1.5.0-5e3764790a5a0190.yaml | 9 + .../nova-bug-2091033-4f54893f67af5c7c.yaml | 7 + ...ova-machine-type-q35-ab363c5f7f3f9807.yaml | 5 + ...a-pci-request-resize-a429c115ba2818e8.yaml | 4 + .../nova-uefi-edk2-ovmf-d636c299e76728e8.yaml | 4 + ...g-noswitchdev-needed-79aaa276bc799923.yaml | 18 + .../notes/ntp-alerts-0d110b4979457165.yaml | 4 + .../nvmemon-update-a326f1b46d95b037.yaml | 4 + .../octavia-alerts-a7f1fe041f5450be.yaml | 6 + .../octavia-dashboard-a5f904b20b8423f5.yaml | 5 + ...nc-healthmonitor-bug-8602034e2185d12c.yaml | 10 + .../notes/ofed-workflow-119225f36a9265c3.yaml | 6 + .../openstack-dash-ff7bf179e37c8e4b.yaml | 5 + ...ck-exporter-nova-api-fa5c2a9663bc97e2.yaml | 5 + .../notes/os-capacity-94006f03f16583e4.yaml | 28 + .../os-capacity-cacert-8b800b22d84ae0b1.yaml | 4 + ...capacity-deploy-hook-b52e87c0819df6fd.yaml | 9 + ...-usage-in-dashboards-597ed10af4fc1aa1.yaml | 9 + .../notes/overcloud-dib-938eac005662a2cb.yaml | 4 + ...dib-ethtool-pciutils-5b3404b146c9f2a0.yaml | 4 + .../notes/ovn-24-03-47c1eb9846f261b0.yaml | 5 + ...chassis-leader-check-551d0e94cbb94ac4.yaml | 6 + ...provider-bug-2049551-c95d69965733cd78.yaml | 5 + ...provider-bug-2072754-5cbd7dc9c366668d.yaml | 6 + .../ovn-sb-db-relay-81f57bc173299913.yaml | 7 + .../notes/ovn_pkg_up-cfb09b7fd17ea786.yaml | 4 + .../ovs-cve-2023-1668-298a2fc907fc07b9.yaml | 9 + ...-passthrough-support-0c7e62585aaf2c23.yaml | 8 + .../notes/pin-baseimage-3d224df5b6f56b10.yaml | 7 + .../notes/pin-deps-dbe52c49e945daf5.yaml | 6 + .../pin-pulp-squeezer-bd3a3c53d9804010.yaml | 5 + ...heus-hostname-labels-82346cb0914680e9.yaml | 4 + ...stackhpc-host-images-a623e4ab2d2a9e5b.yaml | 6 + .../pulp-auth-proxy-24f0b31a4498441b.yaml | 7 + ...pulp-container-extra-9379806192900d22.yaml | 6 + ...ish-promote-playbook-b638be7c77a649dd.yaml | 7 + ...bbimq-too-many-ready-c18ecea9a13c8f2a.yaml | 5 + ...tmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml | 7 + ...tmq-connection-alert-85cc7b29ddf8e3c3.yaml | 5 + ...eue-migration-script-b6d3abebbebf8087.yaml | 6 + ...orter-behind-haproxy-e4371d2732f2a081.yaml | 12 + .../notes/rated-dwpd-40526e85e24ef7ea.yaml | 8 + ...le-pulp-ubuntu-repos-d46ff8eca4a5aac3.yaml | 4 + ...hedocs-configuration-17cf3e6a63f571fa.yaml | 4 + ...eboot-default-serial-5944a2a648da71c7.yaml | 6 + ...heat-with-yaql-3.0.0-4415d8232bc547df.yaml | 7 + ...board-fix-inlet-temp-91a7018adb2e1763.yaml | 5 + .../redfish-exporter-36cbdabf103ca93d.yaml | 7 + ...redfish-uuid-grafana-2f50c76bbff6eba9.yaml | 6 + ...ce-seed-pulp-workers-b085c86bfe7ab5b6.yaml | 6 + ...edfish-target-alerts-a3d77a3f0c3dac8a.yaml | 6 + ...factor-tag-overrides-1ad66680fd801345.yaml | 6 + ...ron-bgp-dragent-tags-39b9c2595ae11872.yaml | 4 + ...2023-package-refresh-9de8049f8cb08a5e.yaml | 25 + ...ckage-refresh-ubuntu-a9fe8a1c3c2f2796.yaml | 20 + ...remove-swap-playbook-f62a41945e29816d.yaml | 7 + ...ve-tmout-for-dev-env-0778550b353dce03.yaml | 6 + ...nerVolumeUsage-alert-c973b61f598d85e5.yaml | 5 + .../reset-bls-entries-b2bded62c5887937.yaml | 7 + ...estore-kolla-mirrors-9c0c01735d7f43df.yaml | 6 + ...rfaces-in-keepalived-41ef4834b7659af4.yaml | 7 + ...ter-deployment-fixes-0196167326dbe456.yaml | 8 + ...e-minimal-cloud-init-83576172be8febca.yaml | 7 + ...common-cve-2024-6409-d36de799a29c3f74.yaml | 6 + .../notes/rl9.6-f46ea1230214a657.yaml | 8 + .../notes/rl9.7-c8d033f11097b76b.yaml | 11 + releasenotes/notes/rl95-6cbdf902e30502bf.yaml | 7 + .../notes/rmq-dashboard-5bb5eb36ff72100c.yaml | 5 + .../notes/rocky-87-rpms-999565263dacde42.yaml | 4 + .../notes/rocky-crb-b6a6cecfdf4d93a4.yaml | 4 + ...urity-common-openssh-6fbd5a1e95fd66ae.yaml | 6 + .../selinux-host-image-ff0fed2583cae7b0.yaml | 5 + .../notes/smart-mon-db8fa642c3af74b1.yaml | 3 + .../notes/smartmontools-bc8176f45d58a75d.yaml | 6 + ...dm-commands-pre-post-bddd25b8bd171ab0.yaml | 8 + ...khpc-openstack-tests-0224908453ae72cc.yaml | 8 + ...ackhpc-repos-enabled-10c8a698991e53c2.yaml | 7 + ...op-services-playbook-b85b53d1a7571009.yaml | 5 + .../tempest-cacert-33e2ae1cf0ba88cf.yaml | 4 + .../tempest-concurrency-1c86c0ef90e104f6.yaml | 5 + ...tu-fix-cve-2024-6387-648efedaeb288023.yaml | 5 + ...tu-image-early-repos-08b4da6165f5fa11.yaml | 6 + .../notes/ubuntu-jammy-cd109d12954d7626.yaml | 8 + ...ntu-refresh-december-998f4c2d2bd0032b.yaml | 5 + .../update-blazar-image-d176c27d55716469.yaml | 5 + .../update-cis-roles-8c73975c37188071.yaml | 9 + ...date-grafana-version-ed01e7bd04782ff7.yaml | 4 + ...proxy-alerting-rules-5594547666659c0f.yaml | 5 + ...te-haproxy-dashboard-b8807c2a0645ca5e.yaml | 6 + ...pdate-hcp-collection-39acaafe76c95b3f.yaml | 4 + ...te-kayobe-automation-55fc9c5c380d819c.yaml | 23 + ...e-magnum-capi-driver-c85135340731a86a.yaml | 6 + ...ate-magnum-capi-helm-129ea84a8903eb10.yaml | 4 + ...te-magnum-containers-8814b1425ec45fc0.yaml | 5 + ...vs-ovn-rocky9-2024.1-a1025a7130d9c647.yaml | 12 + .../update-prometheus-3519aa260d5f24ef.yaml | 4 + .../update-pulp-3.22-aa485b7e619cd380.yaml | 7 + .../update-pulp-3.23-b4193a417a219d26.yaml | 3 + .../update-pulp-3.24-d770db48deaf6076.yaml | 3 + ...te-smartmon-playbook-6bdf9624c579d2e2.yaml | 5 + .../update-ubuntu-repos-4fd1b3083ee7b245.yaml | 4 + ...tags-for-client-pulp-cf46d328b30162be.yaml | 6 + ...bitmq-node-down-rule-c9e9c6b09f57954d.yaml | 8 + ...ault-without-haproxy-db55707e1cc53896.yaml | 8 + .../notes/wallaby-c3c55c39c7b118f2.yaml | 25 + ...azuh-ansible-v4.10.0-ed5209199194cddf.yaml | 5 + ...-custom-sca-policies-a5f15818948928b0.yaml | 6 + .../notes/xena-batch-bc1da4e4d0f6257e.yaml | 21 + releasenotes/requirements.txt | 4 + releasenotes/source/2023.1.rst | 6 + releasenotes/source/2024.1.rst | 6 + releasenotes/source/_static/.placeholder | 0 releasenotes/source/_templates/.placeholder | 0 releasenotes/source/conf.py | 225 + releasenotes/source/index.rst | 22 + releasenotes/source/wallaby.rst | 6 + releasenotes/source/xena.rst | 6 + releasenotes/source/yoga.rst | 6 + releasenotes/source/zed.rst | 6 + requirements.txt | 2 + terraform/.gitignore | 2 + terraform/aio/.terraform.lock.hcl | 23 + terraform/aio/README.rst | 191 + terraform/aio/outputs.tf | 15 + terraform/aio/provider.tf | 14 + terraform/aio/templates/userdata.cfg.tpl | 17 + terraform/aio/vm.tf | 99 + test-requirements.txt | 2 + tools/diagnostics.sh | 124 + tools/kolla-images.py | 431 + tools/loadbalancer-failover.sh | 18 + tools/merge.py | 116 + tools/multiarch-manifests.sh | 37 + tools/rabbitmq-queue-migration.sh | 77 + tools/scan-images.sh | 213 + tools/ubuntu-upgrade-infra-vm.sh | 35 + tools/ubuntu-upgrade-overcloud.sh | 38 + tools/ubuntu-upgrade-seed-hypervisor.sh | 30 + tools/ubuntu-upgrade-seed.sh | 30 + tools/upgrade-prerequisites.sh | 53 + tox.ini | 25 +- 862 files changed, 129835 insertions(+), 139 deletions(-) create mode 100644 .ansible-lint create mode 100644 .ansible-lint-ignore create mode 160000 .automation create mode 100644 .automation.conf/config.sh create mode 100755 .automation.conf/run-books/pulp-sync-content.sh create mode 100644 .automation.conf/tempest/load-lists/barbican create mode 100644 .automation.conf/tempest/load-lists/cinder create mode 100644 .automation.conf/tempest/load-lists/cloudkitty create mode 120000 .automation.conf/tempest/load-lists/default create mode 100644 .automation.conf/tempest/load-lists/designate create mode 100644 .automation.conf/tempest/load-lists/glance create mode 100644 .automation.conf/tempest/load-lists/ironic create mode 100644 .automation.conf/tempest/load-lists/keystone create mode 100644 .automation.conf/tempest/load-lists/magnum create mode 100644 .automation.conf/tempest/load-lists/manila create mode 100644 .automation.conf/tempest/load-lists/neutron create mode 100644 .automation.conf/tempest/load-lists/octavia create mode 100644 .automation.conf/tempest/load-lists/octavia-full create mode 100644 .automation.conf/tempest/load-lists/platform.2022.11-test-list.txt create mode 100644 .automation.conf/tempest/load-lists/tempest-full create mode 100644 .automation.conf/tempest/skip-lists/ci-multinode-tempest-full create mode 100644 .automation.conf/tempest/tempest-ci-aio.overrides.conf create mode 100644 .automation.conf/tempest/tempest-ci-multinode.overrides.conf create mode 100644 .github/CODEOWNERS create mode 100644 .github/auto-label.yaml create mode 100644 .github/cherry-pick-bot.yml create mode 100644 .github/path-filters.yml create mode 100644 .github/workflows/amphora-image-build.yml create mode 100644 .github/workflows/amphora-image-promote.yml create mode 100644 .github/workflows/ipa-image-build.yml create mode 100644 .github/workflows/ipa-image-promote.yml create mode 100644 .github/workflows/multinode-inputs.py create mode 100644 .github/workflows/overcloud-host-image-build.yml create mode 100644 .github/workflows/overcloud-host-image-promote.yml create mode 100644 .github/workflows/overcloud-host-image-upload.yml create mode 100644 .github/workflows/package-build-ofed.yml create mode 100644 .github/workflows/runner-selector.yml create mode 100644 .github/workflows/stackhpc-all-in-one.yml create mode 100644 .github/workflows/stackhpc-build-kayobe-image.yml create mode 100644 .github/workflows/stackhpc-check-tags.yml create mode 100644 .github/workflows/stackhpc-ci-cleanup.yml create mode 100644 .github/workflows/stackhpc-container-image-build.yml create mode 100644 .github/workflows/stackhpc-multinode-periodic.yml create mode 100644 .github/workflows/stackhpc-multinode.yml create mode 100644 .github/workflows/stackhpc-promote.yml create mode 100644 .github/workflows/stackhpc-pull-request.yml create mode 100644 .github/workflows/stackhpc-update-kolla.yml create mode 100644 .github/workflows/tag-and-release.yml create mode 100644 .github/workflows/update-dependencies.yml create mode 100644 .github/workflows/upstream-sync.yml create mode 100644 .gitmodules create mode 100644 .pre-commit-config.yaml create mode 100644 .readthedocs.yaml create mode 100644 doc/requirements.txt create mode 100644 doc/source/_static/images/capi-architecture-diagram.png create mode 100644 doc/source/_static/images/release-train.svg create mode 100644 doc/source/conf.py create mode 100644 doc/source/configuration/cephadm.rst create mode 100644 doc/source/configuration/ci-cd.rst create mode 100644 doc/source/configuration/cloudkitty.rst create mode 100644 doc/source/configuration/firewall.rst create mode 100644 doc/source/configuration/host-images.rst create mode 100644 doc/source/configuration/index.rst create mode 100644 doc/source/configuration/ipa.rst create mode 100644 doc/source/configuration/lvm.rst create mode 100644 doc/source/configuration/magnum-capi.rst create mode 100644 doc/source/configuration/monitoring.rst create mode 100644 doc/source/configuration/openbao.rst create mode 100644 doc/source/configuration/release-train.rst create mode 100644 doc/source/configuration/security-hardening.rst create mode 100644 doc/source/configuration/vault.rst create mode 100644 doc/source/configuration/walled-garden.rst create mode 100644 doc/source/configuration/wazuh.rst create mode 100644 doc/source/contributor/documentation.rst create mode 100644 doc/source/contributor/environments/aufn-ceph.rst create mode 100644 doc/source/contributor/environments/ci-aio.rst create mode 100644 doc/source/contributor/environments/ci-builder.rst create mode 100644 doc/source/contributor/environments/ci-multinode.rst create mode 100644 doc/source/contributor/environments/index.rst create mode 100644 doc/source/contributor/index.rst create mode 100644 doc/source/contributor/ofed.rst create mode 100644 doc/source/contributor/package-updates.rst create mode 100644 doc/source/contributor/pre-commit.rst create mode 100644 doc/source/contributor/release-notes.rst create mode 100644 doc/source/contributor/testing-ci-automation.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/operations/bifrost-hardware-inventory-management.rst create mode 100644 doc/source/operations/ceph-management.rst create mode 100644 doc/source/operations/control-plane-operation.rst create mode 100644 doc/source/operations/customising-horizon.rst create mode 100644 doc/source/operations/gpu-in-openstack.rst create mode 100644 doc/source/operations/hotfix-playbook.rst create mode 100644 doc/source/operations/index.rst create mode 100644 doc/source/operations/migrating-vm.rst create mode 100644 doc/source/operations/nova-compute-ironic.rst create mode 100644 doc/source/operations/octavia.rst create mode 100644 doc/source/operations/openstack-projects-and-users-management.rst create mode 100644 doc/source/operations/openstack-reconfiguration.rst create mode 100644 doc/source/operations/secret-rotation.rst create mode 100644 doc/source/operations/tempest.rst create mode 100644 doc/source/operations/ubuntu-noble.rst create mode 100644 doc/source/operations/upgrading-ceph.rst create mode 100644 doc/source/operations/upgrading-openstack.rst create mode 100644 doc/source/release-notes.rst create mode 100644 doc/source/usage.rst create mode 100644 etc/kayobe/ansible/advise-run.yml create mode 100644 etc/kayobe/ansible/build-ofed-rocky.yml create mode 100644 etc/kayobe/ansible/ceph-enter-maintenance.yml create mode 100644 etc/kayobe/ansible/ceph-exit-maintenance.yml create mode 100644 etc/kayobe/ansible/check-kayobe-version.yml create mode 100644 etc/kayobe/ansible/check-kolla-ansible-version.yml create mode 100644 etc/kayobe/ansible/check-kolla-images-py.yml create mode 100644 etc/kayobe/ansible/check-tags.yml create mode 100644 etc/kayobe/ansible/cis.yml create mode 100644 etc/kayobe/ansible/configure-aio-resources.yml create mode 100644 etc/kayobe/ansible/configure-vxlan.yml create mode 100644 etc/kayobe/ansible/deploy-github-runner.yml create mode 100644 etc/kayobe/ansible/deploy-gitlab-runner.yml create mode 100644 etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml create mode 100644 etc/kayobe/ansible/deploy-os-capacity-exporter.yml create mode 100644 etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml create mode 100644 etc/kayobe/ansible/diagnostics.yml create mode 100644 etc/kayobe/ansible/docker-registry-login.yml create mode 100644 etc/kayobe/ansible/firewalld-watchdog.yml create mode 100644 etc/kayobe/ansible/fix-grub-rl9.yml create mode 100644 etc/kayobe/ansible/fix-hostname.yml create mode 100644 etc/kayobe/ansible/fix-houston.yml create mode 100644 etc/kayobe/ansible/fix-networking.yml create mode 100644 etc/kayobe/ansible/get-cloud-facts.yml create mode 100644 etc/kayobe/ansible/get-nvme-drives.yml create mode 100644 etc/kayobe/ansible/growroot.yml create mode 100644 etc/kayobe/ansible/hotfix-containers.yml create mode 100644 etc/kayobe/ansible/install-doca.yml create mode 100644 etc/kayobe/ansible/install-pre-commit-hooks.yml create mode 100644 etc/kayobe/ansible/nova-compute-disable.yml create mode 100644 etc/kayobe/ansible/nova-compute-drain.yml create mode 100644 etc/kayobe/ansible/nova-compute-enable.yml create mode 100644 etc/kayobe/ansible/octavia-amphora-image-build.yml create mode 100644 etc/kayobe/ansible/octavia-amphora-image-register.yml create mode 100644 etc/kayobe/ansible/openbao-deploy-barbican.yml create mode 100644 etc/kayobe/ansible/openbao-deploy-overcloud.yml create mode 100644 etc/kayobe/ansible/openbao-deploy-seed.yml create mode 100644 etc/kayobe/ansible/openbao-generate-backend-tls.yml create mode 100644 etc/kayobe/ansible/openbao-generate-internal-tls.yml create mode 100644 etc/kayobe/ansible/openbao-generate-test-external-tls.yml create mode 100644 etc/kayobe/ansible/openbao-unseal-overcloud.yml create mode 100644 etc/kayobe/ansible/openbao-unseal-seed.yml create mode 100644 etc/kayobe/ansible/openstack-host-image-upload.yml create mode 100644 etc/kayobe/ansible/ovn-fix-chassis-priorities.yml create mode 100644 etc/kayobe/ansible/pci-passthrough.yml create mode 100644 etc/kayobe/ansible/prometheus-network-names.yml create mode 100644 etc/kayobe/ansible/prometheus.yml.j2 create mode 100644 etc/kayobe/ansible/pulp-amphora-image-download.yml create mode 100644 etc/kayobe/ansible/pulp-artifact-promote.yml create mode 100644 etc/kayobe/ansible/pulp-artifact-upload.yml create mode 100644 etc/kayobe/ansible/pulp-auth-proxy.yml create mode 100644 etc/kayobe/ansible/pulp-container-publish.yml create mode 100644 etc/kayobe/ansible/pulp-container-sync.yml create mode 100644 etc/kayobe/ansible/pulp-host-image-download.yml create mode 100644 etc/kayobe/ansible/pulp-repo-promote-production.yml create mode 100644 etc/kayobe/ansible/pulp-repo-publish.yml create mode 100644 etc/kayobe/ansible/pulp-repo-sync.yml create mode 100644 etc/kayobe/ansible/pulp-sync-publish-promote.yml create mode 100644 etc/kayobe/ansible/purge-command-not-found.yml create mode 100644 etc/kayobe/ansible/push-ofed.yml create mode 100644 etc/kayobe/ansible/rabbitmq-reset.yml create mode 100644 etc/kayobe/ansible/reboot.yml create mode 100644 etc/kayobe/ansible/rekey-hosts.yml create mode 100644 etc/kayobe/ansible/reset-bls-entries.yml create mode 100644 etc/kayobe/ansible/rocky-97-ofed-upgrade.yml create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/README.md create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml create mode 100644 etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 create mode 100644 etc/kayobe/ansible/rsyslog.yml create mode 100644 etc/kayobe/ansible/run-container-hotfix.yml create mode 100644 etc/kayobe/ansible/scripts/aio-init.sh create mode 100644 etc/kayobe/ansible/scripts/generate_fixtures.py create mode 100644 etc/kayobe/ansible/scripts/nvmemon.sh create mode 100644 etc/kayobe/ansible/scripts/smartmon.py create mode 100644 etc/kayobe/ansible/scripts/test_smartmon.py create mode 100644 etc/kayobe/ansible/scripts/tests/Dell_ENT_NVMe_CM6.json create mode 100644 etc/kayobe/ansible/scripts/tests/nvme.json create mode 100644 etc/kayobe/ansible/secret-store-deploy-barbican.yml create mode 100644 etc/kayobe/ansible/secret-store-deploy-overcloud.yml create mode 100644 etc/kayobe/ansible/secret-store-deploy-seed.yml create mode 100644 etc/kayobe/ansible/secret-store-generate-backend-tls.yml create mode 100644 etc/kayobe/ansible/secret-store-generate-internal-tls.yml create mode 100644 etc/kayobe/ansible/secret-store-generate-test-external-tls.yml create mode 100644 etc/kayobe/ansible/secret-store-unseal-overcloud.yml create mode 100644 etc/kayobe/ansible/secret-store-unseal-seed.yml create mode 100644 etc/kayobe/ansible/smartmon-tools.yml create mode 100644 etc/kayobe/ansible/stackhpc-cloud-tests.yml create mode 100644 etc/kayobe/ansible/stop-openstack-services.yml create mode 100644 etc/kayobe/ansible/templates/fix-houston-interface.service.j2 create mode 100644 etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 create mode 100644 etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 create mode 100644 etc/kayobe/ansible/ubuntu-upgrade.yml create mode 100644 etc/kayobe/ansible/wazuh-agent.yml create mode 100644 etc/kayobe/ansible/wazuh-manager.yml create mode 100644 etc/kayobe/ansible/wazuh-secrets.yml create mode 100644 etc/kayobe/ansible/write-github-workflows.yml create mode 100644 etc/kayobe/ansible/write-gitlab-pipelines.yml create mode 100644 etc/kayobe/containers/pulp/post.yml create mode 100644 etc/kayobe/containers/pulp/pre.yml create mode 100644 etc/kayobe/containers/pulp/settings.py create mode 100644 etc/kayobe/containers/redfish_exporter/post.yml create mode 100644 etc/kayobe/containers/redfish_exporter/pre.yml create mode 100644 etc/kayobe/containers/redfish_exporter/redfish_exporter.yml create mode 100644 etc/kayobe/containers/squid_proxy/pre.yml create mode 100644 etc/kayobe/containers/squid_proxy/squid.conf create mode 100755 etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh create mode 100644 etc/kayobe/environments/aufn-ceph/cephadm.yml create mode 100755 etc/kayobe/environments/aufn-ceph/configure-local-networking.sh create mode 100755 etc/kayobe/environments/aufn-ceph/configure-openstack.sh create mode 100644 etc/kayobe/environments/aufn-ceph/globals.yml create mode 100644 etc/kayobe/environments/aufn-ceph/inventory/group_vars/compute/network-interfaces create mode 100644 etc/kayobe/environments/aufn-ceph/inventory/group_vars/controllers/network-interfaces create mode 100644 etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed-hypervisor/network-interfaces create mode 100644 etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed/network-interfaces create mode 100644 etc/kayobe/environments/aufn-ceph/inventory/group_vars/storage/network-interfaces create mode 100644 etc/kayobe/environments/aufn-ceph/inventory/groups create mode 100644 etc/kayobe/environments/aufn-ceph/inventory/hosts create mode 100644 etc/kayobe/environments/aufn-ceph/ipa.yml create mode 100644 etc/kayobe/environments/aufn-ceph/kolla.yml create mode 100644 etc/kayobe/environments/aufn-ceph/kolla/config/bifrost/bifrost.yml create mode 100644 etc/kayobe/environments/aufn-ceph/kolla/config/neutron.conf create mode 100644 etc/kayobe/environments/aufn-ceph/kolla/config/neutron/ml2_conf.ini create mode 100644 etc/kayobe/environments/aufn-ceph/kolla/globals.yml create mode 100644 etc/kayobe/environments/aufn-ceph/kolla/inventory/overcloud-services.j2 create mode 100644 etc/kayobe/environments/aufn-ceph/kolla/kolla-build.conf create mode 100644 etc/kayobe/environments/aufn-ceph/network-allocation.yml create mode 100644 etc/kayobe/environments/aufn-ceph/networks.yml create mode 100644 etc/kayobe/environments/aufn-ceph/neutron.yml create mode 100644 etc/kayobe/environments/aufn-ceph/overcloud.yml create mode 100644 etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml create mode 100644 etc/kayobe/environments/aufn-ceph/seed-vm.yml create mode 100644 etc/kayobe/environments/aufn-ceph/stackhpc.yml create mode 100644 etc/kayobe/environments/aufn-ceph/storage.yml create mode 100644 etc/kayobe/environments/aufn-ceph/tenks.yml create mode 100644 etc/kayobe/environments/ci-aio/automated-setup.sh create mode 100644 etc/kayobe/environments/ci-aio/compute.yml create mode 100644 etc/kayobe/environments/ci-aio/controllers.yml create mode 100644 etc/kayobe/environments/ci-aio/globals.yml create mode 100644 etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml create mode 100644 etc/kayobe/environments/ci-aio/inventory/group_vars/cis-hardening/cis create mode 100644 etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml create mode 100644 etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces create mode 100644 etc/kayobe/environments/ci-aio/inventory/groups create mode 100644 etc/kayobe/environments/ci-aio/inventory/hosts create mode 100644 etc/kayobe/environments/ci-aio/inventory/kayobe-automation create mode 100644 etc/kayobe/environments/ci-aio/kolla/config/ironic.conf create mode 100644 etc/kayobe/environments/ci-aio/kolla/globals.yml create mode 100644 etc/kayobe/environments/ci-aio/network-allocation.yml create mode 100644 etc/kayobe/environments/ci-aio/networks.yml create mode 100644 etc/kayobe/environments/ci-aio/stackhpc-ci.yml create mode 100644 etc/kayobe/environments/ci-aio/stackhpc.yml create mode 100644 etc/kayobe/environments/ci-aio/tempest.yml create mode 100644 etc/kayobe/environments/ci-aio/time.yml create mode 100644 etc/kayobe/environments/ci-builder/globals.yml create mode 120000 etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml create mode 120000 etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml create mode 120000 etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml create mode 100644 etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces create mode 100644 etc/kayobe/environments/ci-builder/inventory/groups create mode 100644 etc/kayobe/environments/ci-builder/inventory/hosts create mode 100644 etc/kayobe/environments/ci-builder/kolla/.keep create mode 100644 etc/kayobe/environments/ci-builder/network-allocation.yml create mode 100644 etc/kayobe/environments/ci-builder/networks.yml create mode 100644 etc/kayobe/environments/ci-builder/overcloud-dib.yml create mode 100644 etc/kayobe/environments/ci-builder/seed.yml create mode 100644 etc/kayobe/environments/ci-builder/stackhpc-ci.yml create mode 100644 etc/kayobe/environments/ci-doca-builder/.kayobe-environment create mode 100644 etc/kayobe/environments/ci-doca-builder/controllers.yml create mode 100644 etc/kayobe/environments/ci-doca-builder/globals.yml create mode 100644 etc/kayobe/environments/ci-doca-builder/inventory/group_vars/ofed-builder/lvm.yml create mode 100644 etc/kayobe/environments/ci-doca-builder/inventory/hosts create mode 100644 etc/kayobe/environments/ci-doca-builder/seed.yml create mode 100644 etc/kayobe/environments/ci-doca-builder/stackhpc-ci.yml create mode 100644 etc/kayobe/environments/ci-multinode/cephadm.yml create mode 100644 etc/kayobe/environments/ci-multinode/compute.yml create mode 100644 etc/kayobe/environments/ci-multinode/controllers.yml create mode 100644 etc/kayobe/environments/ci-multinode/globals.yml create mode 100644 etc/kayobe/environments/ci-multinode/growroot.yml create mode 120000 etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/pre.d/10-fix-hostname.yml create mode 100644 etc/kayobe/environments/ci-multinode/infra-vms.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/cis-hardening/cis create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/lvm.yml create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/network-interfaces create mode 100644 etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/wazuh-manager create mode 100644 etc/kayobe/environments/ci-multinode/inventory/groups create mode 100644 etc/kayobe/environments/ci-multinode/inventory/kayobe-automation create mode 100644 etc/kayobe/environments/ci-multinode/kolla.yml create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/barbican.conf create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/haproxy/services.d/vault.cfg create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/magnum.conf create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf create mode 100644 etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini create mode 100644 etc/kayobe/environments/ci-multinode/kolla/globals.yml create mode 100644 etc/kayobe/environments/ci-multinode/monitoring.yml create mode 100644 etc/kayobe/environments/ci-multinode/networks.yml create mode 100644 etc/kayobe/environments/ci-multinode/neutron.yml create mode 100644 etc/kayobe/environments/ci-multinode/secrets.yml create mode 100644 etc/kayobe/environments/ci-multinode/seed-hypervisor.yml create mode 100644 etc/kayobe/environments/ci-multinode/seed.yml create mode 100644 etc/kayobe/environments/ci-multinode/stackhpc-ci.yml create mode 100644 etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml create mode 100644 etc/kayobe/environments/ci-multinode/stackhpc.yml create mode 100644 etc/kayobe/environments/ci-multinode/storage.yml create mode 100644 etc/kayobe/environments/ci-multinode/tempest.yml create mode 120000 etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-ansible-version.yml create mode 120000 etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-ansible-version.yml create mode 100644 etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml create mode 120000 etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-ansible-version.yml create mode 120000 etc/kayobe/hooks/overcloud-host-configure/pre.d/pci-passthrough.yml create mode 120000 etc/kayobe/hooks/overcloud-host-package-update/post.d/10-fix-grub-rl9.yml create mode 120000 etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-ansible-version.yml create mode 100644 etc/kayobe/hooks/overcloud-service-deploy/post.d/.gitkeep create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-ansible-version.yml create mode 120000 etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-ansible-version.yml create mode 120000 etc/kayobe/hooks/overcloud-service-upgrade/post.d/deploy-os-capacity-exporter.yml create mode 120000 etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-ansible-version.yml create mode 120000 etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-ansible-version.yml create mode 120000 etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-ansible-version.yml create mode 120000 etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml create mode 120000 etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-ansible-version.yml create mode 100644 etc/kayobe/inventory/group_vars/all/growroot create mode 100644 etc/kayobe/inventory/group_vars/all/openbao.yml create mode 100644 etc/kayobe/inventory/group_vars/all/selinux create mode 100644 etc/kayobe/inventory/group_vars/all/stackhpc create mode 100644 etc/kayobe/inventory/group_vars/all/stackhpc-lvm create mode 100644 etc/kayobe/inventory/group_vars/all/stackhpc-repos create mode 100644 etc/kayobe/inventory/group_vars/all/wazuh create mode 100644 etc/kayobe/inventory/group_vars/cis-hardening/cis create mode 100644 etc/kayobe/inventory/group_vars/controllers/cis create mode 100644 etc/kayobe/inventory/group_vars/github-runners/runners.yml create mode 100644 etc/kayobe/inventory/group_vars/github-writer/writer.yml create mode 100644 etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml create mode 100644 etc/kayobe/inventory/group_vars/gitlab-writer/writer.yml create mode 100644 etc/kayobe/inventory/group_vars/overcloud/stackhpc-repos create mode 100644 etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent create mode 100644 etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager create mode 100644 etc/kayobe/kolla-image-tags.yml create mode 100644 etc/kayobe/kolla/config/bifrost/bifrost.yml create mode 100644 etc/kayobe/kolla/config/fluentd/output/00-local.conf create mode 100644 etc/kayobe/kolla/config/fluentd/output/03-opensearch.conf create mode 100644 etc/kayobe/kolla/config/grafana.ini create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/loadbalancer.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/nvme.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/opensearch.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json create mode 100644 etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json create mode 100644 etc/kayobe/kolla/config/grafana/provisioning.yaml create mode 100644 etc/kayobe/kolla/config/haproxy/services.d/os_capacity.cfg create mode 100644 etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg create mode 100644 etc/kayobe/kolla/config/ironic/policy.yaml create mode 100644 etc/kayobe/kolla/config/nova.conf create mode 100644 etc/kayobe/kolla/config/nova/nova-api.conf create mode 100644 etc/kayobe/kolla/config/nova/nova-compute-ironic.conf create mode 100644 etc/kayobe/kolla/config/nova/nova-compute.conf create mode 100644 etc/kayobe/kolla/config/nova/nova-scheduler.conf create mode 100644 etc/kayobe/kolla/config/prometheus/blackbox.rules create mode 100644 etc/kayobe/kolla/config/prometheus/cadvisor.rules create mode 100644 etc/kayobe/kolla/config/prometheus/ceph.rules create mode 100644 etc/kayobe/kolla/config/prometheus/elasticsearch.rules create mode 100644 etc/kayobe/kolla/config/prometheus/fluentd.rules create mode 100644 etc/kayobe/kolla/config/prometheus/haproxy.rules create mode 100644 etc/kayobe/kolla/config/prometheus/loadbalancer.rules create mode 100644 etc/kayobe/kolla/config/prometheus/mysql.rules create mode 100644 etc/kayobe/kolla/config/prometheus/notifications.tmpl create mode 100644 etc/kayobe/kolla/config/prometheus/openstack.rules create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus-blackbox-exporter.yml create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus.rules create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus.yml.d/60-redfish.yml create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml create mode 100644 etc/kayobe/kolla/config/prometheus/rabbitmq.rules create mode 100644 etc/kayobe/kolla/config/prometheus/smart.rules create mode 100644 etc/kayobe/kolla/config/prometheus/system.rules create mode 100644 etc/kayobe/kolla/inventory/group_vars/prometheus create mode 100644 etc/kayobe/kolla/kolla-build.conf create mode 100644 etc/kayobe/ofed.yml create mode 100644 etc/kayobe/openbao.yml create mode 100644 etc/kayobe/pulp-host-image-versions.yml create mode 100644 etc/kayobe/pulp-ipa-image-versions.yml create mode 100644 etc/kayobe/pulp-repo-versions.yml create mode 100644 etc/kayobe/pulp.yml create mode 100644 etc/kayobe/stackhpc-compute.yml create mode 100644 etc/kayobe/stackhpc-ipa-images.yml create mode 100644 etc/kayobe/stackhpc-monitoring.yml create mode 100644 etc/kayobe/stackhpc-overcloud-dib.yml create mode 100644 etc/kayobe/stackhpc-overcloud-host-images.yml create mode 100644 etc/kayobe/stackhpc.yml create mode 100644 etc/kayobe/trivy/allowed-vulnerabilities.yml create mode 100644 etc/kayobe/vault.yml create mode 100644 releasenotes/config.yaml create mode 100644 releasenotes/notes/2.11.1-opensearch-bump-11fc566cd57a5f39.yaml create mode 100644 releasenotes/notes/2023.1-update-packages-and-containers-q2-2024-02ed375b9c73ac75.yaml create mode 100644 releasenotes/notes/NetworkManager-config-server-5423b1ba9e3bc165.yaml create mode 100644 releasenotes/notes/OSSA-2024-005-74fc3c536cde645f.yaml create mode 100644 releasenotes/notes/add-alerts-for-swap-availability-75e28ed7f913d1ec.yaml create mode 100644 releasenotes/notes/add-blazar-fa6ecce8b21c73b4.yaml create mode 100644 releasenotes/notes/add-caso-f36b98453be10169.yaml create mode 100644 releasenotes/notes/add-ceph-mgr-exporter-endpoints-f7d3c8b1131f6982.yaml create mode 100644 releasenotes/notes/add-confirmation-prompt-to-reboot.yml-4fd1ae8e8d360e57.yaml create mode 100644 releasenotes/notes/add-disk-utils-58b871d9afc9cbe8.yaml create mode 100644 releasenotes/notes/add-friendly-interface-names-d95bba2f9af28302.yaml create mode 100644 releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml create mode 100644 releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml create mode 100644 releasenotes/notes/add-hotfix-containers-playbook-5a6d3d48067cf0b6.yaml create mode 100644 releasenotes/notes/add-iscsid-support-e05d5dfe76de235d.yaml create mode 100644 releasenotes/notes/add-jammy-repo-yoga-f58bfb9792a3c6a7.yaml create mode 100644 releasenotes/notes/add-letsencrypt-eab-support-807bdc4078d3114d.yaml create mode 100644 releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml create mode 100644 releasenotes/notes/add-more-tempest-tests-4c162a50378f9c5a.yaml create mode 100644 releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml create mode 100644 releasenotes/notes/add-openbao-for-tls-698ae3834ed5c67f.yaml create mode 100644 releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml create mode 100644 releasenotes/notes/add-opensearch-bdc4a9d230ffef40.yaml create mode 100644 releasenotes/notes/add-post-reboot-delay-2ac3fa99badf4601.yaml create mode 100644 releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml create mode 100644 releasenotes/notes/add-rabbitmq-4-1-support-caracal-ddb7271c83b75897.yaml create mode 100644 releasenotes/notes/add-radosgw-usage-exporter-support-93d55c544418b05a.yaml create mode 100644 releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml create mode 100644 releasenotes/notes/add-rocky-9-packages-43074cb3367171a5.yaml create mode 100644 releasenotes/notes/add-rocky-9.2-repos-eff5e5b0b07111f5.yaml create mode 100644 releasenotes/notes/add-rocky-9.3-repos-454c2f825e088417.yaml create mode 100644 releasenotes/notes/add-rocky-94-repos-27caa21c77252866.yaml create mode 100644 releasenotes/notes/add-stack-user-for-pulp-c96041e82c13aa10.yaml create mode 100644 releasenotes/notes/add-ubuntu-noble-support-caracal-1b9e64d4aa2e1ff7.yaml create mode 100644 releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml create mode 100644 releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml create mode 100644 releasenotes/notes/adds-alert-for-active-routers-91281912213692c9.yaml create mode 100644 releasenotes/notes/adds-amphora-utility-playbooks-21f7c523f4960f89.yaml create mode 100644 releasenotes/notes/adds-cis-hardening-for-ubuntu-jammy-d9bf23a34c08f5be.yaml create mode 100644 releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml create mode 100644 releasenotes/notes/adds-kolla-config-merging-options-6f75d63b4b784d7f.yaml create mode 100644 releasenotes/notes/adds-mdraid-alerts-309fb79e61389325.yaml create mode 100644 releasenotes/notes/adds-networking-generic-switch-batching-support-adffe038ea2441d0.yaml create mode 100644 releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml create mode 100644 releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml create mode 100644 releasenotes/notes/aio-lvm-script-7de0c919f312040b.yaml create mode 100644 releasenotes/notes/aio-script-8221b9dcbcc40c55.yaml create mode 100644 releasenotes/notes/allow-admins-to-list-bms-ff0bc40d605d4ca2.yaml create mode 100644 releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml create mode 100644 releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml create mode 100644 releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml create mode 100644 releasenotes/notes/backend-tls-network-aa7b09008a2e1914.yaml create mode 100644 releasenotes/notes/bifrost-disable-ironic-debug-5427c6f1160779aa.yaml create mode 100644 releasenotes/notes/bifrost-fix-ipa-auth-9d0e0c3b948b5850.yaml create mode 100644 releasenotes/notes/bifrost-update-discovery-fix-220d745e22d6fbe0.yaml create mode 100644 releasenotes/notes/blackbox-backend-4415919e10a1aa4e.yaml create mode 100644 releasenotes/notes/bug-2019977-456bbf34fed3f090.yaml create mode 100644 releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml create mode 100644 releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml create mode 100644 releasenotes/notes/bump-all-tags-58e916ff669d87e2.yaml create mode 100644 releasenotes/notes/bump-ansible-collection-cephadm-2a6c988a34b192a6.yaml create mode 100644 releasenotes/notes/bump-bifrost-xena-c0c65bd823ed15fa.yaml create mode 100644 releasenotes/notes/bump-centos8-stream-snapshots-2023-09-04-a473edfd3f3b2298.yaml create mode 100644 releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml create mode 100644 releasenotes/notes/bump-focal-7d99ewj84r7h70sq1.yaml create mode 100644 releasenotes/notes/bump-glance-to-fix-bug-2086675-303542d3208103b9.yaml create mode 100644 releasenotes/notes/bump-hashicorp-images-2c23873ad5eeb719.yaml create mode 100644 releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml create mode 100644 releasenotes/notes/bump-horizon-694d426decbf7df3.yaml create mode 100644 releasenotes/notes/bump-horizon-cloudkitty-dashboards-e94f706d5a993cc9.yaml create mode 100644 releasenotes/notes/bump-horizon-dae0cd4cfffe7d7e.yaml create mode 100644 releasenotes/notes/bump-keystone-image-ec2token-98caa61ccf86c944.yaml create mode 100644 releasenotes/notes/bump-magnum-51e03a61ae8aa5a4.yaml create mode 100644 releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml create mode 100644 releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml create mode 100644 releasenotes/notes/bump-magnum-capi-helm-7e4ad37d3d9eecce.yaml create mode 100644 releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml create mode 100644 releasenotes/notes/bump-nova-ede0cd4cfffe7d4e.yaml create mode 100644 releasenotes/notes/bump-octavia-273fb50e05836f21.yaml create mode 100644 releasenotes/notes/bump-ovs-ovn-rocky9-2024.1-eb5a42d9f890d3cd.yaml create mode 100644 releasenotes/notes/bump-qemu-kvm-pdcm-e890d516f1efc690.yaml create mode 100644 releasenotes/notes/bump-rocky-9-90412343a29bc45b.yaml create mode 100644 releasenotes/notes/bump-rocky8-snapshots-2023-09-29-e115427edd3334c7.yaml create mode 100644 releasenotes/notes/bump-rocky9-snapshots-2023-09-29-c736c3d37afd7e5c.yaml create mode 100644 releasenotes/notes/bump-ubuntu-snapshots-2023-09-15-22ca5250d40bd5b6.yaml create mode 100644 releasenotes/notes/bumps-redfish-exporter-to-v2-11032fb9dde36283.yaml create mode 100644 releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml create mode 100644 releasenotes/notes/capi-bump-78cd6f89a0eff0d7.yaml create mode 100644 releasenotes/notes/capi-changes-3275e667680af8e1.yaml create mode 100644 releasenotes/notes/caracal-bump-container-images-083b691e83816b3b.yaml create mode 100644 releasenotes/notes/ceph-config-append-1cc6146d3241b63e.yaml create mode 100644 releasenotes/notes/ceph-ingress-images-e756bde5087460ad.yaml create mode 100644 releasenotes/notes/ceph-maintenance-4c4eb0a4f7149665.yaml create mode 100644 releasenotes/notes/ceph-pacific-16.2.11-a675ccb4e4d70de7.yaml create mode 100644 releasenotes/notes/ceph-quincy-e110f3ba270c3a52.yaml create mode 100644 releasenotes/notes/ceph-reef-18.2.7-9d8d3ad3a1ea517b.yaml create mode 100644 releasenotes/notes/ceph-reef-f447dde8c397f91b.yaml create mode 100644 releasenotes/notes/ceph-squid-19.2.3-35c1a6b914d31388.yaml create mode 100644 releasenotes/notes/cephadm-bump-294148e93121da48.yaml create mode 100644 releasenotes/notes/cephadm-pre-post-commands-now-have-defaults-b42ce9b13fa6d63b.yaml create mode 100644 releasenotes/notes/cinder-bug-1823445-5d07997488bc2416.yaml create mode 100644 releasenotes/notes/cis-do-not-disable-ipv6-98bd79ad86555f51.yaml create mode 100644 releasenotes/notes/cis-hardening-no-longer-sets-permissions-on-logs-81a48ab8ed2d6b5f.yaml create mode 100644 releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml create mode 100644 releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml create mode 100644 releasenotes/notes/cve-2024-32498-2cbd205129351766.yaml create mode 100644 releasenotes/notes/deduplicate-pulp-yml-a65b3e949662a284.yaml create mode 100644 releasenotes/notes/dell-sonic-support-a1dd2edd958bbe9d.yaml create mode 100644 releasenotes/notes/diagnostics-378a6693a64d0b3c.yaml create mode 100644 releasenotes/notes/disable-heat-03f3ac52ac610223.yaml create mode 100644 releasenotes/notes/disable-ironic-prometheus-exporter-894d98022a1e926d.yaml create mode 100644 releasenotes/notes/disable-ironic-prometheus-exporter-daf2dda149466a2e.yaml create mode 100644 releasenotes/notes/disable-non-overcloud-apt-config-5b79b5b63c78779c.yaml create mode 100644 releasenotes/notes/disable-password-expiry-3c296c72d36cd9da.yaml create mode 100644 releasenotes/notes/doca-2-9-3-238838fb78e0c7d9.yaml create mode 100644 releasenotes/notes/doca-workflow-017931dbfd580b41.yaml create mode 100644 releasenotes/notes/docker-bump-dc4e85668ee69a21.yaml create mode 100644 releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml create mode 100644 releasenotes/notes/drop-skyline-support-31d683f58f125335.yaml create mode 100644 releasenotes/notes/dwpd-6b9fb0c8d6d3a570.yaml create mode 100644 releasenotes/notes/elrepo-9-9d88af1dea541155.yaml create mode 100644 releasenotes/notes/enable-build-of-neutron-dragent-14897d9744c7de1f.yaml create mode 100644 releasenotes/notes/enable-mariabackup-97834ea8d87e14a9.yaml create mode 100644 releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml create mode 100644 releasenotes/notes/enable_ovn-b84974f398242dcb.yaml create mode 100644 releasenotes/notes/erlang-aarch64-repo-change-a77c49c408413857.yaml create mode 100644 releasenotes/notes/etcd-e4927551bf5d020c.yaml create mode 100644 releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml create mode 100644 releasenotes/notes/feature-smartmon-65cacfe893f0eb47.yaml create mode 100644 releasenotes/notes/fix-2024.1-kolla-image-build-f78a5524381fa4da.yaml create mode 100644 releasenotes/notes/fix-bifrost-httpboot-config-templating-0488dd14d15d65e4.yaml create mode 100644 releasenotes/notes/fix-blackbox-exporter-config-with-no-external-grafana-de1db02c540af6d8.yaml create mode 100644 releasenotes/notes/fix-blackbox-opensearch-dashboard-d973b43c557c8103.yaml create mode 100644 releasenotes/notes/fix-broken-links-dcd128c8e211b2b8.yaml create mode 100644 releasenotes/notes/fix-ceph-pools-top-capacity-used-panel-26d495c45f2678c8.yaml create mode 100644 releasenotes/notes/fix-ceph-rgw-blackbox-endpoint-7af02679b3fd093d.yaml create mode 100644 releasenotes/notes/fix-cephadm-facts-2ee6dc9a1c617944.yaml create mode 100644 releasenotes/notes/fix-cloudkitty-opensearch-backend-743501558e69e712.yaml create mode 100644 releasenotes/notes/fix-cve-2024-40767-24b9b3c35f61a0c8.yaml create mode 100644 releasenotes/notes/fix-cve-2024-44082-122ef225f674d864.yaml create mode 100644 releasenotes/notes/fix-cve-2026-33551-e37bc84ad753d740.yaml create mode 100644 releasenotes/notes/fix-fix-houston-notify-165a7871bda48f68.yaml create mode 100644 releasenotes/notes/fix-fluentd-opensearch-template-14bde9b6d5a723f8.yaml create mode 100644 releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml create mode 100644 releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml create mode 100644 releasenotes/notes/fix-horizon-bug-2055784-3bb729919d345e77.yaml create mode 100644 releasenotes/notes/fix-houston-tc-mirred-bfb16c89f63b472a.yaml create mode 100644 releasenotes/notes/fix-ipa-collectors-f7afc852d5005282.yaml create mode 100644 releasenotes/notes/fix-issue-with-redfish-exporter-scrape-group-b10eaac6ee1e6af3.yaml create mode 100644 releasenotes/notes/fix-kayobe-version-checks-d1fb3e09391e4a3e.yaml create mode 100644 releasenotes/notes/fix-keystone-multiple-cves-cdb096a2fbb94bba.yaml create mode 100644 releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml create mode 100644 releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml create mode 100644 releasenotes/notes/fix-libvirt-dashboard-b33327f804bea5f9.yaml create mode 100644 releasenotes/notes/fix-local-image-build-59967df0a2bceddf.yaml create mode 100644 releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml create mode 100644 releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml create mode 100644 releasenotes/notes/fix-neutron-keepalived-pid-cleanup-5faae4cf78b07f89.yaml create mode 100644 releasenotes/notes/fix-octavia-tls-terminated-pkcs12-4f7e32a6f5ca0143.yaml create mode 100644 releasenotes/notes/fix-openbao-include-tag-dfef2a0e731674f0.yaml create mode 100644 releasenotes/notes/fix-opensearch-heap-size-0cd31054927d4bbd.yaml create mode 100644 releasenotes/notes/fix-os-capacity-prometheus-job-name-for-azimuth-cloud-metrics-39cf81eebe16b879.yaml create mode 100644 releasenotes/notes/fix-ossa-2024-004-f732e58c12e26785.yaml create mode 100644 releasenotes/notes/fix-ossa-2026-002-65bd525d6aee93cc.yaml create mode 100644 releasenotes/notes/fix-over-1TBram-vm-amd-iommu-rl9-6ddb3d7c030a1674.yaml create mode 100644 releasenotes/notes/fix-ovn-log-permissions-4b067cc1b1a7c3b8.yaml create mode 100644 releasenotes/notes/fix-pci-default-template-8660ab2a7a106376.yaml create mode 100644 releasenotes/notes/fix-radosgw-usage-exporter-duplicate-metrics-ca96b3e4b9d6afe9.yaml create mode 100644 releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml create mode 100644 releasenotes/notes/fix-smart-metrics-permissions-8b1183de434fafcc.yaml create mode 100644 releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml create mode 100644 releasenotes/notes/fix-squid-issue-11b5f03719ab8b45.yaml create mode 100644 releasenotes/notes/fix-ssh-in-ubuntu-overcloud-host-image-b395c8afb6d5b820.yaml create mode 100644 releasenotes/notes/fix-vault-standby-alert-501b743a40971926.yaml create mode 100644 releasenotes/notes/fix-wazuh-custom-sca-policies-f16c7f0eae8265a5.yaml create mode 100644 releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml create mode 100644 releasenotes/notes/fixes-blackbox-with-custom-placement-e98f1621dc5c873d.yaml create mode 100644 releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml create mode 100644 releasenotes/notes/fixes-growroot-for-software-raid-3852bdea5415a0be.yaml create mode 100644 releasenotes/notes/fixes-ironic-rebuild-api-version-2.93-4fb297328d55bff8.yaml create mode 100644 releasenotes/notes/fixes-mdev-uuid-parsing-4c9e972c3c424760.yaml create mode 100644 releasenotes/notes/fixes-osd-size-summary-9924ef4aac61d2b6.yaml create mode 100644 releasenotes/notes/fluentd-buffer-too-large-alert-682893410855387f.yaml create mode 100644 releasenotes/notes/generic-blackbox-588859073c7441ca.yaml create mode 100644 releasenotes/notes/glance-s3-multipart-bug-2043740-e7a122a6453c45e4.yaml create mode 100644 releasenotes/notes/grafana-elasticsearch-cluster-health-154275e8d39dd89f.yaml create mode 100644 releasenotes/notes/grafana-openstack-dashboard-e99a6a44e927363f.yaml create mode 100644 releasenotes/notes/grafana-remove-angular-plugins-1c10e83ddb6556f4.yaml create mode 100644 releasenotes/notes/grub_timeout_values-c39b8b1a2c34a602.yaml create mode 100644 releasenotes/notes/haproxy-dashboard-instance-label-836b93921e964680.yaml create mode 100644 releasenotes/notes/hashicorp-sync-images-96dc726e85323104.yaml create mode 100644 releasenotes/notes/hcp-2.5.0-8e30c7b1910f2bd2.yaml create mode 100644 releasenotes/notes/horizon-fix-multiple-domains-51086e9ba3d197b7.yaml create mode 100644 releasenotes/notes/host-image-build-sms-upload-4829088864b15c5b.yaml create mode 100644 releasenotes/notes/host-image-rebuild-957b0a69de1044fc.yaml create mode 100644 releasenotes/notes/improve-ceph-pgs-unclean-alert-98306d397344b572.yaml create mode 100644 releasenotes/notes/increase-pulp-retries-79cc258da9aabb4f.yaml create mode 100644 releasenotes/notes/increase-tls-ttl-c1eba5cca7767d0f.yaml create mode 100644 releasenotes/notes/infra-vms-ntp-group-45ade04fe885c3f8.yaml create mode 100644 releasenotes/notes/ipa-build-workflow-cde24f3b9e3e0667.yaml create mode 100644 releasenotes/notes/ipa-inspection-settings-133fe91b1d855fa0.yaml create mode 100644 releasenotes/notes/ironic-expand-check_cipher_suite_errors-c8915956be17a28c.yaml create mode 100644 releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml create mode 100644 releasenotes/notes/ironic-inspector-blackbox-backend-ff1bd68297435a36.yaml create mode 100644 releasenotes/notes/ironic-patch-cve-2024-44082-87c70b333bc28ea0.yaml create mode 100644 releasenotes/notes/ironic-prometheus-exporter-f9301ef233d47291.yaml create mode 100644 releasenotes/notes/kayobe-image-f1227adacb28f40b.yaml create mode 100644 releasenotes/notes/kolla-bump-cve-2024-36039-07f18e18b5c86980.yaml create mode 100644 releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml create mode 100644 releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml create mode 100644 releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml create mode 100644 releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml create mode 100644 releasenotes/notes/kolla-update-fixes-5ff55225ce85441f.yaml create mode 100644 releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml create mode 100644 releasenotes/notes/logs-in-openstack-dashboard-6e345ff7f16c0658.yaml create mode 100644 releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml create mode 100644 releasenotes/notes/lvm-3c785770722f768a.yaml create mode 100644 releasenotes/notes/magnum-bump-tags-42u298jfj93r3ok32b.yaml create mode 100644 releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml create mode 100644 releasenotes/notes/magnum-capi-1-1-0-68f14759413316c4.yaml create mode 100644 releasenotes/notes/magnum-capi-driver-2dd8cf6e68a1919c.yaml create mode 100644 releasenotes/notes/magnum-container-footer-dcf6dca0667bcb94.yaml create mode 100644 releasenotes/notes/magnum-pin-958b39182e565697.yaml create mode 100644 releasenotes/notes/magnum-remove-cert-append-8797b640f25644ea.yaml create mode 100644 releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml create mode 100644 releasenotes/notes/merge-openbao-vault-playbooks-0c78364f4a86f6e0.yaml create mode 100644 releasenotes/notes/moved-wazuh-certs-b5173d264f543b14.yaml create mode 100644 releasenotes/notes/multiarch-manifests-pipefail-9b860f83d0f93f1e.yaml create mode 100644 releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml create mode 100644 releasenotes/notes/multinode-wazuh-c9edd65ab5cdbcb8.yaml create mode 100644 releasenotes/notes/network-bond-degraded-alert-d2a0b05002609ac1.yaml create mode 100644 releasenotes/notes/network-bond-single-link-766adf41a3c2fd4e.yaml create mode 100644 releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml create mode 100644 releasenotes/notes/neutron-dadfailed-workaround-9cd82d29ee1fba59.yaml create mode 100644 releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml create mode 100644 releasenotes/notes/nova-bug-2091033-4f54893f67af5c7c.yaml create mode 100644 releasenotes/notes/nova-machine-type-q35-ab363c5f7f3f9807.yaml create mode 100644 releasenotes/notes/nova-pci-request-resize-a429c115ba2818e8.yaml create mode 100644 releasenotes/notes/nova-uefi-edk2-ovmf-d636c299e76728e8.yaml create mode 100644 releasenotes/notes/nova-vflag-noswitchdev-needed-79aaa276bc799923.yaml create mode 100644 releasenotes/notes/ntp-alerts-0d110b4979457165.yaml create mode 100644 releasenotes/notes/nvmemon-update-a326f1b46d95b037.yaml create mode 100644 releasenotes/notes/octavia-alerts-a7f1fe041f5450be.yaml create mode 100644 releasenotes/notes/octavia-dashboard-a5f904b20b8423f5.yaml create mode 100644 releasenotes/notes/octavia-sync-healthmonitor-bug-8602034e2185d12c.yaml create mode 100644 releasenotes/notes/ofed-workflow-119225f36a9265c3.yaml create mode 100644 releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml create mode 100644 releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml create mode 100644 releasenotes/notes/os-capacity-94006f03f16583e4.yaml create mode 100644 releasenotes/notes/os-capacity-cacert-8b800b22d84ae0b1.yaml create mode 100644 releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml create mode 100644 releasenotes/notes/osd-usage-in-dashboards-597ed10af4fc1aa1.yaml create mode 100644 releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml create mode 100644 releasenotes/notes/overcloud-dib-ethtool-pciutils-5b3404b146c9f2a0.yaml create mode 100644 releasenotes/notes/ovn-24-03-47c1eb9846f261b0.yaml create mode 100644 releasenotes/notes/ovn-fix-chassis-leader-check-551d0e94cbb94ac4.yaml create mode 100644 releasenotes/notes/ovn-octavia-provider-bug-2049551-c95d69965733cd78.yaml create mode 100644 releasenotes/notes/ovn-octavia-provider-bug-2072754-5cbd7dc9c366668d.yaml create mode 100644 releasenotes/notes/ovn-sb-db-relay-81f57bc173299913.yaml create mode 100644 releasenotes/notes/ovn_pkg_up-cfb09b7fd17ea786.yaml create mode 100644 releasenotes/notes/ovs-cve-2023-1668-298a2fc907fc07b9.yaml create mode 100644 releasenotes/notes/pci-passthrough-support-0c7e62585aaf2c23.yaml create mode 100644 releasenotes/notes/pin-baseimage-3d224df5b6f56b10.yaml create mode 100644 releasenotes/notes/pin-deps-dbe52c49e945daf5.yaml create mode 100644 releasenotes/notes/pin-pulp-squeezer-bd3a3c53d9804010.yaml create mode 100644 releasenotes/notes/prometheus-hostname-labels-82346cb0914680e9.yaml create mode 100644 releasenotes/notes/pull-stackhpc-host-images-a623e4ab2d2a9e5b.yaml create mode 100644 releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml create mode 100644 releasenotes/notes/pulp-container-extra-9379806192900d22.yaml create mode 100644 releasenotes/notes/pulp-sync-publish-promote-playbook-b638be7c77a649dd.yaml create mode 100644 releasenotes/notes/rabbimq-too-many-ready-c18ecea9a13c8f2a.yaml create mode 100644 releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml create mode 100644 releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml create mode 100644 releasenotes/notes/rabbitmq-queue-migration-script-b6d3abebbebf8087.yaml create mode 100644 releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml create mode 100644 releasenotes/notes/rated-dwpd-40526e85e24ef7ea.yaml create mode 100644 releasenotes/notes/re-enable-pulp-ubuntu-repos-d46ff8eca4a5aac3.yaml create mode 100644 releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml create mode 100644 releasenotes/notes/reboot-default-serial-5944a2a648da71c7.yaml create mode 100644 releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml create mode 100644 releasenotes/notes/redfish-dashboard-fix-inlet-temp-91a7018adb2e1763.yaml create mode 100644 releasenotes/notes/redfish-exporter-36cbdabf103ca93d.yaml create mode 100644 releasenotes/notes/redfish-uuid-grafana-2f50c76bbff6eba9.yaml create mode 100644 releasenotes/notes/reduce-seed-pulp-workers-b085c86bfe7ab5b6.yaml create mode 100644 releasenotes/notes/reduces-sensitivity-of-redfish-target-alerts-a3d77a3f0c3dac8a.yaml create mode 100644 releasenotes/notes/refactor-tag-overrides-1ad66680fd801345.yaml create mode 100644 releasenotes/notes/release-neutron-bgp-dragent-tags-39b9c2595ae11872.yaml create mode 100644 releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml create mode 100644 releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml create mode 100644 releasenotes/notes/remove-swap-playbook-f62a41945e29816d.yaml create mode 100644 releasenotes/notes/remove-tmout-for-dev-env-0778550b353dce03.yaml create mode 100644 releasenotes/notes/removes-bogus-ContainerVolumeUsage-alert-c973b61f598d85e5.yaml create mode 100644 releasenotes/notes/reset-bls-entries-b2bded62c5887937.yaml create mode 100644 releasenotes/notes/restore-kolla-mirrors-9c0c01735d7f43df.yaml create mode 100644 releasenotes/notes/revert-track-all-interfaces-in-keepalived-41ef4834b7659af4.yaml create mode 100644 releasenotes/notes/rgw-usage-exporter-deployment-fixes-0196167326dbe456.yaml create mode 100644 releasenotes/notes/rl9-image-minimal-cloud-init-83576172be8febca.yaml create mode 100644 releasenotes/notes/rl9-security-common-cve-2024-6409-d36de799a29c3f74.yaml create mode 100644 releasenotes/notes/rl9.6-f46ea1230214a657.yaml create mode 100644 releasenotes/notes/rl9.7-c8d033f11097b76b.yaml create mode 100644 releasenotes/notes/rl95-6cbdf902e30502bf.yaml create mode 100644 releasenotes/notes/rmq-dashboard-5bb5eb36ff72100c.yaml create mode 100644 releasenotes/notes/rocky-87-rpms-999565263dacde42.yaml create mode 100644 releasenotes/notes/rocky-crb-b6a6cecfdf4d93a4.yaml create mode 100644 releasenotes/notes/security-common-openssh-6fbd5a1e95fd66ae.yaml create mode 100644 releasenotes/notes/selinux-host-image-ff0fed2583cae7b0.yaml create mode 100644 releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml create mode 100644 releasenotes/notes/smartmontools-bc8176f45d58a75d.yaml create mode 100644 releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml create mode 100644 releasenotes/notes/stackhpc-openstack-tests-0224908453ae72cc.yaml create mode 100644 releasenotes/notes/stackhpc-repos-enabled-10c8a698991e53c2.yaml create mode 100644 releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml create mode 100644 releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml create mode 100644 releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml create mode 100644 releasenotes/notes/ubuntu-fix-cve-2024-6387-648efedaeb288023.yaml create mode 100644 releasenotes/notes/ubuntu-image-early-repos-08b4da6165f5fa11.yaml create mode 100644 releasenotes/notes/ubuntu-jammy-cd109d12954d7626.yaml create mode 100644 releasenotes/notes/ubuntu-refresh-december-998f4c2d2bd0032b.yaml create mode 100644 releasenotes/notes/update-blazar-image-d176c27d55716469.yaml create mode 100644 releasenotes/notes/update-cis-roles-8c73975c37188071.yaml create mode 100644 releasenotes/notes/update-grafana-version-ed01e7bd04782ff7.yaml create mode 100644 releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml create mode 100644 releasenotes/notes/update-haproxy-dashboard-b8807c2a0645ca5e.yaml create mode 100644 releasenotes/notes/update-hcp-collection-39acaafe76c95b3f.yaml create mode 100644 releasenotes/notes/update-kayobe-automation-55fc9c5c380d819c.yaml create mode 100644 releasenotes/notes/update-magnum-capi-driver-c85135340731a86a.yaml create mode 100644 releasenotes/notes/update-magnum-capi-helm-129ea84a8903eb10.yaml create mode 100644 releasenotes/notes/update-magnum-containers-8814b1425ec45fc0.yaml create mode 100644 releasenotes/notes/update-ovs-ovn-rocky9-2024.1-a1025a7130d9c647.yaml create mode 100644 releasenotes/notes/update-prometheus-3519aa260d5f24ef.yaml create mode 100644 releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml create mode 100644 releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml create mode 100644 releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml create mode 100644 releasenotes/notes/update-smartmon-playbook-6bdf9624c579d2e2.yaml create mode 100644 releasenotes/notes/update-ubuntu-repos-4fd1b3083ee7b245.yaml create mode 100644 releasenotes/notes/use-include-tags-for-client-pulp-cf46d328b30162be.yaml create mode 100644 releasenotes/notes/use-length-for-rabbitmq-node-down-rule-c9e9c6b09f57954d.yaml create mode 100644 releasenotes/notes/vault-without-haproxy-db55707e1cc53896.yaml create mode 100644 releasenotes/notes/wallaby-c3c55c39c7b118f2.yaml create mode 100644 releasenotes/notes/wazuh-ansible-v4.10.0-ed5209199194cddf.yaml create mode 100644 releasenotes/notes/wazuh-custom-sca-policies-a5f15818948928b0.yaml create mode 100644 releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml create mode 100644 releasenotes/requirements.txt create mode 100644 releasenotes/source/2023.1.rst create mode 100644 releasenotes/source/2024.1.rst create mode 100644 releasenotes/source/_static/.placeholder create mode 100644 releasenotes/source/_templates/.placeholder create mode 100644 releasenotes/source/conf.py create mode 100644 releasenotes/source/index.rst create mode 100644 releasenotes/source/wallaby.rst create mode 100644 releasenotes/source/xena.rst create mode 100644 releasenotes/source/yoga.rst create mode 100644 releasenotes/source/zed.rst create mode 100644 terraform/.gitignore create mode 100644 terraform/aio/.terraform.lock.hcl create mode 100644 terraform/aio/README.rst create mode 100644 terraform/aio/outputs.tf create mode 100644 terraform/aio/provider.tf create mode 100644 terraform/aio/templates/userdata.cfg.tpl create mode 100644 terraform/aio/vm.tf create mode 100644 tools/diagnostics.sh create mode 100755 tools/kolla-images.py create mode 100755 tools/loadbalancer-failover.sh create mode 100755 tools/merge.py create mode 100755 tools/multiarch-manifests.sh create mode 100755 tools/rabbitmq-queue-migration.sh create mode 100755 tools/scan-images.sh create mode 100755 tools/ubuntu-upgrade-infra-vm.sh create mode 100755 tools/ubuntu-upgrade-overcloud.sh create mode 100755 tools/ubuntu-upgrade-seed-hypervisor.sh create mode 100755 tools/ubuntu-upgrade-seed.sh create mode 100755 tools/upgrade-prerequisites.sh diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 000000000..a9a2aad61 --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,10 @@ +--- +skip_list: + - no-changed-when + - risky-file-permissions + - run-once + - name[template] + - package-latest + - yaml + - role-name[path] + - yaml[line-length] diff --git a/.ansible-lint-ignore b/.ansible-lint-ignore new file mode 100644 index 000000000..4f302a2de --- /dev/null +++ b/.ansible-lint-ignore @@ -0,0 +1,9 @@ +# This file contains ignores to rule violations for ansible-lint +etc/kayobe/ansible/vault-deploy-barbican.yml fqcn[action-core] +etc/kayobe/ansible/vault-generate-backend-tls.yml fqcn[action-core] +etc/kayobe/ansible/vault-generate-internal-tls.yml fqcn[action-core] +etc/kayobe/ansible/vault-generate-test-external-tls.yml fqcn[action-core] +etc/kayobe/ansible/rabbitmq-reset.yml command-instead-of-module +etc/kayobe/ansible/ubuntu-upgrade.yml syntax-check[missing-file] +etc/kayobe/ansible/check-kayobe-version.yml command-instead-of-module +etc/kayobe/ansible/check-kolla-ansible-version.yml command-instead-of-module diff --git a/.automation b/.automation new file mode 160000 index 000000000..8a24af6f8 --- /dev/null +++ b/.automation @@ -0,0 +1 @@ +Subproject commit 8a24af6f8ba1447382804a2b3e30a16f5b40bace diff --git a/.automation.conf/config.sh b/.automation.conf/config.sh new file mode 100644 index 000000000..76f2a59a0 --- /dev/null +++ b/.automation.conf/config.sh @@ -0,0 +1,45 @@ +# This file is used to configure kayobe-automation. +# https://github.com/stackhpc/kayobe-automation/blob/main/README.md + +# See: https://github.com/stackhpc/docker-rally/blob/master/bin/rally-verify-wrapper.sh for a full list of tempest parameters that can be overriden. +# You can override tempest parameters like so: + +# The Tempest concurrency determines how many tests can be running at once. +# Higher values run tests faster but risk running out of resources and failing tests +# On production systems, Tempest concurrency can usually be set to a high number e.g. 16-64. It is often limited by the number of available floating IPs. +# On virtualised test environments, compute and networking speeds often limit the concurrency to 1-16 before tests begin to fail due to timeouts. +export TEMPEST_CONCURRENCY=16 + + +# Specify single test whilst experimenting +#export TEMPEST_PATTERN="${TEMPEST_PATTERN:-tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name}" + +if [ ! -z ${KAYOBE_ENVIRONMENT:+x} ]; then + KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES="${KAYOBE_AUTOMATION_CONFIG_PATH}/tempest/tempest-${KAYOBE_ENVIRONMENT}-${KAYOBE_AUTOMATION_TEMPEST_LOADLIST:-}.overrides.conf" + + # Check if loadlist specific overrides exist, if not fallback to environment overrides. + if [ ! -e "${KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES}" ]; then + KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES="${KAYOBE_AUTOMATION_CONFIG_PATH}/tempest/tempest-${KAYOBE_ENVIRONMENT}.overrides.conf" + fi + + if [[ "$KAYOBE_ENVIRONMENT" =~ "aio" ]]; then + # Seem to get servers failing to spawn with higher concurrency + export TEMPEST_CONCURRENCY=1 + fi + + if [[ "$KAYOBE_ENVIRONMENT" =~ "ci-multinode" ]]; then + export TEMPEST_CONCURRENCY=4 + # Uncomment this to perform a full tempest test + # export KAYOBE_AUTOMATION_TEMPEST_LOADLIST=tempest-full + # export KAYOBE_AUTOMATION_TEMPEST_SKIPLIST=ci-multinode-tempest-full + fi + +fi + +if [[ -z "${KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES:+x}" ]] || [[ ! -e "${KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES}" ]]; then + KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES="${KAYOBE_AUTOMATION_CONFIG_PATH}/tempest/tempest.overrides.conf" +fi + +if [[ -f ${KAYOBE_AUTOMATION_REPO_ROOT}/etc/kolla/public-openrc.sh ]]; then + export TEMPEST_OPENRC="$(< ${KAYOBE_AUTOMATION_REPO_ROOT}/etc/kolla/public-openrc.sh)" +fi diff --git a/.automation.conf/run-books/pulp-sync-content.sh b/.automation.conf/run-books/pulp-sync-content.sh new file mode 100755 index 000000000..97ff18e9f --- /dev/null +++ b/.automation.conf/run-books/pulp-sync-content.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -euE +set -o pipefail + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KAYOBE_AUTOMATION_DIR="$(realpath "${PARENT}/../../.automation")" + +function main { + if [ "${PULP_DO_CONTAINER_SYNC:-}" = true ]; then + ${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}" + fi + if [ "${PULP_DO_CONTAINER_PUBLISH:-}" = true ]; then + ${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml' -e stackhpc_pulp_images_kolla_filter="${PULP_KOLLA_FILTER:-}" + fi + if [ "${PULP_DO_REPO_SYNC:-}" = true ]; then + ${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml' + fi + if [ "${PULP_DO_REPO_PUBLISH:-}" = true ]; then + ${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml' + fi + if [ "${PULP_DO_REPO_PROMOTE:-}" = true ]; then + ${KAYOBE_AUTOMATION_DIR}/scripts/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml' + fi +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main +fi diff --git a/.automation.conf/tempest/load-lists/barbican b/.automation.conf/tempest/load-lists/barbican new file mode 100644 index 000000000..a40ed6759 --- /dev/null +++ b/.automation.conf/tempest/load-lists/barbican @@ -0,0 +1,335 @@ +barbican_tempest_plugin.tests.api.test_consumers.ConsumersTest.test_add_delete_consumers_in_container[id-7d46a170-6b3b-4f4d-903a-b29aebb93289] +barbican_tempest_plugin.tests.api.test_containers.ContainersTest.test_add_to_delete_from_container[id-af10a78d-b1f8-440d-8b89-639861f16fd0] +barbican_tempest_plugin.tests.api.test_containers.ContainersTest.test_create_list_delete_empty_container[id-2e13d4bb-54de-463a-a358-0fb9a221d8f3] +barbican_tempest_plugin.tests.api.test_cve_2022_3100.CVE20223100Test.test_cve_2022_3100[id-459159ef-9670-4c59-8528-09466185c84e] +barbican_tempest_plugin.tests.api.test_orders.OrdersTest.test_create_list_delete_orders[id-077c1729-1950-4e62-a29c-daba4aa186ad] +barbican_tempest_plugin.tests.api.test_quotas.ProjectQuotasTest.test_manage_project_quotas[id-07dec492-7f19-4d94-a9d7-28c0643db1bc] +barbican_tempest_plugin.tests.api.test_quotas.QuotasTest.test_get_effective_quota[id-47ebc42b-0e53-4060-b1a1-55bee2c7c43f] +barbican_tempest_plugin.tests.api.test_secret_consumers.SecretConsumersTest.test_add_delete_consumers_in_secret[id-07a47f8b-e454-4dd0-afb6-bfa12677cd8e] +barbican_tempest_plugin.tests.api.test_secret_metadata.SecretMetadataTest.test_secret_metadata[id-2b0c1707-afc3-4674-a6c6-4dc42f318117] +barbican_tempest_plugin.tests.api.test_secrets.SecretsTest.test_create_delete_empty_secret[id-d5fb4ae4-c418-4405-9701-95fc6877aeb9] +barbican_tempest_plugin.tests.api.test_secrets.SecretsTest.test_create_delete_symmetric_key[id-9aee2ad3-5b61-4451-8ccc-a727bbe4618a] +barbican_tempest_plugin.tests.api.test_secrets.SecretsTest.test_get_and_put_payload[id-c5caa619-1e43-4724-8d94-a61ff7025a07] +barbican_tempest_plugin.tests.api.test_secrets.SecretsTest.test_get_secret_metadata[id-f5608620-f1f7-45a5-ac0a-e1c17d1f2f42] +barbican_tempest_plugin.tests.api.test_secrets.SecretsTest.test_list_secrets[id-79ec555d-215d-4006-bcf0-ab4c6cb0b9ff] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_add_secret_to_container[id-64f5bc21-51b5-451a-932b-e6091bf113c3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_create_container_acl[id-072fd676-36da-44e5-ab53-29a0f1dfe3f1] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_create_container_consumer[id-a7f3b9cb-7be1-49bb-9505-e8b16e30b24f] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_create_container[id-db79462a-e6fd-44f5-9d4d-da3c7fda6896] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_delete_container_acl[id-03a94e74-13da-40d5-ac63-19e9e7552379] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_delete_container_consumer[id-b28e5ed5-797a-46fc-81c4-96a8bfa80757] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_delete_container[id-3c96cec6-02c9-4cae-bea9-be11b22d21aa] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_delete_secret_from_container[id-8efb0a35-9b6d-4bde-84f7-43a72ef86bf3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_get_container_acl[id-167a7fd7-3a08-4d9b-8e5f-2e0d438deea4] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_get_container_consumer +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_get_container[id-8d47fec6-487e-43a5-984d-0c3ef08d23e3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_list_container_consumers[id-7d171230-0ec5-443c-84df-ba79bea52064] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_list_containers[id-028f8858-d463-4f51-befc-63a350e87da2] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTests.test_update_container_acl[id-f5fb9d69-5942-45a3-8c71-2d723e816d52] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_add_secret_to_container[id-96cfab13-67f6-488b-868d-f0609c26553a] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_create_container_acl[id-f78cd73f-e01f-47d3-be5a-83f56837efa1] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_create_container_consumer[id-7c64295e-5584-4673-99f8-a87275072f95] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_create_container[id-cff55c34-fdca-44ab-a136-3b70c75fd8ff] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_delete_container_acl[id-79ef4f66-4a77-44fd-b014-94c075c26bd6] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_delete_container_consumer[id-f3dac21e-cfc2-4a97-bd0e-2a207241e9cf] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_delete_container[id-af342d75-67de-4341-9e96-6a58c75a3226] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_delete_secret_from_container[id-d9590a9d-a8be-481e-b50f-22f26973cba4] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_get_container_acl[id-baa049d5-72a3-4848-8997-a4abd43e00b3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_get_container_consumer +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_get_container[id-c2f5e209-82db-48c7-b11b-2719d3305753] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_list_container_consumers[id-71b335fc-8cb6-4229-bd02-866df4ede926] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_list_containers[id-23857de9-c335-4778-b10b-75191990e20c] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectAdminTestsAcrossProjects.test_update_container_acl[id-6b330e3b-4696-496c-9d65-1cdff39439b9] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_add_secret_to_container[id-64f5bc21-51b5-451a-932b-e6091bf113c3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_create_container_acl[id-072fd676-36da-44e5-ab53-29a0f1dfe3f1] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_create_container_consumer[id-a7f3b9cb-7be1-49bb-9505-e8b16e30b24f] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_create_container[id-db79462a-e6fd-44f5-9d4d-da3c7fda6896] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_delete_container_acl[id-03a94e74-13da-40d5-ac63-19e9e7552379] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_delete_container_consumer[id-b28e5ed5-797a-46fc-81c4-96a8bfa80757] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_delete_container[id-3c96cec6-02c9-4cae-bea9-be11b22d21aa] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_delete_secret_from_container[id-8efb0a35-9b6d-4bde-84f7-43a72ef86bf3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_get_container_acl[id-167a7fd7-3a08-4d9b-8e5f-2e0d438deea4] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_get_container_consumer +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_get_container[id-8d47fec6-487e-43a5-984d-0c3ef08d23e3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_list_container_consumers[id-7d171230-0ec5-443c-84df-ba79bea52064] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_list_containers[id-028f8858-d463-4f51-befc-63a350e87da2] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTests.test_update_container_acl[id-f5fb9d69-5942-45a3-8c71-2d723e816d52] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_add_secret_to_container[id-96cfab13-67f6-488b-868d-f0609c26553a] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_create_container_acl[id-f78cd73f-e01f-47d3-be5a-83f56837efa1] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_create_container_consumer[id-7c64295e-5584-4673-99f8-a87275072f95] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_create_container[id-cff55c34-fdca-44ab-a136-3b70c75fd8ff] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_delete_container_acl[id-79ef4f66-4a77-44fd-b014-94c075c26bd6] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_delete_container_consumer[id-f3dac21e-cfc2-4a97-bd0e-2a207241e9cf] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_delete_container[id-af342d75-67de-4341-9e96-6a58c75a3226] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_delete_secret_from_container[id-d9590a9d-a8be-481e-b50f-22f26973cba4] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_get_container_acl[id-baa049d5-72a3-4848-8997-a4abd43e00b3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_get_container_consumer +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_get_container[id-c2f5e209-82db-48c7-b11b-2719d3305753] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_list_container_consumers[id-71b335fc-8cb6-4229-bd02-866df4ede926] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_list_containers[id-23857de9-c335-4778-b10b-75191990e20c] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectMemberTestsAcrossProjects.test_update_container_acl[id-6b330e3b-4696-496c-9d65-1cdff39439b9] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_add_secret_to_container[id-96cfab13-67f6-488b-868d-f0609c26553a] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_create_container_acl[id-f78cd73f-e01f-47d3-be5a-83f56837efa1] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_create_container_consumer[id-7c64295e-5584-4673-99f8-a87275072f95] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_create_container[id-91406373-339d-4322-bf69-e2842564b9d1] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_delete_container_acl[id-79ef4f66-4a77-44fd-b014-94c075c26bd6] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_delete_container_consumer[id-f3dac21e-cfc2-4a97-bd0e-2a207241e9cf] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_delete_container[id-af342d75-67de-4341-9e96-6a58c75a3226] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_delete_secret_from_container[id-d9590a9d-a8be-481e-b50f-22f26973cba4] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_get_container_acl[id-baa049d5-72a3-4848-8997-a4abd43e00b3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_get_container_consumer +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_get_container[id-c2f5e209-82db-48c7-b11b-2719d3305753] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_list_container_consumers[id-71b335fc-8cb6-4229-bd02-866df4ede926] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_list_containers[id-1f0879ae-7e02-4d37-a37c-ee9bc11b40ef] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTests.test_update_container_acl[id-6b330e3b-4696-496c-9d65-1cdff39439b9] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_add_secret_to_container[id-96cfab13-67f6-488b-868d-f0609c26553a] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_create_container_acl[id-f78cd73f-e01f-47d3-be5a-83f56837efa1] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_create_container_consumer[id-7c64295e-5584-4673-99f8-a87275072f95] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_create_container[id-cff55c34-fdca-44ab-a136-3b70c75fd8ff] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_delete_container_acl[id-79ef4f66-4a77-44fd-b014-94c075c26bd6] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_delete_container_consumer[id-f3dac21e-cfc2-4a97-bd0e-2a207241e9cf] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_delete_container[id-af342d75-67de-4341-9e96-6a58c75a3226] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_delete_secret_from_container[id-d9590a9d-a8be-481e-b50f-22f26973cba4] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_get_container_acl[id-baa049d5-72a3-4848-8997-a4abd43e00b3] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_get_container_consumer +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_get_container[id-c2f5e209-82db-48c7-b11b-2719d3305753] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_list_container_consumers[id-71b335fc-8cb6-4229-bd02-866df4ede926] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_list_containers[id-23857de9-c335-4778-b10b-75191990e20c] +barbican_tempest_plugin.tests.rbac.v1.test_containers.ProjectReaderTestsAcrossProjects.test_update_container_acl[id-6b330e3b-4696-496c-9d65-1cdff39439b9] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectAdminTests.test_create_order[id-898154b7-b4e0-44d8-bf84-e87de5d0b48b] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectAdminTests.test_delete_order[id-fe26f0a1-bcfe-449d-8fd1-ccc4c28f13c2] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectAdminTests.test_delete_other_project_order[id-ee9d022c-90d5-427b-b430-b10323270a49] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectAdminTests.test_get_order[id-798c715d-37fb-4c8b-89c5-e679b016fde7] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectAdminTests.test_get_other_project_order[id-9689c961-2b91-4d4d-b3f3-4f185e7ae1cc] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectAdminTests.test_list_orders[id-789262f2-34fd-46c3-824a-f780d5b5c603] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectMemberTests.test_create_order[id-898154b7-b4e0-44d8-bf84-e87de5d0b48b] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectMemberTests.test_delete_order[id-fe26f0a1-bcfe-449d-8fd1-ccc4c28f13c2] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectMemberTests.test_delete_other_project_order[id-ee9d022c-90d5-427b-b430-b10323270a49] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectMemberTests.test_get_order[id-798c715d-37fb-4c8b-89c5-e679b016fde7] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectMemberTests.test_get_other_project_order[id-9689c961-2b91-4d4d-b3f3-4f185e7ae1cc] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectMemberTests.test_list_orders[id-789262f2-34fd-46c3-824a-f780d5b5c603] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectReaderTests.test_create_order[id-fa5f861d-a376-437d-ab88-b3eea9a20403] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectReaderTests.test_delete_order[id-ca5ef19c-19f3-45fd-a20d-920c1bb6414c] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectReaderTests.test_delete_other_project_order[id-ee9d022c-90d5-427b-b430-b10323270a49] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectReaderTests.test_get_order[id-39227b64-4d99-42ce-9acb-0fc4df2949ab] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectReaderTests.test_get_other_project_order[id-9689c961-2b91-4d4d-b3f3-4f185e7ae1cc] +barbican_tempest_plugin.tests.rbac.v1.test_orders.ProjectReaderTests.test_list_orders[id-78c66385-c8ae-44a7-942e-5e1f87072198] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_get_custom_quota_for_other_project[id-17936c5b-5e89-4717-9826-a22243b947cb] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_get_custom_quota_for_project[id-dc5125a6-bc94-4426-ba9d-03e2b03c81a8] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_get_effective_project_quota[id-936e480a-f603-4d9e-bc77-9a69ef05ac28] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_list_project_quotas[id-9099766d-96b3-448d-b400-3b08563d15fc] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_remove_custom_quota_for_other_project[id-89fb47fd-bf05-4df0-bd47-282417c110b9] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_remove_custom_quota_for_project[id-7a16b9d6-cfdc-4eb7-9e89-b824c612609e] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_set_new_quota_for_other_project[id-d41c97e6-e77d-4bc4-a72d-b068294a0ef6] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectAdminTests.test_set_new_quota_for_project[id-6b169f51-9b17-4d05-aee8-849b94101246] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_get_custom_quota_for_other_project[id-5d062790-6754-4d21-bd0c-08d4a74fa6f3] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_get_custom_quota_for_project[id-948afb0d-35e8-4a23-880f-b2dc3eebf1bd] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_get_effective_project_quota[id-936e480a-f603-4d9e-bc77-9a69ef05ac28] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_list_project_quotas[id-528907ad-efd8-481e-b57e-7faed7198405] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_remove_custom_quota_for_other_project[id-7a763152-c64b-42d5-9669-213681327c58] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_remove_custom_quota_for_project[id-7382fb20-01f6-4322-9ebc-5bf6038e3e1b] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_set_new_quota_for_other_project[id-26bd25ab-92c2-4437-a742-f301703ce523] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectMemberTests.test_set_new_quota_for_project[id-c29af3ed-561e-48da-8a1b-81ba19c943bb] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_get_custom_quota_for_other_project[id-5d062790-6754-4d21-bd0c-08d4a74fa6f3] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_get_custom_quota_for_project[id-948afb0d-35e8-4a23-880f-b2dc3eebf1bd] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_get_effective_project_quota[id-936e480a-f603-4d9e-bc77-9a69ef05ac28] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_list_project_quotas[id-528907ad-efd8-481e-b57e-7faed7198405] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_remove_custom_quota_for_other_project[id-7a763152-c64b-42d5-9669-213681327c58] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_remove_custom_quota_for_project[id-7382fb20-01f6-4322-9ebc-5bf6038e3e1b] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_set_new_quota_for_other_project[id-26bd25ab-92c2-4437-a742-f301703ce523] +barbican_tempest_plugin.tests.rbac.v1.test_quotas.ProjectReaderTests.test_set_new_quota_for_project[id-c29af3ed-561e-48da-8a1b-81ba19c943bb] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_create_key_value_pair_on_other_secret[id-7c6223b6-5a2f-4989-ae54-db253702af98] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_create_key_value_pair[id-73d4c9cc-b7ad-4706-bec4-e7dc70698711] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_delete_other_secret_metadata_by_key[id-fb581f6c-1d59-420c-b05b-227514c75789] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_delete_secret_metadata_by_key[id-3740d6ec-304f-43ce-aa54-af62006715d8] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_get_other_secret_metadata_by_key[id-f2cbaec3-94ff-4a9c-bc2d-4c728f46313b] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_get_secret_metadata_by_key[id-26160af4-ff17-4023-9238-a2a9dca9946c] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_get_secret_metadata_from_other_secret[id-efb8d207-5418-4c3a-bb30-d1d5e1695c41] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_get_secret_metadata[id-054348d4-c4b4-446a-ad98-c79f6de42eec] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_put_secret_metadata_on_other_secret[id-c1738dd5-e67a-4b48-8064-c198ca0a7970] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_put_secret_metadata[id-f7178757-5ae4-4041-a57c-322f7665f055] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_update_other_secret_metadata_by_key[id-489d52ae-3324-420b-b69c-938f2eb41f6f] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectAdminTests.test_update_secret_metadata_by_key[id-7ed07736-39e7-4c4f-b5c9-f59017f3e80b] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_create_key_value_pair_on_other_secret[id-7c6223b6-5a2f-4989-ae54-db253702af98] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_create_key_value_pair[id-73d4c9cc-b7ad-4706-bec4-e7dc70698711] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_delete_other_secret_metadata_by_key[id-fb581f6c-1d59-420c-b05b-227514c75789] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_delete_secret_metadata_by_key[id-3740d6ec-304f-43ce-aa54-af62006715d8] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_get_other_secret_metadata_by_key[id-f2cbaec3-94ff-4a9c-bc2d-4c728f46313b] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_get_secret_metadata_by_key[id-26160af4-ff17-4023-9238-a2a9dca9946c] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_get_secret_metadata_from_other_secret[id-efb8d207-5418-4c3a-bb30-d1d5e1695c41] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_get_secret_metadata[id-054348d4-c4b4-446a-ad98-c79f6de42eec] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_put_secret_metadata_on_other_secret[id-c1738dd5-e67a-4b48-8064-c198ca0a7970] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_put_secret_metadata[id-f7178757-5ae4-4041-a57c-322f7665f055] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_update_other_secret_metadata_by_key[id-489d52ae-3324-420b-b69c-938f2eb41f6f] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectMemberTests.test_update_secret_metadata_by_key[id-7ed07736-39e7-4c4f-b5c9-f59017f3e80b] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_create_key_value_pair_on_other_secret[id-7c6223b6-5a2f-4989-ae54-db253702af98] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_create_key_value_pair[id-2dd1cb04-67e9-4c3c-a40e-12184eff5bc6] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_delete_other_secret_metadata_by_key[id-fb581f6c-1d59-420c-b05b-227514c75789] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_delete_secret_metadata_by_key[id-0d80db72-fc6f-445f-b402-f86c91233b4f] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_get_other_secret_metadata_by_key[id-f2cbaec3-94ff-4a9c-bc2d-4c728f46313b] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_get_secret_metadata_by_key[id-b2f3bafb-4a21-47da-966f-3e3010571596] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_get_secret_metadata_from_other_secret[id-efb8d207-5418-4c3a-bb30-d1d5e1695c41] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_get_secret_metadata[id-8f22488b-52d5-4a28-ae32-faf1514ef390] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_put_secret_metadata_on_other_secret[id-c1738dd5-e67a-4b48-8064-c198ca0a7970] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_put_secret_metadata[id-33c1d271-5367-4f0e-a2e7-c28ea1130fa6] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_update_other_secret_metadata_by_key[id-489d52ae-3324-420b-b69c-938f2eb41f6f] +barbican_tempest_plugin.tests.rbac.v1.test_secret_metadata.ProjectReaderTests.test_update_secret_metadata_by_key[id-df6bbafc-f836-44b4-a0a7-3d0f94f5b9ac] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectAdminTests.test_get_global_secret_store[id-e97d0cbe-112c-490e-b3d7-02981161d471] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectAdminTests.test_get_preferred_secret_store[id-c3554210-8960-4d09-a1ba-369b8df6ca1f] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectAdminTests.test_get_secret_store[id-28d8e43b-af38-4985-8f70-3c1098116561] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectAdminTests.test_list_secret_stores[id-bf31560c-42e5-4afc-b1d6-a3a8aa7773d3] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectAdminTests.test_set_preferred_secret_store[id-c459deb6-6447-43c4-820b-384903e700cb] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectAdminTests.test_unset_preferred_secret_store[id-d21ca9b6-b62e-43d1-9c9f-a6e16c939c01] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectMemberTests.test_get_global_secret_store[id-e97d0cbe-112c-490e-b3d7-02981161d471] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectMemberTests.test_get_preferred_secret_store[id-c3554210-8960-4d09-a1ba-369b8df6ca1f] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectMemberTests.test_get_secret_store[id-28d8e43b-af38-4985-8f70-3c1098116561] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectMemberTests.test_list_secret_stores[id-bf31560c-42e5-4afc-b1d6-a3a8aa7773d3] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectMemberTests.test_set_preferred_secret_store[id-ada28e3a-ec67-4994-9dde-410463d6d06e] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectMemberTests.test_unset_preferred_secret_store[id-c3f52fd1-5d18-498f-81b2-45df5cb09a87] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectReaderTests.test_get_global_secret_store[id-e97d0cbe-112c-490e-b3d7-02981161d471] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectReaderTests.test_get_preferred_secret_store[id-c3554210-8960-4d09-a1ba-369b8df6ca1f] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectReaderTests.test_get_secret_store[id-28d8e43b-af38-4985-8f70-3c1098116561] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectReaderTests.test_list_secret_stores[id-bf31560c-42e5-4afc-b1d6-a3a8aa7773d3] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectReaderTests.test_set_preferred_secret_store[id-ada28e3a-ec67-4994-9dde-410463d6d06e] +barbican_tempest_plugin.tests.rbac.v1.test_secret_stores.ProjectReaderTests.test_unset_preferred_secret_store[id-c3f52fd1-5d18-498f-81b2-45df5cb09a87] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_create_secret[id-69f24625-0d8a-4412-b9c2-5a96fc689c87] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_delete_other_project_secret +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_delete_other_secret_acl[id-1470e2fc-46ce-4d06-a11a-201e9b5950c6] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_delete_secret_acl[id-d566f1ab-c318-42cc-80d9-1ff9178d2c63] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_delete_secret[id-bffcf1e6-b9a2-43d8-b95c-b3683d6c4549] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_get_other_project_secret +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_get_other_project_secret_payload[id-dff3d49e-9e31-46bb-b069-d4c72f591718] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_get_other_secret_acl[id-434187eb-1dd2-4544-bb1e-6be0dca8cd25] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_get_secret_acl[id-63482006-18b5-40d3-82da-fdc078d6e5fe] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_get_secret_payload[id-cb848266-0172-4f93-add3-9d6f41a3bc46] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_get_secret[id-72a59d44-1967-44fc-84a1-157a7bf124fa] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_list_secrets[id-ff89eaa4-0014-4935-80cf-f8b7970387e1] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_patch_other_secret_acl[id-ad95395f-45b0-4b34-b92a-e6c25f90e798] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_patch_secret_acl[id-d6128f75-e7af-43dc-bf43-cf5fdc5f83be] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_put_other_project_secret_payload[id-fb2fe2a4-2ca9-4b64-b18f-cc0877eb27bc] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_put_other_secret_acl[id-32312a70-8f02-4663-b7e0-4432950c2c11] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_put_secret_acl[id-915bdc2a-94d2-4835-a46e-a27f16ae57a2] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminTests.test_put_secret_payload[id-f6c58ca1-50f2-4454-b3b2-b338a7dcf3cb] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminV1_1Tests.test_create_secret_consumer[id-2da9bfb4-f53b-45c0-b8c9-f657ced99bd4] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminV1_1Tests.test_delete_secret_consumer[id-2a4eaac5-76a1-48e2-b648-b1b02344130b] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectAdminV1_1Tests.test_list_secret_consumers[id-34e2ded6-30ea-4f8a-b4e1-3aecac8fdd49] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_create_secret[id-69f24625-0d8a-4412-b9c2-5a96fc689c87] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_delete_other_project_secret[id-fc2f42ec-6bf4-4121-9698-4f0a7d01d8f3] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_delete_other_secret_acl[id-1470e2fc-46ce-4d06-a11a-201e9b5950c6] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_delete_secret_acl[id-d566f1ab-c318-42cc-80d9-1ff9178d2c63] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_delete_secret[id-bffcf1e6-b9a2-43d8-b95c-b3683d6c4549] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_get_other_project_secret_payload[id-dff3d49e-9e31-46bb-b069-d4c72f591718] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_get_other_project_secret[id-5219c830-fe82-4f3b-9eda-e3b5e918ba60] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_get_other_secret_acl[id-434187eb-1dd2-4544-bb1e-6be0dca8cd25] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_get_secret_acl[id-63482006-18b5-40d3-82da-fdc078d6e5fe] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_get_secret_payload[id-cb848266-0172-4f93-add3-9d6f41a3bc46] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_get_secret[id-72a59d44-1967-44fc-84a1-157a7bf124fa] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_list_secrets[id-ff89eaa4-0014-4935-80cf-f8b7970387e1] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_patch_other_secret_acl[id-ad95395f-45b0-4b34-b92a-e6c25f90e798] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_patch_secret_acl[id-d6128f75-e7af-43dc-bf43-cf5fdc5f83be] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_put_other_project_secret_payload[id-fb2fe2a4-2ca9-4b64-b18f-cc0877eb27bc] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_put_other_secret_acl[id-32312a70-8f02-4663-b7e0-4432950c2c11] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_put_secret_acl[id-915bdc2a-94d2-4835-a46e-a27f16ae57a2] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberTests.test_put_secret_payload[id-f6c58ca1-50f2-4454-b3b2-b338a7dcf3cb] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberV1_1Tests.test_create_secret_consumer[id-86cadb1e-f748-4d99-9477-4d171d4e9240] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberV1_1Tests.test_delete_secret_consumer[id-f56c4c14-e8c7-4335-8d84-00f34355b53c] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectMemberV1_1Tests.test_list_secret_consumers[id-ca2bbfa3-90b2-4f4e-8e57-3a3562d202a6] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_create_secret[id-e4dfbae6-faca-42a7-a06b-2655e29df193] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_delete_other_project_secret[id-fc2f42ec-6bf4-4121-9698-4f0a7d01d8f3] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_delete_other_secret_acl[id-1470e2fc-46ce-4d06-a11a-201e9b5950c6] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_delete_secret_acl[id-07104bf1-104b-4fa5-b855-82cba69bf24c] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_delete_secret[id-6a4cfca5-1841-49f4-ae1d-bbde0fa94bd7] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_get_other_project_secret_payload[id-dff3d49e-9e31-46bb-b069-d4c72f591718] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_get_other_project_secret[id-5219c830-fe82-4f3b-9eda-e3b5e918ba60] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_get_other_secret_acl[id-434187eb-1dd2-4544-bb1e-6be0dca8cd25] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_get_secret_acl[id-effafb29-fd10-41fb-9404-585af3de3602] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_get_secret_payload[id-b2760216-e492-4081-b981-a5d40bcc6a0e] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_get_secret[id-9c5b46b4-8f0b-4f75-b751-61ddf943fbf3] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_list_secrets[id-f2649794-10d2-4742-a81c-af78eb3d9c0e] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_patch_other_secret_acl[id-ad95395f-45b0-4b34-b92a-e6c25f90e798] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_patch_secret_acl[id-3350274a-b3f4-4178-927f-2591ef2dbea8] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_put_other_project_secret_payload[id-fb2fe2a4-2ca9-4b64-b18f-cc0877eb27bc] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_put_other_secret_acl[id-32312a70-8f02-4663-b7e0-4432950c2c11] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_put_secret_acl[id-d5058429-4e98-43ac-bda4-8160b2b95ef7] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderTests.test_put_secret_payload[id-64b4e2e7-0121-46e7-949f-34332efdec6f] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderV1_1Tests.test_create_secret_consumer[id-d7389369-62e9-4a25-b759-2a64f72fcba2] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderV1_1Tests.test_delete_secret_consumer[id-dbfba5e4-cd52-4ce9-bf50-a7e933ce5dcc] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.ProjectReaderV1_1Tests.test_list_secret_consumers[id-be85626b-ca83-4c90-9bf0-b918b9de21b6] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_create_secret[id-104f71f0-8099-43ae-b4d9-cce5781a79b9] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_delete_other_project_secret[id-ce878824-d424-4abb-8217-068c9a99333b] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_delete_other_secret_acl[id-65cfefb4-69a6-4f31-b8e1-defad1b57645] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_delete_secret_acl[id-d55d5798-c23f-4108-8005-963d350d9d41] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_delete_secret[id-b637c7db-64a9-46c8-b322-c4d282e05164] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_get_other_project_secret_payload[id-cc021881-7fba-48a2-aa6e-68c426b382f9] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_get_other_project_secret[id-be392729-af43-4aab-bbc7-43fcd5df9140] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_get_other_secret_acl[id-5490d517-ba6c-4e28-8712-07dbc9bb9ada] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_get_secret_acl[id-be04944b-b4e2-4f66-b58a-3d047c99d939] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_get_secret_payload[id-1bc76c3a-a69e-4285-8f8e-f7bacd01fef8] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_get_secret[id-8d0a7f54-61f3-432e-8f4b-c04945b40373] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_list_secrets[id-5a29b825-4f28-4733-90fa-579b63ae2b96] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_patch_other_secret_acl[id-5ec28567-111b-405f-93c9-ed5d4259e918] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_patch_secret_acl[id-81423acc-240c-46f0-8de9-4cf6ab5d4bc4] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_put_other_project_secret_payload[id-e87f5e40-7bb3-4fc8-aa5a-23cc1a8850f5] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_put_other_secret_acl[id-1ae61619-104a-4497-999c-00671335bc4f] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_put_secret_acl[id-65ce0063-d6f1-463c-b752-d4871a9df684] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemAdminTests.test_put_secret_payload[id-c2f38e3d-cc52-43c0-9fb3-8065797c40da] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_create_secret[id-104f71f0-8099-43ae-b4d9-cce5781a79b9] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_delete_other_project_secret[id-ce878824-d424-4abb-8217-068c9a99333b] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_delete_other_secret_acl[id-65cfefb4-69a6-4f31-b8e1-defad1b57645] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_delete_secret_acl[id-d55d5798-c23f-4108-8005-963d350d9d41] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_delete_secret[id-b637c7db-64a9-46c8-b322-c4d282e05164] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_get_other_project_secret_payload[id-cc021881-7fba-48a2-aa6e-68c426b382f9] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_get_other_project_secret[id-be392729-af43-4aab-bbc7-43fcd5df9140] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_get_other_secret_acl[id-5490d517-ba6c-4e28-8712-07dbc9bb9ada] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_get_secret_acl[id-be04944b-b4e2-4f66-b58a-3d047c99d939] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_get_secret_payload[id-1bc76c3a-a69e-4285-8f8e-f7bacd01fef8] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_get_secret[id-8d0a7f54-61f3-432e-8f4b-c04945b40373] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_list_secrets[id-5a29b825-4f28-4733-90fa-579b63ae2b96] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_patch_other_secret_acl[id-5ec28567-111b-405f-93c9-ed5d4259e918] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_patch_secret_acl[id-81423acc-240c-46f0-8de9-4cf6ab5d4bc4] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_put_other_project_secret_payload[id-e87f5e40-7bb3-4fc8-aa5a-23cc1a8850f5] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_put_other_secret_acl[id-1ae61619-104a-4497-999c-00671335bc4f] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_put_secret_acl[id-65ce0063-d6f1-463c-b752-d4871a9df684] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemMemberTests.test_put_secret_payload[id-c2f38e3d-cc52-43c0-9fb3-8065797c40da] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_create_secret[id-104f71f0-8099-43ae-b4d9-cce5781a79b9] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_delete_other_project_secret[id-ce878824-d424-4abb-8217-068c9a99333b] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_delete_other_secret_acl[id-65cfefb4-69a6-4f31-b8e1-defad1b57645] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_delete_secret_acl[id-d55d5798-c23f-4108-8005-963d350d9d41] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_delete_secret[id-b637c7db-64a9-46c8-b322-c4d282e05164] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_get_other_project_secret_payload[id-cc021881-7fba-48a2-aa6e-68c426b382f9] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_get_other_project_secret[id-be392729-af43-4aab-bbc7-43fcd5df9140] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_get_other_secret_acl[id-5490d517-ba6c-4e28-8712-07dbc9bb9ada] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_get_secret_acl[id-be04944b-b4e2-4f66-b58a-3d047c99d939] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_get_secret_payload[id-1bc76c3a-a69e-4285-8f8e-f7bacd01fef8] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_get_secret[id-8d0a7f54-61f3-432e-8f4b-c04945b40373] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_list_secrets[id-5a29b825-4f28-4733-90fa-579b63ae2b96] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_patch_other_secret_acl[id-5ec28567-111b-405f-93c9-ed5d4259e918] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_patch_secret_acl[id-81423acc-240c-46f0-8de9-4cf6ab5d4bc4] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_put_other_project_secret_payload[id-e87f5e40-7bb3-4fc8-aa5a-23cc1a8850f5] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_put_other_secret_acl[id-1ae61619-104a-4497-999c-00671335bc4f] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_put_secret_acl[id-65ce0063-d6f1-463c-b752-d4871a9df684] +barbican_tempest_plugin.tests.rbac.v1.test_secrets.SystemReaderTests.test_put_secret_payload[id-c2f38e3d-cc52-43c0-9fb3-8065797c40da] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectAdminTests.test_create_transport_key[id-bed9bace-9b44-448b-b68e-bff46c4c181e] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectAdminTests.test_delete_transport_key[id-aeae2541-af87-40d6-a4b6-767fcb7416d4] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectAdminTests.test_get_transport_key[id-4d4d46df-ec90-4755-a6e3-aa0ff9204113] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectAdminTests.test_list_transport_keys[id-dc647930-d89d-449c-8bf6-b0aff0cd95da] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectMemberTests.test_create_transport_key[id-981d9ec3-7974-40ab-abf0-d19cd17311e0] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectMemberTests.test_delete_transport_key[id-aeae2541-af87-40d6-a4b6-767fcb7416d4] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectMemberTests.test_get_transport_key[id-4d4d46df-ec90-4755-a6e3-aa0ff9204113] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectMemberTests.test_list_transport_keys[id-dc647930-d89d-449c-8bf6-b0aff0cd95da] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectReaderTests.test_create_transport_key[id-981d9ec3-7974-40ab-abf0-d19cd17311e0] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectReaderTests.test_delete_transport_key[id-aeae2541-af87-40d6-a4b6-767fcb7416d4] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectReaderTests.test_get_transport_key[id-4d4d46df-ec90-4755-a6e3-aa0ff9204113] +barbican_tempest_plugin.tests.rbac.v1.test_transport_keys.ProjectReaderTests.test_list_transport_keys[id-dc647930-d89d-449c-8bf6-b0aff0cd95da] +barbican_tempest_plugin.tests.scenario.test_certificate_validation.CertificateValidationTest.test_signed_image_invalid_cert_boot_failure[compute,id-6d354881-35a6-4568-94b8-2204bbf67b29,image] +barbican_tempest_plugin.tests.scenario.test_certificate_validation.CertificateValidationTest.test_signed_image_upload_and_boot[compute,id-b41bc663-5662-4b1e-b8f1-27b2876f16a6,image] +barbican_tempest_plugin.tests.scenario.test_certificate_validation.CertificateValidationTest.test_signed_image_upload_and_hard_reboot[compute,id-aed5254d-1e7a-46b6-8cb0-ef5fd798671a,image] +barbican_tempest_plugin.tests.scenario.test_certificate_validation.CertificateValidationTest.test_signed_image_upload_and_server_rebuild[compute,id-f9c6de51-b027-476f-a6e3-847bb39cfa02,image] +barbican_tempest_plugin.tests.scenario.test_ephemeral_disk_encryption.EphemeralStorageEncryptionTest.test_encrypted_ephemeral_lvm_storage[compute,id-afe720b9-8b35-4a3c-8ff3-15841c2d3148,image] +barbican_tempest_plugin.tests.scenario.test_image_signing.ImageSigningSnapshotTest.test_signed_image_upload_boot_snapshot[compute,id-f0603dfd-8b2c-44e2-8b0f-d65c87aab257,image] +barbican_tempest_plugin.tests.scenario.test_image_signing.ImageSigningTest.test_signed_image_upload_and_boot[compute,id-4343df3c-5553-40ea-8705-0cce73b297a9,image] +barbican_tempest_plugin.tests.scenario.test_image_signing.ImageSigningTest.test_signed_image_upload_boot_failure[compute,id-74f022d6-a6ef-4458-96b7-541deadacf99,image] +barbican_tempest_plugin.tests.scenario.test_image_signing.ImageSigningVolumeTest.test_extend_encrypted_signed_volume_attach_to_instance[compute,id-72ca044d-39a4-4966-b302-f53a446d3e29,image,slow,volume] +barbican_tempest_plugin.tests.scenario.test_image_signing.ImageSigningVolumeTest.test_signed_volume_attach_to_instance[compute,id-9f28ce2e-362e-46ec-bf56-aebce9cc49fb,image,slow,volume] +barbican_tempest_plugin.tests.scenario.test_volume_encryption.VolumeEncryptionTest.test_encrypted_cinder_volumes_cryptsetup[compute,id-cbc752ed-b716-4727-910f-956ccf965723,image,volume] +barbican_tempest_plugin.tests.scenario.test_volume_encryption.VolumeEncryptionTest.test_encrypted_cinder_volumes_luks[compute,id-89165fb4-5534-4b9d-8429-97ccffb8f86f,image,volume] diff --git a/.automation.conf/tempest/load-lists/cinder b/.automation.conf/tempest/load-lists/cinder new file mode 100644 index 000000000..26dbaeeb4 --- /dev/null +++ b/.automation.conf/tempest/load-lists/cinder @@ -0,0 +1,160 @@ +cinder_tempest_plugin.api.volume.admin.test_consistencygroups.ConsistencyGroupsV2Test.test_consistencygroup_cgsnapshot_create_delete[id-2134dd52-f333-4456-bb05-6cb0f009a44f] +cinder_tempest_plugin.api.volume.admin.test_consistencygroups.ConsistencyGroupsV2Test.test_consistencygroup_create_delete[id-3fe776ba-ec1f-4e6c-8d78-4b14c3a7fc44] +cinder_tempest_plugin.api.volume.admin.test_consistencygroups.ConsistencyGroupsV2Test.test_create_consistencygroup_from_cgsnapshot[id-3a6a5525-25ca-4a6c-aac4-cac6fa8f5b43] +cinder_tempest_plugin.api.volume.admin.test_consistencygroups.ConsistencyGroupsV2Test.test_create_consistencygroup_from_consistencygroup[id-556121ae-de9c-4342-9897-e54260447a19] +cinder_tempest_plugin.api.volume.admin.test_volume_backup.VolumesBackupsTest.test_backup_crossproject_admin_negative[id-2daadb2e-409a-4ede-a6ce-6002ec324372] +cinder_tempest_plugin.api.volume.admin.test_volume_backup.VolumesBackupsTest.test_backup_crossproject_user_negative[id-b9feb593-5809-4207-90d3-28e627730f13] +cinder_tempest_plugin.api.volume.admin.test_volume_backup.VolumesBackupsTest.test_incremental_backup_respective_parents[id-ce15f528-bfc1-492d-81db-b6168b631587] +cinder_tempest_plugin.api.volume.test_create_from_image.VolumeAndVolumeTypeFromImageTest.test_create_from_image_with_volume_type_image_property[id-6e9266ff-a917-4dd5-aa4a-c36e59e7a2a6] +cinder_tempest_plugin.api.volume.test_multiple_volume_from_resource.CreateVolumesFromBackupTest.test_create_multiple_volume_from_backup[id-9db67083-bf1a-486c-8f77-3778467f39a1] +cinder_tempest_plugin.api.volume.test_multiple_volume_from_resource.CreateVolumesFromImageTest.test_create_from_image_multiple[id-8976a11b-1ddc-49b6-b66f-8c26adf3fa9e] +cinder_tempest_plugin.api.volume.test_multiple_volume_from_resource.CreateVolumesFromSnapshotTest.test_create_multiple_volume_from_snapshot[id-3b879ad1-d861-4ad3-b2c8-c89162e867c3] +cinder_tempest_plugin.api.volume.test_multiple_volume_from_resource.CreateVolumesFromSourceVolumeTest.test_create_multiple_volume_from_source_volume[id-b4a250d1-3ffd-4727-a2f5-9d858b298558] +cinder_tempest_plugin.api.volume.test_volume_backup.VolumesBackupsTest.test_backup_create_and_restore_to_an_existing_volume[id-b5d837b0-7066-455d-88fc-4a721a899306] +cinder_tempest_plugin.api.volume.test_volume_backup.VolumesBackupsTest.test_incr_backup_create_and_restore_to_an_existing_volume[id-457359e2-a663-4758-8f76-06d392dfd7c7] +cinder_tempest_plugin.api.volume.test_volume_backup.VolumesBackupsTest.test_incremental_backup[id-c810fe2c-cb40-43ab-96aa-471b74516a98] +cinder_tempest_plugin.api.volume.test_volume_backup.VolumesBackupsTest.test_volume_snapshot_backup[id-885410c6-cd1d-452c-a409-7c32b7e0be15] +cinder_tempest_plugin.api.volume.test_volume_dependency.VolumeDependencyTests.test_delete_dep_chain_2[id-63447ef8-e667-4796-ba66-1b9b883af1f1] +cinder_tempest_plugin.api.volume.test_volume_dependency.VolumeDependencyTests.test_delete_dep_chain[id-f8278e5c-50ff-4a1d-8670-3ca0866d411a] +cinder_tempest_plugin.api.volume.test_volume_dependency.VolumeDependencyTests.test_delete_source_snapshot[id-900d8ea5-2afd-4fe5-a0c3-fab4744f0d40] +cinder_tempest_plugin.api.volume.test_volume_dependency.VolumeDependencyTests.test_delete_source_volume[id-42e9df95-854b-4840-9d55-ae62f65e9b8e] +cinder_tempest_plugin.api.volume.test_volume_dependency.VolumeImageDependencyTests.test_image_volume_dependencies_1[id-7a9fba78-2e4b-42b1-9898-bb4a60685320,image,volume] +cinder_tempest_plugin.api.volume.test_volume_dependency.VolumeImageDependencyTests.test_image_volume_dependencies_2[id-0e20bd6e-440f-41d8-9b5d-fc047ac00423,image,volume] +cinder_tempest_plugin.api.volume.test_volume_dependency.VolumeImageDependencyTests.test_image_volume_dependencies_3[id-e6050452-06bd-4c7f-9912-45178c83e379,image,volume] +cinder_tempest_plugin.api.volume.test_volume_revert.VolumeRevertTests.test_volume_revert_to_snapshot_after_extended_negative[id-4e8b0788-87fe-430d-be7a-444d7f8e0347] +cinder_tempest_plugin.api.volume.test_volume_revert.VolumeRevertTests.test_volume_revert_to_snapshot[id-87b7dcb7-4950-4a3a-802c-ece55491846d] +cinder_tempest_plugin.api.volume.test_volume_unicode.CinderUnicodeTest.test_create_delete_unicode_volume_name[id-2d7e2e49-150e-4849-a18e-79f9777c9a96] +cinder_tempest_plugin.api.volume.test_volume_unicode.CinderUnicodeTest.test_snapshot_create_volume_description_non_ascii_code[id-332be44d-5418-4fb3-a8f0-a3587de6929f] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_create_backup[id-d347fa21-a5bf-4ce5-ab6b-246c3a06a735] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_delete_backup[id-b77a8d69-1d12-480d-a83e-5f712d7c2b74] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_export_backup[id-e179a062-47d5-4fa8-b359-dedab2afddd8] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_import_backup[id-1be80834-2463-49fb-a763-906e8c672fd5] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_list_backups[id-81c579bc-db98-4773-9590-b742d0b00b89] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_list_project_backups[id-602dd42d-10df-4eb2-9664-3c9c44e3b35e] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_reset_backup_status[id-88db5943-0053-489a-af30-12b139a38a0b] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_restore_backup[id-7221d2df-338c-4932-be40-ad7166c03db1] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests33.test_show_backup[id-2094dcee-9585-4745-b045-a0f8c79fbe52] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectAdminTests39.test_update_backup[id-2686eecf-e3cd-4f23-8771-aa040ed9be4b] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_create_backup[id-44644140-4d05-4725-9a4b-6d1a71eda9b7] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_delete_backup[id-c98dfea8-b9f2-4a84-947b-1d857c707789] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_export_backup[id-71c7cfaf-7809-4872-b1b2-3feb90b939d4] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_import_backup[id-f1c03c1b-2b48-4be0-8b6a-81df8a75f78c] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_list_backups[id-5a23c53c-924b-47f6-a5d1-ab6327391c12] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_list_project_backups[id-c737bd7b-293c-4d8f-ada9-3b00f7e1adce] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_reset_backup_status[id-307f6fe9-81ed-444a-9aae-99a571d24bf5] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_restore_backup[id-7a6fd066-00e7-4140-866c-8195fbd71e87] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests33.test_show_backup[id-9944bb15-02fa-4321-97a4-ef8cb5b5fec2] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectMemberTests39.test_update_backup[id-a1cdd6f2-e9bc-4f6a-a0e6-2493ac6f9f27] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_create_backup[id-bad2514e-18c0-4fa0-9e35-221182ee24cf] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_delete_backup[id-a9ab3279-aa5e-4ad8-b740-b80a7769d3f9] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_export_backup[id-ab74b8cc-5005-49b4-94f4-994567171b07] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_import_backup[id-caaa5756-261a-4d9c-bfc2-788719630a06] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_list_backups[id-9dd02d4b-d6f8-45ca-a95e-534dbd586aab] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_list_project_backups[id-9ba2e970-c08b-4c1c-b912-2f3b1373ae6e] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_reset_backup_status[id-c832ff77-8f22-499f-a7a3-0834972a1507] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_restore_backup[id-0566fa4a-4e03-4cca-822f-d5a4922da2ab] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests33.test_show_backup[id-e88f8971-2892-4a54-80bb-dd21b18f19e9] +cinder_tempest_plugin.rbac.v3.test_backups.ProjectReaderTests39.test_update_backup[id-50ccc892-6ed0-4015-b181-9f64ffa45f33] +cinder_tempest_plugin.rbac.v3.test_capabilities.ProjectAdminTests.test_get_capabilities[id-1fdbe493-e58f-48bf-bb38-52003eeef8cb] +cinder_tempest_plugin.rbac.v3.test_capabilities.ProjectMemberTests.test_get_capabilities[id-dbaf51de-fafa-4f55-875f-7537524489ab] +cinder_tempest_plugin.rbac.v3.test_capabilities.ProjectReaderTests.test_get_capabilities[id-d16034fc-4204-4ea8-94b3-714de59fdfbf] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_create_snapshot[id-43f77b31-aab4-46d0-b76f-e17000d23589] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_delete_snapshot[id-22939122-8b4e-47d5-abaa-774bc55c07fc] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_force_delete_snapshot[id-e7cb3eb0-d607-4c90-995d-df82d030eca8] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_list_snapshots[id-5b3ec87f-443f-42f7-bd3c-ab05ea30c5e1] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_manage_snapshot[id-c2501d05-9bca-42d7-9ab5-c0d9133e762f] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_reset_snapshot_status[id-da391afd-8baa-458b-b222-f6ab42ab47c3] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_show_snapshot[id-6fee8967-951c-4957-b51b-97b83c13c7c3] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_unmanage_snapshot[id-dd7da3da-68ef-42f5-af1d-29803a4a04fd] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_update_snapshot_status[id-12e00e1b-bf84-41c1-8a1e-8625d1317789] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectMemberTests.test_update_snapshot[id-a774bdca-bfbe-477d-9711-5fb64d7e34ea] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_create_snapshot[id-13ae344f-fa01-44cc-b9f1-d04452940dc1] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_delete_snapshot[id-5b58f647-da0f-4d2a-bf68-680fc692efb4] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_force_delete_snapshot[id-63aa8184-897d-4e00-9b80-d2e7828f1b13] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_list_snapshots[id-dd8e19dc-c8fd-443c-8aed-cdffe07fa6be] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_manage_snapshot[id-d2d1326d-fb47-4448-a1e1-2d1219d30fd5] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_reset_snapshot_status[id-809d8c8c-25bf-4f1f-9b77-1a81ce4292d1] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_show_snapshot[id-6f69e8ed-4e11-40a1-9620-258cf3c45872] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_unmanage_snapshot[id-35495666-b663-4c68-ba44-0695e30a6838] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_update_snapshot_status[id-c90f98d7-3665-4c9f-820f-3f4c2adfdbf5] +cinder_tempest_plugin.rbac.v3.test_snapshots.ProjectReaderTests.test_update_snapshot[id-c46f5df8-9a6f-4ed6-b94c-3b65ef05ee9e] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectAdminTests.test_delete_message[id-1f2db6f2-148f-44c2-97ef-dcff0fccd49a] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectAdminTests.test_list_messages[id-f3567efc-863c-4668-8fb1-6aa3f836451d] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectAdminTests.test_show_message[id-eecc7045-017b-492c-8594-2d40f5fda139] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectMemberTests.test_delete_message[id-a77cd089-cb74-4b44-abcb-06f1a6f80378] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectMemberTests.test_list_messages[id-fb470249-a482-49c6-84af-eda34891a714] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectMemberTests.test_show_message[id-43d248ef-008d-4aff-8c7f-37959a0fa195] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectReaderTests.test_delete_message[id-c6e8744b-7749-425f-81b6-b1c3df6c7162] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectReaderTests.test_list_messages[id-1bef8bf9-6457-40f8-ada2-bc4d27602a07] +cinder_tempest_plugin.rbac.v3.test_user_messages.ProjectReaderTests.test_show_message[id-689c53a9-6db9-44a8-9878-41d28899e0af] +cinder_tempest_plugin.rbac.v3.test_volume_actions.ProjectReaderTests.test_extend_volume[id-4d721c58-2f6f-4857-8f4f-0664d5f7bf49] +cinder_tempest_plugin.rbac.v3.test_volume_actions.ProjectReaderTests.test_force_delete_volume[id-b025ff12-73a4-4f15-af55-876cd43cade3] +cinder_tempest_plugin.rbac.v3.test_volume_actions.ProjectReaderTests.test_reserve_volume[id-d2c13bf9-267a-4a71-be5c-391f22e9b433] +cinder_tempest_plugin.rbac.v3.test_volume_actions.ProjectReaderTests.test_reset_volume_status[id-434b454a-5cbe-492d-a416-70b8ff41f636] +cinder_tempest_plugin.rbac.v3.test_volume_actions.ProjectReaderTests.test_retype_volume[id-4675295a-7c72-4b04-8a43-03d7c88ab6bf] +cinder_tempest_plugin.rbac.v3.test_volume_actions.ProjectReaderTests.test_unreserve_volume[id-725d85cf-96b2-4338-98f4-2f468099c4ed] +cinder_tempest_plugin.rbac.v3.test_volume_actions.ProjectReaderTests.test_update_volume_readonly[id-3beecd52-e314-40d8-875d-a0e7db8dd88f] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_create_encryption_type[id-e7c4e925-6ce6-439b-8be8-6df4cbc32cdc] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_create_or_update_extra_specs_for_volume_type[id-422271a7-0128-4fd6-9f60-aeb4a1ce16ea] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_create_volume_type[id-57384db2-9408-4a31-8c15-022eea5f9b76] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_delete_encryption_type[id-e622af7d-a412-4903-9256-256d8e3cc560] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_delete_extra_spec_for_volume_type[id-4661cc2f-8727-4998-a427-8cb1d512b68a] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_delete_volume_type[id-7486259d-5c40-4fb3-8a95-491c45a0a872] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_list_all_extra_specs_for_volume_type[id-5c491d13-df15-4721-812e-2ed473b86a12] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_list_volume_types[id-e075e8ff-bb05-4c84-b2ab-0205ef3e8dbd] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_show_default_volume_type[id-93886ad8-5cd0-4def-8b0e-40418e55050d] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_show_encryption_spec_item[id-4ff57649-bfe1-48f4-aaac-4577affba8d7] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_show_encryption_type[id-46fc49a3-f76f-4c22-ac83-8d1665437810] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_show_extra_spec_for_volume_type[id-a2cca7b6-0af9-47e5-b8c1-4e0f01822d4e] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_show_volume_type_detail[id-7f794e33-b5cf-4172-b39e-a56cd9c18a2e] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_update_encryption_type[id-90beb71d-93fa-4252-8566-192bdd517715] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_update_extra_spec_for_volume_type[id-d0ff17d3-2c47-485f-b2f1-d53ec32c32e2] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesAdminTests.test_update_volume_type[id-77d065ef-ffdd-4749-b326-d64fbf5d0432] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_create_encryption_type[id-65d86181-905a-4aa6-a9e5-672415d819a0] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_create_or_update_extra_specs_for_volume_type[id-fda21e7e-9292-49b8-9754-f3c25b8e5f57] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_create_volume_type[id-81cebbb8-fa0d-4bd8-a433-e43c7b187456] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_delete_encryption_type[id-c0163522-524f-4dfb-a3d4-6648f58ce99c] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_delete_extra_spec_for_volume_type[id-3695be33-bd22-4090-8252-9c42eb7eeef6] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_delete_volume_type[id-6847c211-647b-4d02-910c-773e76b99fcd] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_list_all_extra_specs_for_volume_type[id-82fd0d34-17b3-4f45-bd2e-728c9a8bff8c] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_list_volume_types[id-308f80c9-6342-45a1-8e6e-9e400b510013] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_show_default_volume_type[id-2e990c61-a2ea-4a01-a2dc-1f483c934e8d] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_show_encryption_spec_item[id-387974ce-3544-48e3-81c0-3f86a5b60b93] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_show_encryption_type[id-7c84b013-c5a8-434f-8ea7-23c5b2d46d5e] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_show_extra_spec_for_volume_type[id-67aa0b40-7c0a-4ae7-8682-fb4f20abd390] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_show_volume_type_detail[id-319f3ca1-bdd7-433c-9bed-03c7b093e7a2] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_update_encryption_type[id-2633f1d3-e648-4d12-86b9-e7f72b41ec68] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_update_extra_spec_for_volume_type[id-65470a71-254d-4152-bdaa-6b7f43e9c74f] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesMemberTests.test_update_volume_type[id-e5e642bf-2f31-4d04-ad43-6ad75562b7e4] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_create_encryption_type[id-c7c0892e-08d1-45e0-8ebf-be949cb4ab02] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_create_or_update_extra_specs_for_volume_type[id-b046a4d7-79a0-436b-9075-863e2299b73d] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_create_volume_type[id-3c3a39b1-fff5-492b-8c1c-9520063901ef] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_delete_encryption_type[id-8d85ec39-bc32-4f49-88e6-63adc7e1f832] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_delete_extra_spec_for_volume_type[id-a7cdd9ae-f389-48f6-b144-abf336b1637b] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_delete_volume_type[id-35581811-6288-4698-aaaf-7f5a4fe662e8] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_list_all_extra_specs_for_volume_type[id-9499752c-3b27-41a3-8f55-4bdba7297f92] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_list_volume_types[id-e8a438f9-e9c1-4f3f-8ae3-ad80ee02cd6a] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_show_default_volume_type[id-aceab52a-c503-4081-936e-b9df1c31046d] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_show_encryption_spec_item[id-ab9c7149-fab7-4584-b4ff-8b997cd62e75] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_show_encryption_type[id-84bd20f1-621c-416d-add2-fbae57137239] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_show_extra_spec_for_volume_type[id-a38f7248-3a5b-4e51-8e32-d2dcf9c771ea] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_show_volume_type_detail[id-7ea28fc2-ce5a-48c9-8d03-31c2826fe566] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_update_encryption_type[id-8186d5bc-183a-4fcc-9c6a-e2b247a0caee] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_update_extra_spec_for_volume_type[id-68689644-22a8-4ba6-a642-db4258681586] +cinder_tempest_plugin.rbac.v3.test_volume_types.VolumeTypesReaderTests.test_update_volume_type[id-e3fdabf0-fd8c-4bab-9870-5a67fe25c6e4] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_create_volume[id-3d87f960-6210-45f5-b70b-679d67a4e17e] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_delete_volume[id-4fd4dce8-ed8a-4f05-8aac-da99858b563d] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_list_volumes_detail[id-b11e59cd-d1dd-43e4-9676-22ab394f5d18] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_list_volumes[id-2f4da8f9-cdc5-4a6e-9143-8237634a629c] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_set_bootable_volume[id-9970b57d-8d5d-460e-931b-28a112df81e0] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_show_volume_summary[id-ef347930-54dc-432f-b742-0a060fc37ae8] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_show_volume[id-9b2667f2-744e-4d1f-8c39-17060010f19f] +cinder_tempest_plugin.rbac.v3.test_volumes.ProjectReaderTests.test_update_volume[id-cda92972-7213-4fa0-bc14-ab012dc95931] +cinder_tempest_plugin.scenario.test_encrypted_volume_transfer.TransferEncryptedVolumeTest.test_create_accept_volume_transfer[compute,id-a694dc4d-d11b-45cb-b268-62e76cc1b4f4,image,network,volume] +cinder_tempest_plugin.scenario.test_encrypted_volume_transfer.TransferEncryptedVolumeTest.test_create_delete_volume_transfer[compute,id-00c04d27-b3c6-454c-a0b4-223a195c4a89,image,network,volume] +cinder_tempest_plugin.scenario.test_snapshots.SnapshotDataIntegrityTests.test_snapshot_data_integrity[compute,id-ff10644e-5a70-4a9f-9801-8204bb81fb61,image,network,volume] +cinder_tempest_plugin.scenario.test_snapshots.SnapshotDependencyTests.test_snapshot_removal[id-e7028f52-f6d4-479c-8809-6f6cf96cfe0f,image,volume] +cinder_tempest_plugin.scenario.test_volume_encrypted.TestEncryptedCinderVolumes.test_attach_cloned_encrypted_volume[compute,id-5bb622ab-5060-48a8-8840-d589a548b9e4,volume] +cinder_tempest_plugin.scenario.test_volume_encrypted.TestEncryptedCinderVolumes.test_boot_cloned_encrypted_volume[compute,id-5bb622ab-5060-48a8-8840-d589a548b7e4,image,volume] +cinder_tempest_plugin.scenario.test_volume_multiattach.VolumeMultiattachTests.test_multiattach_data_integrity[id-e6604b85-5280-4f7e-90b5-186248fd3423] +cinder_tempest_plugin.scenario.test_volume_multiattach.VolumeMultiattachTests.test_volume_multiattach_same_host_negative[id-53514da8-f49c-4cda-8792-ff4a2fa69977] diff --git a/.automation.conf/tempest/load-lists/cloudkitty b/.automation.conf/tempest/load-lists/cloudkitty new file mode 100644 index 000000000..4c78aeb7e --- /dev/null +++ b/.automation.conf/tempest/load-lists/cloudkitty @@ -0,0 +1,34 @@ +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_create_get_delete_collector_mapping[id-af902c86-6022-4b94-a716-ec7932d5ae78] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_get_collector_mappings[id-9c1d4c27-6e7c-42d7-b663-d88f097b7131] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_get_collector_state[id-3fd83647-3058-4450-9588-a528557585c5] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_get_rated_tenants[id-e439019e-9e8a-4bcd-aa83-95bdba6e6115] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_get_rating_modules[id-fba44b6a-6ca4-4155-b5c6-c4eb2465e4fb] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_reload_rating_modules[id-daeef22b-d52d-4e89-abb0-ae492e4648d4] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_set_collector_state[id-71131104-fdae-43ec-9bed-c8d1d5ba7eb0] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyAdminAPITest.test_update_rating_module[id-7fc9e020-9547-4a66-a691-94cab7181358] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyPrimaryAPITest.test_get_config[id-3285bccf-d043-4ad1-b64f-af4db8317cf9] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyPrimaryAPITest.test_get_metric[id-64ecae87-0138-41bd-829f-91302dae7802] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyPrimaryAPITest.test_get_metrics[id-43b03099-0493-4291-9749-85cd8d512811] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyPrimaryAPITest.test_get_storage_dataframes[id-e233139b-3c75-4b70-b1f5-0776ef32c916] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyPrimaryAPITest.test_report_summary[id-cccbff8a-24b2-4251-8f7b-ea941d048b9d] +cloudkitty_tempest_plugin.tests.api.v1.test_cloudkitty_api.CloudkittyPrimaryAPITest.test_report_total[id-2492dfd7-0688-4957-93ac-8c91933c28f5] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_create_delete_hashmap_field[id-974669f9-392e-4aec-8e15-d24db23f08d4] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_create_delete_hashmap_group[id-0f9200ab-146b-4349-a579-ce12062f465b] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_create_delete_hashmap_mapping[id-b7ad24e5-c72c-469a-886b-db43aab8f328] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_create_delete_hashmap_service[id-9e968284-7209-46e1-9742-4882b6e2cf2f] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_create_delete_update_hashmap_threshold[id-d2b3dba3-91df-4aa7-9ae2-96d971df2dbf] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_field[id-9ae98590-ff55-47f9-885e-baa4da1957d1] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_group_mappings[id-92860fc8-596a-42fd-b0d5-97e0f5a7bd2c] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_group[id-858a019a-fb64-4656-b7a6-c92917f641ab] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_mapping_group[id-98ca42dd-a9e2-477a-8e42-16389aed1f44] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_mappings[id-4deb6914-7ba0-4219-a119-61b39bd58807] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_rating_types[id-7037a3f8-b462-4243-a0bc-ffa3b4700397] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_service[id-6c4260c0-8701-4959-b00e-4789d31715a7] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_services[id-9e9a67d1-e53d-46cf-8e13-8a332c40c32f] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_threshold[id-dc463432-3b92-44ac-8caf-b789857f9db7] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_get_hashmap_thresholds[id-d04caad5-eb18-40ce-817e-13c257633cca] +cloudkitty_tempest_plugin.tests.api.v1.test_hashmap_api.CloudkittyHashmapAPITest.test_update_hashmap_mapping[id-6a04634d-2d3a-406e-980a-e7c7c9cc081b] +cloudkitty_tempest_plugin.tests.api.v1.test_pyscript_api.CloudkittyPyscriptAPITest.test_create_update_delete_pyscript[id-9e78cccc-ca85-42ce-8648-4cd9682375df] +cloudkitty_tempest_plugin.tests.api.v1.test_pyscript_api.CloudkittyPyscriptAPITest.test_get_pyscript[id-3fbaf8b4-c472-4509-8d73-55dc4a87a442] +cloudkitty_tempest_plugin.tests.api.v1.test_pyscript_api.CloudkittyPyscriptAPITest.test_get_pyscripts[id-2015c966-b707-40f7-b84d-9aa6550b9e41] +cloudkitty_tempest_plugin.tests.api.v1.test_pyscript_api.CloudkittyPyscriptAPITestNegative.test_update_script_negative[id-999c97cc-1d71-43b8-988f-d89b8fac4040,negative] diff --git a/.automation.conf/tempest/load-lists/default b/.automation.conf/tempest/load-lists/default new file mode 120000 index 000000000..3bc4bc8ed --- /dev/null +++ b/.automation.conf/tempest/load-lists/default @@ -0,0 +1 @@ +platform.2022.11-test-list.txt \ No newline at end of file diff --git a/.automation.conf/tempest/load-lists/designate b/.automation.conf/tempest/load-lists/designate new file mode 100644 index 000000000..ac8390871 --- /dev/null +++ b/.automation.conf/tempest/load-lists/designate @@ -0,0 +1,370 @@ +designate_tempest_plugin.tests.api.admin.test_quotas.QuotasAdminTest.test_delete_quotas[id-33e0affb-5d66-4216-881c-f101a779851a] +designate_tempest_plugin.tests.api.admin.test_quotas.QuotasAdminTest.test_show_quotas[id-ed42f367-e5ba-40d7-a08d-366ad787d21c] +designate_tempest_plugin.tests.api.admin.test_quotas.QuotasAdminTest.test_update_quotas[id-4f2b65b7-c4e1-489c-9047-755e42ba0985] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_create_blacklist_as_primary_fails[id-fe9de464-d8d1-11eb-bcdc-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_create_blacklist_huge_size_description[id-664bdaa0-da47-11eb-b8b8-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_create_blacklist_invalid_pattern[id-ea608152-da3c-11eb-b8b8-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_create_blacklist[id-3a7f7564-6bdd-446e-addc-a3475b4c3f71] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_delete_blacklist[id-dcea40d9-8d36-43cb-8440-4a842faaef0d] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_list_blacklists[id-3a2a1e6c-8176-428c-b5dd-d85217c0209d] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_show_blacklist[id-5bc02942-6225-4619-8f49-2105581a8dd6] +designate_tempest_plugin.tests.api.v2.test_blacklists.BlacklistsAdminTest.test_update_blacklist[id-0063d6ad-9557-49c7-b521-e64a14d4d0d0] +designate_tempest_plugin.tests.api.v2.test_blacklists.TestBlacklistInvalidIdAdmin.test_delete_blacklist_invalid_uuid[id-5d62a026-13e4-48b9-9773-1780660c5920] +designate_tempest_plugin.tests.api.v2.test_blacklists.TestBlacklistInvalidIdAdmin.test_show_blacklist_invalid_uuid[id-c7bae53f-2edc-45d8-b254-8a81482728c1] +designate_tempest_plugin.tests.api.v2.test_blacklists.TestBlacklistInvalidIdAdmin.test_update_blacklist_invalid_uuid[id-c57b97da-ca87-44b5-9f40-a099937433bf] +designate_tempest_plugin.tests.api.v2.test_blacklists.TestBlacklistNotFoundAdmin.test_delete_blacklist_404[id-b1132586-bf06-47a6-9f6f-3bab6a2c1932] +designate_tempest_plugin.tests.api.v2.test_blacklists.TestBlacklistNotFoundAdmin.test_show_blacklist_404[id-9d65b638-fe98-47a8-853f-fa9244d144cc] +designate_tempest_plugin.tests.api.v2.test_blacklists.TestBlacklistNotFoundAdmin.test_update_blacklist_404[id-a9e12415-5040-4fba-905c-95d201fcfd3b] +designate_tempest_plugin.tests.api.v2.test_designate_limits.DesignateLimit.test_list_designate_impersonate_another_user_as_admin[id-828572be-8662-11eb-8ff2-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_designate_limits.DesignateLimit.test_list_designate_impersonate_another_user_as_alt[id-5975fee0-d430-11eb-aa4d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_designate_limits.DesignateLimit.test_list_designate_limits_all_projects[id-828572be-8662-11eb-8ff2-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_designate_limits.DesignateLimit.test_list_designate_limits_as_primary_user[id-828572be-8662-11eb-8ff2-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_designate_limits.DesignateLimit.test_list_designate_limits_RBAC[id-fc57fa6b-5280-4186-9be9-ff4da0961db0] +designate_tempest_plugin.tests.api.v2.test_enabled_api_version.DesignateApiVersion.test_list_enabled_api_versions[id-aa84986e-f2ad-11eb-b58d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_pool.PoolAdminTest.test_create_pool[id-69257f7c-b3d5-4e1b-998e-0677ad12f125] +designate_tempest_plugin.tests.api.v2.test_pool.PoolAdminTest.test_delete_pool[id-d8c4c377-5d88-452d-a4d2-c004d72e1abe] +designate_tempest_plugin.tests.api.v2.test_pool.PoolAdminTest.test_list_pools_dot_json_fails[id-41ad6a84-00ce-4a04-9fd5-b7c15c31e2db] +designate_tempest_plugin.tests.api.v2.test_pool.PoolAdminTest.test_list_pools[id-77c85b40-83b2-4c17-9fbf-e6d516cfce90] +designate_tempest_plugin.tests.api.v2.test_pool.PoolAdminTest.test_show_pool[id-e80eb70a-8ee5-40eb-b06e-599597a8ab7e] +designate_tempest_plugin.tests.api.v2.test_pool.PoolAdminTest.test_update_pool[id-fdcc84ce-af65-4af6-a5fc-6c50acbea0f0] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolAdminNegative.test_create_pool_invalid_hostname_in_ns_records[id-9a787d0e-ac04-11eb-ae06-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolAdminNegative.test_create_pool_invalid_name[id-0a8cdc1e-ac02-11eb-ae06-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolAdminNegative.test_create_pool_invalid_priority_in_ns_records[id-9a787d0e-ac04-11eb-ae06-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolAdminNegative.test_update_pool_with_invalid_hostname_in_ns_records[id-2e496596-ac07-11eb-ae06-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolAdminNegative.test_update_pool_with_invalid_name[id-cc378e4c-ac05-11eb-ae06-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolAdminNegative.test_update_pool_with_invalid_priority_in_ns_records[id-3e934624-ac07-11eb-ae06-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolInvalidIdAdmin.test_delete_pool_invalid_uuid[id-bf5ad3be-2e79-439d-b247-902fe198143b] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolInvalidIdAdmin.test_show_pool_invalid_uuid[id-081d0188-42a7-4953-af0e-b022960715e2] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolInvalidIdAdmin.test_update_pool_invalid_uuid[id-f4ab4f5a-d7f0-4758-b232-8338f02d7c5c] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolNotFoundAdmin.test_delete_pool_404[id-96132295-896b-4de3-8f86-cc2ee513fdad] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolNotFoundAdmin.test_show_pool_404[id-56281b2f-dd5a-4376-8c32-aba771062fa5] +designate_tempest_plugin.tests.api.v2.test_pool.TestPoolNotFoundAdmin.test_update_pool_404[id-10fba3c2-9972-479c-ace1-8f7eac7c159f] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecord.test_list_floatingip_ptr_all_projects[id-a108d6f2-e3c0-11eb-a097-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecord.test_list_floatingip_ptr_records[id-9187a9c6-87d4-11eb-9f9a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecord.test_set_floatingip_ptr[id-2fb9d6ea-871d-11eb-9f9a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecord.test_show_floatingip_ptr_impersonate_another_project[id-d3128a92-e3bd-11eb-a097-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecord.test_show_floatingip_ptr[id-9179325a-87d0-11eb-9f9a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecord.test_unset_floatingip_ptr[id-499b5a7e-87e1-11eb-b412-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecordNegative.test_list_floatingip_ptr_all_projects_no_header[id-0d132ff0-51ad-11ec-8edf-201e8823901f,negative] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecordNegative.test_set_floatingip_ptr_huge_size_description[id-df217902-e3b2-11eb-a097-74e5f9e2a801,negative] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecordNegative.test_set_floatingip_ptr_invalid_name[id-cb2264e2-e3b3-11eb-a097-74e5f9e2a801,negative] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecordNegative.test_set_floatingip_ptr_invalid_ttl[id-8392db50-cdd0-11eb-a00f-74e5f9e2a801,negative] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecordNegative.test_set_floatingip_ptr_not_existing_fip_id[id-0c9349ae-e2e8-11eb-a097-74e5f9e2a801,negative] +designate_tempest_plugin.tests.api.v2.test_ptrs.DesignatePtrRecordNegative.test_show_floatingip_ptr_impersonate_another_project_no_header[id-f616d216-51ac-11ec-8edf-201e8823901f,negative] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2Test.test_reset_quotas_other_project[id-21e45d30-dbc1-4173-9d6b-9b6813ef514b] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2Test.test_reset_quotas[id-0448b089-5803-4ce3-8a6c-5c15ff75a2cc] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2Test.test_show_quotas[id-1dac991a-9e2e-452c-a47a-26ac37381ec5] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2Test.test_update_quotas_invalid_project[id-9b09b3e2-7e88-4569-bce3-9be2f7ac70c4] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2Test.test_update_quotas_other_project[id-9b09b3e2-7e88-4569-bce3-9be2f7ac70c3] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2Test.test_update_quotas[id-76d24c87-1b39-4e19-947c-c08e1380dc61] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2TestNegative.test_admin_sets_invalid_quota_values[id-a6ce5b46-dcce-11eb-903e-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2TestNegative.test_admin_sets_not_existing_quota_type[id-ac212fd8-c602-11ec-b042-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2TestNegative.test_admin_sets_quota_for_a_project[id-ae82a0ba-da60-11eb-bf12-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_quotas.QuotasV2TestNegative.test_primary_fails_to_set_quota[id-40b9d7ac-da5f-11eb-bf12-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_cannot_create_MX_with[id-7fa7783f-1624-4122-bfb2-6cfbf7a5b49b] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_cannot_create_SSHFP_with[id-3016f998-4e4a-4712-b15a-4e8dfbc5a60b] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_cannot_create_TXT_with[id-22a9544b-2382-4ed2-ba12-4dbaedb8e880] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_cannot_create_wildcard_NS_recordset[id-61da1015-291f-43d1-a1a8-345cff12d201] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_cname_recordsets_cannot_have_more_than_one_record[id-92f681aa-d953-4d18-b12e-81a9149ccfd9] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_create_invalid[id-c5ef87e2-cb79-4758-b968-18eef2c251df] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_create_SPF_with[id-775b3db5-ec60-4dd7-85d2-f05a9c544978] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_create_TXT_with[id-03e4f811-0c37-4ce2-8b16-662c824f8f18] +designate_tempest_plugin.tests.api.v2.test_recordset_validation.RecordsetValidationTest.test_update_invalid[id-1164c826-dceb-4557-9a22-7d65c4a4f5f4] +designate_tempest_plugin.tests.api.v2.test_recordset.AdminManagedRecordsetTest.test_admin_updates_soa_and_ns_recordsets[id-84164ff4-8e68-11ec-983f-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetOwnershipTest.test_list_all_projects_recordsets[id-bc0af248-7b4f-11eb-98a5-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetOwnershipTest.test_list_all_recordsets_for_project[id-4d0ff972-7c19-11eb-b331-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetOwnershipTest.test_list_recordsets_impersonate_project[id-910eb17e-7c3a-11eb-a40b-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetOwnershipTest.test_no_create_recordset_by_alt_tenant[id-9c0f58ad-1b31-4899-b184-5380720604e5] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetOwnershipTest.test_no_create_recordset_via_alt_domain[id-3dbe244d-fa85-4afc-869b-0306388d8746] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetOwnershipTest.test_no_create_super_recordsets[id-d4a9aad9-c778-429b-9a0c-4cd2b61a0a01] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsManagedRecordsNegativeTest.test_create_soa_record_not_permitted[id-1e78a742-66ee-11ec-8dc3-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsManagedRecordsNegativeTest.test_delete_ns_record_not_permitted[id-083fa738-bb1b-11ec-b581-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_create_recordset_for_other_tenant[id-c1d9f046-a2b1-11eb-aad4-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_create_recordset_invalid_CNAME_at_Apex[id-43385ce8-0dd3-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_create_recordset_invalid_CNAME_multiple[id-98c94f8c-217a-4056-b996-b1f856d0753e] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_delete_nonexistent_recordset_invalid_id[id-5948b599-a332-4dcb-840b-afc825075ba3] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_delete_nonexistent_recordset[id-64bd94d4-54bd-4bee-b6fd-92ede063234e] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_get_nonexistent_recordset_invalid_id[id-93d744a8-0dfd-4650-bcef-1e6ad632ad72] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_get_nonexistent_recordset[id-b6dad57e-5ce9-4fa5-8d66-aebbcd23b4ad] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_show_recordsets_invalid_ids[id-64e01dc4-a2a8-11eb-aad4-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_update_nonexistent_recordset_invalid_id[id-158340a1-3f69-4aaa-9968-956190563768] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsNegativeTest.test_update_nonexistent_recordset[id-da08f19a-7f10-47cc-8b41-994507190812] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_admin_list_all_recordsets_for_a_project[id-9f364a64-a2aa-11eb-aad4-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_A_recordset_multiply_ips[id-48013b7c-f526-11eb-b04f-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_A[id-d03b69a5-5052-43bc-a38a-b511b6b34304] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_AAAA[id-ac110198-d58a-4a18-aceb-414d7e513d49] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_CNAME[id-1ac46f94-f03a-4f85-b84f-826a2660b927] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_MX_at_APEX[id-bf872487-7975-4a96-bb03-d24e393a0ce8] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_MX_under_APEX[id-96fe72a4-a81c-4a01-a81f-39ebafad115c] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_SRV_TCP[id-6c22a3f9-3f4d-4b32-bdf2-5237851ed25e] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_SRV_UDP[id-59c1aa42-278e-4f7b-a6a1-4320d5daf1fd] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_SSHFP[id-481496f1-917a-40d5-89fd-4a3794c24215] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset_type_TXT[id-8e7ecedb-5c35-46f8-ae0e-39e4aaabc97d] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_recordset[id-631d74fd-6909-4684-a61b-5c4d2f92c3e7,smoke] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_A_at_APEX[id-69f002e5-6511-43d3-abae-7abdd45ae03e] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_A_under_APEX[id-d97ee452-0dc3-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_AAAA_at_APEX[id-1b3c1cc0-0dc4-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_AAAA_under_APEX[id-928e735e-0dc4-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_MX_at_APEX[id-d96138f2-0dc4-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_MX_under_APEX[id-ff273c94-0dc4-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_SPF_at_APEX[id-3097f16a-0dc5-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_SPF_under_APEX[id-50b3f390-0dc5-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_TXT_at_APEX[id-5b73981c-0dc5-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_create_wildcard_recordset_TXT_under_APEX[id-5b73981c-0dc5-11ee-8b75-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_delete_recordset[id-855399c1-8806-4ae5-aa31-cb8a6f35e218] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_delete_zone_with_existing_recordset[id-f15e583e-e479-11eb-8e5a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_list_recordsets[id-5964f730-5546-46e6-9105-5030e9c492b2] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_show_recordsets_impersonate_another_project[id-3f3575a0-a28b-11eb-ab42-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_show_recordsets[id-84c13cb2-9020-4c1e-aeb0-c348d9a70caa] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_update_recordset_one_field[id-60904cc5-148b-4e3b-a0c6-35656dc8d44c] +designate_tempest_plugin.tests.api.v2.test_recordset.RecordsetsTest.test_update_recordset[id-8d41c85f-09f9-48be-a202-92d1bdf5c796] +designate_tempest_plugin.tests.api.v2.test_recordset.RootRecordsetsTests.test_get_single_zones_recordsets[id-65ec0495-81d9-4cfb-8007-9d93b32ae883] +designate_tempest_plugin.tests.api.v2.test_recordset.RootRecordsetsTests.test_list_filter_zones_recordsets[id-a8e41020-65be-453b-a8c1-2497d539c345] +designate_tempest_plugin.tests.api.v2.test_recordset.RootRecordsetsTests.test_list_zones_recordsets_zone_names[id-7f4970bf-9aeb-4a3c-9afd-02f5a7178d35] +designate_tempest_plugin.tests.api.v2.test_recordset.RootRecordsetsTests.test_list_zones_recordsets[id-48a081b9-4474-4da0-9b1a-6359a80456ce] +designate_tempest_plugin.tests.api.v2.test_service_statuses.ServiceStatusAdmin.test_admin_list_service_statuses[id-bf277a76-8583-11eb-a557-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_service_statuses.ServiceStatusAdmin.test_admin_show_service_status[id-fce0f704-c0ae-11ec-8213-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_service_statuses.ServiceStatusNegative.test_primary_is_forbidden_to_list_service_statuses[id-d4753f76-de43-11eb-91d1-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTest.test_create_zone_share_all_projects_header[id-2bb7bcb2-b824-11ed-9e56-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTest.test_create_zone_share_sudo_project_header[id-f26cd3ac-b8fa-11ed-b4ca-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTest.test_delete_zone_share_sudo_project_header[id-46f7db22-b90c-11ed-b4ca-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTest.test_list_zone_shares_all_projects_header[id-2eedfd60-b90f-11ed-b4ca-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTest.test_show_shared_zone_all_projects_header[id-ce2688e8-b90a-11ed-b4ca-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_create_zone_share_invalid_project_id[id-595ae1fc-bce4-11ed-baf2-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_create_zone_share_invalid_zone_id[id-aa42d82e-bcf6-11ed-baf2-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_delete_zone_share_invalid_project_id[id-7136b430-bdb2-11ed-80f5-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_delete_zone_share_invalid_shared_zone_id[id-d44c65e2-bdc1-11ed-80f5-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_delete_zone_share_invalid_zone_id[id-06de2342-bdc2-11ed-80f5-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_list_zone_shares_invalid_project_id[id-e71068c8-bdb1-11ed-80f5-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_list_zone_shares_invalid_zone_id[id-871e7e1c-bd9a-11ed-80f5-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_show_shared_zone_invalid_project_id[id-8852329c-bd95-11ed-80f5-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.AdminSharedZonesTestNegative.test_show_shared_zone_invalid_shared_zone_id[id-9e7202ba-bd94-11ed-80f5-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.NegativeSharedZonesTest.test_share_zone_with_yourself_is_not_allowed[id-957ba3f8-c250-11ed-a8b1-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_shared_zones.NegativeSharedZonesTest.test_target_project_cannot_delete_zone[id-4389a12b-8609-493c-9640-d3c67b625022] +designate_tempest_plugin.tests.api.v2.test_shared_zones.NegativeSharedZonesTest.test_target_project_cannot_subzone[id-abc0f820-ae27-4e85-8f00-0b8e8abf3ae9] +designate_tempest_plugin.tests.api.v2.test_shared_zones.NegativeSharedZonesTest.test_target_project_cannot_update_zone[id-f4354b5c-8dbb-4bb9-8025-f65f8f2b21fb] +designate_tempest_plugin.tests.api.v2.test_shared_zones.NegativeSharedZonesTest.test_target_project_share_permissions[id-4389a12b-8609-493c-9640-d3c67b625022] +designate_tempest_plugin.tests.api.v2.test_shared_zones.SharedZonesTest.test_create_zone_share[id-982a7780-a460-4c13-97df-b4855bf19c7b] +designate_tempest_plugin.tests.api.v2.test_shared_zones.SharedZonesTest.test_delete_zone_share[id-a18a8577-9d02-492a-a869-4ff7d6f4f89b] +designate_tempest_plugin.tests.api.v2.test_shared_zones.SharedZonesTest.test_list_zone_shares[id-707bfa4f-f15b-4486-ba5c-0e5991f0f3a5] +designate_tempest_plugin.tests.api.v2.test_shared_zones.SharedZonesTest.test_show_shared_zone[id-0edecb9b-4890-433c-8195-0935271efc9a] +designate_tempest_plugin.tests.api.v2.test_tld.TestTldInvalidIdAdmin.test_delete_tld_invalid_uuid[id-6a6fc9db-9a73-4ffc-831a-172e1cbc7394] +designate_tempest_plugin.tests.api.v2.test_tld.TestTldInvalidIdAdmin.test_show_tld_invalid_uuid[id-f9ec0730-57ff-4720-8d06-e11d377c7cfc] +designate_tempest_plugin.tests.api.v2.test_tld.TestTldInvalidIdAdmin.test_update_tld_invalid_uuid[id-13dc6518-b479-4502-90f5-f5a5ecc8b1fb] +designate_tempest_plugin.tests.api.v2.test_tld.TestTldNotFoundAdmin.test_delete_tld_404[id-18e465e7-5c7d-4775-acef-bd12a8db1095] +designate_tempest_plugin.tests.api.v2.test_tld.TestTldNotFoundAdmin.test_show_tld_404[id-b237d5ee-0d76-4294-a3b6-c2f8bf4b0e30] +designate_tempest_plugin.tests.api.v2.test_tld.TestTldNotFoundAdmin.test_update_tld_404[id-3d128772-7f52-4473-b569-51ae8294667b] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_create_duplicated_tlds[id-961bd2e8-d4d0-11eb-b8ee-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_create_invalid_tld[id-52a4bb4b-4eff-4591-9dd3-ad98316806c3] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_create_multiply_tlds[id-0c0ab92e-d4db-11eb-b8ee-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_create_tld_as_primary_user[id-757019c0-d4e2-11eb-b8ee-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_create_tld[id-52a4bb4b-4eff-4591-9dd3-ad98316806c3] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_delete_tld[id-26708cb8-7126-48a7-9424-1c225e56e609] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_list_tlds_dot_json_fails[id-8116dcf5-a329-47d1-90be-5ff32f299c53] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_list_tlds[id-95b13759-c85c-4791-829b-9591ca15779d] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_show_tld[id-271af08c-2603-4f61-8eb1-05887b74e25a] +designate_tempest_plugin.tests.api.v2.test_tld.TldAdminTest.test_update_tld[id-1a233812-48d9-4d15-af5e-9961744286ff] +designate_tempest_plugin.tests.api.v2.test_transfer_accepts.TransferAcceptTest.test_create_transfer_accept[id-1c6baf97-a83e-4d2e-a5d8-9d37fb7808f3] +designate_tempest_plugin.tests.api.v2.test_transfer_accepts.TransferAcceptTest.test_list_transfer_accepts[id-0fcd314c-8cae-11eb-a322-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_transfer_accepts.TransferAcceptTest.test_ownership_transferred_zone[id-89b516f0-8c9f-11eb-a322-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_transfer_accepts.TransferAcceptTest.test_show_transfer_accept_impersonate_another_project[id-b6ac770e-a1d3-11eb-b534-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_transfer_accepts.TransferAcceptTest.test_show_transfer_accept[id-37c6afbb-3ea3-4fd8-94ea-a426244f019a] +designate_tempest_plugin.tests.api.v2.test_transfer_accepts.TransferAcceptTestNegative.test_create_transfer_accept_using_deleted_transfer_request_id[id-23afb948-a1ce-11eb-b534-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_transfer_accepts.TransferAcceptTestNegative.test_create_transfer_accept_using_invalid_key[id-324a3e80-a1cc-11eb-b534-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TestTransferRequestInvalidId.test_delete_transfer_request_invalid_uuid[id-1728dca5-01f1-45f4-b59d-7a981d479394] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TestTransferRequestInvalidId.test_show_transfer_request_invalid_uuid[id-2205dd19-ecc7-4c68-9e89-63c47d642b07] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TestTransferRequestInvalidId.test_update_transfer_request_invalid_uuid[id-af0ce46f-10be-4cce-a1d5-1b5c2a39fb97] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TestTransferRequestNotFound.test_create_transfer_request_404[id-39131f7c-e9bb-4f92-a325-444a675e1b3d] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TestTransferRequestNotFound.test_delete_transfer_request_404[id-5a4a0755-c01d-448f-b856-b081b96ae77e] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TestTransferRequestNotFound.test_show_transfer_request_404[id-d255f72f-ba24-43df-9dba-011ed7f4625d] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TestTransferRequestNotFound.test_update_transfer_request_404[id-9ff383fb-c31d-4c6f-8085-7b261e401223] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_create_transfer_request_empty_body[id-4505152f-0a9c-4f02-b385-2216c914a0be] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_create_transfer_request_scoped[id-5deae1ac-7c14-42dc-b14e-4e4b2725beb7] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_create_transfer_request[id-2381d489-ad84-403d-b0a2-8b77e4e966bf] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_delete_transfer_request[id-7d81c487-aa15-44c4-b3e5-424ab9e6a3e5] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_list_transfer_requests_all_projects[id-db985892-9d02-11eb-a160-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_list_transfer_requests_dot_json_fails[id-73b754a9-e856-4fd6-80ba-e8d1b80f5dfa] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_list_transfer_requests_impersonate_another_project[id-bee42f38-e666-4b85-a710-01f40ea1e56a] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_list_transfer_requests[id-ddd42a19-1768-428c-846e-32f9d6493011] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_show_transfer_request_as_target[id-235ded87-0c47-430b-8cad-4f3194b927a6] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_show_transfer_request_impersonate_another_project[id-5bed4582-9cfb-11eb-a160-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_show_transfer_request[id-64a7be9f-8371-4ce1-a242-c1190de7c985] +designate_tempest_plugin.tests.api.v2.test_transfer_request.TransferRequestTest.test_update_transfer_request[id-de5e9d32-c723-4518-84e5-58da9722cc13] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_create_tsigkey_for_pool_with_scope_zone[id-0dfbc2f8-d8bb-11eb-b95a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_create_tsigkey_for_zone_empty_secret[id-5d6b8a84-d745-11eb-beba-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_create_tsigkey_for_zone_invalid_algorithm[id-f94af13a-d743-11eb-beba-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_create_tsigkey_for_zone_invalid_name[id-4df903d8-d745-11eb-beba-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_create_tsigkey_for_zone_invalid_scope[id-dfca9268-d745-11eb-beba-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_create_tsigkey_for_zone_invalid_zone_id[id-57255858-d74a-11eb-beba-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_delete_tsigkey_invalid_uuid[id-55c2fea0-ead6-44c7-8bb1-05111412afdd] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_show_tsigkey_invalid_uuid[id-2a8dfc75-9884-4b1c-8f1f-ed835d96f2fe] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyInvalidIdAdmin.test_update_tsigkey_invalid_uuid[id-2befa10f-fc42-4ae9-9276-672e23f045f2] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyNotFoundAdmin.test_delete_tsigkey_404[id-ba438ede-4823-4922-8f4c-8de278f3d454] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyNotFoundAdmin.test_show_tsigkey_404[id-824c9b49-edc5-4282-929e-467a158d23e4] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TestTsigkeyNotFoundAdmin.test_update_tsigkey_404[id-4ef3493a-ee66-4c62-b070-c57fa9568b69] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_create_tsigkey_for_pool[id-45975fa6-d726-11eb-beba-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_create_tsigkey_for_zone[id-e7b484e3-7ed5-4840-89d7-1e696986f8e4] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_delete_tsigkey[id-9cdffbd2-bc67-4a25-8eb7-4be8635c88a3] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkey_filter_by_algorithm[id-794554f0-d8b8-11eb-b95a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkey_filter_by_name[id-4162a840-d8b2-11eb-b95a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkey_filter_by_scope[id-e8bcf80a-d8b4-11eb-b95a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkey_sort_key_with_sort_direction[id-d5c6dfcc-d8af-11eb-b95a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkey[id-d46e5e86-a18c-4315-aa0c-95a00e816fbf] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkeys_dot_json_fails[id-4bdc20ef-96f9-47f6-a1aa-275159af326b] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkeys_limit_results[id-d46e5e86-a18c-4315-aa0c-95a00e816fbf] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_list_tsigkeys_using_marker[id-f31447b0-d817-11eb-b95a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_show_tsigkey[id-c5d7facf-0f05-47a2-a4fb-87f203860880] +designate_tempest_plugin.tests.api.v2.test_tsigkey.TsigkeyAdminTest.test_update_tsigkey[id-d09dc0dd-dd72-41ee-9085-2afb2bf35459] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_create_blacklist[id-65cbc2ee-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_create_pool[id-c77d62f4-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_create_recordset[id-05c0236e-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_create_tld[id-ef3ae024-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_create_zone[id-56e899c0-de93-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_delete_blacklist[id-66321184-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_delete_pool[id-c7e07682-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_delete_recordset[id-063c95b6-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_delete_tld[id-efb982e4-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_delete_zone[id-57b5cef4-de93-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_list_blacklists[id-654e7596-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_list_pools[id-c7048d66-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_list_recordsets[id-05099b62-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_list_tlds[id-ee9dae6c-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_list_zones[id-b18827ac-de92-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_show_blacklist[id-658ea9cc-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_show_pool[id-c74581cc-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_show_recordset[id-0573ca32-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_show_tld[id-eef1e5f4-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_show_zone[id-f60c32ce-de92-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_update_blacklist[id-66032676-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_update_pool[id-c7ada040-de95-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_update_recordset[id-0600f628-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_update_tld[id-ef7cfda6-de94-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_unauthed.TestDnsUnauthed.test_update_zone[id-5765af6e-de93-11ed-8334-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_zone_tasks.ZoneTasks.test_zone_abandon_forbidden[id-90b21d1a-a1ba-11eb-84fa-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zone_tasks.ZoneTasks.test_zone_abandon[id-287e2cd0-a0e7-11eb-b962-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_create_zone_export[id-2dd8a9a0-98a2-4bf6-bb51-286583b30f40] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_delete_zone_export[id-97234f00-8bcb-43f8-84dd-874f8bc4a80e] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_list_zone_exports_all_projects[id-f34e7f34-9613-11eb-b1cd-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_list_zone_exports_filter_results[id-e4a11a14-9aaa-11eb-be59-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_list_zone_exports[id-476bfdfe-58c8-46e2-b376-8403c0fff440] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_show_zone_export_impersonate_another_project[id-fb04507c-9600-11eb-b1cd-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTest.test_show_zone_export[id-2d29a2a9-1941-4b7e-9d8a-ad6c2140ea68,smoke] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTestNegative.test_create_zone_export_using_deleted_zone[id-518dc308-9604-11eb-b1cd-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTestNegative.test_create_zone_export_using_invalid_zone_id[id-76ab8ec4-95fd-11eb-b1cd-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTestNegative.test_export_not_your_zone[id-943dad4a-9617-11eb-b1cd-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTestNegative.test_show_zonefile_not_supported_accept_value[id-52a1fee0-c338-4ed9-b9f9-41ee7fd73375] +designate_tempest_plugin.tests.api.v2.test_zones_exports.ZonesExportTestNegative.test_show_zonefile_using_not_existing_zone_export_id[id-9a878646-f66b-4fa4-ae95-f3ac3f8e3d31] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_create_zone_import_invalid_name[id-31eaf25a-9532-11eb-a55d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_create_zone_import_invalid_ttl[id-31eaf25a-9532-11eb-a55d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_create_zone_import[id-2e2d907d-0609-405b-9c96-3cb2b87e3dce] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_delete_zone_import[id-56a16e68-b241-4e41-bc5c-c40747fa68e3] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_list_import_zones_all_projects[id-7bd06ec6-9556-11eb-a55d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_list_zones_imports[id-9eab76af-1995-485f-a2ef-8290c1863aba] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_show_import_impersonate_another_project[id-2c1fa20e-9554-11eb-a55d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTest.test_show_zone_import[id-c8909558-0dc6-478a-9e91-eb97b52e59e0,smoke] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTestNegative.test_create_zone_import_invalid_content_type[id-5fa8016e-6ed1-11ec-9bd7-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTestNegative.test_create_zone_import_invalid_file_data[id-8fd744d2-9dff-11ec-9fb6-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTestNegative.test_create_zone_import_invalid_name[id-31eaf25a-9532-11eb-a55d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTestNegative.test_create_zone_import_invalid_ttl[id-31eaf25a-9532-11eb-a55d-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones_imports.ZonesImportTestNegative.test_zone_cannot_be_update_by_import[id-4fb9494e-9e23-11ec-8378-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_zones.ZoneOwnershipTest.test_no_create_duplicate_domain[id-5d28580a-a012-4b57-b211-e077b1a01340] +designate_tempest_plugin.tests.api.v2.test_zones.ZoneOwnershipTest.test_no_create_subdomain_by_alt_user[id-a48776fd-b1aa-4a25-9f09-d1d34cfbb175] +designate_tempest_plugin.tests.api.v2.test_zones.ZoneOwnershipTest.test_no_create_superdomain_by_alt_user[id-f1723d48-c082-43cd-94bf-ebeb5b8c9458] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesAdminTest.test_list_all_projects_zones[id-e1cf7104-8b06-11eb-a861-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesAdminTest.test_show_zone_impersonate_another_project[id-f6fe8cce-8b04-11eb-a861-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_delete_non_existing_zone[id-79921370-92e1-11eb-9d02-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_huge_size_description[id-a3b0a928-a682-11eb-9899-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_list_zones_dot_json_fails[id-925192f2-0ed8-4591-8fe7-a9fa028f90a0] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_no_valid_email[id-551853c0-8593-11eb-8c8a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_no_valid_ttl[id-551853c0-8593-11eb-8c8a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_no_valid_zone_name[id-551853c0-8593-11eb-8c8a-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_not_existing_project_id[id-c4d4b92a-86e3-11ee-965f-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_show_not_existing_zone[id-49268b24-92de-11eb-9d02-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_update_non_existing_zone[id-e391e30a-92e0-11eb-9d02-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesNegativeTest.test_use_invalid_id_to_show_zone[id-736e3b50-92e0-11eb-9d02-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_create_zone_ttl_zero[id-9970b632-f2db-11ec-a757-201e8823901f] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_create_zone_validate_recordsets_created[id-ec150c22-f52e-11eb-b09b-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_create_zones[id-9d2e20fc-e56f-4a62-9c61-9752a9ec615c] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_delete_zone_with_shares[id-bf2ee5c1-67b5-47dc-9902-ddb5b0e03e37] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_delete_zone[id-a4791906-6cd6-4d27-9f15-32273db8bb3d,smoke] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_get_primary_zone_nameservers[id-d4ce813e-64a5-11eb-9f43-74e5f9e2a801] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_shared_zone[id-ad2eed2f-6335-4bc0-87b2-7df7fc4cd82d] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_list_zones[id-5bfa3cfe-5bc8-443b-bf48-cfba44cbb247] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_serial_changes_on_update[id-3acddc86-62cc-4bfa-8589-b99e5d239bf2] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_shared_zone[id-81bff0fb-a5d1-4c64-84db-56ca751c17fc] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_show_zone[id-02ca5d6a-86ce-4f02-9d94-9e5db55c3055] +designate_tempest_plugin.tests.api.v2.test_zones.ZonesTest.test_update_zone[id-123f51cb-19d5-48a9-aacc-476742c02141] +designate_tempest_plugin.tests.scenario.v2.test_blacklists.BlacklistE2E.test_admin_creates_zone_matches_blacklist_name_or_regex[id-de030088-d97e-11eb-8ab8-74e5f9e2a801] +designate_tempest_plugin.tests.scenario.v2.test_blacklists.BlacklistE2E.test_primary_fails_to_create_zone_matches_blacklist_name[id-6956f20c-d8d5-11eb-bcdc-74e5f9e2a801] +designate_tempest_plugin.tests.scenario.v2.test_blacklists.BlacklistE2E.test_primary_fails_to_create_zone_matches_blacklist_regex[id-22b1ee72-d8d2-11eb-bcdc-74e5f9e2a801] +designate_tempest_plugin.tests.scenario.v2.test_classless_ptr.ClasslessPTRTest.test_classless_ptr_delegation_shared_zone[id-0110e7b1-9582-410e-b3d5-bd38a1265222,slow] +designate_tempest_plugin.tests.scenario.v2.test_classless_ptr.ClasslessPTRTest.test_classless_ptr_delegation[id-f2d9596c-ce87-4dfd-9bb4-ad2430fd3fe6,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.QuotasBoundary.test_zone_quota_boundary[id-e4981eb2-3803-11ed-9d3c-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.QuotasV2Test.test_api_export_size_quota[id-41d9cf2c-866a-11ec-8ccb-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.QuotasV2Test.test_recordset_records_quota[id-2513cb6e-85ec-11ec-bf7f-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.QuotasV2Test.test_zone_records_quota[id-893dc648-868d-11ec-8ccb-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.QuotasV2Test.test_zone_recordsets_quota[id-f567bdda-86b3-11ec-8ccb-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.QuotasV2Test.test_zones_quota[id-6987953a-dccf-11eb-903e-74e5f9e2a801,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.SharedZonesQuotaTest.test_recordset_records_enforced_against_owner[id-40b436f0-c895-11ed-8900-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.SharedZonesQuotaTest.test_zone_recocrds_enforced_against_owner[id-cbb8d6b4-c64e-11ed-80d8-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_quotas.SharedZonesQuotaTest.test_zone_recordsets_enforced_against_owner[id-31968688-c0d3-11ed-b04f-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_01_A[id-4664ed66-9ff1-45f2-9e60-d4913195c505,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_02_AAAA[id-cecd9f20-0b62-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_03_SRV[id-f5368d7a-0b62-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_04_SRV[id-74ff9efc-0b63-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_05_CNAME[id-82a14a2e-0b63-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_06_MX[id-ae7a295e-0b63-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_07_MX[id-f9aa8512-0b64-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_08_SSHFP[id-fa6cbd12-0b64-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_09_TXT[id-fa124a1c-0b64-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_10_SPF[id-3e347c28-0b66-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_11_PTR[id-88f6c2ac-0b66-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_12_PTR[id-b9591eea-0b66-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_13_CAA[id-c98cd9b4-0b66-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_create_and_delete_records_on_existing_zone_14_NAPTR[id-f78d6e8c-0b66-11ee-bbcc-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_01_A[id-cbf756b0-ba64-11ec-93d4-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_02_AAAA[id-258f7f57-9a74-4e72-bbfb-c709c411af14,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_03_SRV_TCP[id-304adbc5-668a-457e-9496-8efd20b8ae82,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_04_SRV_UDP[id-bd1283b3-423c-4bb9-8c4f-a205f31f1c2d,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_05_CNAME[id-8b53ae20-d096-4651-a6cf-efd7c98ae8d1,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_06_MX_at_APEX[id-0fd0046a-ac5a-468d-94b3-8a6bde790589,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_07_MX_under_APEX[id-31176def-3f95-459d-8bdd-b9994335b2d9,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_08_SSHFP[id-0009d787-c590-4149-9f30-082195326fad,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_09_TXT[id-af7cec16-dfad-4071-aa05-cafa60bf12a5,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_10_SPF[id-b3fd1f77-c318-4ab0-b18d-34611e51e9e4,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_11_PTR_IPV4[id-c310b94b-f3a5-4d26-bab6-2529e6f29fbf,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_12_PTR_IPV6[id-3e31e406-621f-4f89-b401-b6f38aa63347,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_13_CAA_Record[id-6fd96280-fb62-4eaf-81f9-609cdb7c126e,slow] +designate_tempest_plugin.tests.scenario.v2.test_recordsets.RecordsetsTest.test_update_records_propagated_to_backends_14_NAPTR_Record[id-45a11efe-bee3-4896-ab7c-daee1cb5eb3a,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTest.test_admin_zone_share_CRUD_recordset[id-fb170be8-c0bc-11ed-99a3-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTest.test_create_recordset_for_zone_shared_with_two_projects[id-78b77c6c-18cf-11ee-a872-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTest.test_list_zones_shared_with_more_then_two_projects[id-c5d83684-18cb-11ee-a872-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTest.test_share_imported_zone[id-b7dd37b8-c3ea-11ed-a102-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTest.test_zone_owner_can_delete_shared_recordset[id-de03b4d3-3ccf-4291-a920-89e2694bba22,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTest.test_zone_share_CRUD_recordset[id-b0fad45d-25ec-49b9-89a8-10b0e3c8b14c,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_abandon_shared_zone[id-1702c1d6-c643-11ed-8d86-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_create_export_for_shared_zone[id-1d2c91c2-c328-11ed-a033-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_delete_shared_zone_export[id-cac8ea8e-c33b-11ed-a033-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_fails_to_show_exported_zonefile_for_shared_zone[id-962df772-c33d-11ed-a033-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_list_shared_zone_exports[id-1e74410c-c32c-11ed-a033-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_lists_transfers_of_shared_zone[id-1da0ff64-c3f8-11ed-a102-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_show_delete_transfers_of_shared_zone[id-80ffbd8a-c3f7-11ed-a102-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_shows_shared_zones_nameservers[id-089136f2-c3e4-11ed-a102-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_shared_zones.SharedZonesTestNegative.test_alt_transfers_shared_zone[id-6376d4ca-c3f6-11ed-a102-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_tld.TldZoneTest.test_create_zone_using_existing_tld[id-68b3e7cc-bf0e-11ec-b803-201e8823901f] +designate_tempest_plugin.tests.scenario.v2.test_tld.TldZoneTest.test_create_zone_using_not_existing_tld[id-06deced8-d4de-11eb-b8ee-74e5f9e2a801] +designate_tempest_plugin.tests.scenario.v2.test_zones_export.ZonesExportTest.test_all_recordset_types_exist_in_show_zonefile[id-d8f444aa-a645-4a03-b366-46836f57dc69,slow] +designate_tempest_plugin.tests.scenario.v2.test_zones_export.ZonesExportTest.test_create_zone_export_and_show_exported_zonefile[id-0484c3c4-df57-458e-a6e5-6eb63e0475e0] +designate_tempest_plugin.tests.scenario.v2.test_zones_export.ZonesExportTest.test_missing_accept_headers[id-dc7a9dde-d287-4e22-9788-26578f0d3bf0] +designate_tempest_plugin.tests.scenario.v2.test_zones_export.ZonesExportTest.test_show_exported_zonefile_all_projects[id-c2e55514-ff2e-41d9-a3cc-9e78873254c9] +designate_tempest_plugin.tests.scenario.v2.test_zones_export.ZonesExportTest.test_show_exported_zonefile_any_mime_type[id-9746b7f2-2df4-448c-8a85-5ab6bf74f1fe] +designate_tempest_plugin.tests.scenario.v2.test_zones_export.ZonesExportTest.test_show_exported_zonefile_impersonate_another_project[id-56b8f30e-cd45-4c7a-bc0c-bbf92d7dc697] +designate_tempest_plugin.tests.scenario.v2.test_zones_import.ZonesImportTest.test_create_zone_import_and_wait_for_zone[id-679f38d0-2f2f-49c5-934e-8fe0c452f56e,slow] +designate_tempest_plugin.tests.scenario.v2.test_zones_transfer.ZonesTransferTest.test_zone_transfer_target_project[id-5855b772-a036-11eb-9973-74e5f9e2a801] +designate_tempest_plugin.tests.scenario.v2.test_zones_transfer.ZonesTransferTest.test_zone_transfer[id-60bd80ac-c979-4686-9a03-f2f775f272ab] +designate_tempest_plugin.tests.scenario.v2.test_zones.ZonesTest.test_create_and_delete_zone[id-d0648f53-4114-45bd-8792-462a82f69d32,slow,smoke] +designate_tempest_plugin.tests.scenario.v2.test_zones.ZonesTest.test_create_and_update_zone[id-cabd6334-ba37-11ec-9d8c-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_zones.ZonesTest.test_delete_zone_pending_create[id-c9838adf-14dc-4097-9130-e5cea3727abb,slow] +designate_tempest_plugin.tests.scenario.v2.test_zones.ZonesTest.test_notify_msg_sent_to_nameservers[id-ff9b9fc4-85b4-11ec-bcf5-201e8823901f,slow] +designate_tempest_plugin.tests.scenario.v2.test_zones.ZonesTest.test_zone_create_propagates_to_nameservers[id-ad8d1f5b-da66-46a0-bbee-14dc84a5d791,slow] +designate_tempest_plugin.tests.scenario.v2.test_zones.ZonesTest.test_zone_delete_propagates_to_nameservers[id-d13d3095-c78f-4aae-8fe3-a74ccc335c84,slow] diff --git a/.automation.conf/tempest/load-lists/glance b/.automation.conf/tempest/load-lists/glance new file mode 100644 index 000000000..d8ecbffb0 --- /dev/null +++ b/.automation.conf/tempest/load-lists/glance @@ -0,0 +1,114 @@ +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_add_image_member[id-ec3da4dc-f478-4a70-8799-db0814e340f4] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_create_image[id-025eea27-fa86-44a9-85a2-91295842f808] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_deactivate_image[id-dfc73f6f-bf91-4b6a-8482-acc8c436e066] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_delete_image_member[id-7f0a8e2b-b655-416a-914b-9615cff18bbf] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_delete_image[id-e45899d6-7e16-4cbf-b800-31c05e6caf8c] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_download_image[id-24891c04-28ca-41f9-92d1-c06d8ba4b83d] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_get_image_member[id-8719285b-5b7b-48b8-ba5e-2bc7e3535025] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_get_image[id-61fd8b5e-8a0b-46ca-91c4-6c2c2d35039d] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_list_image_members[id-daaef0c5-1172-457b-b1a3-0736b64c8426] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_list_images[id-d0c18f80-6168-4d98-a86e-c09d28d83bb0] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_reactivate_image[id-3cbef53c-ab8a-4343-b993-8f9e14ab90d1] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_update_image_member[id-2baaaca0-6335-4219-9bd9-207a5cfda6a2] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_update_image[id-9e9f7fd6-e93c-402c-9f3c-177fede8f645] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ImageProjectAdminTests.test_upload_image[id-947f1ae1-c5b6-4552-89e3-1078ca722be4] +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.NamespacesProjectAdminTests.test_create_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.NamespacesProjectAdminTests.test_delete_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.NamespacesProjectAdminTests.test_get_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.NamespacesProjectAdminTests.test_list_namespaces +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.NamespacesProjectAdminTests.test_update_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ObjectsProjectAdminTests.test_create_object +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ObjectsProjectAdminTests.test_delete_object +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ObjectsProjectAdminTests.test_get_object +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ObjectsProjectAdminTests.test_list_objects +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.ObjectsProjectAdminTests.test_update_object +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.PropertiesProjectAdminTests.test_create_property +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.PropertiesProjectAdminTests.test_delete_properties +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.PropertiesProjectAdminTests.test_get_properties +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.PropertiesProjectAdminTests.test_list_properties +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.PropertiesProjectAdminTests.test_update_properties +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.RSTypesProjectAdminTests.test_create_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.RSTypesProjectAdminTests.test_delete_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.RSTypesProjectAdminTests.test_get_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.RSTypesProjectAdminTests.test_list_resource_types +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.TagsProjectAdminTests.test_create_tag +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.TagsProjectAdminTests.test_delete_tags +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.TagsProjectAdminTests.test_get_tags +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.TagsProjectAdminTests.test_list_tags +glance_tempest_plugin.tests.rbac.v2.test_project_admin_tests.TagsProjectAdminTests.test_update_tags +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_add_image_member[id-395579c9-92bb-40a9-a8b6-9daaa20ae610] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_create_image[id-a71e7caf-2403-4fed-a4bf-9717949ecde2] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_deactivate_image[id-abf9fe8b-ada3-4509-b15b-76d04e58f4e8] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_delete_image_member[id-bcfda1fa-ea65-47ce-8434-a64d85512fcf] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_delete_image[id-326d267a-de9d-4217-aadc-a0f2b5993537] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_download_image[id-3dfa6f70-f6fe-4ed5-96eb-5f4634064aa3] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_get_image_member[id-67510e3f-57cd-4a76-9e96-577e49229677] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_get_image[id-2adf7202-7fc9-4a6e-b6dd-fb3d40365ccb] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_list_image_members[id-64ec16c4-3b8d-464d-88fb-f274241b1302] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_list_images[id-259d5578-410e-4b0f-bb2d-cb5b057bc696] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_reactivate_image[id-58558447-8618-4dbe-97ce-bfc39b3743e7] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_update_image_member[id-12636be0-6188-4003-8824-de4f89e3c745] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_update_image[id-13f8949b-3419-4a4a-bd0b-71fa711206fd] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ImagesProjectMemberTests.test_upload_image[id-bd5845dc-d96b-4d83-a8da-7978bd91ddc1] +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.NamespacesProjectMemberTests.test_create_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.NamespacesProjectMemberTests.test_delete_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.NamespacesProjectMemberTests.test_get_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.NamespacesProjectMemberTests.test_list_namespaces +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.NamespacesProjectMemberTests.test_update_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ObjectsProjectMemberTests.test_create_object +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ObjectsProjectMemberTests.test_delete_object +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ObjectsProjectMemberTests.test_get_object +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ObjectsProjectMemberTests.test_list_objects +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.ObjectsProjectMemberTests.test_update_object +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.PropertiesProjectMemberTests.test_create_property +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.PropertiesProjectMemberTests.test_delete_properties +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.PropertiesProjectMemberTests.test_get_properties +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.PropertiesProjectMemberTests.test_list_properties +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.PropertiesProjectMemberTests.test_update_properties +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.RSTypesProjectMemberTests.test_create_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.RSTypesProjectMemberTests.test_delete_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.RSTypesProjectMemberTests.test_get_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.RSTypesProjectMemberTests.test_list_resource_types +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.TagsProjectMemberTests.test_create_tag +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.TagsProjectMemberTests.test_delete_tags +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.TagsProjectMemberTests.test_get_tags +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.TagsProjectMemberTests.test_list_tags +glance_tempest_plugin.tests.rbac.v2.test_project_members_tests.TagsProjectMemberTests.test_update_tags +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_add_image_member[id-8eaddc1a-f9d0-4a68-8fef-1951c328d01c] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_create_image[id-5e151433-5901-45ec-9451-ed3170c299cb] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_deactivate_image[id-310672b0-bf63-4ce0-b3c4-5230b8e7de31] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_delete_image_member[id-864e275e-2238-4b0f-9039-9bc7aed22f92] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_delete_image[id-e8c3382c-c547-49c1-a92e-fa1d6378d4df] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_download_image[id-9067339d-c64b-4e4d-bc0e-a52cd1365ea3] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_get_image_member[id-67510e3f-57cd-4a76-9e96-577e49229677] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_get_image[id-f402e6a2-7cc9-46c0-b6c2-a235f5512788] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_list_image_members[id-64ec16c4-3b8d-464d-88fb-f274241b1302] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_list_images[id-6de1e04c-d0cd-45b3-8007-3712bbca817d] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_reactivate_image[id-0d1fc51c-d4c9-4dd5-9f21-c28b09d3f9ec] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_update_image_member[id-e0feceab-dfc0-4d08-88be-81f5d225c72f] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_update_image[id-f2be46ee-317b-4825-9d3d-bd23a1fb7858] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ImagesProjectReaderTests.test_upload_image[id-b7ac2883-f569-4032-a35b-a79ef1277582] +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.NamespacesProjectReaderTests.test_create_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.NamespacesProjectReaderTests.test_delete_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.NamespacesProjectReaderTests.test_get_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.NamespacesProjectReaderTests.test_list_namespaces +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.NamespacesProjectReaderTests.test_update_namespace +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ObjectsProjectReaderTests.test_create_object +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ObjectsProjectReaderTests.test_delete_object +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ObjectsProjectReaderTests.test_get_object +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ObjectsProjectReaderTests.test_list_objects +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.ObjectsProjectReaderTests.test_update_object +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.PropertiesProjectReaderTests.test_create_property +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.PropertiesProjectReaderTests.test_delete_properties +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.PropertiesProjectReaderTests.test_get_properties +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.PropertiesProjectReaderTests.test_list_properties +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.PropertiesProjectReaderTests.test_update_properties +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.RSTypesProjectReaderTests.test_create_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.RSTypesProjectReaderTests.test_delete_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.RSTypesProjectReaderTests.test_get_resource_type +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.RSTypesProjectReaderTests.test_list_resource_types +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.TagsProjectReaderTests.test_create_tag +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.TagsProjectReaderTests.test_delete_tags +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.TagsProjectReaderTests.test_get_tags +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.TagsProjectReaderTests.test_list_tags +glance_tempest_plugin.tests.rbac.v2.test_project_reader_tests.TagsProjectReaderTests.test_update_tags diff --git a/.automation.conf/tempest/load-lists/ironic b/.automation.conf/tempest/load-lists/ironic new file mode 100644 index 000000000..ca5365841 --- /dev/null +++ b/.automation.conf/tempest/load-lists/ironic @@ -0,0 +1,477 @@ +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_candidate_node[id-12d19297-f35a-408a-8b1e-3cd244e30abe] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_node_mismatch[id-2378727f-77c3-4289-9562-bd2f3b147a60,negative] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_resource_class_mismatch[id-bf7e1375-019a-466a-a294-9c1052827ada,negative] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_traits_mismatch[id-b4eeddee-ca34-44f9-908b-490b78b18486,negative] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_allocation_with_traits[id-eb074d06-e5f4-4fb4-b992-c9929db488ae] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_create_show_allocation[id-9203ea28-3c61-4108-8498-22247b654ff6] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_delete_allocation_by_name[id-5e30452d-ee92-4342-82c1-5eea5e55c937] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_delete_allocation[id-84eb3c21-4e16-4f33-9551-dce0f8689462] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_list_allocations_by_state[id-092b7148-9ff0-4107-be57-2cfcd21eb5d7] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_list_allocations[id-4ca123c4-160d-4d8d-a3f7-15feda812263] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocations.test_show_by_name[id-fbbc13bc-86da-438b-af01-d1bc1bab57d6] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocationsWithJsonExtSupport.test_create_show_allocation_with_json[id-d111b85b-a169-440e-9dcd-9b8b6ad8c917] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocationsWithJsonExtSupport.test_delete_allocation_with_json[id-b94643f4-f789-4c04-a806-bf769e0bc181] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocationsWithoutJsonExtSupport.test_create_show_allocation_with_json[id-6b851ef4-e364-4a3e-af1d-e6b73c1adec6] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestAllocationsWithoutJsonExtSupport.test_delete_allocation_with_json[id-cdf36c46-3da7-4306-b265-bcea3ab3bc3f] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestBackfill.test_backfill_allocation[id-10774c1d-6b79-453a-8e26-9bf04ab580a4] +ironic_tempest_plugin.tests.api.admin.test_allocations.TestBackfill.test_backfill_without_resource_class[id-c33d4b65-1232-4a3f-9aad-942e32f6f7b0] +ironic_tempest_plugin.tests.api.admin.test_api_discovery.TestApiDiscovery.test_api_versions[id-a3c27e94-f56c-42c4-8600-d6790650b9c5] +ironic_tempest_plugin.tests.api.admin.test_api_discovery.TestApiDiscovery.test_default_version[id-896283a6-488e-4f31-af78-6614286cbe0d] +ironic_tempest_plugin.tests.api.admin.test_api_discovery.TestApiDiscovery.test_version_1_resources[id-abc0b34d-e684-4546-9728-ab7a9ad9f174] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_chassis_node_list[id-76305e22-a4e2-4ab3-855c-f4e2368b9335] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_create_chassis_unicode_description[id-cabe9c6f-dc16-41a7-b6b9-0a90c212edd5] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_create_chassis_uuid[id-dd52bd5d-610c-4f2c-8fa3-d5e59269325f] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_create_chassis[id-7c5a2e09-699c-44be-89ed-2bc189992d42] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_delete_chassis[id-5ae649ad-22d1-4fe1-bbc6-97227d199fb3] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_list_chassis[id-29c9cd3f-19b5-417b-9864-99512c3b33b3] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_show_chassis[id-c84644df-31c4-49db-a307-8942881f41c0] +ironic_tempest_plugin.tests.api.admin.test_chassis.TestChassis.test_update_chassis[id-cda8a41f-6be2-4cbf-840c-994b00a89b44] +ironic_tempest_plugin.tests.api.admin.test_conductor.TestConductors.test_list_conductors_detail[id-ca3de366-d80a-4e97-b19b-42d594e8d148] +ironic_tempest_plugin.tests.api.admin.test_conductor.TestConductors.test_list_conductors[id-b6d62be4-53a6-43c6-ae78-bff9d1b4efc1] +ironic_tempest_plugin.tests.api.admin.test_conductor.TestConductors.test_show_conductor[id-7e1829e2-3945-4508-a3d9-c8ebe9463fd8] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_create_deploy_template_specifying_uuid[id-8f39c794-4e6f-42fc-bfcb-e1f6eafff2e9] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_delete_deploy_template[id-7ac8fdd5-928b-4fbc-8341-3ebaad9def5e] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_list_deploy_templates_detail[id-c09c917e-c4d2-4148-9b6a-459bd126ed7c] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_list_deploy_templates[id-cec2a01d-07af-4062-a8b0-9a1703f65bcf] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_list_with_limit[id-89aea2bf-c094-445f-b869-9fd56d1dfe5a] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_show_deploy_template_with_links[id-2fd98e9a-10ce-405a-a32c-0d6079766183] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_show_deploy_template[id-f424dd67-46c3-4169-b8f0-7e0c18b70437] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_update_deploy_template_add[id-bb168c63-452b-4065-9a81-77853ca9540a] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_update_deploy_template_mixed_ops[id-2aa204a2-1d50-48fd-8b76-d2ed15586d50] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplates.test_update_deploy_template_replace[id-a6cf1ade-e19a-41e2-b151-13ecf0d8f08c] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_create_deploy_template_duplicated_deploy_template_name[id-6390acc4-9490-4b23-8b4c-41888a78c9b7,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_create_deploy_template_invalid_name[id-a4085c08-e718-4c2f-a796-0e115b659243,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_create_deploy_template_malformed_deploy_template_uuid[id-2a562fca-f377-4a6e-b332-37ee82d3a983,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_create_deploy_template_malformed_steps[id-cbd33bc5-7602-40b7-943e-3e92217567a3,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_create_deploy_template_no_mandatory_field_name[id-ed3f0cec-13e8-4175-9fdb-d129e7b7fe10,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_create_deploy_template_no_mandatory_field_steps[id-af5dd0df-d903-463f-9535-9e4e9d6fd576,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_show_deploy_template_nonexistent[id-2c006994-88ca-43b7-b605-897d479229d9,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_update_deploy_template_remove_mandatory_field_name[id-ee852ebb-a601-4593-9d59-063fcbc8f964,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_update_deploy_template_remove_mandatory_field_steps[id-5a815f37-f015-4d68-9b22-099504f74805,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesNegative.test_update_deploy_template_replace_empty_name[id-e59bf38d-272f-4490-b21e-9db217f11378,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesOldAPI.test_create_deploy_template_old_api[id-e9481a0d-23e0-4757-bc11-c3c9ab9d3839,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesOldAPI.test_delete_deploy_template_old_api[id-1646b1e5-ab81-45a8-9ea0-30444a4dcaa2,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesOldAPI.test_list_deploy_templates_old_api[id-819480ac-f36a-4402-b1d5-504d7cf55b1f,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesOldAPI.test_update_deploy_template_old_api[id-0d3af2aa-ba53-4c8a-92d4-91f9b4179fe7,negative] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesWithJsonExtSupport.test_delete_deploy_template_with_json[id-df2fabbd-3c8b-4979-8f13-c58c96122adb] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesWithJsonExtSupport.test_show_deploy_template_with_json[id-8b64a54d-ead7-4b3f-8fae-a1b8bdefe3ee] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesWithoutJsonExtSupport.test_delete_deploy_template_with_json[id-61d05189-9f84-4973-835b-a860b655dfe3] +ironic_tempest_plugin.tests.api.admin.test_deploy_templates.TestDeployTemplatesWithoutJsonExtSupport.test_show_deploy_template_with_json[id-432429dd-b964-4133-aa46-cd43a7cd9c37] +ironic_tempest_plugin.tests.api.admin.test_drivers.TestDrivers.test_driver_logical_disk_properties[id-fdf61f5a-f59d-4235-ad6c-cc718740e3e3] +ironic_tempest_plugin.tests.api.admin.test_drivers.TestDrivers.test_driver_properties[id-6efa976f-78a2-4859-b3aa-97d960d6e5e5] +ironic_tempest_plugin.tests.api.admin.test_drivers.TestDrivers.test_list_drivers[id-5aed2790-7592-4655-9b16-99abcc2e6ec5] +ironic_tempest_plugin.tests.api.admin.test_drivers.TestDrivers.test_show_driver[id-fb3287a3-c4d7-44bf-ae9d-1eef906d78ce] +ironic_tempest_plugin.tests.api.admin.test_microversion_enforcement.TestMicroversionEnforcement.test_list_node_filter_shard[id-5df533c6-7a9c-4639-a47f-1377a2a87e6a] +ironic_tempest_plugin.tests.api.admin.test_microversion_enforcement.TestMicroversionEnforcement.test_shard[id-e5403a31-e12b-4f97-a776-dcb819e5e9a0] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestHardwareInterfaces.test_reset_interfaces +ironic_tempest_plugin.tests.api.admin.test_nodes.TestHardwareInterfaces.test_set_interfaces +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeConductor.test_conductor_exposed[id-1af888b2-2a19-43da-8181-a5381d6ff536] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeConductor.test_list_nodes_by_conductor[id-53bcef99-2989-4755-aa8f-c31037cd15de] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeDescription.test_create_node_with_description[id-3d649bb3-a58b-4b9e-8dfa-41ab634b1153] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeDescription.test_description_exposed[id-66d0da49-e5ac-4f49-b065-9d2207d8a3af] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeDescription.test_node_description_set_unset[id-85b4a4b5-37e5-4b60-8dc7-f5a26dfa78a3] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeFault.test_fault_shown[id-649b4660-4f76-4d67-94df-6631a2cb2cd9] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeFault.test_list_nodes_fault[id-62f453be-8f30-4cfe-a19a-23656068e546] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeFault.test_list_nodes_with_invalid_fault[id-c8fb55f1-873f-4fb9-bd57-6f1de0479873] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected_from_deletion[id-04a21b51-2991-4213-8c2f-a96cfdada802] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected_negative[id-1c819f4c-6c1d-4150-ba4a-3b0dcb3c8694,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected_set_unset[id-52f0cb1c-ad7b-43dc-8e22-a76438b67716] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodeProtected.test_node_protected[id-8fbd101e-90e6-4843-b41a-556b34802972] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_conductor_hidden[id-e5470656-bb65-4173-be83-2df3fc9aed24] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_create_node[id-4e939eb2-8a69-4e84-8652-6fffcbc9db8f] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_delete_node[id-9ade60a4-505e-4259-9ec4-71352cbbaf47] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_description_hidden[id-5e7f4c54-8216-42d3-83cc-7bd776ffd16f] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_fault_hidden[id-b85af8c6-572b-4f20-815e-1cf31844b9f6] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_get_console[id-f63b6288-1137-4426-8cfe-0d5b7eb87c06] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_get_node_boot_device[id-9ea73775-f578-40b9-bc34-efc639c4f21f] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_get_node_by_instance_uuid[id-b02a4f38-5e8b-44b2-aed2-a69a36ecfd69] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_get_node_supported_boot_devices[id-3622bc6f-3589-4bc2-89f3-50419c66b133] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_list_nodes_association[id-85b1f6e0-57fd-424c-aeff-c3422920556f] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_list_nodes[id-4ca123c4-160d-4d8d-a3f7-15feda812263] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_node_port_list_no_ports[id-72591acb-f215-49db-8395-710d14eb86ab] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_node_port_list[id-18c4ebd8-f83a-4df7-9653-9fb33a329730] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_set_console_mode[id-80504575-9b21-4670-92d1-143b948f9437] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_set_node_boot_device[id-5519371c-26a2-46e9-aa1a-f74226e9d71f] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_show_node[id-55451300-057c-4ecf-8255-ba42a83d3a03] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_update_node[id-4fed270a-677a-4d19-be87-fd38ae490320] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodes.test_validate_driver_interface[id-cbf1f515-5f4b-4e49-945c-86bcaccfeb1d] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesProtectedOldApi.test_node_protected_old_api[id-08971546-27cc-40ab-851e-ba7bb52c00ab,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClass.test_create_node_resource_class_long[id-2a00340c-8152-4a61-9fc5-0b3cdefec258] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClass.test_create_node_resource_class_too_long[id-e75136d4-0690-48a5-aef3-75040aee73ad,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClass.test_list_nodes_detail_resource_class[id-40733bad-bb79-445e-a094-530a44042995] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClass.test_list_nodes_resource_class[id-f2bf4465-280c-4fdc-bbf7-fcf5188befa4] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClass.test_show_node_resource_class[id-73e6f7b5-3e51-49ea-af5b-146cd49f40ee] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClass.test_update_node_resource_class_too_long[id-f0aeece4-8671-44ea-a482-b4047fc4cf74,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClass.test_update_node_resource_class[id-142db00d-ac0f-415b-8da8-9095fbb561f7] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClassOldApi.test_create_node_resource_class_old_api[id-2c364408-4746-4b3c-9821-20d47b57bdec,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClassOldApi.test_list_nodes_by_resource_class_old_api[id-95903480-f16d-4774-8775-6c7f87b27c59,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesResourceClassOldApi.test_update_node_resource_class_old_api[id-666f3c1a-4922-4a3d-b6d9-dea7c74d30bc,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_add_node_trait_duplicate_trait[id-4b737e7f-101e-493e-b5ce-494fbffe18fd] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_add_node_trait_long[id-47db09d9-af2b-424d-9d51-7efca2920f20] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_add_node_trait_non_existent_node[id-cca0e831-32af-4ce9-bfce-d3834fea57aa,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_add_node_trait_too_long[id-2a4daa8d-2b85-40ac-a8a0-0462cc9a57ef,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_add_node_trait_too_many[id-65bce181-89ce-435e-a7d8-3ba60aafd08d,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_list_node_traits_non_existent_node[id-3b83dbd3-4a89-4173-920a-ca33ed3aad69,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_list_node_traits[id-5c3a2dd0-af10-474d-a209-d30426e1eb5d] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_list_nodes_detail[id-03f9e57f-e584-448a-926f-53035e583e7e] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_list_nodes_traits_field[id-2b82f704-1580-403a-af92-92c29a7eebb7] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_remove_node_trait_non_existent_node[id-f1469745-7cdf-4cae-9699-73d029c47bc3,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_remove_node_trait_non_existent_trait[id-b50ae543-5e5e-4b1a-b2f2-9e00fe55974b,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_remove_node_trait[id-3591d514-39b9-425e-9afe-ea74ae347486] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_remove_node_traits_no_traits[id-4d8c9a35-0036-4139-85c1-5f242395680f] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_remove_node_traits_non_existent_node[id-625c911a-48e8-4bef-810b-7cf33c0846a2,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_remove_node_traits[id-e4bf8bf0-3004-44bc-8bfe-f9f1a167d999] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_set_node_traits_duplicate_trait[id-d81ceeab-a50f-427a-bc5a-aa916478d0d3] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_set_node_traits_max_traits[id-727a5e11-5654-459f-8af6-e14eb987a283] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_set_node_traits_non_existent_node[id-2fb4c9d9-8e5b-4189-b547-26596014491c,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_set_node_traits_too_many[id-75831f5d-ca44-403b-8fd6-f7cad95b1c54,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_set_node_traits[id-aa961bf6-ea2f-484b-961b-eae2da0e6b7e] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_show_node[id-c83c537a-76aa-4d8a-8673-128d01ee403d] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraits.test_update_node_traits[id-9ab6a19c-83b9-4600-b55b-325a51e2f8f6,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_add_node_trait_old_api[id-a4353f3a-bedc-4579-9c7e-4bebcd95903d,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_list_node_traits_old_api[id-5419af7b-4e27-4be4-88f6-e01c598a8102,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_list_nodes_detail_old_api[id-525eeb59-b7ce-413d-a37b-401e67402a4c,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_list_nodes_traits_field[id-eb75b3c8-ac9c-4399-90a2-c0030bfde7a6,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_remove_node_trait_old_api[id-0f9af890-a57a-4c25-86c8-6418d1b8f4d4,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_remove_node_traits_old_api[id-f8375b3c-1939-4d1c-97c4-d23e10680090,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_set_node_traits_old_api[id-91cc43d8-2f6f-4b1b-95e9-68dedca54e6b,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesTraitsOldApi.test_show_node_old_api[id-214ae7fc-149b-4657-b6bc-66353d49ade8,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_already_attached_on_internal_info[id-628350f8-4498-4204-b546-f3c01b93c7e3,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_already_attached_with_portgroups[id-ec0c14a4-6853-4907-9091-755c9c6c152f,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_already_set[id-a3d319d0-cacb-4e55-a3dc-3fa8b74880f2] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_no_free_port[id-91e08d6a-0438-4171-b404-bc86b0bc8861,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_no_port[id-85b610cd-5ba8-49a7-8ce2-5e364056fd29,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_node_doesnt_exist[id-3affca81-9f3f-4dab-ad3d-77c892d8d0d7,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_port_not_in_portgroup[id-5572fecc-bc22-461a-a891-7f74e2e936bd,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_attach_with_empty_portgroup[id-481777b8-8f0d-4932-af3c-1a8e166949f3,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_detach_no_args[id-ff3c9ce2-4979-4a54-a860-fab088a6669f,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_detach_not_existing[id-da036225-47b0-43b7-9586-0d6390bd3cd9,negative] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_on_port[id-a3d319d0-cacb-4e55-a3dc-3fa8b74880f1] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesVif.test_vif_on_portgroup[id-95279515-7d0a-4f5f-987f-93e36aae5585] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesWithJsonExtSupport.test_delete_node_with_json[id-4c376a23-04b8-4fc8-955f-abc5668d34c3] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesWithJsonExtSupport.test_show_node_with_json[id-bc1134aa-2950-409c-b91c-b2d85df9b065] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesWithJsonExtSupport.test_update_node_with_json[id-2ffec2ec-3dca-4e4f-ac6c-dee7c44cdef5] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesWithoutJsonExtSupport.test_delete_node_with_json[id-594160b6-a608-4024-98d1-818f75d2d895] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesWithoutJsonExtSupport.test_show_node_with_json[id-e83cd539-4d09-46d0-a448-365a219ca353] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestNodesWithoutJsonExtSupport.test_update_node_with_json[id-65505002-3ac7-4a07-9f66-2e3bcb0f5e95] +ironic_tempest_plugin.tests.api.admin.test_nodes.TestResetInterfaces.test_no_reset_by_default +ironic_tempest_plugin.tests.api.admin.test_nodes.TestResetInterfaces.test_reset_all_interfaces +ironic_tempest_plugin.tests.api.admin.test_nodes.TestResetInterfaces.test_reset_interfaces +ironic_tempest_plugin.tests.api.admin.test_nodes.TestResetInterfaces.test_set_interfaces +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_1.test_list_nodestates[id-cd8afa5e-3f57-4e43-8185-beb83d3c9015] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_1.test_set_node_power_state[id-fc5b9320-0c98-4e5a-8848-877fe5a0322c] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_1.test_set_node_provision_state[id-ccb8fca9-2ba0-480c-a037-34c3bd09dc74] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_11.test_list_nodestates[id-cd8afa5e-3f57-4e43-8185-beb83d3c9015] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_11.test_set_node_power_state[id-fc5b9320-0c98-4e5a-8848-877fe5a0322c] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_11.test_set_node_provision_state[id-31f53828-b83d-40c7-98e5-843e28a1b6b9] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_12.test_list_nodestates[id-cd8afa5e-3f57-4e43-8185-beb83d3c9015] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_12.test_set_node_power_state[id-fc5b9320-0c98-4e5a-8848-877fe5a0322c] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_12.test_set_node_raid_config[id-4427b1ca-8e79-4139-83d6-77dfac03e61e] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_2.test_list_nodestates[id-cd8afa5e-3f57-4e43-8185-beb83d3c9015] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_2.test_set_node_power_state[id-fc5b9320-0c98-4e5a-8848-877fe5a0322c] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_2.test_set_node_provision_state[id-9c414984-f3b6-4b3d-81da-93b60d4662fb] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_4.test_list_nodestates[id-cd8afa5e-3f57-4e43-8185-beb83d3c9015] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_4.test_set_node_power_state[id-fc5b9320-0c98-4e5a-8848-877fe5a0322c] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_4.test_set_node_provision_state[id-3d606003-05ce-4b5a-964d-bdee382fafe9] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_6.test_list_nodestates[id-cd8afa5e-3f57-4e43-8185-beb83d3c9015] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_6.test_set_node_power_state[id-fc5b9320-0c98-4e5a-8848-877fe5a0322c] +ironic_tempest_plugin.tests.api.admin.test_nodestates.TestNodeStatesV1_6.test_set_node_provision_state[id-6c9ce4a3-713b-4c76-91af-18c48d01f1bb] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_create_portgroup_no_address[id-4336fa0f-da86-4cec-b788-89f59a7635a5] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_create_portgroup_with_address[id-110cd302-256b-4ddc-be10-fc6c9ad8e649] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_delete_portgroup[id-8378c69f-f806-454b-8ddd-6b7fd93ab12b] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_list_portgroups[id-cf2dfd95-5ea1-4109-8ad3-297cd76aa5d3] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_show_portgroup[id-f6be5e70-3e3b-435c-b2fc-bbb2cc9b3185] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_update_portgroup_add[id-a1123416-7bb6-4a6a-9f14-859c72550552] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_update_portgroup_mixed_ops[id-67d5013e-5158-44e8-8d1c-a01d04542be4] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_update_portgroup_remove_by_key[id-9834a4ec-be41-40b4-a3a4-8e46ad7eb19d] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_update_portgroup_remove_collection[id-5da2f7c7-03e8-4db0-8c3e-2fe6ebcc4439] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroups.test_update_portgroup_replace[id-6a491006-2dd5-4c82-be39-a4fa015071c0] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroupsWithJsonExtSupport.test_delete_portgroup_with_json[id-607ea131-ef5c-44e9-a900-a8e426b804e8] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroupsWithJsonExtSupport.test_show_portgroup_with_json[id-9126ecac-310f-440a-81d4-ba9af1767845] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroupsWithJsonExtSupport.test_update_portgroup_with_json[id-a6f154aa-e447-4576-89e2-9e8951fb7c43] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroupsWithoutJsonExtSupport.test_delete_portgroup_with_json[id-2aede448-7165-4d54-92de-3be31ae19af0] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroupsWithoutJsonExtSupport.test_show_portgroup_with_json[id-be8ade56-8ac1-49f4-981a-93b403fa1d79] +ironic_tempest_plugin.tests.api.admin.test_portgroups.TestPortGroupsWithoutJsonExtSupport.test_update_portgroup_with_json[id-f624352a-6560-4ec1-b4ff-ccfe7cda08f2] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_duplicated_mac[id-e27f8b2e-42c6-4a43-a3cd-accff716bc5c,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_duplicated_port_uuid[id-89a34380-3c61-4c32-955c-2cd9ce94da21,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_malformed_mac[id-0a6ee1f7-d0d9-4069-8778-37f3aa07303a,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_malformed_node_id[id-583a6856-6a30-4ac4-889f-14e2adff8105,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_malformed_port_uuid[id-2b51cd18-fb95-458b-9780-e6257787b649,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_no_mandatory_field_mac[id-bcea3476-7033-4183-acfe-e56a30809b46,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_no_mandatory_field_node_id[id-65e84917-733c-40ae-ae4b-96a4adff931c,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_create_port_nonexsistent_node_id[id-30277ee8-0c60-4f1d-b125-0e51c2f43369,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_delete_port_by_mac_not_allowed[id-03d42391-2145-4a6c-95bf-63fe55eb64fd,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_show_port_by_mac_not_allowed[id-4ad85266-31e9-4942-99ac-751897dc9e23,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_show_port_malformed_uuid[id-029190f6-43e1-40a3-b64a-65173ba653a3,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_show_port_nonexistent_uuid[id-0d00e13d-e2e0-45b1-bcbc-55a6d90ca793,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_add_nonexistent_property[id-f8f15803-34d6-45dc-b06f-e5e04bf1b38b,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_by_mac_not_allowed[id-8907082d-ac5e-4be3-b05f-d072ede82020,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_malformed_port_uuid[id-c701e315-aa52-41ea-817c-65c5ca8ca2a8,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_mixed_ops_integrity[id-0629e002-818e-4763-b25b-ae5e07b1cb23,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_nonexistent[id-df1ac70c-db9f-41d9-90f1-78cd6b905718,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_remove_mandatory_field_mac[id-ae2696ca-930a-4a7f-918f-30ae97c60f56,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_remove_mandatory_field_port_uuid[id-5392c1f0-2071-4697-9064-ec2d63019018,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_remove_nonexistent_property[id-06b50d82-802a-47ef-b079-0a3311cf85a2,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_replace_mac_with_duplicated[id-2949f30f-5f59-43fa-a6d9-4eac578afab4,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_replace_mac_with_malformed[id-375022c5-9e9e-4b11-9ca4-656729c0c9b2,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_replace_node_id_with_malformed[id-898ec904-38b1-4fcb-9584-1187d4263a2a,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_replace_node_id_with_nonexistent[id-97f6e048-6e4f-4eba-a09d-fbbc78b77a77,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegative.test_update_port_replace_nonexistent_property[id-5722b853-03fc-4854-8308-2036a1b67d85,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegativeWithPhysicalNetwork.test_create_ports_in_portgroup_with_inconsistent_physical_network[id-e20156fb-956b-4d5b-89a4-f379044a1d3c,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegativeWithPhysicalNetwork.test_update_ports_in_portgroup_with_inconsistent_physical_network_2[id-3cd1c8ec-57d1-40cb-922b-dd02431beea3,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsNegativeWithPhysicalNetwork.test_update_ports_in_portgroup_with_inconsistent_physical_network[id-050e792c-22c9-4e4a-ae89-dfbfc52ad00d,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsWithPhysicalNetworkOldAPI.test_create_port_with_physical_network_old_api[id-307e57e9-082f-4830-9480-91affcbfda08,negative] +ironic_tempest_plugin.tests.api.admin.test_ports_negative.TestPortsWithPhysicalNetworkOldAPI.test_update_port_replace_physical_network_old_api[id-0b278c0a-d334-424e-a5c5-b6d001c2a715,negative] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_create_port_specifying_uuid[id-d1f6b249-4cf6-4fe6-9ed6-a6e84b1bf67b] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_create_port_with_extra[id-4a02c4b0-6573-42a4-a513-2e36ad485b62] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_create_port[id-83975898-2e50-42ed-b5f0-e510e36a0b56] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_delete_port[id-1bf257a9-aea3-494e-89c0-63f657ab4fdd] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_list_ports_details_with_address[id-8a03f688-7d75-4ecd-8cbc-e06b8f346738] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_list_ports_details[id-8a94b50f-9895-4a63-a574-7ecff86e5875] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_list_ports[id-b5e91854-5cd7-4a8e-bb35-3e0a1314606d] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_list_with_limit[id-324a910e-2f80-4258-9087-062b5ae06240] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_show_port_by_address[id-7c1114ff-fc3f-47bb-bc2f-68f61620ba8b] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_show_port_with_links[id-bd773405-aea5-465d-b576-0ab1780069e5] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_show_port[id-9fa77ab5-ce59-4f05-baac-148904ba1597] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_update_port_add[id-241288b3-e98a-400f-a4d7-d1f716146361] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_update_port_mixed_ops[id-5309e897-0799-4649-a982-0179b04c3876] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_update_port_remove[id-d7e7fece-6ed9-460a-9ebe-9267217e8580] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPorts.test_update_port_replace[id-9c26298b-1bcb-47b7-9b9e-8bdd6e3c4aba] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPortsWithPhysicalNetwork.test_create_port_with_physical_network[id-f1a5d279-c456-4311-ad31-fea09f61c22b] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPortsWithPhysicalNetwork.test_create_ports_in_portgroup_with_physical_network[id-4155c24d-8474-4b53-a320-aee475f85a68] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPortsWithPhysicalNetwork.test_update_port_remove_physical_network[id-6503309c-b2c7-4f59-b15a-0d92b5de9210] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPortsWithPhysicalNetwork.test_update_port_replace_physical_network[id-9c26298b-1bcb-47b7-9b9e-8bdd6e3c4aba] +ironic_tempest_plugin.tests.api.admin.test_ports.TestPortsWithPhysicalNetwork.test_update_ports_in_portgroup_with_physical_network[id-cf05a3ef-3bc4-4db7-bb4c-4eb871eb9f81] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_combining_runbook_and_explicit_steps[id-5c7f0aca-cee3-d083-ef2a-e33a8dc467c5] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_create_runbook_specifying_uuid[id-5a0cfe27-b7e8-d4f0-d3c5-e6e1eef63241] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_create_runbook_with_invalid_step_format[id-ddf4a9b7-144d-386a-fd3fcf8960d77199] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_delete_runbook[id-b8bfb388-97b0-aa6e-5130-1a1ed34f94db] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_list_runbooks_detail[id-ebd762c3-c10e-b71f-5efd-af14ac9f6092] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_list_runbooks[id-7b7951b3-e177-21d7-933a-1f29891dea52] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_list_with_limit[id-6aafc619-0d98-5341-7d94-b293e194dcf7] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_show_runbook_with_links[id-c95e2631-24e2-914b-db00-c8dafc35a677] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_show_runbook[id-f1cba93a-2894-7296-0f4b-58867359480b] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_update_runbook_add[id-99f52546-9906-6ded-9186-7262591b99ec] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_update_runbook_mixed_ops[id-ec1550c3-264e-fcce-b131-d2815fdb733b] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooks.test_update_runbook_replace[id-fb192fdd-ea6a-c637-a36e-390c46a7663b] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_create_runbook_duplicated_name[id-6390acc4-9490-4b23-8b4c-41888a78c9b7,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_create_runbook_invalid_name[id-a4085c08-e718-4c2f-a796-0e115b659243,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_create_runbook_malformed_runbook_uuid[id-2a562fca-f377-4a6e-b332-37ee82d3a983,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_create_runbook_malformed_steps[id-cbd33bc5-7602-40b7-943e-3e92217567a3,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_create_runbook_no_mandatory_field_name[id-ed3f0cec-13e8-4175-9fdb-d129e7b7fe10,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_create_runbook_no_mandatory_field_steps[id-af5dd0df-d903-463f-9535-9e4e9d6fd576,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_show_runbook_nonexistent[id-2c006994-88ca-43b7-b605-897d479229d9,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_update_runbook_remove_mandatory_field_name[id-ee852ebb-a601-4593-9d59-063fcbc8f964,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_update_runbook_remove_mandatory_field_steps[id-5a815f37-f015-4d68-9b22-099504f74805,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksNegative.test_update_runbook_replace_empty_name[id-e59bf38d-272f-4490-b21e-9db217f11378,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksOldAPI.test_create_runbook_old_api[id-e9481a0d-23e0-4757-bc11-c3c9ab9d3839,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksOldAPI.test_delete_runbook_old_api[id-1646b1e5-ab81-45a8-9ea0-30444a4dcaa2,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksOldAPI.test_list_runbooks_old_api[id-819480ac-f36a-4402-b1d5-504d7cf55b1f,negative] +ironic_tempest_plugin.tests.api.admin.test_runbooks.TestRunbooksOldAPI.test_update_runbook_old_api[id-0d3af2aa-ba53-4c8a-92d4-91f9b4179fe7,negative] +ironic_tempest_plugin.tests.api.admin.test_shards.TestAddShardsToNode.test_add_shard_to_node_at_create[id-6f1e241d-4386-4730-b9ff-28c6a3dcad31] +ironic_tempest_plugin.tests.api.admin.test_shards.TestAddShardsToNode.test_add_shard_to_node_post_create[id-2eb91d29-e0a5-472b-aeb8-ef6d98eb0f3c] +ironic_tempest_plugin.tests.api.admin.test_shards.TestGetAllShards.test_get_all_shards[id-fc786196-63c7-4e0d-bd14-3e478d4d1e3e] +ironic_tempest_plugin.tests.api.admin.test_shards.TestNodeShardQueries.test_only_show_multiple_requested_shards[id-6f1e241d-4386-4730-b9ff-28c6a3dcad31] +ironic_tempest_plugin.tests.api.admin.test_shards.TestNodeShardQueries.test_only_show_requested_shard[id-6f1e241d-4386-4730-b9ff-28c6a3dcad31] +ironic_tempest_plugin.tests.api.admin.test_shards.TestNodeShardQueries.test_show_all_nodes[id-df74c989-6972-4104-a8d6-bd8e8d811353] +ironic_tempest_plugin.tests.api.admin.test_shards.TestNodeShardQueries.test_show_sharded_nodes[id-f7a2eeb7-d16e-480c-b698-3448491c73a1] +ironic_tempest_plugin.tests.api.admin.test_shards.TestNodeShardQueries.test_show_unsharded_nodes[id-f7a2eeb7-d16e-480c-b698-3448491c73a1] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_create_volume_connector_error[id-3c3cbf45-488a-4386-a811-bf0aa2589c58,negative] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_delete_volume_connector_error[id-ccbda5e6-52b7-400c-94d7-25eec1d590f0,negative] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_delete_volume_connector[id-5795f816-0789-42e6-bb9c-91b4876ad13f] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_list_volume_connectors[id-a4725778-e164-4ee5-96a0-66119a35f783] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_list_with_limit[id-1d0459ad-01c0-46db-b930-7301bc2a3c98] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_show_volume_connector[id-6e4f50b7-0f4f-41c2-971e-d751abcac4e0] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_update_volume_connector_add[id-bfb0ca6b-086d-4663-9b25-e0eaf42da55b] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_update_volume_connector_remove_collection[id-8de03acd-532a-476f-8bc9-0e8b23bfe609] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_update_volume_connector_remove_item[id-b95c75eb-4048-482e-99ff-fe1d32538383] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_update_volume_connector_replace_error[id-5af8dc7a-9965-4787-8184-e60aeaf30957,negative] +ironic_tempest_plugin.tests.api.admin.test_volume_connector.TestVolumeConnector.test_update_volume_connector_replace[id-3c6f8354-e9bd-4f21-aae2-6deb96b04be7] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_create_volume_target_error[id-da5c27d4-68cc-499f-b8ab-3048b87d3bca,negative] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_delete_volume_target_error[id-532a06bc-a9b2-44b0-828a-c53279c87cb2,negative] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_delete_volume_target[id-ea3a9b2e-8971-4830-9274-abaf0239f1ce] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_list_volume_targets[id-ae99a986-d93c-4324-9cdc-41d89e3a659f] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_list_with_limit[id-9da25447-0370-4b33-9c1f-d4503f5950ae] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_show_volume_target[id-a2598388-8f61-4b7e-944f-f37e4f60e1e2] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_update_volume_target_add[id-9629715d-57ba-423b-b985-232674cc3a25] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_update_volume_target_remove_collection[id-6784ddb0-9144-41ea-b8a0-f888ad5c5b62] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_update_volume_target_remove_item[id-1c13a4ee-1a49-4739-8c19-77960fbd1af8] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_update_volume_target_replace_error[id-fd5266d3-4f3c-4dce-9c87-bfdea2b756c7,negative] +ironic_tempest_plugin.tests.api.admin.test_volume_target.TestVolumeTarget.test_update_volume_target_replace[id-8559cd08-feae-4f1a-a0ad-5bad8ea12b76] +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_bios_get +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_create_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_delete_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_disable_cleaning +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_get_boot_device +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_get_console +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_get_indicator_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_get_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_get_states +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_history_get +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_inject_nmi +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_ipa_heartbeat +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_boot_device +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_boot_mode +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_console_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_indicator_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_maintenance +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_power_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_provision_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_raid_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_set_secure_boot +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_traits_delete +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_traits_list +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_traits_set +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_unset_maintenance +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_instance_info +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_chassis_uuid +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_conductor_group +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_driver_info +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_driver_interfaces +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_extra +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_instance_uuid +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_lessee +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_name +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_network_data +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_owner +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_properties +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_node_retired +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_update_owner_provisioned +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_validate_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_vendor_passthru +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_vif_attach +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_vif_detach +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_cannot_vif_list +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeProjectReader.test_reader_list_is_empty +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_can_bios_get +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_can_get_indicator_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_can_get_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_can_get_states +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_can_traits_list +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_can_vif_list +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_create_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_delete_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_disable_cleaning +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_get_boot_device +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_get_console +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_history_get +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_inject_nmi +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_ipa_heartbeat +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_boot_device +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_boot_mode +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_console_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_indicator_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_maintenance +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_power_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_provision_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_raid_state +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_set_secure_boot +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_traits_delete +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_traits_set +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_unset_maintenance +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_instance_info +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_chassis_uuid +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_conductor_group +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_driver_info +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_driver_interfaces +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_extra +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_instance_uuid +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_lessee +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_name +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_network_data +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_owner +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_properties +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_node_retired +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_update_owner_provisioned +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_validate_node +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_vendor_passthru +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_vif_attach +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_cannot_vif_detach +ironic_tempest_plugin.tests.api.rbac_defaults.test_nodes.TestNodeSystemReader.test_reader_list_is_not_empty +ironic_tempest_plugin.tests.api.rbac_defaults.test_runbooks.TestRunbookRBAC.test_runbook_ownership_and_public_flag[id-903d0027-9265-9f87-c86d-09867aa24edd] +ironic_tempest_plugin.tests.api.rbac_defaults.test_runbooks.TestRunbookRBAC.test_runbook_visibility_and_access_control[id-0410fbed-3454-ded3-6f8b3192abfd3fcd] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_adoption.BaremetalAdoptionDriverWholedisk.test_adoption[id-2f51890e-20d9-43ef-af39-41b335ec066b,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_advanced_ops.BaremetalRedfishDHCPLessDeploy.test_ip_access_to_server[id-1f420ef3-99bd-46c7-b859-ce9c2892697f,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalDriverAnsibleWholedisk.test_ip_access_to_server[id-cde532cc-81ba-4489-b374-b4a85cc203eb,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalDriverDirectPartitioned.test_ip_access_to_server[id-7b4b2dcd-2bbb-44f5-991f-0964300af6b7,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalDriverDirectWholedisk.test_ip_access_to_server[id-c2db24e7-07dc-4a20-8f93-d4efae2bfd4e,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalDriverIscsiPartitioned.test_ip_access_to_server[id-7d0b205e-edbc-4e2d-9f6d-95cd74eefecb,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalDriverIscsiWholedisk.test_ip_access_to_server[id-f25b71df-2150-45d7-a780-7f5b07124808,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalHTTPBootTestClass.test_ip_access_to_server[id-782c43db-77a1-4a3a-b46e-0ce9cbb7fba5,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalHttpIPXEBootTestClass.test_ip_access_to_server[id-45400d8e-55a5-4ba6-81f5-935a4183ed90,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracDirectIPxeWholedisk.test_deploy_node[id-b467889b-aba4-4696-9e6f-4ef60a3f78a4,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracDirectPxeWholedisk.test_deploy_node[id-20c0f2f4-723b-4d1e-a4f2-bea11ffb1513,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracRedfishDirectIPxeWholedisk.test_deploy_node[id-b467889b-aba4-4696-9e6f-4ef60a3f78a4,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracRedfishDirectPxeWholedisk.test_deploy_node[id-20c0f2f4-723b-4d1e-a4f2-bea11ffb1513,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracSyncBootModeDirectWholedisk.test_sync_boot_mode[id-c2bebda2-fd27-4b10-9015-f7d877f0eb60,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracVirtualMediaWholedisk.test_deploy_virtual_media_boot[id-b0bc87a5-4324-4134-bd5f-4bb1cf549e5c,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracWSManDirectIPxeWholedisk.test_deploy_node[id-b467889b-aba4-4696-9e6f-4ef60a3f78a4,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIdracWSManDirectPxeWholedisk.test_deploy_node[id-20c0f2f4-723b-4d1e-a4f2-bea11ffb1513,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIlo5UefiHTTPSWholediskHttpsLink.test_ip_access_to_server[id-d926c683-1a32-edbc-07dc-95cd74eefecb,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIloDirectPartitioned.test_ip_access_to_server[id-ea85e19c-d8dc-4577-4d05-fbbbe4ce3f8c,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIloDirectWholediskHttpLink.test_ip_access_to_server[id-c2db24e7-b9bb-44df-6765-e60134346fd0,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIloIPxeWholediskHttpLink.test_ip_access_to_server[id-d926c683-1a32-edbc-07dc-95cd74eefecb,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIloIscsiPartitioned.test_ip_access_to_server[id-d926c683-4d05-8252-b9bb-35babbed6e86,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIloIscsiWholediskHttpLink.test_ip_access_to_server[id-71ccf06f-45db-8f93-afd0-d4efae2bfd4e,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIloPxePartitioned.test_ip_access_to_server[id-71ccf06f-07dc-4577-6869-1b1bfa423b9b,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIloPxeWholediskHttpLink.test_ip_access_to_server[id-d926c683-1a32-edbc-07dc-95cd74eefecb,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIpmiDirectIPxeWholedisk.test_deploy_node[id-9cd5a47b-ef96-4d64-9118-be467b24d1bf,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIpmiDirectPxeWholedisk.test_deploy_node[id-d68a38aa-b731-403a-9d40-b3b49ea75e9b,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIpmiRescuePartitioned.test_rescue_mode[id-113acd0a-9872-4631-b3ee-54da7e3bb262,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIpmiRescueWholedisk.test_rescue_mode[id-d6a1780f-c4bb-4136-8144-29e822e14d66,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalIPXEBootTestClass.test_ip_access_to_server[id-113acd0a-9872-4631-b3ee-54da7e3bb262,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalPXEBootTestClass.test_ip_access_to_server[id-62c12d2c-8c9f-4526-b9ab-b9cd63e0ea8a,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalRedfishDirectPxeWholedisk.test_deploy_node[id-2bf34541-5514-4ea7-b073-7707d408ce99,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalRedfishDirectWholediskHttpLink.test_ip_access_to_server[id-113acd0a-9872-4631-b3ee-54da7e3bb262,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalRedfishIPxeAnacondaNoGlance.test_ip_access_to_server_using_stage2 +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalRedfishIPxeAnacondaNoGlance.test_ip_access_to_server_without_stage2 +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalRedfishIPxeWholediskHttpLink.test_ip_access_to_server[id-113acd0a-9872-4631-b3ee-54da7e3bb262,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_basic_ops.BaremetalRedfishVmediaBootTestClass.test_ip_access_to_server[id-10535270-27e5-4616-9013-9507b1960dfa,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_bios.BaremetalFakeBios.test_bios_apply_and_reset_configuration[id-ef55c44a-cc10-4cf6-8fda-85f0c0793150] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_bios.BaremetalIdracBiosCleaning.test_bios_apply_configuration[id-6ded82ab-b444-436b-bb78-06fa5957d6c3] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_bios.BaremetalIdracRedfishBios.test_bios_apply_configuration[id-6ded82ab-b444-436b-bb78-06fa5957d6c3] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_bios.BaremetalIdracWSManBios.test_bios_apply_configuration[id-6ded82ab-b444-436b-bb78-06fa5957d6c3] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalCleaningIpmiWholedisk.test_manual_cleaning[id-065238db-1b6d-4d75-a9da-c240f8cbd956,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracManagementCleaning.test_clear_job_queue[id-9252ec6f-6b5b-447e-a323-c52775b88b4e] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracManagementCleaning.test_known_good_state[id-7baeff52-7d6e-4dea-a48f-a85a6bfc9f62] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracManagementCleaning.test_reset_idrac[id-d085ff72-abef-4931-a5b0-06efd5f9a037] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRaidCleaning.test_hardware_raid[id-8a908a3c-f2af-48fb-8553-9163715aa403,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRaidCleaning.test_raid_cleaning_max_size_raid_10[id-92fe534d-77f1-422d-84e4-e30fe9e3d928,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishConfigurationMolds.test_configuration_molds_export[id-69386cf6-bbf2-451e-b927-f68c66075f02,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishConfigurationMolds.test_configuration_molds_import[id-cad15719-c293-4338-8fa9-986ef02b4682,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishFirmwareUpdate.test_firmware_update[id-360e0c0e-3c17-4d2e-b052-55a932c1a4c7,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishManagementCleaning.test_clear_job_queue[id-9252ec6f-6b5b-447e-a323-c52775b88b4e] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishManagementCleaning.test_known_good_state[id-7baeff52-7d6e-4dea-a48f-a85a6bfc9f62] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishManagementCleaning.test_reset_idrac[id-d085ff72-abef-4931-a5b0-06efd5f9a037] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishRaidCleaning.test_hardware_raid[id-8a908a3c-f2af-48fb-8553-9163715aa403,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracRedfishRaidCleaning.test_raid_cleaning_max_size_raid_10[id-92fe534d-77f1-422d-84e4-e30fe9e3d928,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracWSManManagementCleaning.test_clear_job_queue[id-9252ec6f-6b5b-447e-a323-c52775b88b4e] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracWSManManagementCleaning.test_known_good_state[id-7baeff52-7d6e-4dea-a48f-a85a6bfc9f62] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracWSManManagementCleaning.test_reset_idrac[id-d085ff72-abef-4931-a5b0-06efd5f9a037] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracWSManRaidCleaning.test_hardware_raid[id-8a908a3c-f2af-48fb-8553-9163715aa403,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalIdracWSManRaidCleaning.test_raid_cleaning_max_size_raid_10[id-92fe534d-77f1-422d-84e4-e30fe9e3d928,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.BaremetalRedfishFirmwareUpdate.test_firmware_update[id-360e0c0e-3c17-4d2e-b052-55a932c1a4c7,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.SoftwareRaidDirect.test_software_raid[id-125361ac-0eb3-4d79-8be2-a91936aa3f46,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_cleaning.SoftwareRaidIscsi.test_software_raid[id-7ecba4f7-98b8-4ea1-b95e-3ec399f46798,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_inspection_basic.BaremetalIdracInspect.test_baremetal_inspect[id-47ea4487-4720-43e8-a024-53ae82f8c264] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_inspection_basic.BaremetalIdracRedfishInspect.test_baremetal_inspect[id-47ea4487-4720-43e8-a024-53ae82f8c264] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_inspection_basic.BaremetalIdracWSManInspect.test_baremetal_inspect[id-47ea4487-4720-43e8-a024-53ae82f8c264] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_inspection_basic.BaremetalRedfishAgentInspect.test_baremetal_inspect[id-47ea4487-4720-43e8-a024-53ae82f8c264] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_inspection_basic.BaremetalRedfishAgentInspect.test_inspect_abort[id-82670a85-49b7-4d28-b8a8-6e18b0d4c8d1] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_ramdisk_iso.BaremetalRamdiskBootIsoIPMIIPXE.test_ramdisk_boot[id-2859d115-9266-4461-9286-79b146e65dc9,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_ramdisk_iso.BaremetalRamdiskBootIsoRedfishVMedia.test_ramdisk_boot[id-2859d115-9266-4461-9286-79b146e65dc9,image,network] +ironic_tempest_plugin.tests.scenario.ironic_standalone.test_ramdisk_iso.BaremetalRamdiskBootIsoSNMPIPXE.test_ramdisk_boot[id-2859d115-9266-4461-9286-79b146e65dc9,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_basic_ops.BaremetalBasicOps.test_baremetal_server_ops_partition_image[compute,id-549173a5-38ec-42bb-b0e2-c8b9f4a08943,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_basic_ops.BaremetalBasicOps.test_baremetal_server_ops_wholedisk_image[compute,id-d7d48aa1-0395-4f31-a908-60969adc4322,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_basic_ops.BaremetalBasicOpsAndRescue.test_baremetal_server_ops_partition_image[compute,id-549173a5-38ec-42bb-b0e2-c8b9f4a08943,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_basic_ops.BaremetalBasicOpsAndRescue.test_baremetal_server_ops_wholedisk_image[compute,id-d7d48aa1-0395-4f31-a908-60969adc4322,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_boot_from_volume.BaremetalBFV.test_baremetal_boot_from_volume[compute,id-d6e05e61-8221-44ac-b785-57545f8e0fcf,image,network,volume] +ironic_tempest_plugin.tests.scenario.test_baremetal_multitenancy.BaremetalMultitenancy.test_baremetal_multitenancy[compute,id-26e2f145-2a8e-4dc7-8457-7f2eb2c6749d,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_multitenancy.BaremetalMultitenancy.test_baremetal_vm_multitenancy[compute,id-9e38631a-2df2-11e9-810e-8c16450ea513,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_single_tenant.BaremetalSingleTenant.test_baremetal_single_tenant[compute,id-8fe15552-3788-11e9-b599-74e5f9e2a801,image,network] +ironic_tempest_plugin.tests.scenario.test_baremetal_single_tenant.BaremetalSingleTenant.test_baremetal_vm_single_tenant[compute,id-90b3b6be-3788-11e9-b599-74e5f9e2a801,image,network] +ironic_tempest_plugin.tests.scenario.test_introspection_basic.InspectorBasicTest.test_baremetal_introspection[compute,id-03bf7990-bee0-4dd7-bf74-b97ad7b52a4b,image,network,smoke] +ironic_tempest_plugin.tests.scenario.test_introspection_basic.InspectorBasicTest.test_introspection_abort[id-70ca3070-184b-4b7d-8892-e977d2bc2870] +ironic_tempest_plugin.tests.scenario.test_introspection_discovery.InspectorDiscoveryTest.test_bearmetal_auto_discovery[id-dd3abe5e-0d23-488d-bb4e-344cdeff7dcb] diff --git a/.automation.conf/tempest/load-lists/keystone b/.automation.conf/tempest/load-lists/keystone new file mode 100644 index 000000000..e9134b96c --- /dev/null +++ b/.automation.conf/tempest/load-lists/keystone @@ -0,0 +1,1626 @@ +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_add_protocol_to_identity_provider_unknown_mapping[id-04fdf262-af91-4a68-a1cf-794c6d2f2eeb] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_add_protocol_to_identity_provider[id-f5bdf482-1ad3-4aad-b52e-8fe7c1361104] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_get_protocol_from_identity_provider[id-613d073b-0db7-41aa-b4df-5a1848cde0b3] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_identity_provider_create_with_enabled_true[id-f430a337-545d-455e-bb6c-cb0fdf4be5c1] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_identity_provider_create_with_remote_ids[id-238e6163-d600-4f59-9982-c621f057221d] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_identity_provider_create[id-09450910-b816-4150-8513-a2fd4628a0c3] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_identity_provider_get[id-8a7817ad-27f8-436b-9cbe-46aa20989beb] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_identity_provider_list[id-cbfe5de9-c58a-4810-950c-2acdf985879d] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_identity_provider_update[id-36a0d9f0-9517-4139-85d0-f78d905aece5] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_list_protocols_from_identity_provider[id-6e6501ac-edae-4dc2-bb68-7a2b58602383] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_update_mapping_from_identity_provider_protocol[id-680d36df-d6b9-4695-be29-6fc2065c5dde] +keystone_tempest_plugin.tests.api.identity.v3.test_identity_providers.IndentityProvidersTest.test_update_protocol_from_identity_provider_unknown_mapping[id-c73311e7-c207-4c11-998f-532a91f1b0d1] +keystone_tempest_plugin.tests.api.identity.v3.test_mapping_rules.MappingRulesTest.test_mapping_rule_update[id-1fc5d104-faf5-4809-8c89-29b5c1666a96] +keystone_tempest_plugin.tests.api.identity.v3.test_mapping_rules.MappingRulesTest.test_mapping_rules_create_without_mandatory_attributes_fails[id-341dac45-ce1f-4f15-afdc-1f9a7d7d7c40,negative] +keystone_tempest_plugin.tests.api.identity.v3.test_mapping_rules.MappingRulesTest.test_mapping_rules_create[id-4ca48c01-b6da-4759-acb6-007e15ad712a] +keystone_tempest_plugin.tests.api.identity.v3.test_mapping_rules.MappingRulesTest.test_mapping_rules_get[id-8db213e3-1db0-48c6-863c-7a3ed23577ec] +keystone_tempest_plugin.tests.api.identity.v3.test_mapping_rules.MappingRulesTest.test_mapping_rules_list[id-bb80b242-2a6a-4d29-b45f-4035be574a6e] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_authorize_request_token[id-607aecc4-a623-4566-a3a5-bb0e2a6fc9c5] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_create_access_token[id-7d488fc9-342c-4c12-b6b8-b158e2183925] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_create_and_show_consumer[id-23d2fe8d-fc8d-4bef-8aaa-289400732c3f] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_create_request_token[id-a17d60e4-7cb5-4e06-9e16-b044f3ee6d94] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_delete_consumer[id-3820f3d0-9b06-4d15-8f01-c7dd4eea30a2] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_list_access_tokens[id-5929055d-7c0f-4661-a9a5-15c4b95082dc] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_list_consumers[id-6da689b1-39a0-44ee-9624-445159119c57] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_list_roles_for_access_token[id-0075f413-e249-42e5-9bc9-d6e3aecf6cbc] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_revoke_access_token[id-1b802896-91a0-4cbb-a8b9-860c7087fad8] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_show_role_for_access_token[id-28aee994-86b1-4596-a652-572f558045e7] +keystone_tempest_plugin.tests.api.identity.v3.test_oauth1_tokens.OAUTH1TokensTest.test_update_consumer[id-5a03fa78-3a03-449b-a04c-ef9de80eb6c4] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_create_with_bad_attributes[id-f77ed1c0-c428-44a7-9364-e8e4362c360a,negative] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_create_with_enabled_true[id-8550b419-f212-4e34-a8fa-7ff64f8a7fd3] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_create_with_relay_state_prefix[id-0e319a14-1548-474e-a406-273c6b1c1f2d] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_create_without_mandatory_attributes[id-d9d7454c-50b7-4966-aedb-b9d520a41409,negative] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_create[id-6fae0971-5acb-4559-ba25-96f1fd7e5385] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_get[id-7df78c7a-9265-4b4f-9630-193b7f07d9eb] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_list[id-9237cea0-fbeb-4d64-8347-46c567e1d78f] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_update_with_bad_attributes_fails[id-91ce1183-1a15-4598-ae5f-85cfa98a1c77,negative] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_provider_update[id-bb68653f-fbba-4f20-ac1b-7b318a557366] +keystone_tempest_plugin.tests.api.identity.v3.test_service_providers.ServiceProvidersTest.test_service_providers_in_token[id-7553579b-9a9e-45dd-9ada-70d906b516c0] +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainAdminTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainAdminTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainAdminTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainMemberTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainMemberTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainMemberTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainReaderTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainReaderTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.DomainReaderTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectAdminTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectAdminTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectAdminTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectMemberTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectMemberTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectMemberTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectReaderTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectReaderTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.ProjectReaderTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemAdminTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemAdminTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemAdminTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemMemberTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemMemberTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemMemberTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemReaderTests.test_identity_delete_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemReaderTests.test_identity_get_access_rule +keystone_tempest_plugin.tests.rbac.v3.test_access_rule.SystemReaderTests.test_identity_list_access_rules +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainAdminTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainAdminTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainAdminTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainAdminTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainMemberTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainMemberTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainMemberTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainMemberTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainReaderTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainReaderTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainReaderTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.DomainReaderTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectAdminTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectAdminTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectAdminTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectAdminTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectMemberTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectMemberTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectMemberTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectMemberTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectReaderTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectReaderTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectReaderTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.ProjectReaderTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemAdminTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemAdminTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemAdminTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemAdminTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemMemberTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemMemberTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemMemberTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemMemberTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemReaderTests.test_identity_create_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemReaderTests.test_identity_delete_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemReaderTests.test_identity_get_application_credential +keystone_tempest_plugin.tests.rbac.v3.test_application_credential.SystemReaderTests.test_identity_list_application_credentials +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainAdminTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainAdminTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainAdminTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainAdminTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainAdminTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainMemberTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainMemberTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainMemberTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainMemberTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainMemberTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainReaderTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainReaderTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainReaderTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainReaderTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.DomainReaderTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectAdminTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectAdminTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectAdminTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectAdminTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectAdminTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectMemberTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectMemberTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectMemberTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectMemberTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectMemberTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectReaderTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectReaderTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectReaderTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectReaderTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.ProjectReaderTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemAdminTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemAdminTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemAdminTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemAdminTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemAdminTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemMemberTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemMemberTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemMemberTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemMemberTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemMemberTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemReaderTests.test_identity_create_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemReaderTests.test_identity_delete_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemReaderTests.test_identity_get_consumer +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemReaderTests.test_identity_list_consumers +keystone_tempest_plugin.tests.rbac.v3.test_consumer.SystemReaderTests.test_identity_update_consumer +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainAdminTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainAdminTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainAdminTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainAdminTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainAdminTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainMemberTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainMemberTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainMemberTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainMemberTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainMemberTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainReaderTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainReaderTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainReaderTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainReaderTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.DomainReaderTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectAdminTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectAdminTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectAdminTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectAdminTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectAdminTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectMemberTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectMemberTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectMemberTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectMemberTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectMemberTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectReaderTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectReaderTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectReaderTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectReaderTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.ProjectReaderTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemAdminTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemAdminTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemAdminTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemAdminTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemAdminTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemMemberTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemMemberTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemMemberTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemMemberTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemMemberTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemReaderTests.test_identity_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemReaderTests.test_identity_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemReaderTests.test_identity_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemReaderTests.test_identity_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_credential.SystemReaderTests.test_identity_update_credential +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainAdminTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainAdminTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainAdminTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainAdminTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainAdminTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainAdminTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainMemberTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainMemberTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainMemberTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainMemberTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainMemberTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainMemberTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainReaderTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainReaderTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainReaderTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainReaderTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainReaderTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.DomainReaderTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectAdminTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectAdminTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectAdminTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectAdminTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectAdminTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectAdminTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectMemberTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectMemberTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectMemberTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectMemberTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectMemberTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectMemberTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectReaderTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectReaderTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectReaderTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectReaderTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectReaderTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.ProjectReaderTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemAdminTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemAdminTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemAdminTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemAdminTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemAdminTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemAdminTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemMemberTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemMemberTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemMemberTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemMemberTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemMemberTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemMemberTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemReaderTests.test_identity_create_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemReaderTests.test_identity_delete_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemReaderTests.test_identity_get_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemReaderTests.test_identity_get_domain_config_default +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemReaderTests.test_identity_get_security_compliance_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain_config.SystemReaderTests.test_identity_update_domain_config +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainAdminTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainAdminTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainAdminTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainAdminTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainAdminTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainMemberTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainMemberTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainMemberTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainMemberTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainMemberTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainReaderTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainReaderTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainReaderTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainReaderTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.DomainReaderTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectAdminTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectAdminTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectAdminTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectAdminTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectAdminTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectMemberTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectMemberTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectMemberTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectMemberTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectMemberTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectReaderTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectReaderTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectReaderTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectReaderTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.ProjectReaderTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemAdminTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemAdminTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemAdminTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemAdminTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemAdminTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemMemberTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemMemberTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemMemberTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemMemberTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemMemberTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemReaderTests.test_identity_create_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemReaderTests.test_identity_delete_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemReaderTests.test_identity_get_domain +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemReaderTests.test_identity_list_domains +keystone_tempest_plugin.tests.rbac.v3.test_domain.SystemReaderTests.test_identity_update_domain +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainAdminTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainAdminTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainAdminTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainAdminTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainMemberTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainMemberTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainMemberTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainMemberTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainReaderTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainReaderTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainReaderTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.DomainReaderTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectAdminTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectAdminTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectAdminTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectAdminTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectMemberTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectMemberTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectMemberTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectMemberTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectReaderTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectReaderTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectReaderTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.ProjectReaderTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemAdminTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemAdminTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemAdminTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemAdminTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemMemberTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemMemberTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemMemberTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemMemberTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemReaderTests.test_identity_ec2_create_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemReaderTests.test_identity_ec2_delete_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemReaderTests.test_identity_ec2_get_credential +keystone_tempest_plugin.tests.rbac.v3.test_ec2_credential.SystemReaderTests.test_identity_ec2_list_credentials +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainAdminTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainMemberTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.DomainReaderTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectAdminTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectMemberTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.ProjectReaderTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemAdminTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemMemberTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_add_endpoint_group_to_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_create_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_delete_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_get_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_get_endpoint_group_in_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_list_endpoint_groups +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_list_endpoint_groups_for_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_list_endpoints_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_list_projects_associated_with_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_remove_endpoint_group_from_project +keystone_tempest_plugin.tests.rbac.v3.test_endpoint_group.SystemReaderTests.test_identity_update_endpoint_group +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainAdminTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainAdminTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainAdminTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainAdminTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainAdminTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainMemberTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainMemberTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainMemberTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainMemberTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainMemberTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainReaderTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainReaderTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainReaderTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainReaderTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.DomainReaderTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectAdminTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectAdminTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectAdminTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectAdminTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectAdminTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectMemberTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectMemberTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectMemberTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectMemberTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectMemberTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectReaderTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectReaderTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectReaderTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectReaderTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.ProjectReaderTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemAdminTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemAdminTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemAdminTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemAdminTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemAdminTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemMemberTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemMemberTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemMemberTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemMemberTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemMemberTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemReaderTests.test_identity_create_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemReaderTests.test_identity_delete_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemReaderTests.test_identity_get_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemReaderTests.test_identity_list_endpoints +keystone_tempest_plugin.tests.rbac.v3.test_endpoint.SystemReaderTests.test_identity_update_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainAdminTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainMemberTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.DomainReaderTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectAdminTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectMemberTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.ProjectReaderTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemAdminTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemMemberTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_check_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_check_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_check_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_create_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_create_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_create_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_list_grants +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_list_system_grants_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_list_system_grants_for_user +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_revoke_grant +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_revoke_system_grant_for_group +keystone_tempest_plugin.tests.rbac.v3.test_grant.SystemReaderTests.test_identity_revoke_system_grant_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainAdminTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainMemberTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.DomainReaderTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectAdminTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectMemberTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.ProjectReaderTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemAdminTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemMemberTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_add_user_to_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_check_user_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_create_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_delete_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_get_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_list_groups +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_list_groups_for_user +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_list_users_in_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_remove_user_from_group +keystone_tempest_plugin.tests.rbac.v3.test_group.SystemReaderTests.test_identity_update_group +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainAdminTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainAdminTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainAdminTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainAdminTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainAdminTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainMemberTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainMemberTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainMemberTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainMemberTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainMemberTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainReaderTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainReaderTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainReaderTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainReaderTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.DomainReaderTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectAdminTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectAdminTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectAdminTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectAdminTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectAdminTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectMemberTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectMemberTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectMemberTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectMemberTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectMemberTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectReaderTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectReaderTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectReaderTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectReaderTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.ProjectReaderTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemAdminTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemAdminTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemAdminTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemAdminTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemAdminTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemMemberTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemMemberTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemMemberTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemMemberTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemMemberTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemReaderTests.test_identity_create_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemReaderTests.test_identity_delete_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemReaderTests.test_identity_get_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemReaderTests.test_identity_list_identity_providers +keystone_tempest_plugin.tests.rbac.v3.test_identity_provider.SystemReaderTests.test_identity_update_identity_provider +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainAdminTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainAdminTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainAdminTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainAdminTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainAdminTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainAdminTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainMemberTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainMemberTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainMemberTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainMemberTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainMemberTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainMemberTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainReaderTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainReaderTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainReaderTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainReaderTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainReaderTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.DomainReaderTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectAdminTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectAdminTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectAdminTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectAdminTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectAdminTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectAdminTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectMemberTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectMemberTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectMemberTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectMemberTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectMemberTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectMemberTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectReaderTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectReaderTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectReaderTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectReaderTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectReaderTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.ProjectReaderTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemAdminTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemAdminTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemAdminTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemAdminTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemAdminTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemAdminTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemMemberTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemMemberTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemMemberTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemMemberTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemMemberTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemMemberTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemReaderTests.test_identity_check_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemReaderTests.test_identity_create_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemReaderTests.test_identity_delete_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemReaderTests.test_identity_get_implied_role +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemReaderTests.test_identity_list_implied_roles +keystone_tempest_plugin.tests.rbac.v3.test_implied_role.SystemReaderTests.test_identity_list_role_inference_rules +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainAdminTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainAdminTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainAdminTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainAdminTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainAdminTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainAdminTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainMemberTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainMemberTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainMemberTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainMemberTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainMemberTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainMemberTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainReaderTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainReaderTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainReaderTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainReaderTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainReaderTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.DomainReaderTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectAdminTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectAdminTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectAdminTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectAdminTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectAdminTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectAdminTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectMemberTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectMemberTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectMemberTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectMemberTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectMemberTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectMemberTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectReaderTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectReaderTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectReaderTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectReaderTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectReaderTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.ProjectReaderTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemAdminTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemAdminTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemAdminTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemAdminTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemAdminTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemAdminTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemMemberTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemMemberTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemMemberTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemMemberTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemMemberTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemMemberTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemReaderTests.test_identity_create_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemReaderTests.test_identity_delete_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemReaderTests.test_identity_get_limit +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemReaderTests.test_identity_get_limit_model +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemReaderTests.test_identity_list_limits +keystone_tempest_plugin.tests.rbac.v3.test_limit.SystemReaderTests.test_identity_update_limit +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainAdminTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainAdminTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainAdminTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainAdminTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainAdminTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainMemberTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainMemberTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainMemberTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainMemberTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainMemberTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainReaderTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainReaderTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainReaderTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainReaderTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.DomainReaderTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectAdminTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectAdminTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectAdminTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectAdminTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectAdminTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectMemberTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectMemberTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectMemberTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectMemberTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectMemberTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectReaderTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectReaderTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectReaderTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectReaderTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.ProjectReaderTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemAdminTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemAdminTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemAdminTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemAdminTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemAdminTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemMemberTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemMemberTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemMemberTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemMemberTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemMemberTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemReaderTests.test_identity_create_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemReaderTests.test_identity_delete_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemReaderTests.test_identity_get_mapping +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemReaderTests.test_identity_list_mappings +keystone_tempest_plugin.tests.rbac.v3.test_mapping.SystemReaderTests.test_identity_update_mapping +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainAdminTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainMemberTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.DomainReaderTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectAdminTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectMemberTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.ProjectReaderTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemAdminTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemMemberTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_check_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_check_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_check_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_create_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_create_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_create_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_delete_policy_association_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_delete_policy_association_for_region_and_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_delete_policy_association_for_service +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_get_policy_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_policy_association.SystemReaderTests.test_identity_list_endpoints_for_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainAdminTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainAdminTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainAdminTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainAdminTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainAdminTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainMemberTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainMemberTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainMemberTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainMemberTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainMemberTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainReaderTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainReaderTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainReaderTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainReaderTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.DomainReaderTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectAdminTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectAdminTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectAdminTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectAdminTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectAdminTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectMemberTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectMemberTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectMemberTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectMemberTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectMemberTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectReaderTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectReaderTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectReaderTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectReaderTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.ProjectReaderTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemAdminTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemAdminTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemAdminTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemAdminTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemAdminTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemMemberTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemMemberTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemMemberTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemMemberTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemMemberTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemReaderTests.test_identity_create_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemReaderTests.test_identity_delete_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemReaderTests.test_identity_get_policy +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemReaderTests.test_identity_list_policies +keystone_tempest_plugin.tests.rbac.v3.test_policy.SystemReaderTests.test_identity_update_policy +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainAdminTests.test_identity_add_endpoint_to_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainAdminTests.test_identity_check_endpoint_in_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainAdminTests.test_identity_list_endpoints_for_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainAdminTests.test_identity_list_projects_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainAdminTests.test_identity_remove_endpoint_from_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainMemberTests.test_identity_add_endpoint_to_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainMemberTests.test_identity_check_endpoint_in_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainMemberTests.test_identity_list_endpoints_for_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainMemberTests.test_identity_list_projects_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainMemberTests.test_identity_remove_endpoint_from_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainReaderTests.test_identity_add_endpoint_to_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainReaderTests.test_identity_check_endpoint_in_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainReaderTests.test_identity_list_endpoints_for_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainReaderTests.test_identity_list_projects_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.DomainReaderTests.test_identity_remove_endpoint_from_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectMemberTests.test_identity_add_endpoint_to_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectMemberTests.test_identity_check_endpoint_in_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectMemberTests.test_identity_list_endpoints_for_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectMemberTests.test_identity_list_projects_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectMemberTests.test_identity_remove_endpoint_from_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectReaderTests.test_identity_add_endpoint_to_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectReaderTests.test_identity_check_endpoint_in_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectReaderTests.test_identity_list_endpoints_for_project +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectReaderTests.test_identity_list_projects_for_endpoint +keystone_tempest_plugin.tests.rbac.v3.test_project_endpoint.ProjectReaderTests.test_identity_remove_endpoint_from_project +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainAdminTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainAdminTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainAdminTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainAdminTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainAdminTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainAdminTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainMemberTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainMemberTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainMemberTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainMemberTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainMemberTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainMemberTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainReaderTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainReaderTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainReaderTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainReaderTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainReaderTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.DomainReaderTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectAdminTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectAdminTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectAdminTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectAdminTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectAdminTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectAdminTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectMemberTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectMemberTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectMemberTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectMemberTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectMemberTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectMemberTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectReaderTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectReaderTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectReaderTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectReaderTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectReaderTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.ProjectReaderTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemAdminTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemAdminTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemAdminTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemAdminTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemAdminTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemAdminTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemMemberTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemMemberTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemMemberTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemMemberTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemMemberTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemMemberTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemReaderTests.test_identity_create_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemReaderTests.test_identity_delete_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemReaderTests.test_identity_delete_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemReaderTests.test_identity_get_project_tag +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemReaderTests.test_identity_list_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project_tag.SystemReaderTests.test_identity_update_project_tags +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainAdminTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainAdminTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainAdminTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainAdminTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainAdminTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainAdminTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainMemberTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainMemberTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainMemberTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainMemberTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainMemberTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainMemberTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainReaderTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainReaderTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainReaderTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainReaderTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainReaderTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.DomainReaderTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectAdminTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectAdminTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectAdminTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectAdminTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectAdminTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectAdminTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectMemberTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectMemberTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectMemberTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectMemberTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectMemberTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectMemberTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectReaderTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectReaderTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectReaderTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectReaderTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectReaderTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.ProjectReaderTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemAdminTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemAdminTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemAdminTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemAdminTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemAdminTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemAdminTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemMemberTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemMemberTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemMemberTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemMemberTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemMemberTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemMemberTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemReaderTests.test_identity_create_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemReaderTests.test_identity_delete_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemReaderTests.test_identity_get_project +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemReaderTests.test_identity_list_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemReaderTests.test_identity_list_user_projects +keystone_tempest_plugin.tests.rbac.v3.test_project.SystemReaderTests.test_identity_update_project +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainAdminTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainAdminTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainAdminTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainAdminTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainAdminTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainMemberTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainMemberTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainMemberTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainMemberTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainMemberTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainReaderTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainReaderTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainReaderTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainReaderTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.DomainReaderTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectAdminTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectAdminTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectAdminTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectAdminTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectAdminTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectMemberTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectMemberTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectMemberTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectMemberTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectMemberTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectReaderTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectReaderTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectReaderTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectReaderTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.ProjectReaderTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemAdminTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemAdminTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemAdminTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemAdminTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemAdminTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemMemberTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemMemberTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemMemberTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemMemberTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemMemberTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemReaderTests.test_identity_create_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemReaderTests.test_identity_delete_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemReaderTests.test_identity_get_protocol +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemReaderTests.test_identity_list_protocols +keystone_tempest_plugin.tests.rbac.v3.test_protocol.SystemReaderTests.test_identity_update_protocol +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainAdminTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainAdminTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainAdminTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainAdminTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainAdminTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainMemberTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainMemberTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainMemberTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainMemberTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainMemberTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainReaderTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainReaderTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainReaderTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainReaderTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.DomainReaderTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectAdminTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectAdminTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectAdminTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectAdminTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectAdminTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectMemberTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectMemberTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectMemberTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectMemberTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectMemberTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectReaderTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectReaderTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectReaderTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectReaderTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.ProjectReaderTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemAdminTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemAdminTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemAdminTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemAdminTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemAdminTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemMemberTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemMemberTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemMemberTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemMemberTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemMemberTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemReaderTests.test_identity_create_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemReaderTests.test_identity_delete_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemReaderTests.test_identity_get_region +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemReaderTests.test_identity_list_regions +keystone_tempest_plugin.tests.rbac.v3.test_region.SystemReaderTests.test_identity_update_region +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainAdminTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainAdminTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainAdminTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainAdminTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainAdminTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainMemberTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainMemberTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainMemberTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainMemberTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainMemberTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainReaderTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainReaderTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainReaderTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainReaderTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.DomainReaderTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectAdminTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectAdminTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectAdminTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectAdminTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectAdminTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectMemberTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectMemberTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectMemberTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectMemberTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectMemberTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectReaderTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectReaderTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectReaderTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectReaderTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.ProjectReaderTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemAdminTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemAdminTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemAdminTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemAdminTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemAdminTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemMemberTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemMemberTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemMemberTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemMemberTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemMemberTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemReaderTests.test_identity_create_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemReaderTests.test_identity_delete_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemReaderTests.test_identity_get_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemReaderTests.test_identity_list_registered_limits +keystone_tempest_plugin.tests.rbac.v3.test_registered_limit.SystemReaderTests.test_identity_update_registered_limit +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.DomainAdminTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.DomainAdminTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.DomainMemberTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.DomainMemberTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.DomainReaderTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.DomainReaderTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.ProjectAdminTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.ProjectAdminTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.ProjectMemberTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.ProjectMemberTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.ProjectReaderTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.ProjectReaderTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.SystemAdminTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.SystemAdminTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.SystemMemberTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.SystemMemberTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.SystemReaderTests.test_identity_list_role_assignments +keystone_tempest_plugin.tests.rbac.v3.test_role_assignment.SystemReaderTests.test_identity_list_role_assignments_for_tree +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainAdminTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainMemberTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.DomainReaderTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectAdminTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectMemberTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.ProjectReaderTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemAdminTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemMemberTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_create_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_create_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_delete_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_delete_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_get_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_get_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_list_domain_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_list_roles +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_update_domain_role +keystone_tempest_plugin.tests.rbac.v3.test_role.SystemReaderTests.test_identity_update_role +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainAdminTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainAdminTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainAdminTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainAdminTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainAdminTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainMemberTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainMemberTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainMemberTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainMemberTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainMemberTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainReaderTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainReaderTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainReaderTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainReaderTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.DomainReaderTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectAdminTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectAdminTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectAdminTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectAdminTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectAdminTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectMemberTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectMemberTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectMemberTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectMemberTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectMemberTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectReaderTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectReaderTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectReaderTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectReaderTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.ProjectReaderTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemAdminTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemAdminTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemAdminTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemAdminTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemAdminTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemMemberTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemMemberTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemMemberTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemMemberTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemMemberTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemReaderTests.test_identity_create_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemReaderTests.test_identity_delete_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemReaderTests.test_identity_get_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemReaderTests.test_identity_list_service_providers +keystone_tempest_plugin.tests.rbac.v3.test_service_provider.SystemReaderTests.test_identity_update_service_provider +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainAdminTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainAdminTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainAdminTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainAdminTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainAdminTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainMemberTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainMemberTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainMemberTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainMemberTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainMemberTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainReaderTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainReaderTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainReaderTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainReaderTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.DomainReaderTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectAdminTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectAdminTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectAdminTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectAdminTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectAdminTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectMemberTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectMemberTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectMemberTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectMemberTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectMemberTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectReaderTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectReaderTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectReaderTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectReaderTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.ProjectReaderTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemAdminTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemAdminTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemAdminTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemAdminTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemAdminTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemMemberTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemMemberTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemMemberTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemMemberTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemMemberTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemReaderTests.test_identity_create_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemReaderTests.test_identity_delete_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemReaderTests.test_identity_get_service +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemReaderTests.test_identity_list_services +keystone_tempest_plugin.tests.rbac.v3.test_service.SystemReaderTests.test_identity_update_service +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainAdminTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainAdminTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainAdminTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainMemberTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainMemberTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainMemberTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainReaderTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainReaderTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.DomainReaderTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectAdminTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectAdminTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectAdminTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectMemberTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectMemberTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectMemberTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectReaderTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectReaderTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.ProjectReaderTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemAdminTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemAdminTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemAdminTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemMemberTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemMemberTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemMemberTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemReaderTests.test_identity_check_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemReaderTests.test_identity_revoke_token +keystone_tempest_plugin.tests.rbac.v3.test_token.SystemReaderTests.test_identity_validate_token +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainAdminTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainMemberTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.DomainReaderTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectAdminTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectMemberTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.ProjectReaderTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemAdminTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemMemberTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_create_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_delete_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_get_role_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_get_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_list_roles_for_trust +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_list_trusts +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_list_trusts_for_trustee +keystone_tempest_plugin.tests.rbac.v3.test_trust.SystemReaderTests.test_identity_list_trusts_for_trustor +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainAdminTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainAdminTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainAdminTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainAdminTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainAdminTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainMemberTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainMemberTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainMemberTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainMemberTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainMemberTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainReaderTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainReaderTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainReaderTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainReaderTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.DomainReaderTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectAdminTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectAdminTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectAdminTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectAdminTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectAdminTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectMemberTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectMemberTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectMemberTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectMemberTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectMemberTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectReaderTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectReaderTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectReaderTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectReaderTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.ProjectReaderTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemAdminTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemAdminTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemAdminTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemAdminTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemAdminTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemMemberTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemMemberTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemMemberTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemMemberTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemMemberTests.test_identity_update_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemReaderTests.test_identity_create_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemReaderTests.test_identity_delete_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemReaderTests.test_identity_get_user +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemReaderTests.test_identity_list_users +keystone_tempest_plugin.tests.rbac.v3.test_user.SystemReaderTests.test_identity_update_user +keystone_tempest_plugin.tests.scenario.test_federated_authentication.TestK2KFederatedAuthentication.test_request_scoped_token +keystone_tempest_plugin.tests.scenario.test_federated_authentication.TestK2KFederatedAuthentication.test_request_unscoped_token +keystone_tempest_plugin.tests.scenario.test_federated_authentication.TestSaml2FederatedExternalAuthentication.test_request_scoped_token +keystone_tempest_plugin.tests.scenario.test_federated_authentication.TestSaml2FederatedExternalAuthentication.test_request_unscoped_token +keystone_tempest_plugin.tests.scenario.test_oidc_federated_authentication.TestOidcFederatedAuthentication.test_request_scoped_token +keystone_tempest_plugin.tests.scenario.test_oidc_federated_authentication.TestOidcFederatedAuthentication.test_request_unscoped_token diff --git a/.automation.conf/tempest/load-lists/magnum b/.automation.conf/tempest/load-lists/magnum new file mode 100644 index 000000000..afaa6785f --- /dev/null +++ b/.automation.conf/tempest/load-lists/magnum @@ -0,0 +1,60 @@ +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template_admin.ClusterTemplateAdminTest.test_create_get_public_cluster_template[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template_admin.ClusterTemplateAdminTest.test_update_cluster_template_public_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_invalid_network_driver[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_invalid_volume_driver[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_missing_flavor[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_missing_image[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_missing_keypair[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_create_cluster_template[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_create_get_public_cluster_template[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_404[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_by_name_404[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_by_name[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_get_cluster_template_by_name_404[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_get_cluster_template_by_uuid_404[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_list_cluster_templates[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_404[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_invalid_patch[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_name_not_found[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_public_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster.ClusterTest.test_create_cluster_for_nonexisting_cluster_template[id-1c81a8b2-de86-4494-a8a7-b194f4c1a1f9,negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor[id-5e4e68d7-6b0a-4548-a462-2f5c83004509,negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster.ClusterTest.test_create_cluster_with_zero_masters[id-395288a7-dc5d-474b-ba2b-09158553d549,negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes[id-6201ac61-fd28-43a9-824a-2d452b2fe5c5,positive,slow] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster.ClusterTest.test_create_list_sign_delete_clusters[id-82bdca24-d732-41d1-bfc0-983ec8229ca0,positive,slow] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster.ClusterTest.test_delete_cluster_for_nonexisting_cluster[id-208fabdb-f7b8-4bdd-8762-fdf88be0fbca,negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_cluster.ClusterTest.test_update_cluster_for_nonexisting_cluster[id-71c75c03-090c-4f33-9aad-f2919a3b63d2,negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_magnum_service.MagnumServiceTest.test_magnum_service_list_needs_admin[negative] +magnum_tempest_plugin.tests.api.v1.rbac.test_magnum_service.MagnumServiceTest.test_magnum_service_list[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin.ClusterTemplateAdminTest.test_create_get_public_cluster_template[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template_admin.ClusterTemplateAdminTest.test_update_cluster_template_public_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_invalid_network_driver[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_invalid_volume_driver[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_missing_flavor[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_missing_image[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_create_cluster_template_missing_keypair[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_create_cluster_template[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_create_get_public_cluster_template[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_404[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_by_name_404[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_by_name[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_delete_cluster_template_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_get_cluster_template_by_name_404[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_get_cluster_template_by_uuid_404[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_list_cluster_templates[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_404[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_invalid_patch[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_name_not_found[negative] +magnum_tempest_plugin.tests.api.v1.test_cluster_template.ClusterTemplateTest.test_update_cluster_template_public_by_uuid[positive] +magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_for_nonexisting_cluster_template[id-1c81a8b2-de86-4494-a8a7-b194f4c1a1f9,negative] +magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_nonexisting_flavor[id-5e4e68d7-6b0a-4548-a462-2f5c83004509,negative] +magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_masters[id-395288a7-dc5d-474b-ba2b-09158553d549,negative] +magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes[id-6201ac61-fd28-43a9-824a-2d452b2fe5c5,positive,slow] +magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_list_sign_delete_clusters[id-82bdca24-d732-41d1-bfc0-983ec8229ca0,positive,slow] +magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_delete_cluster_for_nonexisting_cluster[id-208fabdb-f7b8-4bdd-8762-fdf88be0fbca,negative] +magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_update_cluster_for_nonexisting_cluster[id-71c75c03-090c-4f33-9aad-f2919a3b63d2,negative] +magnum_tempest_plugin.tests.api.v1.test_magnum_service.MagnumServiceTest.test_magnum_service_list_needs_admin[negative] +magnum_tempest_plugin.tests.api.v1.test_magnum_service.MagnumServiceTest.test_magnum_service_list[positive] diff --git a/.automation.conf/tempest/load-lists/manila b/.automation.conf/tempest/load-lists/manila new file mode 100644 index 000000000..9faa74d08 --- /dev/null +++ b/.automation.conf/tempest/load-lists/manila @@ -0,0 +1,1956 @@ +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_force_delete_nonexistent_share_instance[api,id-7cbfc035-12ea-4e2c-8da1-baf261e45f03,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_force_delete_nonexistent_share[api,id-dedca5c1-151d-40f7-bb7f-8913d51c05a9,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_force_delete_nonexistent_snapshot[api,id-e31d2d7b-7202-4699-9423-72f710e72181,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_migration_API_invalid_microversion_1_migrate_share[api,id-59b09ad2-d405-4762-a253-d7b7cf56f0a5,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_migration_API_invalid_microversion_2_migration_complete[api,id-59b09ad2-d405-4762-a253-d7b7cf56f0a5,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_migration_API_invalid_microversion_3_reset_task_state[api,id-59b09ad2-d405-4762-a253-d7b7cf56f0a5,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_migration_API_invalid_microversion_4_migration_get_progress[api,id-59b09ad2-d405-4762-a253-d7b7cf56f0a5,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_migration_API_invalid_microversion_5_migration_cancel[api,id-59b09ad2-d405-4762-a253-d7b7cf56f0a5,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_reset_nonexistent_share_instance_state[api,id-26ce6f02-98eb-435a-9065-2e5bbcac87c5,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_reset_nonexistent_share_state[api,id-17e7eb3c-dbe6-4667-b838-663211365d44,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_reset_nonexistent_snapshot_state[api,id-7e07f684-b68f-4b0e-89cc-05d70a67dd69,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_reset_task_state_share_not_found[api,id-aba8638c-bfed-4c3e-994b-5309fcd912b2,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsAPIOnlyNegativeTest.test_try_list_share_instance_with_member[api,id-1c928920-1538-400a-ab28-c58dd75503c3,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_reset_share_instance_state_to_unacceptable_state[api_with_backend,id-3bfa9555-9c7e-45a2-b5bd-384329cb6fda,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_reset_share_state_to_unacceptable_state[api_with_backend,id-f730c395-a501-44cf-90d9-a3273771b895,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_reset_snapshot_state_to_unacceptable_state[api_with_backend,id-02e0d0d5-ac66-4d24-9aa7-568f75944a05,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_reset_task_state_invalid_state[api_with_backend,id-d662457c-2b84-4f13-aee7-5ffafe2552f1,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_force_delete_share_instance_with_member[api_with_backend,id-257da3e0-9460-4d97-8a56-c86c0427cc64,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_force_delete_share_with_member[api_with_backend,id-7cd0b48e-2815-4f8c-8718-3c071ff9701f,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_force_delete_snapshot_with_member[api_with_backend,id-c9a1894f-d58f-4885-86ba-736e9ab8428a,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_get_instances_of_share_with_member[api_with_backend,id-ab361521-adc9-4fe3-9699-a5ccc49b579b,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_get_share_instance_with_member[api_with_backend,id-821da7c8-3501-44ba-9ffe-45f485a6e573,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_reset_share_instance_state_with_member[api_with_backend,id-d4abddba-1c20-49e1-85b1-5452f0faceb0,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_reset_share_state_with_member[api_with_backend,id-3b525c29-b657-493f-aa41-b17676a95fd2,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions_negative.AdminActionsNegativeTest.test_try_reset_snapshot_state_with_member[api_with_backend,id-48dfb1ec-6db6-4022-8a41-2eb2883e0988,negative] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_ensure_share_server_creation_when_dhss_enabled[api_with_backend,id-4233b941-a909-4f35-9ec9-753736949dd2,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_extend_share_force_1_True[backend,id-83d94560-e9b4-47c1-b21e-400531528e27,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_extend_share_force_2_False[backend,id-83d94560-e9b4-47c1-b21e-400531528e27,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_force_delete_share_instance[api_with_backend,id-382fca90-746e-4ad1-a509-b82a643d4a03,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_force_delete_share[api_with_backend,id-2e8fee75-6b7f-4b69-8f68-0646ce6a96e9,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_force_delete_snapshot[api_with_backend,id-d5a48182-ecd7-463e-a31a-148c81d3c5ed,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_instance_state_1_error[api_with_backend,id-13075b2d-fe83-41bf-b6ef-99cfcc00257d,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_instance_state_2_available[api_with_backend,id-13075b2d-fe83-41bf-b6ef-99cfcc00257d,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_instance_state_3_error_deleting[api_with_backend,id-13075b2d-fe83-41bf-b6ef-99cfcc00257d,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_instance_state_4_deleting[api_with_backend,id-13075b2d-fe83-41bf-b6ef-99cfcc00257d,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_instance_state_5_creating[api_with_backend,id-13075b2d-fe83-41bf-b6ef-99cfcc00257d,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_state_1_error[api_with_backend,id-4f8c6ae9-0656-445f-a911-fbf98fe761d0,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_state_2_available[api_with_backend,id-4f8c6ae9-0656-445f-a911-fbf98fe761d0,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_state_3_error_deleting[api_with_backend,id-4f8c6ae9-0656-445f-a911-fbf98fe761d0,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_state_4_deleting[api_with_backend,id-4f8c6ae9-0656-445f-a911-fbf98fe761d0,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_state_5_creating[api_with_backend,id-4f8c6ae9-0656-445f-a911-fbf98fe761d0,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_share_task_state[api_with_backend,id-49a576eb-733a-4299-aa6f-918fe7c67a6a,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_snapshot_state_1_error[api_with_backend,id-3e16d990-fa19-45e9-893f-e0b7a90127bd,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_snapshot_state_2_available[api_with_backend,id-3e16d990-fa19-45e9-893f-e0b7a90127bd,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_snapshot_state_3_error_deleting[api_with_backend,id-3e16d990-fa19-45e9-893f-e0b7a90127bd,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_snapshot_state_4_deleting[api_with_backend,id-3e16d990-fa19-45e9-893f-e0b7a90127bd,positive] +manila_tempest_tests.tests.api.admin.test_admin_actions.AdminActionsTest.test_reset_snapshot_state_5_creating[api_with_backend,id-3e16d990-fa19-45e9-893f-e0b7a90127bd,positive] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsAPIOnlyNegativeTest.test_get_export_locations_by_nonexistent_share_instance[api,id-21ba5111-91a8-4ec3-86dc-689fc2fa90e6,negative] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsAPIOnlyNegativeTest.test_get_export_locations_by_nonexistent_share[api,id-4b5b4e89-0c80-4383-b272-62d5e0419d9a,negative] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsNegativeTest.test_get_inexistent_share_export_location[api_with_backend,id-8eac1355-f272-4913-8a49-1a8a9cb086bd,negative] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsNegativeTest.test_get_inexistent_share_instance_export_location[api_with_backend,id-064a18dd-1a00-42f1-84c0-5a3e3b46fb39,negative] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsNegativeTest.test_get_share_export_location_by_different_project_user[api_with_backend,id-0f6823a5-3929-4025-9cd4-b5198b4384dd,negative] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsNegativeTest.test_get_share_instance_export_locations_as_member[api_with_backend,id-abde4357-a26c-4adb-88a6-ece6b0e15b5e,negative] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsNegativeTest.test_list_share_export_locations_by_different_project_user[api_with_backend,id-a3c3d16b-5f62-4089-8f86-efc660592986,negative] +manila_tempest_tests.tests.api.admin.test_export_locations_negative.ExportLocationsNegativeTest.test_list_share_instance_export_locations_as_member[api_with_backend,id-6d0b9d1b-fc87-4b7f-add5-919b0ddcda90,negative] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_get_share_export_location_by_member[api_with_backend,id-66cef86f-5da8-4cb4-bc21-91f6c1e27cb5,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_get_share_export_location[api_with_backend,id-814da2ce-2909-4b02-a92e-12bc1b640580,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_get_share_instance_export_location[api_with_backend,id-59421c43-293f-41fd-8ac6-e856deeceac9,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_list_share_export_locations_by_member[api_with_backend,id-397969c6-7fc8-4bf8-86c7-300b96857c54,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_list_share_export_locations_with_preferred_flag[api_with_backend,id-032173d7-3ddf-4730-8524-d1a96a2a9e16,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_list_share_export_locations[api_with_backend,id-dfcb05af-369a-44c9-a06a-67d12a2a0917,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_list_share_instance_export_locations_with_preferred_flag[api_with_backend,id-b93e4cba-ea98-4b1c-90f8-e0a8763033a3,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_list_share_instance_export_locations[api_with_backend,id-06ea2636-1c9f-4889-8b5f-e10c2c2572cb,positive] +manila_tempest_tests.tests.api.admin.test_export_locations.ExportLocationsTest.test_share_contains_all_export_locations_of_all_share_instances[api_with_backend,id-581acd8d-b89d-4684-8310-b910b46acc7a,positive] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_add_access_rule_during_migration[api_with_backend,id-6e83fc25-4e3e-49a7-93e8-db4e6b355a91,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_create_snapshot_during_share_migration[api_with_backend,id-90cf0ae4-4251-4142-bfa8-41f67a9e5b23,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_change_type_no_valid_host[api_with_backend,id-ee57024c-d00e-4def-8eec-cbc62bae327f,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_host_assisted_not_allowed_API_1[api_with_backend,id-0558e9c4-0416-41d2-b28a-803d4b81521a,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_host_assisted_not_allowed_API_2[api_with_backend,id-0558e9c4-0416-41d2-b28a-803d4b81521a,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_host_assisted_not_allowed_API_3[api_with_backend,id-0558e9c4-0416-41d2-b28a-803d4b81521a,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_host_assisted_not_allowed_API_4[api_with_backend,id-0558e9c4-0416-41d2-b28a-803d4b81521a,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_host_invalid[api_with_backend,id-af17204f-ffab-4ba8-8cb6-032e49216f67,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_invalid_share_network[api_with_backend,id-e8f1e491-697a-4941-bf51-4d37f0a93fa5,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_invalid_share_type[api_with_backend,id-be262d44-2ca2-4b9c-be3a-5a6a98ed871b,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_not_available[api_with_backend,id-86b427a7-27c0-4cd5-8f52-9688b339980b,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_not_found[api_with_backend,id-e2bd0cca-c091-4785-a9dc-7f42d2bb95a5,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_opposite_type_share_network_invalid[api_with_backend,id-16c72693-6f9e-4cb4-a166-c60accd3479b,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_same_host_1_True[api_with_backend,id-78670c24-c4ee-45b5-b166-2d053c333144,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_same_host_2_False[api_with_backend,id-78670c24-c4ee-45b5-b166-2d053c333144,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migrate_share_with_snapshot[api_with_backend,id-6276bea6-6939-4569-930f-218d99c0fa56,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migration_cancel_invalid[api_with_backend,id-8aa1f2a0-bc44-4df5-a556-161590e594a3,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migration_cancel_not_found[api_with_backend,id-8ef562b4-7704-4a78-973f-9bf8d2b6f6a6,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migration_complete_invalid[api_with_backend,id-2ab1fc82-bc13-4c99-8324-c6b23530e8a4,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migration_complete_not_found[api_with_backend,id-a509871a-3f3a-4618-bb60-9661732dd371,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migration_get_progress_None[api_with_backend,id-6d0dfb2e-51a0-4cb7-8c69-6135a49c6057,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_migration_get_progress_not_found[api_with_backend,id-044c792b-63e0-42c3-9f44-dc2280e2af08,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_share_resize_during_share_migration_1___extend_share____3__[api_with_backend,id-20121039-bb11-45d8-9972-d2daff7a779c,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_share_resize_during_share_migration_2___shrink_share____1__[api_with_backend,id-20121039-bb11-45d8-9972-d2daff7a779c,negative] +manila_tempest_tests.tests.api.admin.test_migration_negative.MigrationNegativeTest.test_share_type_azs_share_migrate_unsupported_az[api_with_backend,id-1f529b09-e404-4f0e-9423-bb4b117b5522,negative] +manila_tempest_tests.tests.api.admin.test_migration.MigrationCancelNFSTest.test_migration_cancel_1_True[backend,id-d39dfa1b-6e91-4efc-84f1-76f878b51f2a,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationCancelNFSTest.test_migration_cancel_2_False[backend,id-d39dfa1b-6e91-4efc-84f1-76f878b51f2a,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationCancelNFSTest.test_migration_cancel_share_with_snapshot[backend,id-640dce56-2084-488d-8dea-456840ff397e,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationOfShareWithSnapshotNFSTest.test_migrating_share_with_snapshot[backend,id-a95eb701-626a-4175-967b-4880d3716857,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationOppositeDriverModesNFSTest.test_migration_opposite_driver_modes_1_True[backend,id-d8cce50d-e8da-4fbc-8f94-0827bf277b6c,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationOppositeDriverModesNFSTest.test_migration_opposite_driver_modes_2_False[backend,id-d8cce50d-e8da-4fbc-8f94-0827bf277b6c,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationTwoPhaseNFSTest.test_migration_2phase_1_True[backend,id-e6cf0e4d-bdf3-49c1-b6ba-56d1ad6c81d2,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationTwoPhaseNFSTest.test_migration_2phase_2_False[backend,id-e6cf0e4d-bdf3-49c1-b6ba-56d1ad6c81d2,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithDifferentSnapshotSupportNFSTest.test_migrate_share_to_no_snapshot_capability_share_type_1_True[backend,id-59313673-6576-4163-ab96-41bafcdad63a,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithDifferentSnapshotSupportNFSTest.test_migrate_share_to_no_snapshot_capability_share_type_2_False[backend,id-59313673-6576-4163-ab96-41bafcdad63a,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithDifferentSnapshotSupportNFSTest.test_migrate_share_to_snapshot_capability_share_type_1_True[backend,id-a18b3637-2070-4a1b-acd9-c392eb7963b5,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithDifferentSnapshotSupportNFSTest.test_migrate_share_to_snapshot_capability_share_type_2_False[backend,id-a18b3637-2070-4a1b-acd9-c392eb7963b5,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithShareExtendingNFSTest.test_extend_on_migrated_share_1_True[backend,id-0e3d75e0-385a-4f7a-889f-2a3db79db8c2,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithShareExtendingNFSTest.test_extend_on_migrated_share_2_False[backend,id-0e3d75e0-385a-4f7a-889f-2a3db79db8c2,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithShareShrinkingNFSTest.test_shrink_on_migrated_share_1_True[backend,id-58c72e51-d217-48bc-8155-5a010912312e,positive] +manila_tempest_tests.tests.api.admin.test_migration.MigrationWithShareShrinkingNFSTest.test_shrink_on_migrated_share_2_False[backend,id-58c72e51-d217-48bc-8155-5a010912312e,positive] +manila_tempest_tests.tests.api.admin.test_multi_backend.ShareMultiBackendTest.test_share_backend_name_distinction[api_with_backend,id-bfa0c056-0a15-40e1-bdff-f1e10b95736c,positive] +manila_tempest_tests.tests.api.admin.test_multi_backend.ShareMultiBackendTest.test_share_backend_name_reporting[api_with_backend,id-888add9d-4ad0-4d88-a62f-af7ee1d9f9ab,positive] +manila_tempest_tests.tests.api.admin.test_multi_backend.ShareMultiBackendTest.test_share_share_type_v_2_6[api_with_backend,id-f25e0cb0-d656-4f16-a761-ec23992cd9e7,positive] +manila_tempest_tests.tests.api.admin.test_multi_backend.ShareMultiBackendTest.test_share_share_type[api_with_backend,id-691fbcef-6d8d-4ad9-b493-501bbb3dcf3c,positive] +manila_tempest_tests.tests.api.admin.test_quotas_negative.ReplicaQuotasNegativeTest.test_create_replica_over_replica_limit_1___share_replicas___2_[api_with_backend,id-ad19eaa5-bacd-4bc5-9592-622b0c856cdd,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.ReplicaQuotasNegativeTest.test_create_replica_over_replica_limit_2___replica_gigabytes___None_[api_with_backend,id-ad19eaa5-bacd-4bc5-9592-622b0c856cdd,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.ReplicaQuotasNegativeTest.test_extend_replica_over_limit[api_with_backend,id-0006f1ff-69c0-40b7-8437-55cc0a08a195,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_create_share_group_with_exceeding_quota_limit[api,id-37dd40a8-375e-454b-8b80-229cb0eecb01,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_create_share_over_quota_limit[api_with_backend,id-6396daab-ba73-4140-8a0a-8eef0a01804d,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_create_share_with_size_bigger_than_quota[api,id-75d39eda-a2b5-4271-a61d-9e2c86370b3e,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_create_snapshot_over_quota_limit[api_with_backend,id-a2267f4d-63ef-4631-a01d-3723707e5516,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_get_quotas_with_empty_tenant_id[api,id-c7174059-7172-4cc8-9121-aefe509ef14c,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_quotas_providing_share_type_and_user_id_1_show[api,id-719768d1-d313-40e9-9127-c5777840ecbd,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_quotas_providing_share_type_and_user_id_2_reset[api,id-719768d1-d313-40e9-9127-c5777840ecbd,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_quotas_providing_share_type_and_user_id_3_update[api,id-719768d1-d313-40e9-9127-c5777840ecbd,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_reset_quotas_with_empty_tenant_id[api,id-0fbcbfad-fdb5-42d6-b005-041dc4ddea64,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_group_quotas_using_too_old_microversion_1_share_groups[api,id-1b504c74-2ce9-40f6-87fb-9e643b1b5906,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_group_quotas_using_too_old_microversion_2_share_group_snapshots[api,id-1b504c74-2ce9-40f6-87fb-9e643b1b5906,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_replica_quotas_using_too_old_microversion_1_share_replicas[api,id-66f22d42-37bc-4f9b-8e0b-a679341e1e88,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_replica_quotas_using_too_old_microversion_2_replica_gigabytes[api,id-66f22d42-37bc-4f9b-8e0b-a679341e1e88,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_type_quotas_using_nonexistent_share_type_1_show[api,id-cf45eb7d-7330-4b2d-8214-e4149eb4a398,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_type_quotas_using_nonexistent_share_type_2_reset[api,id-cf45eb7d-7330-4b2d-8214-e4149eb4a398,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_type_quotas_using_nonexistent_share_type_3_update[api,id-cf45eb7d-7330-4b2d-8214-e4149eb4a398,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_type_quotas_using_too_old_microversion_1_show[api,id-acc609c2-f314-4540-984c-33e93d048f6c,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_type_quotas_using_too_old_microversion_2_reset[api,id-acc609c2-f314-4540-984c-33e93d048f6c,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_share_type_quotas_using_too_old_microversion_3_update[api,id-acc609c2-f314-4540-984c-33e93d048f6c,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quota_detail_with_wrong_versions[api,id-50af5d8c-0b30-4e8d-93bf-bb975db58516,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_01___quota_sets____2_0____show_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_02___quota_sets____2_0____default_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_03___quota_sets____2_0____reset_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_04___quota_sets____2_0____update_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_05___quota_sets____2_6____show_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_06___quota_sets____2_6____default_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_07___quota_sets____2_6____reset_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_08___quota_sets____2_6____update_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_09___os_quota_sets____2_7____show_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_10___os_quota_sets____2_7____default_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_11___os_quota_sets____2_7____reset_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_show_quotas_with_wrong_versions_12___os_quota_sets____2_7____update_quotas__[api,id-ed38ab0a-694c-48ea-bce5-5c264f485d5b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_set_user_quota_gigabytes_bigger_than_tenant_quota[api,id-6187050f-f262-48cf-bdd6-32982f860fba,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_set_user_quota_replicas_bigger_than_tenant_quota_1_share_replicas[api,id-40dabf7d-98da-48a9-bb62-285098d5acb4,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_set_user_quota_replicas_bigger_than_tenant_quota_2_replica_gigabytes[api,id-40dabf7d-98da-48a9-bb62-285098d5acb4,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_set_user_quota_share_networks_bigger_than_tenant_quota[api,id-a272a24b-21a6-4f6c-917c-cbe103203a11,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_set_user_quota_shares_bigger_than_tenant_quota[api,id-e039535c-dc4e-497a-ac09-30b3395ba95b,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_set_user_quota_snap_gigabytes_bigger_than_tenant_quota[api,id-8eb4aed8-c239-49d3-aee1-d63d043aa3d1,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_set_user_quota_snaps_bigger_than_tenant_quota[api,id-57495588-d645-402e-9b89-0ab63a58ce3e,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_update_share_type_quota_for_share_groups_1_share_groups[api,id-5eb6ce15-1172-4bcb-9c7b-91543bf714e8,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_update_share_type_quota_for_share_groups_2_share_group_snapshots[api,id-5eb6ce15-1172-4bcb-9c7b-91543bf714e8,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_update_share_type_quota_for_share_networks_1_id[api,id-2ba641a1-100b-417e-80e2-d3f717fd3c7c,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_try_update_share_type_quota_for_share_networks_2_name[api,id-2ba641a1-100b-417e-80e2-d3f717fd3c7c,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_quota_with_wrong_data_1[api,id-07d3e69a-7cda-4ca7-9fea-c32f6830fdd3,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_quota_with_wrong_data_2[api,id-07d3e69a-7cda-4ca7-9fea-c32f6830fdd3,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_quota_with_wrong_data_3[api,id-07d3e69a-7cda-4ca7-9fea-c32f6830fdd3,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_quota_with_wrong_data_4[api,id-07d3e69a-7cda-4ca7-9fea-c32f6830fdd3,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_quota_with_wrong_data_5[api,id-07d3e69a-7cda-4ca7-9fea-c32f6830fdd3,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_replica_quotas_wrong_data_1[api,id-d070ccc6-6685-4f49-a8e5-9b891790881e,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_replica_quotas_wrong_data_2[api,id-d070ccc6-6685-4f49-a8e5-9b891790881e,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_sg_quota_with_wrong_data_1[api,id-b11c5d5f-2b58-4a5f-8469-b22ea51709c0,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_sg_quota_with_wrong_data_2[api,id-b11c5d5f-2b58-4a5f-8469-b22ea51709c0,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_share_type_quotas_bigger_than_project_quota_1_11[api,id-82256511-aa46-4b99-a6e5-8b400534e96d,negative] +manila_tempest_tests.tests.api.admin.test_quotas_negative.SharesAdminQuotasNegativeTest.test_update_share_type_quotas_bigger_than_project_quota_2__1[api,id-82256511-aa46-4b99-a6e5-8b400534e96d,negative] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_default_quotas[api,id-f62c48e3-9736-4f0c-9f9b-f139f393ac0a,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_show_quotas_for_user[api,id-9b96dd45-7c0d-41ee-88e4-600185f61358,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_show_quotas[api,id-1ff57cfa-cd8d-495f-86eb-9fead307428e,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_show_replica_quotas_for_user_using_too_old_microversion[api,id-19fe431b-e83e-4c4e-acb8-018d7a470c8b,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_show_sg_quotas_for_user_using_too_old_microversion[api,id-b8bcbc04-68fb-4c8f-9f4c-a3b6c6b8911c,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_show_sg_quotas_using_too_old_microversion[api,id-2e98a13e-b2ed-4977-bafe-47ea48b504f2,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_show_share_type_quotas_1___id___True_[api,id-836e1725-2853-4d54-b281-8173773d8527,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasTest.test_show_share_type_quotas_2___name___False_[api,id-836e1725-2853-4d54-b281-8173773d8527,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_quotas_usages[api_with_backend,id-4687eb25-17b3-4995-ace2-62f8bda29c57,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_reset_share_type_quotas_1___id___True_[api,id-15e57302-5a14-4be4-8720-95b639c2bfad,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_reset_share_type_quotas_2___name___False_[api,id-15e57302-5a14-4be4-8720-95b639c2bfad,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_reset_tenant_quotas[api,id-84e24c32-ee78-461e-ac1f-f9e4d99f88e2,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_set_share_type_quota_bigger_than_users_quota[api,id-c95be1eb-6331-4c37-9fac-ed6c36270457,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_share_group_quotas_usages[api_with_backend,id-fe357398-12d4-4a63-b5ae-0d5091ba3442,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_gigabytes[api,id-b6a94e87-091a-48dc-9b51-13d81541869c,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_replica_quotas_1_share_replicas[api,id-15aa5df5-b2ae-4a3a-acb8-efbbc84581be,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_replica_quotas_2_replica_gigabytes[api,id-15aa5df5-b2ae-4a3a-acb8-efbbc84581be,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_share_groups[api,id-7c2cd2d4-4352-4811-9e39-70f56e6297c2,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_share_networks[api,id-e35455f5-92db-4669-ac21-9daf170df248,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_shares[api,id-d4bba375-7111-4b93-b6dd-4f0532febc3e,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_snapshot_gigabytes[api,id-0d044db4-ae5b-416d-aa51-098afb72cd6c,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_quota_for_snapshots[api,id-9779d166-09d3-4745-8acc-2243eadec3ea,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_gigabytes[api,id-2de4b7cf-9189-413f-858b-860ecf5fd18b,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_replica_quotas_1_share_replicas[api,id-84b99731-f748-44fe-a291-162d05da9e25,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_replica_quotas_2_replica_gigabytes[api,id-84b99731-f748-44fe-a291-162d05da9e25,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_share_group_snapshots[api,id-76d270d5-f314-47cb-9c3f-409f8ff12ce2,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_share_networks[api,id-66b04887-e611-4d4f-a40b-c8b14766b6af,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_shares[api,id-756ffd0e-a476-49af-ac85-9bb4ce5e29b7,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_snapshot_gigabytes[api,id-cbe63027-1108-4779-9fd3-22f41f60d6bb,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_unlimited_user_quota_for_snapshots[api,id-245b3bf3-09ef-4b6d-8643-f156bf1bf23c,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_share_type_quota_1___id___True_[api,id-155ea3de-b3b5-4aa0-be8b-eebcc19ce874,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_share_type_quota_2___name___False_[api,id-155ea3de-b3b5-4aa0-be8b-eebcc19ce874,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_share_type_quotas_bigger_than_project_quota_1_11[api,id-315cb76f-920d-4cb9-ac7d-16be8e95e1b2,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_share_type_quotas_bigger_than_project_quota_2__1[api,id-315cb76f-920d-4cb9-ac7d-16be8e95e1b2,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_tenant_quota_gigabytes[api,id-37ee5bd2-db07-4817-b71a-7c3e78634399,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_tenant_quota_share_groups_1_share_groups[api,id-cb09de7e-94e9-401a-b82b-8b2de210f8b9,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_tenant_quota_share_groups_2_share_group_snapshots[api,id-cb09de7e-94e9-401a-b82b-8b2de210f8b9,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_tenant_quota_share_networks[api,id-da7f3179-f2f3-402e-82c2-e6855774a99a,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_tenant_quota_shares[api,id-da16e906-e8e6-4aa0-9fc1-76ed48cfd428,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_tenant_quota_snapshot_gigabytes[api,id-284a2e95-48a1-4f1b-b952-f734b1b6238a,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_tenant_quota_snapshots[api,id-78957d97-afad-4371-a21e-79641fff83f6,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_gigabytes[api,id-75977d53-f06b-41a2-8365-0ce549e4a51a,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_replica_related_1___share_replicas___False_[api,id-af16dc89-c93d-43de-8902-2c88c75f107f,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_replica_related_2___share_replicas___True_[api,id-af16dc89-c93d-43de-8902-2c88c75f107f,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_replica_related_3___replica_gigabytes___False_[api,id-af16dc89-c93d-43de-8902-2c88c75f107f,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_replica_related_4___replica_gigabytes___True_[api,id-af16dc89-c93d-43de-8902-2c88c75f107f,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_share_groups_1_share_groups[api,id-c32a716b-f971-4855-97ea-f30d4423d03d,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_share_groups_2_share_group_snapshots[api,id-c32a716b-f971-4855-97ea-f30d4423d03d,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_share_networks[api,id-f75f01a0-5921-44ab-b373-bb9e070f87eb,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_shares[api,id-2c7f9e19-268d-4420-a046-a7faf21174a1,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_snapshot_gigabytes[api,id-00a189fc-93ed-44c2-b9dc-1d9b6c26d005,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quota_snapshots[api,id-53f4fd79-39aa-42be-82ce-e423ebffe837,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quotas_bigger_than_project_quota_1_11[api,id-43f58705-3cad-46bc-816c-41e8fa55dd8d,positive] +manila_tempest_tests.tests.api.admin.test_quotas.SharesAdminQuotasUpdateTest.test_update_user_quotas_bigger_than_project_quota_2__1[api,id-43f58705-3cad-46bc-816c-41e8fa55dd8d,positive] +manila_tempest_tests.tests.api.admin.test_replication_actions.ReplicationActionsAdminTest.test_extend_replicated_share[api_with_backend,id-b39f319e-2515-42c0-85c4-21c2fb2123bf,positive] +manila_tempest_tests.tests.api.admin.test_replication_actions.ReplicationActionsAdminTest.test_manage_replicated_share_snapshot[api_with_backend,id-93220873-c6c4-40f7-840d-d0ff02e7cd7e,negative] +manila_tempest_tests.tests.api.admin.test_replication_actions.ReplicationActionsAdminTest.test_manage_share_for_replication_type[backend,id-84150cd6-2777-4806-8aa3-51359f16816e,positive] +manila_tempest_tests.tests.api.admin.test_replication_actions.ReplicationActionsAdminTest.test_shrink_replicated_share[api_with_backend,id-743bfb8e-a314-4e8e-92b5-079bd3eae72d,positive] +manila_tempest_tests.tests.api.admin.test_replication_actions.ReplicationActionsAdminTest.test_unmanage_replicated_share_snapshot[api_with_backend,id-0b0cd350-8691-477b-adb1-5e79b92e3759,negative] +manila_tempest_tests.tests.api.admin.test_replication_actions.ReplicationActionsAdminTest.test_unmanage_replicated_share_with_no_replica[backend,id-796fb2a8-1ac3-4eee-b12e-da511eb52e87,positive] +manila_tempest_tests.tests.api.admin.test_replication_actions.ReplicationActionsAdminTest.test_unmanage_replicated_share_with_replica[api_with_backend,id-cbbe2650-47bb-456b-8b41-74c66270ea97,negative] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_force_delete_share_replica_1_2_11[backend,id-22a199b7-f4f6-4ede-b09f-8047a9d01cad,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_force_delete_share_replica_2_2_56[backend,id-22a199b7-f4f6-4ede-b09f-8047a9d01cad,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_force_delete_share_replica_3_2_88[backend,id-22a199b7-f4f6-4ede-b09f-8047a9d01cad,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_promote_out_of_sync_share_replica_1_2_11[backend,id-0213cdfd-6a0f-4f24-a154-69796888a64a,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_promote_out_of_sync_share_replica_2_2_56[backend,id-0213cdfd-6a0f-4f24-a154-69796888a64a,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_promote_out_of_sync_share_replica_3_2_88[backend,id-0213cdfd-6a0f-4f24-a154-69796888a64a,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_reset_share_replica_state_1_2_11[backend,id-258844da-a853-42b6-87db-b16e616018c6,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_reset_share_replica_state_2_2_56[backend,id-258844da-a853-42b6-87db-b16e616018c6,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_reset_share_replica_state_3_2_88[backend,id-258844da-a853-42b6-87db-b16e616018c6,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_reset_share_replica_status_1_2_11[backend,id-16bd90f0-c478-4a99-8633-b18703ff56fa,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_reset_share_replica_status_2_2_56[backend,id-16bd90f0-c478-4a99-8633-b18703ff56fa,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_reset_share_replica_status_3_2_88[backend,id-16bd90f0-c478-4a99-8633-b18703ff56fa,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_resync_share_replica_1_2_11[backend,id-2969565a-85e8-4c61-9dfb-cc7f7ca9f6dd,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_resync_share_replica_2_2_56[backend,id-2969565a-85e8-4c61-9dfb-cc7f7ca9f6dd,positive] +manila_tempest_tests.tests.api.admin.test_replication.ReplicationAdminTest.test_resync_share_replica_3_2_88[backend,id-2969565a-85e8-4c61-9dfb-cc7f7ca9f6dd,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_hints.SharesSchedulerHintsAdminTest.test_only_host_scheduler_hint_in_share_creation[api_with_backend,id-54f4dea7-890e-443b-aea5-f6108da893f0,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_hints.SharesSchedulerHintsAdminTest.test_only_host_scheduler_hint_in_share_replica_creation[api_with_backend,id-1dec3306-61f4-41b9-ba4a-572a9e6f5f57,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_detail_with_filters_negative[api,id-56f04711-7b57-4d2a-a4f3-96c82845208e,negative] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_detail_with_filters[api,id-071a1e47-959e-47df-ad19-335064bb25ff,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_detail[api,id-b6fdd156-3923-495a-ab01-f94bfb5d7040,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_filters_negative[api,id-1c2f1b0f-c509-4676-8335-bb6b10debea2,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_filters[api,id-8f4582c4-df82-4b64-be79-a8c6fdc58eac,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_1__True___name__[api_with_backend,id-f027fb62-1ec3-4f52-a782-e9dd9db34fda,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_2__True___id__[api_with_backend,id-f027fb62-1ec3-4f52-a782-e9dd9db34fda,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_3__False___name__[api_with_backend,id-f027fb62-1ec3-4f52-a782-e9dd9db34fda,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_4__False___id__[api_with_backend,id-f027fb62-1ec3-4f52-a782-e9dd9db34fda,positive] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_negative_1__True___name__[api_with_backend,id-31cb2c99-3abf-4dce-8e66-7fd30b168300,negative] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_negative_2__True___id__[api_with_backend,id-31cb2c99-3abf-4dce-8e66-7fd30b168300,negative] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_negative_3__False___name__[api_with_backend,id-31cb2c99-3abf-4dce-8e66-7fd30b168300,negative] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list_with_share_type_filter_with_detail_negative_4__False___id__[api_with_backend,id-31cb2c99-3abf-4dce-8e66-7fd30b168300,negative] +manila_tempest_tests.tests.api.admin.test_scheduler_stats.SchedulerStatsAdminTest.test_pool_list[api,id-dab6d654-7d1c-4bc0-8017-1c4c072b49a1,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_all_tenants[api,id-6f843a0a-21da-4a83-ae3b-3831f99b3678,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_detailed_filter_by_ss_attributes[api,id-f055faad-dd36-4eed-9b50-61280931dea2,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_filter_by_share_network[api,id-88f62835-0aee-4bed-a37f-ffd99430da8a,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_invalid_filters[api,id-fe515fa7-0275-4190-adc7-5321e1047dad,negative] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_with_detail_1_1_0[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_with_detail_2_2_42[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_with_detail_3_2_44[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services_with_detail_4_2_88[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.admin.test_security_services.SecurityServiceAdminTest.test_list_security_services[api,id-f6f5657c-a93c-49ed-86e3-b351a92734d5,positive] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_get_service_by_invalid_binary[api,id-766461b0-e89a-4113-8229-24c4d11d585a,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_get_service_by_invalid_host[api,id-386dc131-39f9-4d5f-af87-edf76e3d0e71,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_get_service_by_invalid_params[api,id-04f59090-069d-4cae-b028-e004ff29126c,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_get_service_by_invalid_state[api,id-41936575-3a96-455b-8069-7f6563abf0e2,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_get_service_by_invalid_status[api,id-da0fef1d-c4d3-4c33-a836-5f836e85df69,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_get_service_by_invalid_zone[api,id-ac570fde-690d-4448-9cce-ce35e0a14b88,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_list_services_with_non_admin_user[api,id-59ab1a7d-817d-42e5-a1b9-adfc2912e08c,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_list_services_with_wrong_versions_1___os_services____2_7__[api,id-3c72227b-7fa1-4294-bdf4-413ec4c324e5,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_list_services_with_wrong_versions_2___services____2_6__[api,id-3c72227b-7fa1-4294-bdf4-413ec4c324e5,negative] +manila_tempest_tests.tests.api.admin.test_services_negative.ServicesAdminNegativeTest.test_list_services_with_wrong_versions_3___services____2_0__[api,id-3c72227b-7fa1-4294-bdf4-413ec4c324e5,negative] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_all_filters_1_shares_client[api,id-b77c4bc4-57d5-4181-9e95-e230ab682b32,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_all_filters_2_shares_v2_client[api,id-b77c4bc4-57d5-4181-9e95-e230ab682b32,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_availability_zone_1_shares_client[api,id-d12ea678-025f-46b4-95c5-3a03b3e440d7,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_availability_zone_2_shares_v2_client[api,id-d12ea678-025f-46b4-95c5-3a03b3e440d7,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_binary_name_1_shares_client[api,id-1dd4d799-b900-4476-9e51-ad9db6ee4435,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_binary_name_2_shares_v2_client[api,id-1dd4d799-b900-4476-9e51-ad9db6ee4435,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_host_name_1_shares_client[api,id-e80f4b89-7280-47eb-9cbf-01a09e04c2d8,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_host_name_2_shares_v2_client[api,id-e80f4b89-7280-47eb-9cbf-01a09e04c2d8,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_state_1_shares_client[api,id-c30234f0-1331-4560-93a7-cbda7d00eb53,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_state_2_shares_v2_client[api,id-c30234f0-1331-4560-93a7-cbda7d00eb53,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_status_1_shares_client[api,id-e82921a7-6c98-4c9c-a47b-34a0badc3b59,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_get_services_by_status_2_shares_v2_client[api,id-e82921a7-6c98-4c9c-a47b-34a0badc3b59,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_list_services_1_shares_client[api,id-74cd12ab-a1f5-40fb-9110-d9035b4b20c5,positive] +manila_tempest_tests.tests.api.admin.test_services.ServicesAdminTest.test_list_services_2_shares_v2_client[api,id-74cd12ab-a1f5-40fb-9110-d9035b4b20c5,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_add_project_access_to_nonexistent_share_group_type[api,id-68e3d7f9-fdc0-44a2-ae84-a72eba314e19,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_add_project_access_to_public_share_group_type[api,id-6c30989d-2e8a-48d3-809a-556dddae51bd,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_create_share_group_type_with_empty_name[api,id-ce6ba41b-4207-4866-8f70-7a89f64b4cd4,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_create_share_group_type_with_nonexistent_share_type[api,id-6c95f5b7-6e30-4e1e-b0fc-a0f05cf2982d,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_create_share_group_type_with_too_big_name[api,id-77159ec6-adb7-4d35-9d4b-039d322f7852,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_create_share_group_type_with_wrong_value_for_group_specs[api,id-43ac388d-05e8-45d5-b633-631cc1f290af,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_create_share_group_type_without_name[api,id-1f8e3f98-4df7-4383-94d6-4ad058ef79c1,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_get_share_group_type_using_nonexistent_id[api,id-8fb8bd73-0219-460d-993e-bff7ddec29e8,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_remove_project_access_from_nonexistent_share_group_type[api,id-94f0c902-ab12-435e-abd5-e09a892db82a,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_remove_project_access_from_public_share_group_type[api,id-3375a1f8-7828-4cbe-89d7-fe11fcf30b21,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_try_create_duplicate_of_share_group_type[api,id-081183cb-0f1c-4b67-8552-909935dc0be2,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_try_delete_share_group_type_using_nonexistent_id[api,id-76b5c302-f492-45b4-b464-7be6efd373be,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_try_get_extra_spec_with_nonexistent_key[api,id-2be79455-0ce7-4ca6-818f-40651ba79c6e,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_try_get_extra_specs_from_nonexistent_share_group_type[api,id-2264f7eb-3ff0-47e9-8ab0-54694113db3d,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types_negative.ShareGroupTypesAdminNegativeTest.test_try_get_share_group_type_extra_specs_with_user[api,id-3e763f5b-6663-4620-9471-ed3050da6201,negative] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_get_delete_share_group_type_1___id____2_88__[api,id-e2ba1754-cecc-4178-ad39-eefbb59e4d6d,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_get_delete_share_group_type_2___id____2_31__[api,id-e2ba1754-cecc-4178-ad39-eefbb59e4d6d,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_get_delete_share_group_type_3___id____2_55__[api,id-e2ba1754-cecc-4178-ad39-eefbb59e4d6d,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_get_delete_share_group_type_4___name____2_88__[api,id-e2ba1754-cecc-4178-ad39-eefbb59e4d6d,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_get_delete_share_group_type_5___name____2_31__[api,id-e2ba1754-cecc-4178-ad39-eefbb59e4d6d,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_get_delete_share_group_type_6___name____2_55__[api,id-e2ba1754-cecc-4178-ad39-eefbb59e4d6d,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_share_group_type_multiple_share_types_min_1_id[api,id-65cf04d6-0620-4e0d-96f2-456946e8a552,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_share_group_type_multiple_share_types_min_2_name[api,id-65cf04d6-0620-4e0d-96f2-456946e8a552,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_share_group_type_with_multiple_specs_min[api,id-ec459ebb-5c45-4d4f-95f0-3bbf2974366c,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_create_share_group_type_with_one_spec_min[api,id-6e103b7b-cd01-4edb-91e7-a16681781b36,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_delete_single_share_group_type_spec_min_1_2_31[api,id-efddee69-ca23-4681-8247-94ded81c4c3a,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_delete_single_share_group_type_spec_min_2_2_55[api,id-efddee69-ca23-4681-8247-94ded81c4c3a,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_delete_single_share_group_type_spec_min_3_2_88[api,id-efddee69-ca23-4681-8247-94ded81c4c3a,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_get_all_share_group_type_extra_specs[api,id-eef69171-9757-423c-8cd4-487cbd84ca24,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_get_one_share_group_type_extra_spec[api,id-dd620bfd-197b-4675-ace6-e26f809bb26e,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_private_share_group_type_access_1_2_31[api,id-c2d34b42-e3ec-404e-8b7a-0fe9b1560507,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_private_share_group_type_access_2_2_55[api,id-c2d34b42-e3ec-404e-8b7a-0fe9b1560507,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_private_share_group_type_access_3_2_88[api,id-c2d34b42-e3ec-404e-8b7a-0fe9b1560507,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_share_group_type_create_show_list_with_is_default_key_1_2_45[api,id-b8b20a96-cecc-4677-8a77-aae3b93e5b96,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_share_group_type_create_show_list_with_is_default_key_2_2_46[api,id-b8b20a96-cecc-4677-8a77-aae3b93e5b96,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_share_group_type_create_show_list_with_is_default_key_3_2_88[api,id-b8b20a96-cecc-4677-8a77-aae3b93e5b96,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_update_all_share_group_type_specs_min[api,id-3ed054c5-d0ab-4a73-b704-994eb7f9c46c,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_update_single_share_group_type_spec_1_2_31[api,id-15b44580-a34d-4e0d-a77b-0e76b45d6199,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_update_single_share_group_type_spec_2_2_55[api,id-15b44580-a34d-4e0d-a77b-0e76b45d6199,positive] +manila_tempest_tests.tests.api.admin.test_share_group_types.ShareGroupTypesTest.test_update_single_share_group_type_spec_3_2_88[api,id-15b44580-a34d-4e0d-a77b-0e76b45d6199,positive] +manila_tempest_tests.tests.api.admin.test_share_groups_negative.ShareGroupsNegativeTest.test_create_share_group_with_wrong_consistent_snapshot_spec[api_with_backend,id-b90537b7-634d-4fca-b451-770fbcca7927,negative] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_create_sg_from_snapshot_verify_share_server_information_min[api_with_backend,id-8ca1f0a0-2a36-4adb-af6b-6741b00307c5,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_create_sg_with_sg_type_but_without_any_group_specs[api_with_backend,id-93fd4a97-a25a-4a17-b5ae-c8894c1adfc5,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_create_share_group_with_multiple_share_types_min[api_with_backend,id-ff6c17af-03ba-4506-923d-b6c229492d0e,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_create_share_group_with_single_share_type_min_1_2_31[api_with_backend,id-79eaa86f-4c8f-49fd-acb2-ec051aa6bf93,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_create_share_group_with_single_share_type_min_2_2_55[api_with_backend,id-79eaa86f-4c8f-49fd-acb2-ec051aa6bf93,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_create_share_group_with_single_share_type_min_3_2_88[api_with_backend,id-79eaa86f-4c8f-49fd-acb2-ec051aa6bf93,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_default_share_group_type_applied_1_2_31[api_with_backend,id-99f0471c-e978-42ac-b50b-848b16692eab,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_default_share_group_type_applied_2_2_55[api_with_backend,id-99f0471c-e978-42ac-b50b-848b16692eab,positive] +manila_tempest_tests.tests.api.admin.test_share_groups.ShareGroupsTest.test_default_share_group_type_applied_3_2_88[api_with_backend,id-99f0471c-e978-42ac-b50b-848b16692eab,positive] +manila_tempest_tests.tests.api.admin.test_share_instances_negative.ShareInstancesNegativeTest.test_list_share_instances_with_export_location_and_invalid_version_1_path[api_with_backend,id-babe885e-a8ab-439d-8b95-e5422983a942,negative] +manila_tempest_tests.tests.api.admin.test_share_instances_negative.ShareInstancesNegativeTest.test_list_share_instances_with_export_location_and_invalid_version_2_id[api_with_backend,id-babe885e-a8ab-439d-8b95-e5422983a942,negative] +manila_tempest_tests.tests.api.admin.test_share_instances_negative.ShareInstancesNegativeTest.test_list_share_instances_with_export_location_not_exist_1_path[api_with_backend,id-ce0d045c-e418-42fa-86e4-ead493fc0663,negative] +manila_tempest_tests.tests.api.admin.test_share_instances_negative.ShareInstancesNegativeTest.test_list_share_instances_with_export_location_not_exist_2_id[api_with_backend,id-ce0d045c-e418-42fa-86e4-ead493fc0663,negative] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_get_instances_of_share_v2_3[api_with_backend,id-b5211759-17ab-4806-a58c-c65bc8a06087,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_get_share_instance_1_2_3[api_with_backend,id-d4b2544a-96ce-41ed-8b8e-e46ceef91462,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_get_share_instance_2_2_9[api_with_backend,id-d4b2544a-96ce-41ed-8b8e-e46ceef91462,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_get_share_instance_3_2_10[api_with_backend,id-d4b2544a-96ce-41ed-8b8e-e46ceef91462,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_get_share_instance_4_2_30[api_with_backend,id-d4b2544a-96ce-41ed-8b8e-e46ceef91462,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_get_share_instance_5_2_54[api_with_backend,id-d4b2544a-96ce-41ed-8b8e-e46ceef91462,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_list_share_instances_v2_3[api_with_backend,id-b2d8ccba-5973-4e72-b3df-52555ce40b4f,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_list_share_instances_with_export_location_path_and_id_1_path[api_with_backend,id-c27b415d-341c-42f0-a269-2c94f69fbee1,positive] +manila_tempest_tests.tests.api.admin.test_share_instances.ShareInstancesTest.test_list_share_instances_with_export_location_path_and_id_2_id[api_with_backend,id-c27b415d-341c-42f0-a269-2c94f69fbee1,positive] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_delete_share_in_manage_error[api_with_backend,id-162b6b96-e801-40f4-b111-7e533cf8b943,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_manage_invalid_param_manage_error[api_with_backend,id-01b36343-c503-4c14-bb74-71f131ae96b6,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_manage_invalid_param_raises_exception[api_with_backend,id-8267161e-4f55-44eb-9af5-30d1a3fb2606,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_manage_share_duplicate[api_with_backend,id-832b1c4d-e6f2-4a75-9ea5-d99f780db743,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_manage_share_without_share_server_id[api_with_backend,id-c2f28981-5758-46e8-9d90-5c3269461d94,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_unmanage_share_transitional_state[api_with_backend,id-f71bcb3c-836e-4e9e-972a-50debeb8c285,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_unmanage_share_with_server_unsupported[api_with_backend,id-916ca8e2-5635-4cc9-8ab6-36b7b53ecd0a,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCephFSShareNegativeTest.test_unmanage_share_with_snapshot[api_with_backend,id-7cad4763-8f78-46a5-aeeb-6ca174284d3d,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_delete_share_in_manage_error[api_with_backend,id-162b6b96-e801-40f4-b111-7e533cf8b943,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_manage_invalid_param_manage_error[api_with_backend,id-01b36343-c503-4c14-bb74-71f131ae96b6,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_manage_invalid_param_raises_exception[api_with_backend,id-8267161e-4f55-44eb-9af5-30d1a3fb2606,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_manage_share_duplicate[api_with_backend,id-832b1c4d-e6f2-4a75-9ea5-d99f780db743,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_manage_share_without_share_server_id[api_with_backend,id-c2f28981-5758-46e8-9d90-5c3269461d94,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_unmanage_share_transitional_state[api_with_backend,id-f71bcb3c-836e-4e9e-972a-50debeb8c285,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_unmanage_share_with_server_unsupported[api_with_backend,id-916ca8e2-5635-4cc9-8ab6-36b7b53ecd0a,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageCIFSShareNegativeTest.test_unmanage_share_with_snapshot[api_with_backend,id-7cad4763-8f78-46a5-aeeb-6ca174284d3d,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_delete_share_in_manage_error[api_with_backend,id-162b6b96-e801-40f4-b111-7e533cf8b943,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_manage_invalid_param_manage_error[api_with_backend,id-01b36343-c503-4c14-bb74-71f131ae96b6,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_manage_invalid_param_raises_exception[api_with_backend,id-8267161e-4f55-44eb-9af5-30d1a3fb2606,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_manage_share_duplicate[api_with_backend,id-832b1c4d-e6f2-4a75-9ea5-d99f780db743,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_manage_share_without_share_server_id[api_with_backend,id-c2f28981-5758-46e8-9d90-5c3269461d94,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_unmanage_share_transitional_state[api_with_backend,id-f71bcb3c-836e-4e9e-972a-50debeb8c285,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_unmanage_share_with_server_unsupported[api_with_backend,id-916ca8e2-5635-4cc9-8ab6-36b7b53ecd0a,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageGLUSTERFSShareNegativeTest.test_unmanage_share_with_snapshot[api_with_backend,id-7cad4763-8f78-46a5-aeeb-6ca174284d3d,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_delete_share_in_manage_error[api_with_backend,id-162b6b96-e801-40f4-b111-7e533cf8b943,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_manage_invalid_param_manage_error[api_with_backend,id-01b36343-c503-4c14-bb74-71f131ae96b6,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_manage_invalid_param_raises_exception[api_with_backend,id-8267161e-4f55-44eb-9af5-30d1a3fb2606,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_manage_share_duplicate[api_with_backend,id-832b1c4d-e6f2-4a75-9ea5-d99f780db743,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_manage_share_without_share_server_id[api_with_backend,id-c2f28981-5758-46e8-9d90-5c3269461d94,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_unmanage_share_transitional_state[api_with_backend,id-f71bcb3c-836e-4e9e-972a-50debeb8c285,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_unmanage_share_with_server_unsupported[api_with_backend,id-916ca8e2-5635-4cc9-8ab6-36b7b53ecd0a,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageHDFSShareNegativeTest.test_unmanage_share_with_snapshot[api_with_backend,id-7cad4763-8f78-46a5-aeeb-6ca174284d3d,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_delete_share_in_manage_error[api_with_backend,id-162b6b96-e801-40f4-b111-7e533cf8b943,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_manage_invalid_param_manage_error[api_with_backend,id-01b36343-c503-4c14-bb74-71f131ae96b6,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_manage_invalid_param_raises_exception[api_with_backend,id-8267161e-4f55-44eb-9af5-30d1a3fb2606,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_manage_share_duplicate[api_with_backend,id-832b1c4d-e6f2-4a75-9ea5-d99f780db743,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_manage_share_without_share_server_id[api_with_backend,id-c2f28981-5758-46e8-9d90-5c3269461d94,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_unmanage_share_transitional_state[api_with_backend,id-f71bcb3c-836e-4e9e-972a-50debeb8c285,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_unmanage_share_with_server_unsupported[api_with_backend,id-916ca8e2-5635-4cc9-8ab6-36b7b53ecd0a,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageMapRFSShareNegativeTest.test_unmanage_share_with_snapshot[api_with_backend,id-7cad4763-8f78-46a5-aeeb-6ca174284d3d,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_delete_share_in_manage_error[api_with_backend,id-162b6b96-e801-40f4-b111-7e533cf8b943,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_manage_invalid_param_manage_error[api_with_backend,id-01b36343-c503-4c14-bb74-71f131ae96b6,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_manage_invalid_param_raises_exception[api_with_backend,id-8267161e-4f55-44eb-9af5-30d1a3fb2606,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_manage_share_duplicate[api_with_backend,id-832b1c4d-e6f2-4a75-9ea5-d99f780db743,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_manage_share_without_share_server_id[api_with_backend,id-c2f28981-5758-46e8-9d90-5c3269461d94,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_unmanage_share_transitional_state[api_with_backend,id-f71bcb3c-836e-4e9e-972a-50debeb8c285,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_unmanage_share_with_server_unsupported[api_with_backend,id-916ca8e2-5635-4cc9-8ab6-36b7b53ecd0a,negative] +manila_tempest_tests.tests.api.admin.test_share_manage_negative.ManageNFSShareNegativeTest.test_unmanage_share_with_snapshot[api_with_backend,id-7cad4763-8f78-46a5-aeeb-6ca174284d3d,negative] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCephFSShareTest.test_manage_show_user_id[api_with_backend,id-da7b7a4f-6693-4460-bdb7-1f8d42032bc6,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCephFSShareTest.test_manage_with_is_public_True[api_with_backend,id-8c0beefb-19da-441e-b73f-d90eb8000ff3,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCephFSShareTest.test_manage_with_os_share_manage_url[api_with_backend,id-15b654d0-34ed-4154-9f5f-b96d2e4e9d1c,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCephFSShareTest.test_manage[backend,id-24ec7840-0174-484f-8b3a-8c6f162bf576,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCIFSShareTest.test_manage_show_user_id[api_with_backend,id-da7b7a4f-6693-4460-bdb7-1f8d42032bc6,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCIFSShareTest.test_manage_with_is_public_True[api_with_backend,id-8c0beefb-19da-441e-b73f-d90eb8000ff3,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCIFSShareTest.test_manage_with_os_share_manage_url[api_with_backend,id-15b654d0-34ed-4154-9f5f-b96d2e4e9d1c,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageCIFSShareTest.test_manage[backend,id-24ec7840-0174-484f-8b3a-8c6f162bf576,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageGLUSTERFSShareTest.test_manage_show_user_id[api_with_backend,id-da7b7a4f-6693-4460-bdb7-1f8d42032bc6,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageGLUSTERFSShareTest.test_manage_with_is_public_True[api_with_backend,id-8c0beefb-19da-441e-b73f-d90eb8000ff3,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageGLUSTERFSShareTest.test_manage_with_os_share_manage_url[api_with_backend,id-15b654d0-34ed-4154-9f5f-b96d2e4e9d1c,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageGLUSTERFSShareTest.test_manage[backend,id-24ec7840-0174-484f-8b3a-8c6f162bf576,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageHDFSShareTest.test_manage_show_user_id[api_with_backend,id-da7b7a4f-6693-4460-bdb7-1f8d42032bc6,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageHDFSShareTest.test_manage_with_is_public_True[api_with_backend,id-8c0beefb-19da-441e-b73f-d90eb8000ff3,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageHDFSShareTest.test_manage_with_os_share_manage_url[api_with_backend,id-15b654d0-34ed-4154-9f5f-b96d2e4e9d1c,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageHDFSShareTest.test_manage[backend,id-24ec7840-0174-484f-8b3a-8c6f162bf576,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageMapRFSShareTest.test_manage_show_user_id[api_with_backend,id-da7b7a4f-6693-4460-bdb7-1f8d42032bc6,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageMapRFSShareTest.test_manage_with_is_public_True[api_with_backend,id-8c0beefb-19da-441e-b73f-d90eb8000ff3,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageMapRFSShareTest.test_manage_with_os_share_manage_url[api_with_backend,id-15b654d0-34ed-4154-9f5f-b96d2e4e9d1c,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageMapRFSShareTest.test_manage[backend,id-24ec7840-0174-484f-8b3a-8c6f162bf576,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageNFSShareTest.test_manage_show_user_id[api_with_backend,id-da7b7a4f-6693-4460-bdb7-1f8d42032bc6,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageNFSShareTest.test_manage_with_is_public_True[api_with_backend,id-8c0beefb-19da-441e-b73f-d90eb8000ff3,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageNFSShareTest.test_manage_with_os_share_manage_url[api_with_backend,id-15b654d0-34ed-4154-9f5f-b96d2e4e9d1c,positive] +manila_tempest_tests.tests.api.admin.test_share_manage.ManageNFSShareTest.test_manage[backend,id-24ec7840-0174-484f-8b3a-8c6f162bf576,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_list_share_networks_all_filter_opts[api,id-27490442-60f8-4514-a10a-194a400b48bb,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_list_share_networks_all_tenants[api,id-983fb22d-3057-402f-8988-62ce41a557fb,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_list_share_networks_filter_by_project_id[api,id-36c26b6b-8984-4255-959b-74f6ef46c37b,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_list_share_networks_filter_by_ss[api,id-6d13a090-0855-40c0-85b1-424f6878c6ce,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_list_share_networks_like_filter[api,id-bff1356e-70aa-4bbe-b398-cb4dadd8fcb1,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_list_share_networks_with_detail[api,id-285c7a91-1703-42a5-86c8-2463edde60e2,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_list_share_networks[api,gate,id-41c635b1-d9ef-4c05-9100-5e4b0034b523,positive,smoke] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_try_list_share_networks_all_tenants[api,id-18fe9031-cefc-4df3-bbb0-6541f5fda12b,positive] +manila_tempest_tests.tests.api.admin.test_share_networks.ShareNetworkAdminTest.test_try_list_share_networks_project_id[api,id-caf6635a-ecb8-4981-9776-11dcfdf3cdbc,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_delete_share_server_invalid_state[api_with_backend,id-0f21aebe-e983-4112-a494-b287293112d2,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_manage_share_server_double_manage[api_with_backend,id-30606085-4b01-4489-a18c-22e67ed0e221,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_manage_share_server_invalid_identifier[api_with_backend,id-d2feb338-5c78-4370-b5ba-c8b16df63b43,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_manage_share_server_invalid_params_1___host____invalid_host__[api_with_backend,id-60b94091-f35d-42ac-a25a-ca1f7e66bb44,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_manage_share_server_invalid_params_2___share_network_id____invalid_share_network_id__[api_with_backend,id-60b94091-f35d-42ac-a25a-ca1f7e66bb44,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_manage_share_server_invalid_params_3___share_network_subnet_id____invalid_share_network_subnet_id__[api_with_backend,id-60b94091-f35d-42ac-a25a-ca1f7e66bb44,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_share_server_reset_state_invalid_state[api_with_backend,id-fbe6cf2d-3e33-40d8-b420-b785d3cb7559,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_unmanage_share_server_invalid_state[api_with_backend,id-84f9a808-8bff-4652-8877-4c9f0dfeed8a,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage_negative.ManageShareServersNegativeTest.test_unmanage_share_server_with_share[api_with_backend,id-c0a13fc4-9f30-483a-b473-0f0ce533f645,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_manage.ManageShareServersTest.test_manage_share_server_1_True[backend,id-2bbe564d-d314-474b-8060-ada30581d0cd,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_manage.ManageShareServersTest.test_manage_share_server_2_False[backend,id-2bbe564d-d314-474b-8060-ada30581d0cd,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationCIFS.test_share_server_invalid_operation_migration_cancel[api_with_backend,id-245f39d7-bcbc-4711-afd7-651a5535a880,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationCIFS.test_share_server_invalid_operation_migration_check[api_with_backend,id-1f8d75c1-aa3c-465a-b2dd-9ad33933944f,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationCIFS.test_share_server_invalid_operation_migration_complete[api_with_backend,id-3db45440-2c70-4fa4-b5eb-75e3cb0204f8,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationCIFS.test_share_server_invalid_operation_migration_get_progress[api_with_backend,id-d2830fe4-8d13-40d2-b987-18d414bb6196,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationCIFS.test_share_server_invalid_operation_migration_start[api_with_backend,id-c256c5f5-b4d1-47b7-a1f4-af21f19ce600,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationNFS.test_share_server_invalid_operation_migration_cancel[api_with_backend,id-245f39d7-bcbc-4711-afd7-651a5535a880,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationNFS.test_share_server_invalid_operation_migration_check[api_with_backend,id-1f8d75c1-aa3c-465a-b2dd-9ad33933944f,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationNFS.test_share_server_invalid_operation_migration_complete[api_with_backend,id-3db45440-2c70-4fa4-b5eb-75e3cb0204f8,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationNFS.test_share_server_invalid_operation_migration_get_progress[api_with_backend,id-d2830fe4-8d13-40d2-b987-18d414bb6196,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerErrorStatusOperationNFS.test_share_server_invalid_operation_migration_start[api_with_backend,id-c256c5f5-b4d1-47b7-a1f4-af21f19ce600,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_host_migration_check[api_with_backend,id-e7e2c19c-a0ed-41ab-b666-b2beae4a690c,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_host_migration_start[api_with_backend,id-2d25cf84-0b5c-4a9f-ae20-9bec09bb6914,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_server_migration_cancel[api,id-2aeffcfa-4e68-40e4-8a75-03b017503501,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_server_migration_check[api,id-1be6ec2a-3118-4033-9cdb-ea6d199d97f4,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_server_migration_complete[api,id-3b464298-a4e4-417b-92d6-acfbd30ac45b,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_server_migration_get_progress[api,id-47795631-eb50-424b-9fac-d2ee832cd01c,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_server_migration_start[api,id-52d23980-80e7-40de-8dba-1bb1382ef995,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_share_network_migration_check[api_with_backend,id-2617e714-7a8e-49a4-8109-beab3ea6527f,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersCIFS.test_share_server_invalid_share_network_migration_start[api_with_backend,id-f0d7a055-3b46-4d2b-9b96-1d719bd323e8,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_host_migration_check[api_with_backend,id-e7e2c19c-a0ed-41ab-b666-b2beae4a690c,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_host_migration_start[api_with_backend,id-2d25cf84-0b5c-4a9f-ae20-9bec09bb6914,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_server_migration_cancel[api,id-2aeffcfa-4e68-40e4-8a75-03b017503501,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_server_migration_check[api,id-1be6ec2a-3118-4033-9cdb-ea6d199d97f4,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_server_migration_complete[api,id-3b464298-a4e4-417b-92d6-acfbd30ac45b,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_server_migration_get_progress[api,id-47795631-eb50-424b-9fac-d2ee832cd01c,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_server_migration_start[api,id-52d23980-80e7-40de-8dba-1bb1382ef995,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_share_network_migration_check[api_with_backend,id-2617e714-7a8e-49a4-8109-beab3ea6527f,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidParametersNFS.test_share_server_invalid_share_network_migration_start[api_with_backend,id-f0d7a055-3b46-4d2b-9b96-1d719bd323e8,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidStatesCIFS.test_share_server_migration_start_invalid_network[api_with_backend,id-bcec0503-b2a9-4514-bf3f-a30d55f41e78,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationInvalidStatesCIFS.test_share_server_migration_start_invalid_share_state[api_with_backend,id-11374277-efcf-4992-ad94-c8f4a393d41b,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartInvalidStatesNFS.test_share_server_migration_start_invalid_network[api_with_backend,id-bcec0503-b2a9-4514-bf3f-a30d55f41e78,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartInvalidStatesNFS.test_share_server_migration_start_invalid_share_state[api_with_backend,id-11374277-efcf-4992-ad94-c8f4a393d41b,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartNegativesCIFS.test_share_server_migration_start_try_create_access_rule[api_with_backend,id-93882b54-78d4-4c4e-95b5-993de0cdb25d,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartNegativesCIFS.test_share_server_migration_start_try_create_snapshot[api_with_backend,id-5b904db3-fc36-4c35-a8ef-cf6b80315388,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartNegativesCIFS.test_share_server_migration_start_try_delete_share_network[api_with_backend,id-7c74a4a8-61b2-4c55-bc4b-02eac73d2c6e,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartNegativesNFS.test_share_server_migration_start_try_create_access_rule[api_with_backend,id-93882b54-78d4-4c4e-95b5-993de0cdb25d,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartNegativesNFS.test_share_server_migration_start_try_create_snapshot[api_with_backend,id-5b904db3-fc36-4c35-a8ef-cf6b80315388,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration_negative.ShareServerMigrationStartNegativesNFS.test_share_server_migration_start_try_delete_share_network[api_with_backend,id-7c74a4a8-61b2-4c55-bc4b-02eac73d2c6e,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicCIFS.test_share_server_migration_cancel[backend,id-5b84bcb6-17d8-4073-8e02-53b54aee6f8b,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicCIFS.test_share_server_migration_check_1_True[backend,id-52e154eb-2d39-45af-b5c1-49ea569ab804,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicCIFS.test_share_server_migration_check_2_False[backend,id-52e154eb-2d39-45af-b5c1-49ea569ab804,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicCIFS.test_share_server_migration_complete_1_False[backend,id-99e439a8-a716-4205-bf5b-af50128cb908,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicCIFS.test_share_server_migration_complete_2_True[backend,id-99e439a8-a716-4205-bf5b-af50128cb908,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicNFS.test_share_server_migration_cancel[backend,id-5b84bcb6-17d8-4073-8e02-53b54aee6f8b,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicNFS.test_share_server_migration_check_1_True[backend,id-52e154eb-2d39-45af-b5c1-49ea569ab804,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicNFS.test_share_server_migration_check_2_False[backend,id-52e154eb-2d39-45af-b5c1-49ea569ab804,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicNFS.test_share_server_migration_complete_1_False[backend,id-99e439a8-a716-4205-bf5b-af50128cb908,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_migration.ShareServerMigrationBasicNFS.test_share_server_migration_complete_2_True[backend,id-99e439a8-a716-4205-bf5b-af50128cb908,positive] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_delete_share_server_with_member[api,id-65e12bf7-2ec6-4a5b-971b-b1ecb67b77b7,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_delete_share_server_with_nonexistent_id[api,id-0acb9107-18b2-4e9d-8432-37fd0d4c79b3,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_list_share_servers_with_fake_host[api,id-e893b32a-124f-4e5c-a425-58c8a4eac4a5,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_list_share_servers_with_fake_project[api,id-2f1162a8-bb52-4e2a-abc0-68d16f769e4f,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_list_share_servers_with_fake_share_network[api,id-ca23f385-56b2-4c02-9797-d88c3b7fb981,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_list_share_servers_with_fake_status[api,id-3e5d6007-5214-4fa2-bd33-dfd3bead67bf,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_list_share_servers_with_wrong_filter_key[api,id-664b5201-7ba9-4e33-9534-5307fc003e44,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_list_share_servers_with_wrong_filter_value[api,id-dcf169c9-1238-40cb-8a5c-ca6aca9d4d6b,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_show_share_server_details_with_inexistent_id[api,id-368c5404-483e-4eee-bb80-86206d153ea2,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_show_share_server_with_inexistent_id[api,id-f7580ef6-f7bb-4b52-ba45-82d2d2d66dbe,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_try_list_share_servers_with_member[api,id-6d55516f-9018-4372-8310-f725c4562961,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_try_show_share_server_details_with_member[api,id-16b6f911-487e-4c25-9241-75de2dbfc8ff,negative] +manila_tempest_tests.tests.api.admin.test_share_servers_negative.ShareServersNegativeAdminTest.test_try_show_share_server_with_member[api,id-d9021f06-c146-4e76-852e-8a1ebb3fa92e,negative] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_delete_share_server_1_True[api_with_backend,id-2fdf8d29-3ab8-4424-b684-6253f45b9666,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_delete_share_server_2_False[api_with_backend,id-2fdf8d29-3ab8-4424-b684-6253f45b9666,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_list_share_servers_with_host_filter[api_with_backend,id-515e3332-cce9-4a28-ad2e-96dbb2cc672e,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_list_share_servers_with_project_id_filter[api_with_backend,id-ed671cbb-7732-4966-9f1e-b997b8aa05ff,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_list_share_servers_with_share_network_id_filter[api_with_backend,id-f64e8024-eb42-4a4c-be6b-97d35633a019,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_list_share_servers_with_share_network_name_filter[api_with_backend,id-70064129-df97-4dba-9482-2819558111ec,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_list_share_servers_with_status_filter[api_with_backend,id-77a5c1c5-db2f-4ff6-872c-ed73a575acd0,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_list_share_servers_without_filters[api_with_backend,id-3f821248-2c05-4323-a95f-a0216a537b0a,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_share_server_reset_state[api_with_backend,id-960d49c5-f8d6-42be-aaf4-b7890a55df03,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_show_share_server_details[api_with_backend,id-782d8f5f-2c02-44dd-8d43-e06b651a71be,positive] +manila_tempest_tests.tests.api.admin.test_share_servers.ShareServersAdminTest.test_show_share_server[api_with_backend,id-e1af24f4-bf63-467d-a857-3a402fa9b65b,positive] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances_negative.SnapshotInstancesNegativeNoResourceTest.test_get_snapshot_instance_with_non_existent_instance[api,id-abded04d-40c0-4eb9-b2be-58b4efb16244,negative] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances_negative.SnapshotInstancesNegativeNoResourceTest.test_list_snapshot_instances_by_non_admin[api,id-1609702b-de87-4d12-9a9c-78077d4676f3,negative] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances_negative.SnapshotInstancesNegativeTest.test_get_snapshot_instance_by_non_admin[api_with_backend,id-d80331e4-8738-46c7-b726-1e716acef738,negative] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances_negative.SnapshotInstancesNegativeTest.test_list_snapshot_instances_with_snapshot_by_non_admin[api_with_backend,id-6e371aac-ff8b-4eac-abc9-b8d777448ff3,negative] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances_negative.SnapshotInstancesNegativeTest.test_reset_snapshot_instance_status_by_non_admin[api_with_backend,id-666a6caf-31b0-45d9-808c-e590250cffd4,negative] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.test_get_snapshot_instance[api_with_backend,id-d21bf82d-2b3b-4061-8db5-502b7376a44d,positive] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.test_list_snapshot_instances_by_snapshot_1_True[api_with_backend,id-bcb29129-9713-4481-8e74-97682c62f218,positive] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.test_list_snapshot_instances_by_snapshot_2_False[api_with_backend,id-bcb29129-9713-4481-8e74-97682c62f218,positive] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.test_list_snapshot_instances[api_with_backend,id-1d0bd333-300d-4af2-81eb-176b550f2c36,positive] +manila_tempest_tests.tests.api.admin.test_share_snapshot_instances.ShareSnapshotInstancesTest.test_reset_snapshot_instance_status_and_delete[api_with_backend,id-c7724848-4e17-4d28-801d-2ec77dfcf502,positive] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_create_es_with_empty_shr_type_id[api,id-430e0bed-9073-4822-8d0b-a3c55a8dfa31,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_create_es_with_empty_specs[api,id-7b9bee14-5ca5-4110-a56a-b3030b6b3948,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_create_es_with_invalid_shr_type_id[api,id-7b43025d-b014-460f-9c9d-1004433fb798,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_create_es_with_invalid_specs[api,id-7f199925-44d2-4d92-bedc-2636c07621fb,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_create_extra_specs_with_user[api,id-195c1cc6-249a-4f82-b420-4901d2557b3a,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_delete_es_key_with_empty_shr_type_id[api,id-0b8f6b51-0583-4b59-a851-9189db657a05,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_delete_es_key_with_invalid_shr_type_id[api,id-9aa8fb2a-28a5-44b7-848b-9b49e1e9d670,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_delete_extra_specs_with_user[api,id-3f43c5d0-23c5-4b76-98c7-a3f9adb33c89,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_delete_required_spec_snapshot_support_version_1_2_0[api,id-6ea50e81-2c93-4258-8358-6f8d354a339a,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_delete_required_spec_snapshot_support_version_2_2_23[api,id-6ea50e81-2c93-4258-8358-6f8d354a339a,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_delete_spec_driver_handles_share_servers[api,id-6849eada-89a8-4009-a91d-87367621f9aa,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_delete_with_invalid_key[api,id-cd68d020-24d2-4f68-8691-782b4815c1b0,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_get_extra_spec_with_empty_key[api,id-51241ed9-350b-4218-bfb0-c446d660d70b,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_get_extra_spec_with_invalid_key[api,id-271d825b-2c57-429a-8dca-2cb9dd140dd0,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_get_extra_spec_with_user[api,id-1d3e687e-b2fb-4b96-8428-324ff881eea2,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_get_extra_specs_with_empty_shr_type_id[api,id-cf07e7bc-cd3a-4c85-a848-786a60ba1f7d,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_get_extra_specs_with_invalid_shr_type_id[api,id-dfe6e34b-c86a-4cf6-9bf3-19de3a886b67,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_get_extra_specs_with_user[api,id-4c9505d9-d4ef-42fa-8410-8ab88ec0c852,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_list_es_with_empty_shr_type_id[api,id-938115f7-512e-49c3-a16a-a6498d6069bb,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_list_es_with_invalid_shr_type_id[api,id-65c3c528-008f-43f5-b367-8bd77cfab4a7,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_list_extra_specs_with_user[api,id-dc883ec3-1bae-4ed7-8bf5-2cdc7027e37b,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_read_extra_specs_on_share_type_with_user[api,id-36c5ada4-9efd-4f6a-b58d-24f08a2433ce,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_set_too_long_key[api,id-d82a7bcc-1dc5-4ef8-87cd-f8dff1574adc,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_set_too_long_value_with_creation[api,id-210faa88-8f2f-45f5-9bcf-54ce81d03788,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_set_too_long_value_with_update_of_one_key[api,id-c512437e-6cfd-4545-859f-554955bd0fc9,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_set_too_long_value_with_update[api,id-890dceaa-22c4-4d2c-99ca-16ac8cdda33c,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_update_extra_spec_with_user[api,id-62a9b77a-f796-4bd9-baf9-7c24b3f55560,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_update_extra_specs_with_user[api,id-207cec3c-8ed9-4d6d-8fc8-3aecaacdff93,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_update_spec_with_empty_key[api,id-eab96e92-9b95-44b0-89a2-e907a103039d,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_update_spec_with_empty_shr_type_id[api,id-1ed5cbc9-21a8-45b7-8069-b7c0aaeede21,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_update_spec_with_invalid_shr_type_id[api,id-704087fa-0397-4d9d-98b7-f9b081f64c86,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_update_with_invalid_shr_type_id[api,id-70af1a8a-ab3e-4c8b-862d-8a36c2d47cb3,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs_negative.ExtraSpecsAdminNegativeTest.test_try_update_with_invalid_specs[api,id-d2595594-eaad-43dc-b847-0a009a17d854,negative] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs.ExtraSpecsReadAdminTest.test_get_all_share_type_extra_specs[api,id-508ceb85-5456-4db9-b33f-00620d7baea1,positive] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs.ExtraSpecsReadAdminTest.test_get_one_share_type_extra_spec[api,id-d4cae104-838b-4248-8c9c-125165710786,positive] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs.ExtraSpecsWriteAdminTest.test_delete_one_share_type_extra_spec[api,id-129db0ce-9a4f-4a40-994f-e492174e0661,positive] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs.ExtraSpecsWriteAdminTest.test_delete_snapshot_support_extra_spec_1_2_24[api,id-1b9f501d-8f34-46d0-b318-83bdfed571ec,positive] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs.ExtraSpecsWriteAdminTest.test_delete_snapshot_support_extra_spec_2_2_88[api,id-1b9f501d-8f34-46d0-b318-83bdfed571ec,positive] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs.ExtraSpecsWriteAdminTest.test_update_all_share_type_extra_specs[api,id-374e7ff5-0d26-4778-bbb5-4ffd28521ad9,positive] +manila_tempest_tests.tests.api.admin.test_share_types_extra_specs.ExtraSpecsWriteAdminTest.test_update_one_share_type_extra_spec[api,id-baba776b-1b05-42ca-a002-bff1f8f14e13,positive] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_add_share_type_allowed_for_public[api,id-c13f54eb-17a4-4403-be87-f6a3ca18de6e,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_add_share_type_by_nonexistent_id[api,id-3d9a7e9d-2b64-422e-b8b6-88c3088564f6,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_create_share_type_with_description_in_wrong_version_1_2_0[api,id-42f7777a-7ee4-4622-80f8-8726b467c4db,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_create_share_type_with_description_in_wrong_version_2_2_6[api,id-42f7777a-7ee4-4622-80f8-8726b467c4db,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_create_share_type_with_description_in_wrong_version_3_2_40[api,id-42f7777a-7ee4-4622-80f8-8726b467c4db,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_create_share_type_with_empty_name[api,id-a1cd6c4f-4dc4-4f45-813b-f1cd3527c614,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_create_share_type_with_too_big_name[api,id-ca59430b-d1fb-4e8f-b1e3-6ab6a6b40984,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_create_share_with_non_allowed_share_type[backend,id-0fd53c51-e1ba-4392-9f4c-5d3bdd157163,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_create_share_with_nonexistent_share_type[api,id-0efe4ed6-9318-4174-aef7-fca4b6aa6444,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_get_share_type_by_nonexistent_id[api,id-43b0ef5a-3a05-4f74-a08e-57efebb7e66e,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_remove_share_type_allowed_for_public[api,id-bf1d68fb-b954-4b3b-af54-115f3b67b3b3,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_remove_share_type_by_nonexistent_id[api,id-13d5c3cd-15cd-4c2e-ace6-46889cf9e0e2,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_try_create_duplicate_of_share_type[api,id-1f481bab-5205-49ee-bf01-b1848a32f9ee,negative] +manila_tempest_tests.tests.api.admin.test_share_types_negative.ShareTypesAdminNegativeTest.test_try_delete_share_type_by_nonexistent_id[api,id-27e00ad7-edc8-4d50-b53e-4da41dd8a5d3,negative] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_get_share_with_share_type[api_with_backend,id-4e2ad320-9f3d-4797-a3ad-bf800bcd1831,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_private_share_type_access[api,id-d2261a27-d4a4-4237-9fad-f6fd8f27783a,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_delete[api,id-34000fb9-b595-4a10-8306-7465f9ebc45d,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_get_1_2_0[api,id-228d8bab-0b31-433e-956f-9db9877e6573,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_get_2_2_6[api,id-228d8bab-0b31-433e-956f-9db9877e6573,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_get_3_2_7[api,id-228d8bab-0b31-433e-956f-9db9877e6573,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_get_4_2_40[api,id-228d8bab-0b31-433e-956f-9db9877e6573,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_get_5_2_41[api,id-228d8bab-0b31-433e-956f-9db9877e6573,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_list_1_2_0[api,id-5cc4c2e5-d2a4-4bfc-9208-3455ac551f20,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_list_2_2_6[api,id-5cc4c2e5-d2a4-4bfc-9208-3455ac551f20,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_list_3_2_7[api,id-5cc4c2e5-d2a4-4bfc-9208-3455ac551f20,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_list_4_2_40[api,id-5cc4c2e5-d2a4-4bfc-9208-3455ac551f20,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_list_5_2_41[api,id-5cc4c2e5-d2a4-4bfc-9208-3455ac551f20,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_show_list_with_is_default_key_1_2_45[api,id-90dca5c5-f28e-4f16-90ed-78f5d725664e,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_show_list_with_is_default_key_2_2_46[api,id-90dca5c5-f28e-4f16-90ed-78f5d725664e,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_show_list_with_is_default_key_3_2_88[api,id-90dca5c5-f28e-4f16-90ed-78f5d725664e,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_01_2_50_name_description_public[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_02_2_50_name[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_03_2_50_description_public[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_04_2_50_public[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_05_2_50[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_06_2_88_name_description_public[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_07_2_88_name[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_08_2_88_description[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_09_2_88_public[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_create_update_10_2_88[api,id-a9af19e1-e789-4c4f-a39b-dd8df6ed00b1,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_unset_description_1___2_50___None______None_[api,id-9019dc61-b2b1-472d-9b15-a3986439d4c3,positive] +manila_tempest_tests.tests.api.admin.test_share_types.ShareTypesAdminTest.test_share_type_unset_description_2___2_88___None______None_[api,id-9019dc61-b2b1-472d-9b15-a3986439d4c3,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_get_share[api_with_backend,id-5f61f5dd-891e-478f-b102-803096820882,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_get_snapshot[api_with_backend,id-d24a438e-4622-48ac-993e-a30d04746745,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_or_with_detail_filter_by_export_location_1___path___True_[api_with_backend,id-a27e5e3f-451f-4200-af38-99a562ccbe86,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_or_with_detail_filter_by_export_location_2___id___True_[api_with_backend,id-a27e5e3f-451f-4200-af38-99a562ccbe86,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_or_with_detail_filter_by_export_location_3___path___False_[api_with_backend,id-a27e5e3f-451f-4200-af38-99a562ccbe86,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_or_with_detail_filter_by_export_location_4___id___False_[api_with_backend,id-a27e5e3f-451f-4200-af38-99a562ccbe86,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_active_status[api_with_backend,id-8eac9b63-666f-4c52-8c5f-58b1fdf201e2,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_existed_name[api_with_backend,id-631f4226-f1ea-47b1-a472-8f12da2d05c4,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_extra_specs[api_with_backend,id-d884c91e-88f5-4e42-83d9-ec3b440af893,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_fake_name[api_with_backend,id-d0dae9e5-a826-48e4-b7b7-24b08ad5a7cb,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_fake_status[api_with_backend,id-e94f41c0-f6c4-4d77-b4f9-2c796c27e348,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_host[api_with_backend,id-04afc330-78ee-494f-a660-7670c877a440,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_metadata[api_with_backend,id-47dad08b-0c36-428f-8ab9-5eba92ffc995,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_share_network_id[api_with_backend,id-4582de51-1dcd-4c44-b550-eca9a9685038,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_share_type_id[api_with_backend,id-76fbe8ba-f1d3-4446-b9b8-55617762a2c7,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_filter_by_snapshot_id[api_with_backend,id-645aebc4-55ac-406d-b7ab-5614c4fc12e6,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail_with_asc_sorting[api_with_backend,id-87659cee-4692-412a-9bfe-06fc97d30ba0,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares_with_detail[api_with_backend,id-85f9438d-d3f6-4f3a-8134-e89915373df3,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_shares[api_with_backend,id-60d34573-8452-47ab-9455-0067bdd3ed9c,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_snapshots_with_detail[api_with_backend,id-84013334-5985-4067-8b54-4c633f6022f3,positive] +manila_tempest_tests.tests.api.admin.test_shares_actions.SharesActionsAdminTest.test_list_snapshots[api_with_backend,id-9fae88a5-dd95-40ba-96e2-ac3694cf455f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsAPIOnlyNegativeTest.test_list_export_locations_by_nonexistent_snapshot_instance[api,id-c2aa3770-c061-4b49-83ac-ab29773c2e0c,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsAPIOnlyNegativeTest.test_list_export_locations_by_nonexistent_snapshot[api,id-37901216-b574-4786-9b1d-9b1ccdf123d2,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsAPIOnlyNegativeTest.test_list_inexistent_snapshot_instance_export_locations_by_member[api,id-74d5d46d-8161-4e17-acbc-812248d6d694,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsNegativeTest.test_get_inexistent_snapshot_export_location[api_with_backend,id-53f0f184-7398-4e7a-ac21-fa432570db7f,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsNegativeTest.test_get_inexistent_snapshot_instance_export_location[api_with_backend,id-52e0b807-7b29-4795-960a-518bcadc1503,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsNegativeTest.test_get_snapshot_export_location_by_different_project_user[api_with_backend,id-66839514-796a-4ee9-a8ed-7614521d01d5,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsNegativeTest.test_get_snapshot_instance_export_location_by_member[api_with_backend,id-3f4e2a0e-1522-47fb-b770-9d7a0651dde2,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations_negative.SnapshotExportLocationsNegativeTest.test_list_snapshot_export_locations_by_different_project_user[api_with_backend,id-43229517-bf93-4be7-9f89-a69034d2f03c,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations.SnapshotExportLocationsTest.test_get_snapshot_export_location[api_with_backend,id-6272b60b-31a1-41c1-86f5-af28926898e6,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations.SnapshotExportLocationsTest.test_get_snapshot_instance_export_location[api_with_backend,id-03be6418-5ba3-4919-a798-89d7e5ffb925,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations.SnapshotExportLocationsTest.test_list_snapshot_export_location[api_with_backend,id-18287f50-0e12-463d-906f-5c7cba256288,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_export_locations.SnapshotExportLocationsTest.test_snapshot_contains_all_export_locations_of_all_snapshot_instances[api_with_backend,id-cdf444ea-95a3-4f7b-ae48-6b027a6b9529,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageCIFSSnapshotNegativeTest.test_manage_already_exists[api_with_backend,id-645a88f7-1eec-49a2-a232-757dc2112dd8,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageCIFSSnapshotNegativeTest.test_manage_invalid_provider_location[api_with_backend,id-e94706f4-9c72-41f1-9ed4-09eb93d0b36f,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageCIFSSnapshotNegativeTest.test_manage_not_found[api_with_backend,id-df7ac126-2c13-4902-b9c0-d103c71680ee,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageCIFSSnapshotNegativeTest.test_unmanage_snapshot_with_server_unsupported[api_with_backend,id-d0049626-a524-4c8a-a58a-006ae2cdfea5,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageGLUSTERFSSnapshotNegativeTest.test_manage_already_exists[api_with_backend,id-645a88f7-1eec-49a2-a232-757dc2112dd8,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageGLUSTERFSSnapshotNegativeTest.test_manage_invalid_provider_location[api_with_backend,id-e94706f4-9c72-41f1-9ed4-09eb93d0b36f,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageGLUSTERFSSnapshotNegativeTest.test_manage_not_found[api_with_backend,id-df7ac126-2c13-4902-b9c0-d103c71680ee,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageGLUSTERFSSnapshotNegativeTest.test_unmanage_snapshot_with_server_unsupported[api_with_backend,id-d0049626-a524-4c8a-a58a-006ae2cdfea5,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageHDFSSnapshotNegativeTest.test_manage_already_exists[api_with_backend,id-645a88f7-1eec-49a2-a232-757dc2112dd8,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageHDFSSnapshotNegativeTest.test_manage_invalid_provider_location[api_with_backend,id-e94706f4-9c72-41f1-9ed4-09eb93d0b36f,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageHDFSSnapshotNegativeTest.test_manage_not_found[api_with_backend,id-df7ac126-2c13-4902-b9c0-d103c71680ee,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageHDFSSnapshotNegativeTest.test_unmanage_snapshot_with_server_unsupported[api_with_backend,id-d0049626-a524-4c8a-a58a-006ae2cdfea5,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageMapRFSSnapshotNegativeTest.test_manage_already_exists[api_with_backend,id-645a88f7-1eec-49a2-a232-757dc2112dd8,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageMapRFSSnapshotNegativeTest.test_manage_invalid_provider_location[api_with_backend,id-e94706f4-9c72-41f1-9ed4-09eb93d0b36f,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageMapRFSSnapshotNegativeTest.test_manage_not_found[api_with_backend,id-df7ac126-2c13-4902-b9c0-d103c71680ee,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageMapRFSSnapshotNegativeTest.test_unmanage_snapshot_with_server_unsupported[api_with_backend,id-d0049626-a524-4c8a-a58a-006ae2cdfea5,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageNFSSnapshotNegativeTest.test_manage_already_exists[api_with_backend,id-645a88f7-1eec-49a2-a232-757dc2112dd8,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageNFSSnapshotNegativeTest.test_manage_invalid_provider_location[api_with_backend,id-e94706f4-9c72-41f1-9ed4-09eb93d0b36f,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageNFSSnapshotNegativeTest.test_manage_not_found[api_with_backend,id-df7ac126-2c13-4902-b9c0-d103c71680ee,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage_negative.ManageNFSSnapshotNegativeTest.test_unmanage_snapshot_with_server_unsupported[api_with_backend,id-d0049626-a524-4c8a-a58a-006ae2cdfea5,negative] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageCIFSSnapshotTest.test_manage_different_versions_1_2_12[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageCIFSSnapshotTest.test_manage_different_versions_2_2_16[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageCIFSSnapshotTest.test_manage_different_versions_3_2_73[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageCIFSSnapshotTest.test_manage_different_versions_4_2_88[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageGLUSTERFSSnapshotTest.test_manage_different_versions_1_2_12[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageGLUSTERFSSnapshotTest.test_manage_different_versions_2_2_16[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageGLUSTERFSSnapshotTest.test_manage_different_versions_3_2_73[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageGLUSTERFSSnapshotTest.test_manage_different_versions_4_2_88[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageHDFSSnapshotTest.test_manage_different_versions_1_2_12[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageHDFSSnapshotTest.test_manage_different_versions_2_2_16[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageHDFSSnapshotTest.test_manage_different_versions_3_2_73[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageHDFSSnapshotTest.test_manage_different_versions_4_2_88[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageMapRFSSnapshotTest.test_manage_different_versions_1_2_12[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageMapRFSSnapshotTest.test_manage_different_versions_2_2_16[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageMapRFSSnapshotTest.test_manage_different_versions_3_2_73[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageMapRFSSnapshotTest.test_manage_different_versions_4_2_88[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageNFSSnapshotTest.test_manage_different_versions_1_2_12[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageNFSSnapshotTest.test_manage_different_versions_2_2_16[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageNFSSnapshotTest.test_manage_different_versions_3_2_73[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_snapshot_manage.ManageNFSSnapshotTest.test_manage_different_versions_4_2_88[backend,id-5fb65a19-fb73-4b5a-9210-010f93e0304f,positive] +manila_tempest_tests.tests.api.admin.test_user_messages_negative.UserMessageNegativeTest.test_delete_message_of_other_tenants[api,id-2f5be1aa-974b-4f6a-ae3a-084578e64f82,negative] +manila_tempest_tests.tests.api.admin.test_user_messages_negative.UserMessageNegativeTest.test_delete_nonexistent_message[api,id-9029cc9f-b904-4e58-b268-adf7a93cc1f1,negative] +manila_tempest_tests.tests.api.admin.test_user_messages_negative.UserMessageNegativeTest.test_list_messages_with_invalid_time_format[api,id-03e80563-1a36-408e-baa8-0e3ed46f7a0a,negative] +manila_tempest_tests.tests.api.admin.test_user_messages_negative.UserMessageNegativeTest.test_show_message_of_other_tenants[api,id-cb6c0dbd-a3cb-404b-a358-7cec3596aff4,negative] +manila_tempest_tests.tests.api.admin.test_user_messages_negative.UserMessageNegativeTest.test_show_nonexistent_message[api,id-f493a1f9-43e1-4a85-a673-8520d5a81f68,negative] +manila_tempest_tests.tests.api.admin.test_user_messages.UserMessageTest.test_delete_message[api,id-88b2db98-8e33-4552-9225-c19a3b6f18e0,positive] +manila_tempest_tests.tests.api.admin.test_user_messages.UserMessageTest.test_list_messages_filtered[api,id-292b0a2c-f066-48a6-9492-8dede87ee7b4,positive] +manila_tempest_tests.tests.api.admin.test_user_messages.UserMessageTest.test_list_messages_sorted_and_paginated[api,id-ff938d3e-9cdd-470a-b091-d389be7be755,positive] +manila_tempest_tests.tests.api.admin.test_user_messages.UserMessageTest.test_list_messages_with_since_and_before_filters[api,id-2ed0c40e-cdaa-471b-97d4-5ebe3fb040e9,positive] +manila_tempest_tests.tests.api.admin.test_user_messages.UserMessageTest.test_list_messages[api,id-37127e11-7aa7-46b2-ab05-e3bf36d94fd8,positive] +manila_tempest_tests.tests.api.admin.test_user_messages.UserMessageTest.test_show_message[api,id-a92971c5-a72a-4de2-8e9b-dc612b798a7a,positive] +manila_tempest_tests.tests.api.test_access_rules_metadata_negative.AccessesMetadataNegativeTest.test_try_delete_unexisting_access_metadata[api_with_backend,id-53dc2752-687d-4895-bc9e-518c3bed29d0,negative] +manila_tempest_tests.tests.api.test_access_rules_metadata_negative.AccessesMetadataNegativeTest.test_try_upd_access_metadata_error_1[api_with_backend,id-d2d41db8-ae00-4641-a5ec-499cee1877f1,negative] +manila_tempest_tests.tests.api.test_access_rules_metadata_negative.AccessesMetadataNegativeTest.test_try_upd_access_metadata_error_2[api_with_backend,id-d2d41db8-ae00-4641-a5ec-499cee1877f1,negative] +manila_tempest_tests.tests.api.test_access_rules_metadata_negative.AccessesMetadataNegativeTest.test_try_upd_access_metadata_error_3[api_with_backend,id-d2d41db8-ae00-4641-a5ec-499cee1877f1,negative] +manila_tempest_tests.tests.api.test_access_rules_metadata.AccessRulesMetadataTest.test_list_access_filter_by_metadata[api_with_backend,id-c97cc976-2c2f-4db3-ba07-cd04af29c9f4,positive] +manila_tempest_tests.tests.api.test_access_rules_metadata.AccessRulesMetadataTest.test_set_get_delete_access_metadata[api_with_backend,id-4c8e0236-2e7b-4337-be3c-17b51a738644,positive] +manila_tempest_tests.tests.api.test_access_rules_metadata.AccessRulesMetadataTest.test_update_metadata_by_key[api_with_backend,id-8c294d7d-0702-49ce-b964-0945ec323370,positive] +manila_tempest_tests.tests.api.test_availability_zones_negative.AvailabilityZonesNegativeTest.test_list_availability_zones_api_not_found_with_legacy_url[api,id-050fa9fb-4cc8-4064-9e91-bb169fca03db,negative] +manila_tempest_tests.tests.api.test_availability_zones_negative.AvailabilityZonesNegativeTest.test_list_availability_zones_api_not_found[api,id-133ffc40-bff1-491a-a230-189cddd66e29,negative] +manila_tempest_tests.tests.api.test_availability_zones.AvailabilityZonesTest.test_list_availability_zones_legacy_url_api_v1[api,id-202f20d3-1afa-40ea-a5e6-8b7bda40e6cf,positive] +manila_tempest_tests.tests.api.test_availability_zones.AvailabilityZonesTest.test_list_availability_zones_legacy_url_api_v2[api,id-7054f2f4-bc77-4d60-82a6-2f23b93d281e,positive] +manila_tempest_tests.tests.api.test_availability_zones.AvailabilityZonesTest.test_list_availability_zones[api,id-4caadb86-2988-4adb-b705-aece99235c1e,positive] +manila_tempest_tests.tests.api.test_backup_negative.ShareBackupNegativeTest.test_create_backup_when_share_has_snapshots[backend,id-58c36c97-faf4-4fec-9a9b-7cff0d2012de,negative] +manila_tempest_tests.tests.api.test_backup_negative.ShareBackupNegativeTest.test_create_backup_when_share_is_in_backup_creating_state[backend,id-58c36c97-faf4-4fec-9a9b-7cff0d2035ab,negative] +manila_tempest_tests.tests.api.test_backup_negative.ShareBackupNegativeTest.test_create_backup_when_share_is_in_error_state[backend,id-58c36c97-faf4-4fec-9a9b-7cff0d2012ab,negative] +manila_tempest_tests.tests.api.test_backup_negative.ShareBackupNegativeTest.test_delete_backup_when_backup_is_not_available[backend,id-58c12c97-faf4-4fec-9a9b-7cff0d2012de,negative] +manila_tempest_tests.tests.api.test_backup_negative.ShareBackupNegativeTest.test_restore_backup_when_backup_is_not_available[backend,id-58c12998-faf4-4fec-9a9b-7cff0d2012de,negative] +manila_tempest_tests.tests.api.test_backup_negative.ShareBackupNegativeTest.test_restore_backup_when_share_is_not_available[backend,id-58c56c97-faf4-4fec-9a9b-7cff0d2012de,negative] +manila_tempest_tests.tests.api.test_backup.ShareBackupTest.test_create_share_backup[backend,id-12c36c97-faf4-4fec-9a9b-7cff0d2035cd,positive] +manila_tempest_tests.tests.api.test_backup.ShareBackupTest.test_delete_share_backup[backend,id-34c36c97-faf4-4fec-9a9b-7cff0d2035cd,positive] +manila_tempest_tests.tests.api.test_backup.ShareBackupTest.test_list_share_backups[backend,id-19c36c97-faf4-4fec-9a9b-7cff0d2045af,positive] +manila_tempest_tests.tests.api.test_backup.ShareBackupTest.test_restore_share_backup[backend,id-56c36c97-faf4-4fec-9a9b-7cff0d2035cd,positive] +manila_tempest_tests.tests.api.test_backup.ShareBackupTest.test_update_share_backup[backend,id-78c36c97-faf4-4fec-9a9b-7cff0d2035cd,positive] +manila_tempest_tests.tests.api.test_extensions.ExtensionsTest.test_extensions[api,id-befd2105-c9f6-4eca-a693-9900eb0fb37a,positive] +manila_tempest_tests.tests.api.test_limits.ShareLimitsTest.test_limits_keys[api,id-239903d2-f1cb-4bec-b456-d57456308244,positive] +manila_tempest_tests.tests.api.test_limits.ShareLimitsTest.test_limits_values[api,id-fea1a10e-6bcb-46eb-bbba-f6f0a8d4f377,positive] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataAPIOnlyNegativeTest.test_try_set_metadata_to_unexisting_share_1_True[api,id-22aecf50-0d98-4b97-82b8-599559f7692f,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataAPIOnlyNegativeTest.test_try_set_metadata_to_unexisting_share_2_False[api,id-22aecf50-0d98-4b97-82b8-599559f7692f,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataAPIOnlyNegativeTest.test_try_update_all_metadata_for_unexisting_share_1_True[api,id-7df0acd7-03f8-45c4-8c72-eb6932af70b1,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataAPIOnlyNegativeTest.test_try_update_all_metadata_for_unexisting_share_2_False[api,id-7df0acd7-03f8-45c4-8c72-eb6932af70b1,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataNegativeTest.test_try_delete_unexisting_metadata[api_with_backend,id-14df3262-5a2b-4de4-b335-422329b22b07,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataNegativeTest.test_try_set_metadata_with_empty_key[api_with_backend,id-0cb7f160-4fa4-4d30-8a46-373ddae5844d,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataNegativeTest.test_try_set_metadata_with_too_big_key[api_with_backend,id-94c7ebb3-14c3-4ff1-9839-ae3acb318cd0,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataNegativeTest.test_try_set_metadata_with_too_big_value[api_with_backend,id-1114970a-1b45-4c56-b20a-e13e1764e3c4,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataNegativeTest.test_try_update_metadata_with_empty_key[api_with_backend,id-759ca34d-1c87-43f3-8da2-8e1d373049ac,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataNegativeTest.test_try_update_metadata_with_too_big_key[api_with_backend,id-33ef3047-6ca3-4547-a681-b52314382dcb,negative] +manila_tempest_tests.tests.api.test_metadata_negative.SharesMetadataNegativeTest.test_try_update_metadata_with_too_big_value[api_with_backend,id-c2eddcf0-cf81-4f9f-b06d-c9165ab8553e,negative] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_and_update_metadata_by_key[api_with_backend,id-c94851f4-2559-4712-9297-9912db1da7ff,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_get_delete_metadata[api_with_backend,id-2725ab8e-cc04-4032-9393-74726ba43eb7,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_metadata_in_share_creation[api_with_backend,id-9070249f-6e94-4a38-a036-08debee547c3,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_metadata_key_already_exist[api_with_backend,id-2ec70ba5-050b-3b17-c862-c149e53543c0,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_metadata_max_size_key[api_with_backend,id-34c5bd96-ced7-42ef-a114-570cc63cf81d,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_metadata_max_size_value[api_with_backend,id-759ec4ab-2537-44ad-852b-1af85c6ca933,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_metadata_min_size_key[api_with_backend,id-698ba406-493f-4c69-a093-273676fed438,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_metadata_min_size_value[api_with_backend,id-c776ab40-f75e-4d9f-abcf-a8f628a25991,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_set_metadata_not_delete_pre_metadata[api_with_backend,id-4e5f8159-62b6-4d5c-f729-d8b1f029d7de,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_upd_metadata_max_size_key[api_with_backend,id-5eff5619-b7cd-42f1-85e0-47d3d47098dd,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_upd_metadata_max_size_value[api_with_backend,id-694d95e1-ba8c-49fc-a888-6f9f0d51d77d,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_upd_metadata_min_size_key[api_with_backend,id-c5ca19ba-3595-414a-8ff9-fbc88cd801ba,positive] +manila_tempest_tests.tests.api.test_metadata.SharesMetadataTest.test_upd_metadata_min_size_value[api_with_backend,id-44a572f1-6b5c-49d0-8f2e-1583ec3428d8,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_root_version[api,id-1b20943e-a398-49dd-a69d-513ce6ea01b4,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_v1_no_version[api,id-6ccaae5f-2382-45a5-a844-6d31837beba8,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_v1_with_version[api,id-7b7e4ddf-08bb-4764-a94f-5f377da3b2cb,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_v2_invalid_version[api,id-038ff428-edde-43ff-ae29-1b18676b61d5,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_v2_max_version[api,id-e0766a36-b57c-46f8-b474-a38b74f3b9ed,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_v2_min_version[api,id-101574fd-9d47-4bd6-97f6-cde219f089ca,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_v2_no_version[api,id-9a8bd7fd-f9d0-4fc6-8e1c-0178d87ec7c1,positive] +manila_tempest_tests.tests.api.test_microversions.MicroversionsTest.test_microversions_v2_unacceptable_version[api,id-0d82fc0b-d24d-4d07-ab43-bd0895ec0c4c,positive] +manila_tempest_tests.tests.api.test_public_shares_negative.PublicSharesNegativeTest.test_delete_metadata_of_other_tenants_public_share[api_with_backend,id-bd62adeb-73c2-4b04-8812-80b479cd5c3b,negative] +manila_tempest_tests.tests.api.test_public_shares_negative.PublicSharesNegativeTest.test_delete_other_tenants_public_share[api_with_backend,id-68d1f1bc-16e4-4086-8982-7e44ca6bdc4d,negative] +manila_tempest_tests.tests.api.test_public_shares_negative.PublicSharesNegativeTest.test_set_metadata_of_other_tenants_public_share[api_with_backend,id-1f9e5d84-0885-4a4b-9196-9031a1c01508,negative] +manila_tempest_tests.tests.api.test_public_shares_negative.PublicSharesNegativeTest.test_update_metadata_of_other_tenants_public_share[api_with_backend,id-fed7a935-9699-43a1-854e-67b61ba6233e,negative] +manila_tempest_tests.tests.api.test_public_shares_negative.PublicSharesNegativeTest.test_update_other_tenants_public_share[api_with_backend,id-3443493b-f56a-4faa-9968-e7cbb0d2802f,negative] +manila_tempest_tests.tests.api.test_public_shares_negative.PublicSharesNegativeTest.test_update_share_with_wrong_public_value[api_with_backend,id-255011c0-4ed9-4174-bb13-8bbd06a62529,negative] +manila_tempest_tests.tests.api.test_public_shares.PublicSharesTest.test_list_shares_public_with_detail[api_with_backend,id-557a0474-9e30-47b4-a766-19e2afb13e66,positive] +manila_tempest_tests.tests.api.test_public_shares.PublicSharesTest.test_update_share_set_is_public[api_with_backend,id-e073182e-459d-4e08-9300-5bc964ca806b,positive] +manila_tempest_tests.tests.api.test_quotas_negative.SharesQuotasNegativeTest.test_get_quotas_detail_with_wrong_version_1_2_6[api,id-0f0033b3-357e-42e6-9c94-cac650e1cd50,negative] +manila_tempest_tests.tests.api.test_quotas_negative.SharesQuotasNegativeTest.test_get_quotas_detail_with_wrong_version_2_2_7[api,id-0f0033b3-357e-42e6-9c94-cac650e1cd50,negative] +manila_tempest_tests.tests.api.test_quotas_negative.SharesQuotasNegativeTest.test_get_quotas_detail_with_wrong_version_3_2_24[api,id-0f0033b3-357e-42e6-9c94-cac650e1cd50,negative] +manila_tempest_tests.tests.api.test_quotas_negative.SharesQuotasNegativeTest.test_get_quotas_with_empty_tenant_id[api,id-d0dfe81d-8e8c-4847-a55f-95ba8a3d922c,negative] +manila_tempest_tests.tests.api.test_quotas_negative.SharesQuotasNegativeTest.test_try_reset_quotas_with_user[api,id-e7dbc580-1857-4f88-8886-988dc2f2c7b9,negative] +manila_tempest_tests.tests.api.test_quotas_negative.SharesQuotasNegativeTest.test_try_update_quotas_with_user[api,id-f1c8e16f-5406-4389-a29c-547cca8a56e0,negative] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_default_quotas_1_shares_client[api,id-a4649f11-41d0-4e73-ab5a-1466f2339b2f,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_default_quotas_2_shares_v2_client[api,id-a4649f11-41d0-4e73-ab5a-1466f2339b2f,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_quota_detail_2_52_no_share_replica_quotas[api,id-7bd5ac42-9fcb-477f-a253-02cde2bde661,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_1_shares_client[api,id-19525293-2b32-48c3-a7cd-cab279ac4631,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_2_shares_v2_client[api,id-19525293-2b32-48c3-a7cd-cab279ac4631,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_detail_1___2_25___True_[api,id-795614f6-4a18-47d5-b817-0b294e9d4c48,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_detail_2___2_25___False_[api,id-795614f6-4a18-47d5-b817-0b294e9d4c48,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_detail_3___2_53___True_[api,id-795614f6-4a18-47d5-b817-0b294e9d4c48,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_detail_4___2_53___False_[api,id-795614f6-4a18-47d5-b817-0b294e9d4c48,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_detail_5___2_88___True_[api,id-795614f6-4a18-47d5-b817-0b294e9d4c48,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_detail_6___2_88___False_[api,id-795614f6-4a18-47d5-b817-0b294e9d4c48,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_for_user_1_shares_client[api,id-1e670af7-9734-42a8-97b7-3080526f3aca,positive] +manila_tempest_tests.tests.api.test_quotas.SharesQuotasTest.test_show_quotas_for_user_2_shares_v2_client[api,id-1e670af7-9734-42a8-97b7-3080526f3aca,positive] +manila_tempest_tests.tests.api.test_replication_export_locations_negative.ReplicationExportLocationsNegativeTest.test_get_replica_export_location_for_invalid_export_id[api_with_backend,id-29ec890e-e324-4487-b2e3-f87642b6a58d,negative] +manila_tempest_tests.tests.api.test_replication_export_locations_negative.ReplicationExportLocationsNegativeTest.test_get_replica_export_location_for_non_replica[api_with_backend,id-f1b89548-4da7-4ce1-a3c0-363a6c1ea726,negative] +manila_tempest_tests.tests.api.test_replication_export_locations_negative.ReplicationExportLocationsNegativeTest.test_get_share_export_location_for_secondary_replica[api_with_backend,id-7a6318c2-71e9-45d2-aafe-5dcc0810a14a,negative] +manila_tempest_tests.tests.api.test_replication_export_locations_negative.ReplicationExportLocationsNegativeTest.test_list_replica_export_locations_for_invalid_replica[api,id-f60d7eec-ec29-4767-985f-ff8c9e771cae,negative] +manila_tempest_tests.tests.api.test_replication_export_locations.ReplicationExportLocationsTest.test_replica_export_locations[api_with_backend,id-10ab6304-a1cd-4e60-90e3-7f9358b8808a,positive] +manila_tempest_tests.tests.api.test_replication_export_locations.ReplicationExportLocationsTest.test_replicated_share_export_locations_1_2_46[api_with_backend,id-da22cfb8-7dd8-4bf1-87fc-a1f7b51ebf8e,positive] +manila_tempest_tests.tests.api.test_replication_export_locations.ReplicationExportLocationsTest.test_replicated_share_export_locations_2_2_47[api_with_backend,id-da22cfb8-7dd8-4bf1-87fc-a1f7b51ebf8e,positive] +manila_tempest_tests.tests.api.test_replication_export_locations.ReplicationExportLocationsTest.test_replicated_share_export_locations_3_2_88[api_with_backend,id-da22cfb8-7dd8-4bf1-87fc-a1f7b51ebf8e,positive] +manila_tempest_tests.tests.api.test_replication_export_locations.ReplicationExportLocationsTest.test_replicated_share_export_locations_with_promotion_1_2_46[api_with_backend,id-58430f57-c6eb-44e2-9583-eecb1dd10594,positive] +manila_tempest_tests.tests.api.test_replication_export_locations.ReplicationExportLocationsTest.test_replicated_share_export_locations_with_promotion_2_2_47[api_with_backend,id-58430f57-c6eb-44e2-9583-eecb1dd10594,positive] +manila_tempest_tests.tests.api.test_replication_export_locations.ReplicationExportLocationsTest.test_replicated_share_export_locations_with_promotion_3_2_88[api_with_backend,id-58430f57-c6eb-44e2-9583-eecb1dd10594,positive] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationAPIOnlyNegativeTest.test_get_replica_by_nonexistent_id[api,id-72395c9b-4432-4a8b-84b4-60303e6bc962,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationAPIOnlyNegativeTest.test_try_delete_replica_by_nonexistent_id[api,id-41d45772-e021-4404-a991-85f7a6e6431b,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_add_access_rule_share_replica_error_status_1_2_11[api_with_backend,id-600a13d2-5cf0-482e-97af-9f598b55a407,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_add_access_rule_share_replica_error_status_2_2_73[api_with_backend,id-600a13d2-5cf0-482e-97af-9f598b55a407,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_add_replica_to_share_with_error_state[api_with_backend,id-d14ba73e-4f13-451e-86d5-ffeb188e51b7,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_migration_of_replicated_share[api_with_backend,id-91b93b71-4048-412b-bb42-0fe88edfb987,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_promote_active_share_replica[api_with_backend,id-e525c3d0-9f45-489f-8e78-ed9e3ac540f7,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_promote_out_of_sync_share_replica[api_with_backend,id-b9c2e57b-f1ae-475c-9d0b-df75dbe93b61,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_promote_share_replica_for_writable_share_type[api_with_backend,id-37dc3f7e-5a36-4020-ae1b-3c5e6771f024,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_try_add_replica_nonexistent_subnet[api_with_backend,id-97fc5ab3-c6e2-4da4-ae45-08d458a5ba0c,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_try_add_replica_share_type_azs_unsupported_az[api_with_backend,id-bf01bcfc-57cb-4e56-957f-8aa9f1b9be1b,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_try_add_replica_to_share_with_no_replication_share_type[api_with_backend,id-90c5947d-b857-406c-b7dc-7010cbee6482,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_try_delete_last_active_replica[api_with_backend,id-94b41c28-e7a9-4f19-9315-4a57210141af,negative] +manila_tempest_tests.tests.api.test_replication_negative.ReplicationNegativeTest.test_try_delete_share_having_replica[api_with_backend,id-6fbf984d-4efa-41d7-88bf-11ef2f078c02,negative] +manila_tempest_tests.tests.api.test_replication_snapshots.ReplicationSnapshotTest.test_create_replica_from_snapshot_share[backend,id-913056a7-d897-4a1b-a7be-b2b2cf5d9572,positive] +manila_tempest_tests.tests.api.test_replication_snapshots.ReplicationSnapshotTest.test_delete_snapshot_after_adding_replica[backend,id-b4cd376b-1eb8-4780-801a-64b0c3bee1e4,positive] +manila_tempest_tests.tests.api.test_replication_snapshots.ReplicationSnapshotTest.test_snapshot_after_share_replica[backend,id-67fce02a-1d46-4186-8260-69e30b26195d,positive] +manila_tempest_tests.tests.api.test_replication_snapshots.ReplicationSnapshotTest.test_snapshot_before_and_after_share_replica[backend,id-a5372844-69ad-40e1-bbf3-265eb66af123,positive] +manila_tempest_tests.tests.api.test_replication_snapshots.ReplicationSnapshotTest.test_snapshot_before_share_replica[backend,id-ced8920b-6e3a-4bb4-9b3d-86e54377fcb7,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationActionsTest.test_detail_list_share_replicas_for_all_shares[api_with_backend,id-e39aeb5d-fe4b-4896-8615-e6e7290bcb56,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationActionsTest.test_detail_list_share_replicas_for_share[api_with_backend,id-f5225fb7-fcbe-4825-bf5b-0e11c2d26e03,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationActionsTest.test_show_share_replica[api_with_backend,id-abe0e49d-0b94-4b81-a220-ab047712492d,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationActionsTest.test_summary_list_share_replicas_for_all_shares[api_with_backend,id-8d11848a-7766-41e5-af09-6121e5bad447,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_active_replication_state[api_with_backend,id-1452156b-75a5-4f3c-a921-834732a03b0a,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_access_rule_create_replica_delete_rule[backend,id-00e12b41-b95d-494a-99be-e584aae10f5c,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_access_rule_share_replica_error_status[api_with_backend,id-600a13d2-5cf0-482e-97af-9f598b55a406,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_delete_share_replica_different_subnet[backend,id-58c3faf4-6c97-4fec-9a9b-7cff0d2035cd,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_delete_share_replica_multiple_subnets[backend,id-4235e789-dbd6-47a8-8f2e-d70edf78e532,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_delete_share_replica_network_allocation_update[backend,id-26694947-d4a0-46c8-99e8-2e0eca1b6a08,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_delete_share_replica[backend,id-8858617f-292d-4e5c-9e15-794b7f1b2e3c,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_multiple_share_replicas[backend,id-a542c179-ea41-4bc0-bd80-e06eaddf5253,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_add_rule_promote_share_replica_verify_rule[backend,id-3af912f4-b5d7-4241-b2b3-bdf12ff398a4,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_create_replica_add_access_rule_delete_replica[backend,id-3af3f19a-1195-464e-870b-1a3918914f1b,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_create_share_replica_with_provided_network_1_2_72[backend,id-c59e3198-062b-4284-8a3b-189a62213573,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_create_share_replica_with_provided_network_2_2_88[backend,id-c59e3198-062b-4284-8a3b-189a62213573,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_promote_and_promote_back[backend,id-b5ade58b-cb81-47eb-966b-28e6d85b5568,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_promote_in_sync_share_replica[backend,id-98b7c1d6-02e8-425a-b697-db2d2671fa11,positive] +manila_tempest_tests.tests.api.test_replication.ReplicationTest.test_share_type_azs_share_replicas[api_with_backend,id-7904e3c7-e6d0-472d-b9c9-c0772b4f9f1b,positive] +manila_tempest_tests.tests.api.test_resource_locks_negative.ResourceLockNegativeTestAPIOnly.test_create_resource_lock_invalid_resource[api,id-dd978cf7-1622-49e8-a6c8-3da4ac6c6f86,negative] +manila_tempest_tests.tests.api.test_resource_locks_negative.ResourceLockNegativeTestAPIOnly.test_delete_resource_lock_invalid[api,id-d5600bdc-72c8-43fd-9900-c112aa6c87fa,negative] +manila_tempest_tests.tests.api.test_resource_locks_negative.ResourceLockNegativeTestWithShares.test_create_resource_lock_invalid_lock_reason_too_long[api_with_backend,id-0057b3e7-c250-492d-805b-e355dff954ed,negative] +manila_tempest_tests.tests.api.test_resource_locks_negative.ResourceLockNegativeTestWithShares.test_create_resource_lock_invalid_resource_action[api_with_backend,id-658297a8-d675-471d-8a19-3d9e9af3a352,negative] +manila_tempest_tests.tests.api.test_resource_locks_negative.ResourceLockNegativeTestWithShares.test_delete_resource_lock_created_by_different_user[api_with_backend,id-00a8ef2b-8769-4aad-aefc-43fc579492f7,negative] +manila_tempest_tests.tests.api.test_resource_locks_negative.ResourceLockNegativeTestWithShares.test_update_resource_lock_created_by_different_user[api_with_backend,id-45b12120-0fc3-461f-8776-fdb92e599394,negative] +manila_tempest_tests.tests.api.test_resource_locks_negative.ResourceLockNegativeTestWithShares.test_update_resource_lock_invalid_param[api_with_backend,id-a2db3d29-b42f-4c0b-b484-afd32f91f747,negative] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockCRUTest.test_create_resource_lock[api_with_backend,id-a7f0fb6a-05ac-4afa-b8d9-04d20549bbd1,positive] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockCRUTest.test_get_resource_lock[api_with_backend,id-8cbf7331-f3a1-4c7b-ab1e-f8b938bf135e,positive] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockCRUTest.test_list_resource_locks_filtered[api_with_backend,id-22831edc-9d99-432d-a0b6-85af8853db98,positive] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockCRUTest.test_list_resource_locks_sorted_and_paginated[api_with_backend,id-72cc0d43-f676-4dd8-8a93-faa71608de98,positive] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockCRUTest.test_list_resource_locks[api_with_backend,id-f3d162a6-2ab4-433b-b8e7-6bf4f0bb6b0e,positive] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockCRUTest.test_update_resource_lock[api_with_backend,id-d7b51cde-ff4f-45ce-a237-401e8be5b4e5,positive] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockDeleteTest.test_delete_lock[api_with_backend,id-835fd617-4600-40a0-9ba1-40e5e0097b01,positive] +manila_tempest_tests.tests.api.test_resource_locks.ResourceLockDeleteTest.test_delete_locked_resource[api_with_backend,id-a96e70c7-0afe-4335-9abc-4b45ef778bd7,positive] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_error_share_to_snapshot_1_2_27[api_with_backend,id-108a451e-a8e9-450f-8f75-53883d58c6be,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_error_share_to_snapshot_2_2_88[api_with_backend,id-108a451e-a8e9-450f-8f75-53883d58c6be,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_error_snapshot_1_2_27[api_with_backend,id-7360ee16-ac7d-46ce-9c81-251d64fb7434,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_error_snapshot_2_2_88[api_with_backend,id-7360ee16-ac7d-46ce-9c81-251d64fb7434,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_invalid_snapshot_1_2_27[api_with_backend,id-ef706fad-5ac4-41dc-af81-5aa0331560cf,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_invalid_snapshot_2_2_88[api_with_backend,id-ef706fad-5ac4-41dc-af81-5aa0331560cf,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_missing_snapshot_1_2_27[api_with_backend,id-29024057-dbbd-4cf6-a796-207dfbd4c3ff,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_missing_snapshot_2_2_88[api_with_backend,id-29024057-dbbd-4cf6-a796-207dfbd4c3ff,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_second_latest_snapshot_1_2_27[api_with_backend,id-21dd8561-8913-42a2-a95c-74b536964c94,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot_negative.RevertToSnapshotNegativeTest.test_revert_to_second_latest_snapshot_2_2_88[api_with_backend,id-21dd8561-8913-42a2-a95c-74b536964c94,negative] +manila_tempest_tests.tests.api.test_revert_to_snapshot.RevertToSnapshotTest.test_revert_to_latest_snapshot_1_2_27[backend,id-196f2bc5-e13a-4730-ac51-61e339068a06,positive] +manila_tempest_tests.tests.api.test_revert_to_snapshot.RevertToSnapshotTest.test_revert_to_latest_snapshot_2_2_88[backend,id-196f2bc5-e13a-4730-ac51-61e339068a06,positive] +manila_tempest_tests.tests.api.test_revert_to_snapshot.RevertToSnapshotTest.test_revert_to_previous_snapshot_1_2_27[backend,id-09bd9942-7ef9-4d24-b2dd-f83bdda27b50,positive] +manila_tempest_tests.tests.api.test_revert_to_snapshot.RevertToSnapshotTest.test_revert_to_previous_snapshot_2_2_88[backend,id-09bd9942-7ef9-4d24-b2dd-f83bdda27b50,positive] +manila_tempest_tests.tests.api.test_revert_to_snapshot.RevertToSnapshotTest.test_revert_to_replicated_snapshot_1_2_27[backend,id-146de138-d351-49dc-a13a-5cdbed40b9ac,positive] +manila_tempest_tests.tests.api.test_revert_to_snapshot.RevertToSnapshotTest.test_revert_to_replicated_snapshot_2_2_88[backend,id-146de138-d351-49dc-a13a-5cdbed40b9ac,positive] +manila_tempest_tests.tests.api.test_rules_negative.ShareCephxRulesForCephFSNegativeTest.test_can_apply_new_cephx_rules_when_one_is_in_error_state[api_with_backend,id-1a9f46f0-d4e1-40ac-8726-aedd0320d583,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCephxRulesForCephFSNegativeTest.test_create_access_rule_cephx_admin_user[api_with_backend,id-16b7d848-2f7c-4709-85a3-2dfb4576cc59,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCephxRulesForCephFSNegativeTest.test_create_access_rule_cephx_with_invalid_cephx_id_1_jane_doe[api_with_backend,id-7b33c073-353e-4952-97dc-c3948a3cd037,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCephxRulesForCephFSNegativeTest.test_create_access_rule_cephx_with_wrong_level[api_with_backend,id-dd8be44c-c7e8-42fe-b81c-095a1c66730c,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCephxRulesForCephFSNegativeTest.test_different_tenants_cannot_use_same_cephx_id[api_with_backend,id-4ffed391-d7cc-481b-bb74-9f3406ddd75f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_to_snapshot_1_shares_client[api_with_backend,id-dd85d5cd-aa83-4f44-8572-bd7e68a84fb2,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_to_snapshot_2_shares_v2_client[api_with_backend,id-dd85d5cd-aa83-4f44-8572-bd7e68a84fb2,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_empty_common_name_1_shares_client[api_with_backend,id-a16d53d5-50d4-4015-912f-2850c5d62690,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_empty_common_name_2_shares_v2_client[api_with_backend,id-a16d53d5-50d4-4015-912f-2850c5d62690,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_too_big_common_name_1_shares_client[api_with_backend,id-1c25c134-92b4-4875-a061-88d394e28bcc,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_too_big_common_name_2_shares_v2_client[api_with_backend,id-1c25c134-92b4-4875-a061-88d394e28bcc,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_whitespace_common_name_1_shares_client[api_with_backend,id-7b5383d8-5bcd-47aa-955b-ed3757a5bdb4,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_whitespace_common_name_2_shares_v2_client[api_with_backend,id-7b5383d8-5bcd-47aa-955b-ed3757a5bdb4,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_wrong_share_id_1_shares_client[api_with_backend,id-eb47a511-7688-4689-a2ad-54ba85b39b07,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareCertRulesForGLUSTERFSNegativeTest.test_create_access_rule_cert_with_wrong_share_id_2_shares_v2_client[api_with_backend,id-eb47a511-7688-4689-a2ad-54ba85b39b07,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_add_access_rule_on_share_with_no_host[api_with_backend,id-d2856c7d-9417-416d-8d08-e68376ee5b2e,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_allow_access_multiple_visibility_locks_not_allowed[api_with_backend,id-f5b9e7c9-7e6b-4918-a1c4-e03c8d82c46a,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_01_1_2_3_256[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_02_1_1_1__[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_03_1_2_3_4_33[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_04_1_2_3__[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_05_1_2_3___23[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_06_1_2_3_1_23[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_07_1_2_3_1__1[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_08_1_2_3_1_[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_09_ad80__abaa_0_c2_2__3[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_10_AD80_ABAA___26[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_ip_with_wrong_target_11_2001_DB8_2de_0_0_0_0_e13_200a[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_with_wrong_level_1_shares_client[api_with_backend,id-e891deff-23d9-4872-911c-bd9b43dc797f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_access_rule_with_wrong_level_2_shares_v2_client[api_with_backend,id-e891deff-23d9-4872-911c-bd9b43dc797f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_duplicate_of_ip_rule_1_1_0[api_with_backend,id-efd594aa-dd24-427e-acdf-10d124afb572,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_duplicate_of_ip_rule_2_2_9[api_with_backend,id-efd594aa-dd24-427e-acdf-10d124afb572,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_duplicate_of_ip_rule_3_2_88[api_with_backend,id-efd594aa-dd24-427e-acdf-10d124afb572,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_duplicate_single_host_rules_1_10_20_30_40[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_duplicate_single_host_rules_2_fd8c_b029_bba6_ac54__1[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_duplicate_single_host_rules_3_fd2c_b029_bba6_df54__1_128[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_create_duplicate_single_host_rules_4_10_10_30_40_32[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_deny_access_unrestrict_other_user_rule[api_with_backend,id-478d3c84-b0ea-41c8-a860-e87f182d991c,positive] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_deny_access_without_unrestrict_as_owner_user_1_True[api_with_backend,id-c107b0b7-7a3e-4114-af64-ca8fe6e836c9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForCIFSNegativeTest.test_deny_access_without_unrestrict_as_owner_user_2_False[api_with_backend,id-c107b0b7-7a3e-4114-af64-ca8fe6e836c9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_add_access_rule_on_share_with_no_host[api_with_backend,id-d2856c7d-9417-416d-8d08-e68376ee5b2e,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_allow_access_multiple_visibility_locks_not_allowed[api_with_backend,id-f5b9e7c9-7e6b-4918-a1c4-e03c8d82c46a,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_01_1_2_3_256[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_02_1_1_1__[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_03_1_2_3_4_33[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_04_1_2_3__[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_05_1_2_3___23[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_06_1_2_3_1_23[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_07_1_2_3_1__1[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_08_1_2_3_1_[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_09_ad80__abaa_0_c2_2__3[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_10_AD80_ABAA___26[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_11_2001_DB8_2de_0_0_0_0_e13_200a[api_with_backend,id-16781b45-d2bb-4891-aa97-c28c0769d5bd,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_with_wrong_level_1_shares_client[api_with_backend,id-e891deff-23d9-4872-911c-bd9b43dc797f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_access_rule_with_wrong_level_2_shares_v2_client[api_with_backend,id-e891deff-23d9-4872-911c-bd9b43dc797f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_duplicate_of_ip_rule_1_1_0[api_with_backend,id-efd594aa-dd24-427e-acdf-10d124afb572,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_duplicate_of_ip_rule_2_2_9[api_with_backend,id-efd594aa-dd24-427e-acdf-10d124afb572,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_duplicate_of_ip_rule_3_2_88[api_with_backend,id-efd594aa-dd24-427e-acdf-10d124afb572,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_duplicate_single_host_rules_1_10_20_30_40[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_duplicate_single_host_rules_2_fd8c_b029_bba6_ac54__1[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_duplicate_single_host_rules_3_fd2c_b029_bba6_df54__1_128[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_create_duplicate_single_host_rules_4_10_10_30_40_32[api_with_backend,id-63932d1d-a60a-4af7-ba3b-7cf6c68aaee9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_deny_access_unrestrict_other_user_rule[api_with_backend,id-478d3c84-b0ea-41c8-a860-e87f182d991c,positive] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_deny_access_without_unrestrict_as_owner_user_1_True[api_with_backend,id-c107b0b7-7a3e-4114-af64-ca8fe6e836c9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareIpRulesForNFSNegativeTest.test_deny_access_without_unrestrict_as_owner_user_2_False[api_with_backend,id-c107b0b7-7a3e-4114-af64-ca8fe6e836c9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesAPIOnlyNegativeTest.test_create_access_rule_ip_with_wrong_share_id_1_shares_client[api,id-01279461-3ccc-49b2-a615-d7984dd0db8c,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesAPIOnlyNegativeTest.test_create_access_rule_ip_with_wrong_share_id_2_shares_v2_client[api,id-01279461-3ccc-49b2-a615-d7984dd0db8c,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesNegativeTest.test_create_access_rule_ip_to_snapshot_1_shares_client[api_with_backend,id-fd6ede10-97d6-4ee8-a661-c516b7421c91,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesNegativeTest.test_create_access_rule_ip_to_snapshot_2_shares_v2_client[api_with_backend,id-fd6ede10-97d6-4ee8-a661-c516b7421c91,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesNegativeTest.test_create_access_rule_ip_with_wrong_type_1_shares_client[api_with_backend,id-13f9329f-12db-467d-9268-a9cca75997d9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesNegativeTest.test_create_access_rule_ip_with_wrong_type_2_shares_v2_client[api_with_backend,id-13f9329f-12db-467d-9268-a9cca75997d9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesNegativeTest.test_delete_access_rule_with_wrong_id_1_shares_client[api_with_backend,id-84da9231-5c4b-4615-8500-8fc6d30ff7ea,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareRulesNegativeTest.test_delete_access_rule_with_wrong_id_2_shares_v2_client[api_with_backend,id-84da9231-5c4b-4615-8500-8fc6d30ff7ea,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_to_snapshot_1_shares_client[api_with_backend,id-bc62ce96-36fe-4c9b-b6b9-4d5a661c8035,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_to_snapshot_2_shares_v2_client[api_with_backend,id-bc62ce96-36fe-4c9b-b6b9-4d5a661c8035,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_empty_key_1_shares_client[api_with_backend,id-a4d8358d-dec0-4c2a-a544-182816a0ba6f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_empty_key_2_shares_v2_client[api_with_backend,id-a4d8358d-dec0-4c2a-a544-182816a0ba6f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_too_big_key_1_shares_client[api_with_backend,id-f8f4d3ee-82b8-4d37-917d-a0cd72073df4,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_too_big_key_2_shares_v2_client[api_with_backend,id-f8f4d3ee-82b8-4d37-917d-a0cd72073df4,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_too_little_key_1_shares_client[api_with_backend,id-f5252e86-4767-48ad-8be5-43e12c93df79,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_too_little_key_2_shares_v2_client[api_with_backend,id-f5252e86-4767-48ad-8be5-43e12c93df79,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_wrong_input_1_1_shares_client[api_with_backend,id-21724a99-0790-49d5-a069-d1df43782965,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_wrong_input_1_2_shares_v2_client[api_with_backend,id-21724a99-0790-49d5-a069-d1df43782965,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_wrong_input_2_1_shares_client[api_with_backend,id-d6148911-3a0c-4e1f-afdb-fcf203fe4a5b,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_wrong_input_2_2_shares_v2_client[api_with_backend,id-d6148911-3a0c-4e1f-afdb-fcf203fe4a5b,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_wrong_share_id_1_shares_client[api_with_backend,id-04d5b25f-b335-4574-82b0-f607c8b3bf25,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_user_with_wrong_share_id_2_shares_v2_client[api_with_backend,id-04d5b25f-b335-4574-82b0-f607c8b3bf25,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_with_wrong_level_1_shares_client[api_with_backend,id-301bdbd5-4398-4320-b334-7370995369e9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_create_access_rule_with_wrong_level_2_shares_v2_client[api_with_backend,id-301bdbd5-4398-4320-b334-7370995369e9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForCIFSNegativeTest.test_update_access_rule_with_wrong_level[api_with_backend,id-d5b1e7c9-7e6b-4918-a1c4-e03c8d82c46a,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_to_snapshot_1_shares_client[api_with_backend,id-bc62ce96-36fe-4c9b-b6b9-4d5a661c8035,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_to_snapshot_2_shares_v2_client[api_with_backend,id-bc62ce96-36fe-4c9b-b6b9-4d5a661c8035,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_empty_key_1_shares_client[api_with_backend,id-a4d8358d-dec0-4c2a-a544-182816a0ba6f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_empty_key_2_shares_v2_client[api_with_backend,id-a4d8358d-dec0-4c2a-a544-182816a0ba6f,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_too_big_key_1_shares_client[api_with_backend,id-f8f4d3ee-82b8-4d37-917d-a0cd72073df4,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_too_big_key_2_shares_v2_client[api_with_backend,id-f8f4d3ee-82b8-4d37-917d-a0cd72073df4,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_too_little_key_1_shares_client[api_with_backend,id-f5252e86-4767-48ad-8be5-43e12c93df79,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_too_little_key_2_shares_v2_client[api_with_backend,id-f5252e86-4767-48ad-8be5-43e12c93df79,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_wrong_input_1_1_shares_client[api_with_backend,id-21724a99-0790-49d5-a069-d1df43782965,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_wrong_input_1_2_shares_v2_client[api_with_backend,id-21724a99-0790-49d5-a069-d1df43782965,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_wrong_input_2_1_shares_client[api_with_backend,id-d6148911-3a0c-4e1f-afdb-fcf203fe4a5b,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_wrong_input_2_2_shares_v2_client[api_with_backend,id-d6148911-3a0c-4e1f-afdb-fcf203fe4a5b,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_wrong_share_id_1_shares_client[api_with_backend,id-04d5b25f-b335-4574-82b0-f607c8b3bf25,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_user_with_wrong_share_id_2_shares_v2_client[api_with_backend,id-04d5b25f-b335-4574-82b0-f607c8b3bf25,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_with_wrong_level_1_shares_client[api_with_backend,id-301bdbd5-4398-4320-b334-7370995369e9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_create_access_rule_with_wrong_level_2_shares_v2_client[api_with_backend,id-301bdbd5-4398-4320-b334-7370995369e9,negative] +manila_tempest_tests.tests.api.test_rules_negative.ShareUserRulesForNFSNegativeTest.test_update_access_rule_with_wrong_level[api_with_backend,id-d5b1e7c9-7e6b-4918-a1c4-e03c8d82c46a,negative] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_01___2_13____alice____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_02___2_13____alice____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_03___2_13____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_04___2_13____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_05___2_13____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_06___2_13____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_07___2_27____alice____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_08___2_27____alice____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_09___2_27____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_10___2_27____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_11___2_27____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_12___2_27____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_13___2_28____alice____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_14___2_28____alice____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_15___2_28____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_16___2_28____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_17___2_28____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_18___2_28____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_19___2_88____alice____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_20___2_88____alice____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_21___2_88____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_22___2_88____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_23___2_88____alice_bob____rw__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_create_delete_cephx_rule_24___2_88____alice_bob____ro__[backend,id-4e636fd2-26ef-4b63-96eb-77860a8b6cdf,positive] +manila_tempest_tests.tests.api.test_rules.ShareCephxRulesForCephFSTest.test_different_users_in_same_tenant_can_use_same_cephx_id[api_with_backend,id-ad907303-a439-4fcb-8845-fe91ecab7dc2,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_ro_access_rule_1_1_0[backend,id-cdd93d8e-7255-4ed4-8ef0-929a62bb302c,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_ro_access_rule_2_2_9[backend,id-cdd93d8e-7255-4ed4-8ef0-929a62bb302c,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_ro_access_rule_3_2_27[backend,id-cdd93d8e-7255-4ed4-8ef0-929a62bb302c,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_ro_access_rule_4_2_28[backend,id-cdd93d8e-7255-4ed4-8ef0-929a62bb302c,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_ro_access_rule_5_2_88[backend,id-cdd93d8e-7255-4ed4-8ef0-929a62bb302c,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_rule_1_1_0[backend,id-775ebc55-4a4d-4012-a030-2eeb7b6d2ce8,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_rule_2_2_9[backend,id-775ebc55-4a4d-4012-a030-2eeb7b6d2ce8,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_rule_3_2_27[backend,id-775ebc55-4a4d-4012-a030-2eeb7b6d2ce8,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_rule_4_2_28[backend,id-775ebc55-4a4d-4012-a030-2eeb7b6d2ce8,positive] +manila_tempest_tests.tests.api.test_rules.ShareCertRulesForGLUSTERFSTest.test_create_delete_cert_rule_5_2_88[backend,id-775ebc55-4a4d-4012-a030-2eeb7b6d2ce8,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rule_with_cidr_1___1_0___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rule_with_cidr_2___2_9___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rule_with_cidr_3___2_37___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rule_with_cidr_4___2_88___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rule_with_cidr_5___2_38___6_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rule_with_cidr_6___2_88___6_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rules_with_one_ip_1___1_0___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rules_with_one_ip_2___2_9___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rules_with_one_ip_3___2_37___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rules_with_one_ip_4___2_88___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rules_with_one_ip_5___2_38___6_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_access_rules_with_one_ip_6___2_88___6_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_ro_access_rule_1_1_0[backend,id-8fa0a15f-c04c-4521-91e7-020943bede8a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_ro_access_rule_2_2_9[backend,id-8fa0a15f-c04c-4521-91e7-020943bede8a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_ro_access_rule_3_2_27[backend,id-8fa0a15f-c04c-4521-91e7-020943bede8a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_ro_access_rule_4_2_28[backend,id-8fa0a15f-c04c-4521-91e7-020943bede8a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_create_delete_ro_access_rule_5_2_88[backend,id-8fa0a15f-c04c-4521-91e7-020943bede8a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForCIFSTest.test_update_access_rule[api_with_backend,id-02940881-6f95-77f8-b47d-0941c4e6bafb,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rule_with_cidr_1___1_0___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rule_with_cidr_2___2_9___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rule_with_cidr_3___2_37___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rule_with_cidr_4___2_88___4_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rule_with_cidr_5___2_38___6_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rule_with_cidr_6___2_88___6_[backend,id-5d25168a-d646-443e-8cf1-3151eb7887f5,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rules_with_one_ip_1___1_0___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rules_with_one_ip_2___2_9___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rules_with_one_ip_3___2_37___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rules_with_one_ip_4___2_88___4_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rules_with_one_ip_5___2_38___6_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_access_rules_with_one_ip_6___2_88___6_[backend,id-3390df2d-f6f8-4634-a562-87c1be994f6a,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_ro_access_rule_1_1_0[backend,id-187a4fb0-ba1d-45b9-83c9-f0272e7e6f3e,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_ro_access_rule_2_2_9[backend,id-187a4fb0-ba1d-45b9-83c9-f0272e7e6f3e,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_ro_access_rule_3_2_27[backend,id-187a4fb0-ba1d-45b9-83c9-f0272e7e6f3e,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_ro_access_rule_4_2_28[backend,id-187a4fb0-ba1d-45b9-83c9-f0272e7e6f3e,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_create_delete_ro_access_rule_5_2_88[backend,id-187a4fb0-ba1d-45b9-83c9-f0272e7e6f3e,positive] +manila_tempest_tests.tests.api.test_rules.ShareIpRulesForNFSTest.test_update_access_rule[api_with_backend,id-01940881-6f95-77f8-b47d-0941c4e6bafb,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_access_rules_deleted_if_share_deleted_1_1_0[api_with_backend,id-b77bcbda-9754-48f0-9be6-79341ad1af64,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_access_rules_deleted_if_share_deleted_2_2_9[api_with_backend,id-b77bcbda-9754-48f0-9be6-79341ad1af64,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_access_rules_deleted_if_share_deleted_3_2_27[api_with_backend,id-b77bcbda-9754-48f0-9be6-79341ad1af64,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_access_rules_deleted_if_share_deleted_4_2_28[api_with_backend,id-b77bcbda-9754-48f0-9be6-79341ad1af64,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_access_rules_deleted_if_share_deleted_5_2_88[api_with_backend,id-b77bcbda-9754-48f0-9be6-79341ad1af64,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_admin_delete_restricted_rules[api_with_backend,id-00202c6c-b4c7-4fa6-933a-562fbffde405,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_admin_listing_restricted_rules[api_with_backend,id-4829265a-eb32-400d-91a0-be06ce31a2ef,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_access_rules_1_1_0[api_with_backend,id-c52e95cc-d6ea-4d02-9b52-cd7c1913dfff,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_access_rules_2_2_9[api_with_backend,id-c52e95cc-d6ea-4d02-9b52-cd7c1913dfff,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_access_rules_3_2_27[api_with_backend,id-c52e95cc-d6ea-4d02-9b52-cd7c1913dfff,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_access_rules_4_2_28[api_with_backend,id-c52e95cc-d6ea-4d02-9b52-cd7c1913dfff,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_access_rules_5_2_45[api_with_backend,id-c52e95cc-d6ea-4d02-9b52-cd7c1913dfff,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_access_rules_6_2_88[api_with_backend,id-c52e95cc-d6ea-4d02-9b52-cd7c1913dfff,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_restricted_rules_from_other_user_1___2_81___True_[api_with_backend,id-3bca373e-e54f-49e1-8789-99a383cf4df3,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_restricted_rules_from_other_user_2___2_81___False_[api_with_backend,id-3bca373e-e54f-49e1-8789-99a383cf4df3,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_restricted_rules_from_other_user_3___2_88___True_[api_with_backend,id-3bca373e-e54f-49e1-8789-99a383cf4df3,positive] +manila_tempest_tests.tests.api.test_rules.ShareRulesTest.test_list_restricted_rules_from_other_user_4___2_88___False_[api_with_backend,id-3bca373e-e54f-49e1-8789-99a383cf4df3,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_ro_access_rule_1_1_0[backend,id-ee11084d-6c1d-4856-8044-9aa9e6c670fb,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_ro_access_rule_2_2_9[backend,id-ee11084d-6c1d-4856-8044-9aa9e6c670fb,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_ro_access_rule_3_2_27[backend,id-ee11084d-6c1d-4856-8044-9aa9e6c670fb,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_ro_access_rule_4_2_28[backend,id-ee11084d-6c1d-4856-8044-9aa9e6c670fb,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_ro_access_rule_5_2_88[backend,id-ee11084d-6c1d-4856-8044-9aa9e6c670fb,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_user_rule_1_1_0[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_user_rule_2_2_9[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_user_rule_3_2_27[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_user_rule_4_2_28[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForCIFSTest.test_create_delete_user_rule_5_2_88[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_ro_access_rule_1_1_0[backend,id-ccb08342-b7ef-4dda-84ba-8de9879d8862,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_ro_access_rule_2_2_9[backend,id-ccb08342-b7ef-4dda-84ba-8de9879d8862,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_ro_access_rule_3_2_27[backend,id-ccb08342-b7ef-4dda-84ba-8de9879d8862,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_ro_access_rule_4_2_28[backend,id-ccb08342-b7ef-4dda-84ba-8de9879d8862,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_ro_access_rule_5_2_88[backend,id-ccb08342-b7ef-4dda-84ba-8de9879d8862,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_user_rule_1_1_0[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_user_rule_2_2_9[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_user_rule_3_2_27[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_user_rule_4_2_28[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_rules.ShareUserRulesForNFSTest.test_create_delete_user_rule_5_2_88[backend,id-1f87565f-c3d9-448d-b89a-387d6c2fdae6,positive] +manila_tempest_tests.tests.api.test_scheduler_hints_negative.SharesSchedulerHintsNegativeTest.test_scheduler_hint_with_invalid_hint[api_with_backend,id-6f0c5561-8a6a-4cfb-bbe7-84ffc39bf78d,negative] +manila_tempest_tests.tests.api.test_scheduler_hints_negative.SharesSchedulerHintsNegativeTest.test_scheduler_hint_with_invalid_share_id[api_with_backend,id-2228a187-4f03-4195-9e23-fa1a42110fdc,negative] +manila_tempest_tests.tests.api.test_scheduler_hints.SharesSchedulerHintsTest.test_different_host_scheduler_hint_in_share_creation[api_with_backend,id-6569e0c3-43c9-4ee2-84ff-ea7fa8da8110,positive] +manila_tempest_tests.tests.api.test_scheduler_hints.SharesSchedulerHintsTest.test_same_host_scheduler_hint_in_share_creation[api_with_backend,id-f96d5836-bfc9-4c22-888e-3f62d731573c,positive] +manila_tempest_tests.tests.api.test_scheduler_stats_negative.SchedulerStatsNegativeTest.test_try_list_pools_detailed_with_user[api,id-f89136cf-34a1-475b-8514-3114f9f159a5,negative] +manila_tempest_tests.tests.api.test_scheduler_stats_negative.SchedulerStatsNegativeTest.test_try_list_pools_with_user[api,id-33db76eb-7501-43f9-8c4f-8e6e35519a4d,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_add_empty_sec_service_id_to_share_network[api,id-85dd5693-a89c-4d05-9416-0e11fbba23f5,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_add_nonexistant_sec_service_to_share_network[api,id-3f7af51f-3afa-495c-94b7-e9d29f06cf1d,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_add_sec_service_to_nonexistant_share_network[api,id-d9af5086-ace9-4be3-8119-e765699c0c91,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_add_sec_service_to_share_network_with_empty_id[api,id-7272426d-ab58-4efb-a490-0c78c07fa7fe,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_add_sec_service_twice_to_share_network[api,id-e3d17444-8ed4-445e-bc65-c748dbc5d21f,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_delete_nonexistant_sec_service_from_share_network[api,id-7f8d7527-2d62-478a-ab19-213156777612,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_delete_nonexistant_ss_from_nonexistant_sn[api,id-c7c2f66f-81f8-4984-b807-2b9520105a33,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_delete_sec_service_from_nonexistant_share_network[api,id-be1c9c79-efa1-471e-920b-da4733ad383e,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_delete_ss_from_sn_used_by_share_server[api_with_backend,id-eb66a8f7-b549-4cf1-8719-30844fb151b6,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_list_sec_services_for_nonexistant_share_network[api,id-f87aefa6-9681-477d-a118-603883849f4f,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_try_delete_ss_that_assigned_to_sn[api,id-d422a15a-1f4c-4531-a092-9216b90c4179,negative] +manila_tempest_tests.tests.api.test_security_services_mapping_negative.SecServicesMappingNegativeTest.test_try_map_two_ss_with_same_type_to_sn[api,id-6a15c8ff-eba3-40e5-8fa1-6eab52338672,negative] +manila_tempest_tests.tests.api.test_security_services_mapping.SecurityServicesMappingTest.test_map_ss_to_sn_and_delete[api,id-9dd352b2-6d47-4cab-aa61-52d8081f67a2,positive] +manila_tempest_tests.tests.api.test_security_services_mapping.SecurityServicesMappingTest.test_map_ss_to_sn_and_list[api,id-e8c5b4d5-7ad2-4aa7-bab0-b454a2e150e9,positive] +manila_tempest_tests.tests.api.test_security_services_mapping.SecurityServicesMappingTest.test_remap_ss_to_sn[api,id-2b0bd5cc-eb35-430f-acfd-f80a2e467667,positive] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_get_deleted_security_service[api,id-288dbf42-ee22-4445-8363-7ebb1c3d89c9,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_create_security_service_with_empty_type[api,id-f5cdf074-f5d4-4d9e-990b-c3d9385dfc2b,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_create_security_service_with_wrong_type[api,id-6a7efebc-989a-42b3-a971-c28af1e209f5,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_delete_security_service_with_wrong_type[api,id-c94eb229-5caf-4e81-b8a9-d73c0b63a93b,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_delete_security_service_without_id[api,id-887e0c2c-6658-442d-92e7-3b0c86a2dfcd,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_get_security_service_with_wrong_id[api,id-34e923a4-e5b4-4375-88b4-8bb05e8f010d,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_get_security_service_without_id[api,id-2935de41-40b1-4907-8ab5-f4921f670bfd,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_update_invalid_keys_sh_server_exists[api_with_backend,id-ab62240a-219c-4fa7-b1ca-08f91bec76f0,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_update_nonexistant_security_service[api,id-60e51b95-af7f-463d-8d8e-6bca4ed3bec8,negative] +manila_tempest_tests.tests.api.test_security_services_negative.SecurityServicesNegativeTest.test_try_update_security_service_with_empty_id[api,id-e4554b3b-bc74-4204-8596-d502149bb408,negative] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_create_delete_security_service[api,id-70927e29-4a6a-431a-bbc1-76bc419e0579,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_get_security_service_1_1_0[api,id-bb052be4-0176-4613-b7d5-e12bef391ddb,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_get_security_service_2_2_43[api,id-bb052be4-0176-4613-b7d5-e12bef391ddb,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_get_security_service_3_2_44[api,id-bb052be4-0176-4613-b7d5-e12bef391ddb,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_get_security_service_4_2_88[api,id-bb052be4-0176-4613-b7d5-e12bef391ddb,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services_detailed_filter_by_ss_attributes[api,id-f055faad-dd36-4eed-9b50-61280931dea2,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services_filter_by_invalid_opt[api,id-8d9af272-df89-470d-9ff8-92ba774c9fff,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services_filter_by_share_network[api,id-88f62835-0aee-4bed-a37f-ffd99430da8a,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services_with_detail_1_1_0[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services_with_detail_2_2_42[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services_with_detail_3_2_44[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services_with_detail_4_2_88[api,id-22b22937-7436-458c-ac22-8ff19feab253,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_list_security_services[api,id-f6f5657c-a93c-49ed-86e3-b351a92734d5,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_try_list_security_services_all_tenants_ignored[api,id-d501710e-4710-4c13-a373-75ed6ababb13,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_try_update_valid_keys_sh_server_exists[api_with_backend,id-c3c04992-da11-4677-9098-eff3f4231a4b,positive] +manila_tempest_tests.tests.api.test_security_services.SecurityServicesTest.test_update_security_service[api,id-84d47747-13c8-4ab9-9fc4-a43fbb29ad18,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_create_share_group_from_populated_share_group_snapshot_1_2_31[api_with_backend,id-650c5fa7-11f2-48bd-b012-fc2e32b6f446,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_create_share_group_from_populated_share_group_snapshot_2_2_55[api_with_backend,id-650c5fa7-11f2-48bd-b012-fc2e32b6f446,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_create_share_group_from_populated_share_group_snapshot_3_2_88[api_with_backend,id-650c5fa7-11f2-48bd-b012-fc2e32b6f446,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_filter_shares_by_share_group_id_min[api_with_backend,id-e72be2f9-56db-467f-89d7-0dddbf7e37e9,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_group_1_2_31[api_with_backend,id-1e359389-09a7-4235-84c9-7b5c83632fff,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_group_2_2_55[api_with_backend,id-1e359389-09a7-4235-84c9-7b5c83632fff,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_group_3_2_88[api_with_backend,id-1e359389-09a7-4235-84c9-7b5c83632fff,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_group_snapshot_1_2_31[api_with_backend,id-5d2ca4f5-04da-4528-af47-ec980b95e884,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_group_snapshot_2_2_55[api_with_backend,id-5d2ca4f5-04da-4528-af47-ec980b95e884,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_group_snapshot_3_2_88[api_with_backend,id-5d2ca4f5-04da-4528-af47-ec980b95e884,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_group_snapshot_members_min[api_with_backend,id-67e8c099-f1c1-4972-9c51-bb7bfe1d7994,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_get_share_min_supported_sg_microversion[api_with_backend,id-45b77673-b1bb-43a1-b4b7-41351930adbd,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_list_share_groups_1_2_31[api_with_backend,id-04fcd695-c5f8-4de7-ab09-131424e6bdfb,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_list_share_groups_2_2_55[api_with_backend,id-04fcd695-c5f8-4de7-ab09-131424e6bdfb,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_list_share_groups_3_2_88[api_with_backend,id-04fcd695-c5f8-4de7-ab09-131424e6bdfb,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_list_share_groups_with_detail_min_1_2_31[api_with_backend,id-16986c21-ecbc-429e-ab3d-8d1596a3eac4,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_list_share_groups_with_detail_min_2_2_36[api_with_backend,id-16986c21-ecbc-429e-ab3d-8d1596a3eac4,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_list_share_groups_with_detail_min_3_2_55[api_with_backend,id-16986c21-ecbc-429e-ab3d-8d1596a3eac4,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupActionsTest.test_list_share_groups_with_detail_min_4_2_88[api_with_backend,id-16986c21-ecbc-429e-ab3d-8d1596a3eac4,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupRenameTest.test_create_update_read_share_group_with_unicode_1_2_31[api_with_backend,id-611b1555-df09-499b-8aef-e8261e3f6863,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupRenameTest.test_create_update_read_share_group_with_unicode_2_2_55[api_with_backend,id-611b1555-df09-499b-8aef-e8261e3f6863,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupRenameTest.test_create_update_read_share_group_with_unicode_3_2_88[api_with_backend,id-611b1555-df09-499b-8aef-e8261e3f6863,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupRenameTest.test_update_share_group_1_2_31[api_with_backend,id-7f0a07ce-afdd-4c51-a29c-d8fe6cb5f6a5,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupRenameTest.test_update_share_group_2_2_55[api_with_backend,id-7f0a07ce-afdd-4c51-a29c-d8fe6cb5f6a5,positive] +manila_tempest_tests.tests.api.test_share_group_actions.ShareGroupRenameTest.test_update_share_group_3_2_88[api_with_backend,id-7f0a07ce-afdd-4c51-a29c-d8fe6cb5f6a5,positive] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_group_with_nonexistent_share_network_id_value_min[api_with_backend,id-ea817e28-08b9-40c1-bbab-a8820ec564ac,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_and_share_with_different_azs[api_with_backend,id-64527564-9cd6-42db-8897-910f4fc1a151,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_snapshot_with_invalid_sg_id_value_min[api_with_backend,id-25a829e2-be7d-4a4d-881e-bc0634515985,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_snapshot_with_nonexistent_sg_id_value_min[api_with_backend,id-16ad5a77-0ef7-4906-8e14-56703c2c9d71,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_with_invalid_share_network_id_value_min[api_with_backend,id-aae1b1db-ea04-4a53-88ad-e5ee648fe938,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_with_invalid_share_type_id_value_min[api_with_backend,id-8bc89858-61de-49f3-868c-394841a93503,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_with_invalid_source_sg_snapshot_id_value_min[api_with_backend,id-7ce3fb52-1bec-42b1-9b4f-671c8465764b,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_with_nonexistent_az_min[api_with_backend,id-8fc20c22-082f-4851-bcc3-d2f3af57f027,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_with_nonexistent_share_type_id_value_min[api_with_backend,id-97183b15-d150-4c6b-b812-734c2500afe7,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_create_sg_with_nonexistent_source_sg_snapshot_id_value_min[api_with_backend,id-43c7a454-06b4-4c6e-8aaf-34709db64e28,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_delete_sg_containing_a_share_min[api_with_backend,id-c2225b19-d5f5-4d15-bb9a-de63bfce6760,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_delete_sg_in_use_by_sg_snapshot_min[api_with_backend,id-18fe2dee-4a07-484e-8f0f-bbc238500dc3,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_delete_sg_with_invalid_id_min[api_with_backend,id-bf8f9865-a906-4d55-b233-fec57cf43b66,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_delete_sg_without_passing_sg_id_min[api_with_backend,id-16431501-800f-4695-bae6-6a4c715c2fbf,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_delete_share_in_use_by_sg_snapshot_min[api_with_backend,id-d2a58f10-cc86-498d-a5e0-1468d4345852,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_filter_shares_on_empty_share_group_id_min[api_with_backend,id-f805f683-fe05-4534-9f40-a74be42ff82b,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_filter_shares_on_invalid_group_id_min[api_with_backend,id-edd329b8-7188-481f-9445-8f6d913538fa,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_filter_shares_on_nonexistent_group_id_min[api_with_backend,id-5dc10968-cbff-46d9-a1aa-bafccc7a1905,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_get_sg_with_invalid_id_min[api_with_backend,id-d486a184-1160-4664-ad9b-6f8974685343,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_get_sg_without_passing_group_id_min[api_with_backend,id-7a5dcec1-27cc-4bfe-a166-d8c1832a4dc7,negative] +manila_tempest_tests.tests.api.test_share_groups_negative.ShareGroupsNegativeTest.test_update_sg_with_invalid_id_min[api_with_backend,id-9127933a-1960-415e-bf21-8bcb5720e371,negative] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_delete_empty_share_group_snapshot_min[api_with_backend,id-cf7984af-1e1d-4eaf-bf9a-d8ddf5cebd01,positive] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_populate_delete_share_group_min[api_with_backend,id-809d5e3d-5a4b-458a-a985-853d59800da5,positive] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_sg_and_share_specifying_az_1_sg[api_with_backend,id-14fd6d88-87ff-4af2-ad17-f95dbd8dcd61,positive] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_sg_and_share_specifying_az_2_sg_and_share[api_with_backend,id-14fd6d88-87ff-4af2-ad17-f95dbd8dcd61,positive] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_sg_and_share_specifying_az_3_none[api_with_backend,id-14fd6d88-87ff-4af2-ad17-f95dbd8dcd61,positive] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_sg_and_share_with_multiple_subnets_1_False[api_with_backend,id-17fd1867-03a3-43d0-9be3-daf90b6c5e02,positive] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_sg_and_share_with_multiple_subnets_2_True[api_with_backend,id-17fd1867-03a3-43d0-9be3-daf90b6c5e02,positive] +manila_tempest_tests.tests.api.test_share_groups.ShareGroupsTest.test_create_share_group_from_empty_share_group_snapshot_min[api_with_backend,id-727d9c69-4c3b-4375-a91b-8b3efd349976,positive] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_add_share_network_subnet_az_not_found[api,id-68264ac2-57e5-47ea-aa71-198c020be975,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_add_share_network_subnet_in_same_az_exists_1_True[api,id-13f397bf-5e3a-42b0-b4f9-9cd2dbbb0955,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_add_share_network_subnet_in_same_az_exists_2_False[api,id-13f397bf-5e3a-42b0-b4f9-9cd2dbbb0955,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_add_share_network_subnet_missing_parameters[api,id-36e991c1-f09b-47dc-adc9-3da388b14612,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_add_share_network_subnet_share_network_not_found[api,id-d20b6105-22d1-4fc0-8468-45dd019240c0,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_delete_contains_shares[api_with_backend,id-5088edfe-2800-4dec-a982-42c864bba954,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_delete_contains_unmanaged_share_servers[api_with_backend,id-bf9d63c4-37fb-4646-add7-905b196316f1,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_delete_subnet_not_found[api,id-a5bdc817-0c34-4856-9732-f561e6ab5a83,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_delete_subnet_share_network_not_found[api,id-1025b69b-f1fd-4e44-b749-33779545ffe8,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_get_deleted_subnet[api,id-9dfc6ba3-c7be-4a6d-b104-ce77d2337bb3,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_show_subnet_not_found[api,id-22d4528b-39ae-4f97-b1b2-1b9d65146c97,negative] +manila_tempest_tests.tests.api.test_share_network_subnets_negative.ShareNetworkSubnetsNegativeTest.test_show_subnet_share_network_not_found[api,id-11e76b07-f3e3-401f-9469-256b93a768e6,negative] +manila_tempest_tests.tests.api.test_share_network_subnets.ShareNetworkSubnetsTest.test_create_delete_subnet[api,id-3e1e4da7-049f-404e-8673-142695a9a785,positive] +manila_tempest_tests.tests.api.test_share_network_subnets.ShareNetworkSubnetsTest.test_create_share_on_share_network_with_multiple_subnets_1_True[api_with_backend,id-043fbe02-466d-4344-8e2f-f02cb65132cb,positive] +manila_tempest_tests.tests.api.test_share_network_subnets.ShareNetworkSubnetsTest.test_create_share_on_share_network_with_multiple_subnets_2_False[api_with_backend,id-043fbe02-466d-4344-8e2f-f02cb65132cb,positive] +manila_tempest_tests.tests.api.test_share_network_subnets.ShareNetworkSubnetsTest.test_create_share_on_subnet_with_availability_zone[api_with_backend,id-89ed6115-eb1d-4a7e-a0a3-9b4a239fadc1,positive] +manila_tempest_tests.tests.api.test_share_network_subnets.ShareNetworkSubnetsTest.test_show_share_network_subnet[api,id-51c6836a-c6d2-4b80-a992-cf91f9a4332b,positive] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_check_add_share_network_subnet_az_not_found[api,id-d9a487fb-6638-4f93-8b69-3e1a85bfbc7d,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_check_add_share_network_subnet_share_network_not_found[api,id-f6f47c64-6821-4d4a-aa7d-3b0244158197,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_create_share_network_inexistent_az[api_with_backend,id-d84c3c5c-5913-42d4-9a66-0d5a78295adb,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_delete_share_network_contains_more_than_one_subnet[api_with_backend,id-8a995305-ede9-4002-a9cd-f24ff4d71f63,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_list_shares_with_like_filter_not_exist[api_with_backend,id-4e71de31-1064-40da-948d-a72063fbd647,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_delete_share_network_with_existing_shares[api_with_backend,id-6e4912fd-ae85-4a43-81e8-e5b340099b64,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_delete_share_network_with_wrong_type[api,id-7e22e8b9-a1ce-480c-89b3-b4edd807285a,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_delete_share_network_without_id[api,id-fe6ac194-5003-404c-b372-7515a58ff969,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_get_deleted_share_network[api,id-9166b81c-d6ab-4592-bcf7-9410250e30dd,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_get_share_network_with_wrong_id[api,id-80397850-2f64-48b3-b19b-79c4ac0bd58f,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_get_share_network_without_id[api,id-66289664-bf01-40dd-a76d-fd2c953bbceb,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_list_share_networks_wrong_created_before_value[api,id-c96dacaf-4cea-4fe9-bbd7-c9b1001f5495,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_list_share_networks_wrong_created_since_value[api,id-0d104b72-aab5-48b5-87f8-847d2155faa9,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_update_invalid_keys_sh_server_exists[api_with_backend,id-211b64b4-4c2b-4b6b-b011-725f40a37b03,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_update_nonexistant_share_network[api,id-a7c55dbe-c23e-403f-b8aa-4aa1128f32a4,negative] +manila_tempest_tests.tests.api.test_share_networks_negative.ShareNetworksNegativeTest.test_try_update_share_network_with_empty_id[api,id-984349ca-df7d-4f85-a45f-948189debb65,negative] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_create_delete_share_network[api,id-b998a594-f630-475d-b46f-e4143caf61fb,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_create_two_share_networks_with_same_net_and_subnet[api,id-be5f4f60-493e-47ea-a5bd-f16dfaa98c5c,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_gateway_mtu_neutron_net_id_with_neutron[api_with_backend,id-2dbf91da-04ae-4f9f-a7b9-0299c6b2e02c,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_get_share_network[api,id-55990ec2-37f0-483f-9c67-76fd6f377cc1,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks_all_filter_opts[api,id-27490442-60f8-4514-a10a-194a400b48bb,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks_filter_by_ss[api,id-6d13a090-0855-40c0-85b1-424f6878c6ce,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks_like_filter[api,id-bff1356e-70aa-4bbe-b398-cb4dadd8fcb1,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks_with_detail[api,id-285c7a91-1703-42a5-86c8-2463edde60e2,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_list_share_networks[api,gate,id-41c635b1-d9ef-4c05-9100-5e4b0034b523,positive,smoke] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_recreate_share_network[api,id-7595a844-a28e-476c-89f1-4d3193ce9d5b,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_try_list_share_networks_all_tenants[api,id-18fe9031-cefc-4df3-bbb0-6541f5fda12b,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_try_list_share_networks_project_id[api,id-caf6635a-ecb8-4981-9776-11dcfdf3cdbc,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_update_share_network[api,id-1837fdd3-8068-4e88-bc50-9224498f84c0,positive] +manila_tempest_tests.tests.api.test_share_networks.ShareNetworksTest.test_update_valid_keys_sh_server_exists[api_with_backend,id-198a5c08-3aaf-4623-9720-95d33ebe3376,positive] +manila_tempest_tests.tests.api.test_share_servers_multiple_subnet_negative.ShareServerMultipleSubNegativeTest.test_create_share_multiple_subnets_to_unsupported_backend[api_with_backend,id-1e2a9415-b02f-4c02-812d-bedc361f92ce,negative] +manila_tempest_tests.tests.api.test_share_servers_multiple_subnet.ShareServerMultipleSubnetTest.test_create_share_on_multiple_subnets_same_az[backend,id-5600bd52-ecb4-47d3-a4e8-3e6565cb0b80,positive] +manila_tempest_tests.tests.api.test_share_servers_multiple_subnet.ShareServerMultipleSubnetTest.test_create_share_with_network_allocation_update[backend,id-2a9debd5-47a3-42cc-823b-2b9de435a5e4,positive] +manila_tempest_tests.tests.api.test_share_servers_multiple_subnet.ShareServerMultipleSubnetTest.test_share_network_subnet_create_check[backend,id-2624f9a7-660b-4f91-89b8-c026b3bb8d1f,positive] +manila_tempest_tests.tests.api.test_share_transfers_negative.ShareTransferNegativeTest.test_accept_transfer_with_incorrect_auth_key[api,id-05a6a345-7609-421f-be21-d79041970674,negative] +manila_tempest_tests.tests.api.test_share_transfers_negative.ShareTransferNegativeTest.test_accept_transfer_without_auth_key[api_with_backend,id-cc7af032-0504-417e-8ab9-73b37bed7f85,negative] +manila_tempest_tests.tests.api.test_share_transfers_negative.ShareTransferNegativeTest.test_delete_nonexistent_transfer[api,id-085d5971-fe6e-4497-93cb-f1eb176a10da,negative] +manila_tempest_tests.tests.api.test_share_transfers_negative.ShareTransferNegativeTest.test_delete_transfer_of_other_tenants[api_with_backend,id-b3e26356-5eb0-4f73-b5a7-d3594cc2f30e,negative] +manila_tempest_tests.tests.api.test_share_transfers_negative.ShareTransferNegativeTest.test_show_nonexistent_transfer[api,id-4b9e75b1-4ac6-4111-b09e-e6dacd0ac2c3,negative] +manila_tempest_tests.tests.api.test_share_transfers_negative.ShareTransferNegativeTest.test_show_transfer_of_other_tenants[api_with_backend,id-baf66f62-253e-40dd-a6a9-109bc7613e52,negative] +manila_tempest_tests.tests.api.test_share_transfers.ShareTransferTest.test_create_and_accept_share_transfer[api_with_backend,id-3c2622ab-3368-4693-afb6-e60bd27e61ef,positive] +manila_tempest_tests.tests.api.test_share_transfers.ShareTransferTest.test_create_and_delete_share_transfer[api_with_backend,id-716e71a0-8265-4410-9170-08714095d9e8,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones_negative.ShareTypeAvailabilityZonesNegativeTest.test_share_type_azs_create_with_invalid_az_spec_1_az1______az2__az_3___[api_with_backend,id-7a23af8d-a995-43ac-a381-2cd5c5f57f39,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones_negative.ShareTypeAvailabilityZonesNegativeTest.test_share_type_azs_create_with_invalid_az_spec_2_az1__az_3[api_with_backend,id-7a23af8d-a995-43ac-a381-2cd5c5f57f39,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones_negative.ShareTypeAvailabilityZonesNegativeTest.test_share_type_azs_create_with_invalid_az_spec_3__az2___az_3[api_with_backend,id-7a23af8d-a995-43ac-a381-2cd5c5f57f39,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones_negative.ShareTypeAvailabilityZonesNegativeTest.test_share_type_azs_filter_by_invalid_azs_extra_spec[api,id-78f1afba-6087-415c-aaa1-a880ae341bfc,negative] +manila_tempest_tests.tests.api.test_share_type_availability_zones_negative.ShareTypeAvailabilityZonesNegativeTest.test_share_type_azs_share_groups_unsupported[api,id-5712abae-52a5-46ec-8850-46e732b18b06,negative] +manila_tempest_tests.tests.api.test_share_type_availability_zones_negative.ShareTypeAvailabilityZonesNegativeTest.test_share_type_azs_shares_unsupported_az[api,id-9285d7d5-cb40-462d-998c-891f0f934fdd,negative] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_create_and_get_share_type_1_az1______az2__az_3__[api_with_backend,id-23005fad-59e1-4a4b-994c-7377e34b4ec6,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_create_and_get_share_type_2_az1_az2_az_3[api_with_backend,id-23005fad-59e1-4a4b-994c-7377e34b4ec6,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_create_and_get_share_type_3_az1____az2____az_3[api_with_backend,id-23005fad-59e1-4a4b-994c-7377e34b4ec6,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_filter_by_availability_zones_1_az1[api_with_backend,id-545f1742-39dc-4bfb-8a81-6ec941b58fef,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_filter_by_availability_zones_2_az2[api_with_backend,id-545f1742-39dc-4bfb-8a81-6ec941b58fef,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_filter_by_availability_zones_3_az_3[api_with_backend,id-545f1742-39dc-4bfb-8a81-6ec941b58fef,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_filter_by_availability_zones_4_az1___az_3[api_with_backend,id-545f1742-39dc-4bfb-8a81-6ec941b58fef,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_filter_by_availability_zones_5_az_3__az1[api_with_backend,id-545f1742-39dc-4bfb-8a81-6ec941b58fef,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_filter_by_availability_zones_6_az2__az_3__az1[api_with_backend,id-545f1742-39dc-4bfb-8a81-6ec941b58fef,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_old_version_api_ignores_spec[api_with_backend,id-00a3e4ab-00a5-4683-a773-166353a69f28,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_share_groups_az_in_create_req_1_True[api_with_backend,id-fa5618ad-c452-41bd-8dc3-cc85abd6cee5,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_share_groups_az_in_create_req_2_False[api_with_backend,id-fa5618ad-c452-41bd-8dc3-cc85abd6cee5,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_shares_az_in_create_req_1_True[api_with_backend,id-3399c706-5dcb-41b2-9701-4550bf4f81cb,positive] +manila_tempest_tests.tests.api.test_share_type_availability_zones.ShareTypeAvailabilityZonesTest.test_share_type_azs_shares_az_in_create_req_2_False[api_with_backend,id-3399c706-5dcb-41b2-9701-4550bf4f81cb,positive] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_bad_request_1_2_50[api,id-4a22945c-8988-43a1-88c9-eb86e6abcd8e,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_bad_request_2_2_88[api,id-4a22945c-8988-43a1-88c9-eb86e6abcd8e,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_bad_request_3_2_50_bad_public[api,id-4a22945c-8988-43a1-88c9-eb86e6abcd8e,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_bad_request_4_2_88_bad_public[api,id-4a22945c-8988-43a1-88c9-eb86e6abcd8e,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_bad_request_5_2_50_description[api,id-4a22945c-8988-43a1-88c9-eb86e6abcd8e,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_bad_request_6_2_88_description[api,id-4a22945c-8988-43a1-88c9-eb86e6abcd8e,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_conflict_1_2_50[api,id-7193465a-ed8e-44d5-9ca9-4e8a3c5958e0,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_share_type_update_conflict_2_2_88[api,id-7193465a-ed8e-44d5-9ca9-4e8a3c5958e0,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_try_add_access_to_share_type_with_user[api,id-06203276-f6a3-4a07-a014-8749763395d6,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_try_create_share_type_with_user[api,id-d6a6ac4d-6582-408d-ba55-6f5128eb940e,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_try_delete_share_type_with_user[api,id-857c664f-e634-4865-ba05-bdcd4336725d,negative] +manila_tempest_tests.tests.api.test_share_types_negative.ShareTypesNegativeTest.test_try_remove_access_from_share_type_with_user[api,id-08b2d093-2ad8-46aa-8112-81d50547f36d,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_get_share_from_other_project[api_with_backend,id-e8f857f1-ec32-4f81-9e09-26065891dc93,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_description_not_exist[api_with_backend,id-5b0ceae1-357f-4b51-81a6-88973ea20c16,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_export_location_and_invalid_version_1_path[api_with_backend,id-ff307c91-3bb9-48b5-926c-5a2747320151,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_export_location_and_invalid_version_2_id[api_with_backend,id-ff307c91-3bb9-48b5-926c-5a2747320151,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_export_location_not_exist_1_path[api_with_backend,id-ffc3dc76-2f92-4308-a125-1d3905ed72ba,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_export_location_not_exist_2_id[api_with_backend,id-ffc3dc76-2f92-4308-a125-1d3905ed72ba,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_like_filter_and_invalid_version[api_with_backend,id-3dbcf17b-cc63-43ea-b45f-eae12300729e,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_like_filter_not_exist[api_with_backend,id-f41c6cd2-62cf-4bba-a26e-21a6e86eae15,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_shares_with_name_not_exist[api_with_backend,id-31e33495-5ec3-4658-bdef-d9d1e034705a,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_snapshots_with_description_not_exist[api_with_backend,id-061ee37a-96b2-4b4f-9cfe-2c8c80ed4370,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_list_snapshots_with_name_not_exist[api_with_backend,id-9d3c3158-1a92-4e37-b00f-a4a40b813109,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_restore_share_with_wrong_id[api,id-87345725-f187-4d7d-86b1-62284e8c75ae,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_extend_over_quota[api_with_backend,id-c4481ba3-0cff-448b-a728-69a9a34e3aa6,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_extend_with_invalid_share_state[api_with_backend,id-067c4b10-4324-45ac-8365-5d446b66c18a,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_extend_with_less_size[api_with_backend,id-3448cd2b-34eb-453f-b72f-39fbea778e42,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_extend_with_same_size[api_with_backend,id-79f2304a-7959-4169-8a76-b67814e0733a,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_force_extend_non_admin_user[api_with_backend,id-f9d2ba94-4032-d17a-b4ab-a2b67f650a39,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_shrink_with_greater_size[api_with_backend,id-99d42f94-8da1-4c04-ad5b-9738d6acc139,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_shrink_with_invalid_share_state[api_with_backend,id-d53ece5c-70e4-4953-a1d7-7d4384510519,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_share_shrink_with_same_size[api_with_backend,id-3d4c8f34-49b8-4628-b1cb-652ae67473a5,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_soft_delete_share_from_other_project[api_with_backend,id-f6106ee4-1a01-444f-b623-912a5e751d49,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_soft_delete_share_has_been_soft_deleted[api_with_backend,id-36cbe23b-08d2-49d9-bb42-f9eb2a804cb1,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_soft_delete_share_with_invalid_share_state[api_with_backend,id-cf675ac9-0970-49fc-a051-8a94555c73b5,negative] +manila_tempest_tests.tests.api.test_shares_actions_negative.SharesActionsNegativeTest.test_soft_delete_share_with_wrong_id[api,id-0ccd44dd-2fda-403e-bc23-7ce428550f36,negative] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_extend_share[backend,id-17e6f579-e0d7-4724-a639-4974e82bb5ed,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_export_locations_removed[api_with_backend,id-c13c1cf5-c708-4e62-8b71-c47dbb45a7a0,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_v2_1[api_with_backend,id-188badb2-0ca3-44e5-abca-3029475d7f73,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_with_access_rules_status[api_with_backend,id-2c439716-c34e-46f1-b055-41e411fb4e66,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_with_create_share_from_snapshot_support[api_with_backend,id-2899055e-607d-4423-b1b7-27eefa6ce5f0,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_with_replication_type_key[api_with_backend,id-67c9ad27-f26d-4ff9-a528-a35b4c5a55c1,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_with_revert_to_snapshot_support[api_with_backend,id-04c359b5-74b3-4a07-9f9e-f0ac1867d58d,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_with_share_type_name_key[api_with_backend,id-b803a076-593a-469d-ab75-7e67ac294dd5,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_with_snapshot_support_key[api_with_backend,id-45ec8e36-cb8a-4a03-8c41-ba5d5dc2a5c5,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_share_with_user_id[api_with_backend,id-2a61703d-5a62-49c5-b143-fea0c9125cd9,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_snapshot_1_None[api_with_backend,id-174829eb-fd3e-46ef-880b-f05c3d44d1fe,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_snapshot_2_2_16[api_with_backend,id-174829eb-fd3e-46ef-880b-f05c3d44d1fe,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_get_snapshot_3_2_88[api_with_backend,id-174829eb-fd3e-46ef-880b-f05c3d44d1fe,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_and_create_share_from_snapshot_support[api_with_backend,id-21410324-e0b5-452a-8c7e-599a27267e1d,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_and_snapshot_support_key[api_with_backend,id-ee57db25-0fbf-40ce-8dc1-070845d245a7,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_export_locations_removed[api_with_backend,id-a313cd85-53e5-48b7-b4a7-14e9273e843d,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_active_status[api_with_backend,id-708e3e2e-8761-4d16-b18d-a834ee7ca69e,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_all_tenants[api_with_backend,id-7609b7bb-613e-474d-a9b3-e41584842503,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_existed_description[api_with_backend,id-f446e8cb-5bef-45ac-8b87-f4136f44ca69,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_existed_name[api_with_backend,id-866194c6-3910-409e-ad81-2cff223cfaaf,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_fake_name[api_with_backend,id-56416b95-949f-4b09-9a5e-377b674efd25,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_fake_status[api_with_backend,id-5ec2fcf8-18d4-4790-95de-45c8a05582c7,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_inexact_name[api_with_backend,id-1276b97b-cf46-4953-973f-f995985a1ce4,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_metadata[api_with_backend,id-bc24dc42-050b-4105-a90a-f5649cbd8e49,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_share_network_id[api_with_backend,id-685286c7-1df6-48c3-839d-8162737446b8,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_filter_by_snapshot_id[api_with_backend,id-fd87884d-71a4-4ca8-8b2b-07b4df2de3bd,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_replication_type_key[api_with_backend,id-70097cf3-76b8-49d2-b1c7-ad59394dd978,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_share_type_name_key[api_with_backend,id-38a4cf70-8ed9-4a82-a338-9f20463b1147,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_v2_1[api_with_backend,id-d88a157a-fe4d-456e-90ba-f0bd9e5d02ec,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_with_access_rules_status[api_with_backend,id-0de4dc84-bace-4b9a-8470-a40c4a9b14b4,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_with_asc_sorting[api_with_backend,id-70a9d947-bd45-46db-b529-6c48c0ff8985,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_with_count[api_with_backend,id-0019afa2-fae2-417f-a7e0-2af665a966b0,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_detail_with_revert_to_snapshot_support[api_with_backend,id-45041fc9-bb0d-4dd6-ac0d-a40a22a91f3f,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares_with_user_id[api_with_backend,id-f8e57190-b105-4937-b96e-55725eb64739,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_shares[api_with_backend,id-7d61311a-81b0-481c-abb9-cfb0b4f82e29,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_1_None[api_with_backend,id-4d717665-e4ca-47df-b4b9-f6d096159779,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_2_2_16[api_with_backend,id-4d717665-e4ca-47df-b4b9-f6d096159779,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_3_2_36[api_with_backend,id-4d717665-e4ca-47df-b4b9-f6d096159779,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_4_2_88[api_with_backend,id-4d717665-e4ca-47df-b4b9-f6d096159779,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_and_asc_sorting[api_with_backend,id-59968026-12af-4029-a3d0-42c291b7db96,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_filter_by_description[api_with_backend,id-f969aba1-d293-48e3-a638-a89785bb41ef,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_filter_by_status_and_name[api_with_backend,id-0a94e996-c4db-4fef-b486-4004ea65c11a,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots_with_detail_use_limit[api_with_backend,id-80497a21-7533-47b5-93aa-29e0f7924cb9,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_list_snapshots[api_with_backend,id-3d5d85ff-6158-4af2-a765-cfa07a46adde,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_shrink_share[backend,id-8f64b930-9b4e-41d7-bbd6-82d9951931f3,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesActionsTest.test_soft_delete_and_restore_share[api_with_backend,id-7a19fb58-b645-44cc-a6d7-b3508ff8754d,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesRenameTest.test_rename_snapshot[api_with_backend,id-20f299f6-2441-4629-b44e-d791d57f413c,positive] +manila_tempest_tests.tests.api.test_shares_actions.SharesRenameTest.test_update_share[api_with_backend,id-7661d042-8222-483c-9249-9f53931e7347,positive] +manila_tempest_tests.tests.api.test_shares_from_snapshot_across_pools.SharesFromSnapshotAcrossPools.test_create_share_from_snapshot_across_pools_within_backend[backend,id-6f1fa7d0-94f2-4373-8730-b0986781cc88,positive] +manila_tempest_tests.tests.api.test_shares_from_snapshot_across_pools.SharesFromSnapshotAcrossPools.test_share_from_snapshot_across_azs[backend,id-46f33338-d70d-4c9a-ae27-3dd5b7bd1531,positive] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_non_existent_az[api,id-2f0df934-b2fb-4ebd-96f7-183cb699dcdd,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_size_greater_than_specified_in_share_type[api,id-b8097d56-067e-4d7c-8401-31bc7021fe86,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_size_less_than_specified_in_share_type[api,id-b8097d56-067e-4d7c-8401-31bc7021fe87,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_with_invalid_protocol[api,id-78e5e327-c68e-4910-82b2-27f5f4d150ac,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_with_invalid_size[api,id-3620a380-f9f9-4521-a468-473581637344,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_with_out_passing_size[api,id-26ed523d-a215-4661-a038-633b74c9cad7,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_with_wrong_public_value[api,id-2b7c7ea8-c0e9-446c-a8e3-add765452c04,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_share_with_zero_size[api,id-5ae2ecd7-a694-4ba9-ae23-cca75580b9d8,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_create_snapshot_with_wrong_id[api,id-08e5a9c7-45cb-414c-b375-28c335f20ff1,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_delete_share_with_wrong_id[api,id-9698d1a3-8ee8-46fa-a46b-1084d98e7149,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_delete_share_without_passing_share_id[api,id-b8097d56-067e-4d7c-8401-31bc7021fd24,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_delete_snapshot_with_wrong_id[api,id-bf303b29-bbcb-4a96-96e9-270e12df58d1,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_get_share_with_wrong_id[api,id-f82d1667-ae39-43bb-b5aa-bfc9b2ec7292,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_get_share_without_passing_share_id[api,id-d03cf44e-6e69-415f-be36-25defb86df56,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_list_by_share_server_by_user[api,id-73f37f33-946d-4213-9d27-25cd4e9a0208,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_list_by_user_with_host_filter[api,id-1a438374-8a91-4566-9cef-0386f6609445,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_list_shares_nonadmin_with_nonexistent_share_server_filter[api,id-a9487254-606b-444f-ba6a-2f461bcaf474,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_manage_share_by_user[api,id-97a4dd2f-7c90-4eb7-bf74-d698c3060833,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesAPIOnlyNegativeTest.test_unmanage_share_by_user[api,id-75837f93-8c2c-40a4-bb9e-d76c53db07c7,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesNegativeTest.test_create_share_from_snap_with_different_share_network[api_with_backend,id-e84ce567-a090-47c7-87c4-6ee427bdee7a,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesNegativeTest.test_create_share_from_snap_with_less_size[api_with_backend,id-3df8e2d8-9b79-428d-9d8b-30bc66b5b40e,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesNegativeTest.test_create_share_with_nonexistant_share_network[api_with_backend,id-3047fb1c-5acc-4ef2-8796-b2d2d49829b5,negative] +manila_tempest_tests.tests.api.test_shares_negative.SharesNegativeTest.test_try_delete_share_with_existing_snapshot[api_with_backend,id-b9bb8dee-0c7c-4e51-909c-028335b1a6a0,negative] +manila_tempest_tests.tests.api.test_shares.SharesCephFSTest.test_create_delete_snapshot[backend,id-775f8f87-5727-4bb7-b69f-9ce6b9bdb140,positive] +manila_tempest_tests.tests.api.test_shares.SharesCephFSTest.test_create_get_delete_share[backend,id-21ad41fb-04cf-493c-bc2f-66c80220898b,positive] +manila_tempest_tests.tests.api.test_shares.SharesCephFSTest.test_create_share_from_snapshot_share_network_not_provided[backend,id-c609c0b2-d649-4ca3-8334-629b213f5c72,positive] +manila_tempest_tests.tests.api.test_shares.SharesCephFSTest.test_create_share_from_snapshot[backend,id-8a14831d-ad1f-447f-b5de-2b8a233b24c0,positive] +manila_tempest_tests.tests.api.test_shares.SharesCIFSTest.test_create_delete_snapshot[backend,id-775f8f87-5727-4bb7-b69f-9ce6b9bdb140,positive] +manila_tempest_tests.tests.api.test_shares.SharesCIFSTest.test_create_get_delete_share[backend,id-21ad41fb-04cf-493c-bc2f-66c80220898b,positive] +manila_tempest_tests.tests.api.test_shares.SharesCIFSTest.test_create_share_from_snapshot_share_network_not_provided[backend,id-c609c0b2-d649-4ca3-8334-629b213f5c72,positive] +manila_tempest_tests.tests.api.test_shares.SharesCIFSTest.test_create_share_from_snapshot[backend,id-8a14831d-ad1f-447f-b5de-2b8a233b24c0,positive] +manila_tempest_tests.tests.api.test_shares.SharesGLUSTERFSTest.test_create_delete_snapshot[backend,id-775f8f87-5727-4bb7-b69f-9ce6b9bdb140,positive] +manila_tempest_tests.tests.api.test_shares.SharesGLUSTERFSTest.test_create_get_delete_share[backend,id-21ad41fb-04cf-493c-bc2f-66c80220898b,positive] +manila_tempest_tests.tests.api.test_shares.SharesGLUSTERFSTest.test_create_share_from_snapshot_share_network_not_provided[backend,id-c609c0b2-d649-4ca3-8334-629b213f5c72,positive] +manila_tempest_tests.tests.api.test_shares.SharesGLUSTERFSTest.test_create_share_from_snapshot[backend,id-8a14831d-ad1f-447f-b5de-2b8a233b24c0,positive] +manila_tempest_tests.tests.api.test_shares.SharesHDFSTest.test_create_delete_snapshot[backend,id-775f8f87-5727-4bb7-b69f-9ce6b9bdb140,positive] +manila_tempest_tests.tests.api.test_shares.SharesHDFSTest.test_create_get_delete_share[backend,id-21ad41fb-04cf-493c-bc2f-66c80220898b,positive] +manila_tempest_tests.tests.api.test_shares.SharesHDFSTest.test_create_share_from_snapshot_share_network_not_provided[backend,id-c609c0b2-d649-4ca3-8334-629b213f5c72,positive] +manila_tempest_tests.tests.api.test_shares.SharesHDFSTest.test_create_share_from_snapshot[backend,id-8a14831d-ad1f-447f-b5de-2b8a233b24c0,positive] +manila_tempest_tests.tests.api.test_shares.SharesMapRFSTest.test_create_delete_snapshot[backend,id-775f8f87-5727-4bb7-b69f-9ce6b9bdb140,positive] +manila_tempest_tests.tests.api.test_shares.SharesMapRFSTest.test_create_get_delete_share[backend,id-21ad41fb-04cf-493c-bc2f-66c80220898b,positive] +manila_tempest_tests.tests.api.test_shares.SharesMapRFSTest.test_create_share_from_snapshot_share_network_not_provided[backend,id-c609c0b2-d649-4ca3-8334-629b213f5c72,positive] +manila_tempest_tests.tests.api.test_shares.SharesMapRFSTest.test_create_share_from_snapshot[backend,id-8a14831d-ad1f-447f-b5de-2b8a233b24c0,positive] +manila_tempest_tests.tests.api.test_shares.SharesNFSTest.test_create_delete_snapshot[backend,id-775f8f87-5727-4bb7-b69f-9ce6b9bdb140,positive] +manila_tempest_tests.tests.api.test_shares.SharesNFSTest.test_create_get_delete_share[backend,id-21ad41fb-04cf-493c-bc2f-66c80220898b,positive] +manila_tempest_tests.tests.api.test_shares.SharesNFSTest.test_create_share_from_snapshot_share_network_not_provided[backend,id-c609c0b2-d649-4ca3-8334-629b213f5c72,positive] +manila_tempest_tests.tests.api.test_shares.SharesNFSTest.test_create_share_from_snapshot[backend,id-8a14831d-ad1f-447f-b5de-2b8a233b24c0,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_delete_unexisting_metadata[api_with_backend,id-9afb381d-c48c-4c2c-a5b5-42463daef5a2,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_set_metadata_to_unexisting_snapshot[api,id-8be4773b-6af9-413f-97e2-8acdb6149e7a,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_set_metadata_with_empty_key[api_with_backend,id-ef0afcc8-7b12-41bc-8aa1-4916ad4b4560,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_set_metadata_with_too_big_key[api_with_backend,id-ef61255e-462c-49fe-8e94-ff3afafcccb3,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_set_metadata_with_too_big_value[api_with_backend,id-1bf97c18-27df-4618-94f4-224d1a98bc0c,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_upd_metadata_with_empty_key[api_with_backend,id-9f2aee7c-ebd6-4516-87f7-bd85453d74c9,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_upd_metadata_with_too_big_key[api_with_backend,id-f896f354-6179-4abb-b0c5-7b7dc96f0870,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_upd_metadata_with_too_big_value[api_with_backend,id-2b9e08fa-b35d-4bfe-9137-e59ab50bd9ef,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata_negative.ShareSnapshotMetadataNegativeTest.test_try_update_all_metadata_to_unexisting_snapshot[api,id-03a7f6e9-de8b-4669-87e1-b179308b477d,negative] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_and_update_metadata_by_key[api_with_backend,id-90120310-07a9-43f4-9d5e-38d0a3f2f5bb,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_get_delete_metadata[api_with_backend,id-7cbdf3c5-fb72-4ea5-9e60-ba50bad68ee9,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_metadata_in_snapshot_creation[api_with_backend,id-5d537913-ce6f-4771-beb2-84e2390b06d3,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_metadata_key_already_exist[api_with_backend,id-b7a00be5-3dd1-4d25-8723-c662581c923f,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_metadata_max_size_key[api_with_backend,id-dc226070-5820-4df2-a30a-9dfb2f037a4b,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_metadata_max_size_value[api_with_backend,id-85c480bc-0ffa-43e1-bc0a-284c5641996d,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_metadata_min_size_key[api_with_backend,id-8963b7ae-db3a-476e-b0c7-29023e7aa321,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_metadata_min_size_value[api_with_backend,id-940c283f-4f43-4122-86e8-32230da81886,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_set_metadata_not_delete_pre_metadata[api_with_backend,id-23ec837d-1b50-499c-bbb9-a7bde843c9e8,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_upd_metadata_max_size_key[api_with_backend,id-1b5f06b0-bbff-49d1-8a4b-6e912039e2ba,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_upd_metadata_max_size_value[api_with_backend,id-fdfbe469-6403-41de-b909-c4c13fc57407,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_upd_metadata_min_size_key[api_with_backend,id-c42335ae-ee90-4b73-b022-51c0a9bc301d,positive] +manila_tempest_tests.tests.api.test_snapshot_metadata.ShareSnapshotMetadataTest.test_upd_metadata_min_size_value[api_with_backend,id-849fdcd4-9b4c-4aea-833a-240d7d06966b,positive] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_01_1_2_3_256[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_02_1_1_1__[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_03_1_2_3_4_33[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_04_1_2_3__[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_05_1_2_3___23[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_06_1_2_3_1_23[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_07_1_2_3_1_[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_08_1_2_3_1__1[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_09_fe80_217_f2ff_fe07_ed62[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_10_2001_db8__1_148[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_11_ffff_ffff_ffff_ffff_ffff_ffff_ffff_ffff_ffff[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_access_rule_ip_with_wrong_target_12_2001_0db8_0000_85a3_0000_0000_ac1f_8001_64[api_with_backend,id-a877276e-03d4-4fc5-b4b4-c8874d23f1dc,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_duplicate_of_ip_rule_1_1_2_3_4[api_with_backend,id-dd93150f-8081-4533-abb3-f99a03652a87,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_duplicate_of_ip_rule_2_fd8c_b029_bba6_ac54__1[api_with_backend,id-dd93150f-8081-4533-abb3-f99a03652a87,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_duplicate_of_ip_rule_3_fd8c_b029_bba6_ac54__1_128[api_with_backend,id-dd93150f-8081-4533-abb3-f99a03652a87,negative] +manila_tempest_tests.tests.api.test_snapshot_rules_negative.SnapshotIpRulesForNFSNegativeTest.test_create_duplicate_of_ip_rule_4_1_2_3_4_32[api_with_backend,id-dd93150f-8081-4533-abb3-f99a03652a87,negative] +manila_tempest_tests.tests.api.test_snapshot_rules.ShareSnapshotIpRulesForNFSTest.test_create_delete_access_rules_1_1_1_1_1[backend,id-bdce2be8-80b9-4f68-bdc0-09a52ba0e6fd,positive] +manila_tempest_tests.tests.api.test_snapshot_rules.ShareSnapshotIpRulesForNFSTest.test_create_delete_access_rules_2_1_2_3_4_32[backend,id-bdce2be8-80b9-4f68-bdc0-09a52ba0e6fd,positive] +manila_tempest_tests.tests.api.test_snapshot_rules.ShareSnapshotUserRulesForCIFSTest.test_create_delete_access_rules[backend,id-c2625cd2-4dc0-431d-b47b-8f097e22f16d,positive] +manila_tempest_tests.tests.rbac.test_availability_zones.TestProjectAdminTestsNFS.test_list_availability_zones[api,id-87d9bb1c-f4de-40e5-8f25-05a6e1055c0b,positive] +manila_tempest_tests.tests.rbac.test_availability_zones.TestProjectMemberTestsNFS.test_list_availability_zones[api,id-ee2db349-176a-47bc-a20d-5ba9b5f8a813,positive] +manila_tempest_tests.tests.rbac.test_availability_zones.TestProjectReaderTestsNFS.test_list_availability_zones[api,id-a095fac8-ae62-4be7-8a3e-b0fc1bc71348,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectAdminTestsCEPHFS.test_list_export_locations[api_with_backend,id-c8d75c9f-104b-48a8-9817-17280ce516a8,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectAdminTestsCEPHFS.test_show_export_location[api_with_backend,id-01ac6355-fcad-4af8-b0ad-748fc111051d,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectAdminTestsCIFS.test_list_export_locations[api_with_backend,id-c8d75c9f-104b-48a8-9817-17280ce516a8,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectAdminTestsCIFS.test_show_export_location[api_with_backend,id-01ac6355-fcad-4af8-b0ad-748fc111051d,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectAdminTestsNFS.test_list_export_locations[api_with_backend,id-c8d75c9f-104b-48a8-9817-17280ce516a8,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectAdminTestsNFS.test_show_export_location[api_with_backend,id-01ac6355-fcad-4af8-b0ad-748fc111051d,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectMemberTestsCEPHFS.test_list_export_locations[api_with_backend,id-b25351bb-102f-437e-a0c1-01d926fa0a7d,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectMemberTestsCEPHFS.test_show_export_location[api_with_backend,id-95297484-1081-4225-986d-e4b3017d992f,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectMemberTestsCIFS.test_list_export_locations[api_with_backend,id-b25351bb-102f-437e-a0c1-01d926fa0a7d,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectMemberTestsCIFS.test_show_export_location[api_with_backend,id-95297484-1081-4225-986d-e4b3017d992f,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectMemberTestsNFS.test_list_export_locations[api_with_backend,id-b25351bb-102f-437e-a0c1-01d926fa0a7d,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectMemberTestsNFS.test_show_export_location[api_with_backend,id-95297484-1081-4225-986d-e4b3017d992f,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectReaderTestsCEPHFS.test_list_export_locations[api_with_backend,id-889ecfa7-89b3-4b16-a368-f55bbc6af228,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectReaderTestsCEPHFS.test_show_export_location[api_with_backend,id-1ab5f0c2-d844-4284-9b66-1ba2aa14b835,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectReaderTestsCIFS.test_list_export_locations[api_with_backend,id-889ecfa7-89b3-4b16-a368-f55bbc6af228,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectReaderTestsCIFS.test_show_export_location[api_with_backend,id-1ab5f0c2-d844-4284-9b66-1ba2aa14b835,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectReaderTestsNFS.test_list_export_locations[api_with_backend,id-889ecfa7-89b3-4b16-a368-f55bbc6af228,positive] +manila_tempest_tests.tests.rbac.test_export_locations.TestProjectReaderTestsNFS.test_show_export_location[api_with_backend,id-1ab5f0c2-d844-4284-9b66-1ba2aa14b835,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectAdminTests.test_default_quotas[api_with_backend,id-e102292f-93f9-4918-96b2-bb270e29e43e,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectAdminTests.test_delete_quotas[api_with_backend,id-fe9ce5ab-4e93-4bdf-bd2d-d1b35a8b19f8,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectAdminTests.test_show_quotas_detail[api_with_backend,id-0bce045c-5575-4301-b526-032812a2e71f,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectAdminTests.test_show_quotas[api_with_backend,id-77c14ee8-9dbc-47dc-a86e-3a26f33beda5,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectAdminTests.test_update_tenant_quota_shares[api_with_backend,id-b055f9ea-6176-45f9-a918-d9120912fcf6,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectMemberTests.test_default_quotas[api_with_backend,id-a81d40fc-04b2-4535-ad44-c989a51e49b9,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectMemberTests.test_delete_quotas[api_with_backend,id-1928eea7-ca78-4004-8e5f-6d58a446503c,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectMemberTests.test_show_quotas_detail[api_with_backend,id-68b3d3e7-8ebd-4b20-bf56-7b4e4c365eda,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectMemberTests.test_show_quotas[api_with_backend,id-52194358-6268-446c-ada4-74fb7e23dbe9,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectMemberTests.test_update_tenant_quota_shares[api_with_backend,id-5a86d62d-5fdf-448e-bd6b-43e26e39201f,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectReaderTests.test_default_quotas_detail[api_with_backend,id-0648bf5f-d8c8-4fd4-9713-27e9b5a1cda8,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectReaderTests.test_default_quotas[api_with_backend,id-51ec3c23-8c3b-45ff-9e41-38141ac82145,positive] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectReaderTests.test_delete_quotas[api_with_backend,id-8185210d-edf4-40e7-840a-484ab21bf7bd,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectReaderTests.test_show_quotas_detail[api_with_backend,id-68b3d3e7-8ebd-4b20-bf56-7b4e4c365eda,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectReaderTests.test_show_quotas[api_with_backend,id-48ca6e6b-6ad1-43b6-bdb7-848fe6a4d0fb,negative] +manila_tempest_tests.tests.rbac.test_quotas.TestProjectReaderTests.test_update_tenant_quota_shares[api_with_backend,id-4051f57d-3d79-4007-8b90-b5abf744b4b3,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCEPHFS.test_delete_access_rule_metadata[api_with_backend,id-fd580d91-1d8d-4dd0-8484-01c412ddb768,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCEPHFS.test_delete_access[api_with_backend,id-e24d7018-cb49-4306-9947-716b4e4250c5,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCEPHFS.test_get_access[api_with_backend,id-5b6897d1-4b2a-490c-990e-941ea4893f47,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCEPHFS.test_grant_access_rule[api_with_backend,id-b4d7a91c-a75e-4ad9-93cb-8e5234fea97a,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCEPHFS.test_list_access[api_with_backend,id-f8e9a2bb-ccff-4fc5-8d61-2930f87406cd,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCEPHFS.test_update_access_rule_metadata[api_with_backend,id-ffc07445-d0d1-4bf9-9fbc-4f409d48bccd,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCEPHFS.test_update_access[api_with_backend,id-01939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCIFS.test_delete_access_rule_metadata[api_with_backend,id-fd580d91-1d8d-4dd0-8484-01c412ddb768,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCIFS.test_delete_access[api_with_backend,id-e24d7018-cb49-4306-9947-716b4e4250c5,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCIFS.test_get_access[api_with_backend,id-5b6897d1-4b2a-490c-990e-941ea4893f47,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCIFS.test_grant_access_rule[api_with_backend,id-b4d7a91c-a75e-4ad9-93cb-8e5234fea97a,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCIFS.test_list_access[api_with_backend,id-f8e9a2bb-ccff-4fc5-8d61-2930f87406cd,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCIFS.test_update_access_rule_metadata[api_with_backend,id-ffc07445-d0d1-4bf9-9fbc-4f409d48bccd,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsCIFS.test_update_access[api_with_backend,id-01939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsNFS.test_delete_access_rule_metadata[api_with_backend,id-fd580d91-1d8d-4dd0-8484-01c412ddb768,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsNFS.test_delete_access[api_with_backend,id-e24d7018-cb49-4306-9947-716b4e4250c5,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsNFS.test_get_access[api_with_backend,id-5b6897d1-4b2a-490c-990e-941ea4893f47,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsNFS.test_grant_access_rule[api_with_backend,id-b4d7a91c-a75e-4ad9-93cb-8e5234fea97a,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsNFS.test_list_access[api_with_backend,id-f8e9a2bb-ccff-4fc5-8d61-2930f87406cd,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsNFS.test_update_access_rule_metadata[api_with_backend,id-ffc07445-d0d1-4bf9-9fbc-4f409d48bccd,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectAdminTestsNFS.test_update_access[api_with_backend,id-01939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCEPHFS.test_delete_access_rule_metadata[api_with_backend,id-abb17315-6510-4b6e-ae6c-dd99a6088954,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCEPHFS.test_delete_access[api_with_backend,id-8665d1b1-de4c-42d4-93ff-8dc6d2b73a2d,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCEPHFS.test_get_access[api_with_backend,id-de643909-88a2-470b-8a14-0417696ec451,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCEPHFS.test_grant_access_rule[api_with_backend,id-61cf6f6c-5d7c-48d7-9d5a-e6ea288afdbc,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCEPHFS.test_list_access[api_with_backend,id-7c6c4262-5095-4cd7-9d9c-8064009a9055,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCEPHFS.test_update_access_rule_metadata[api_with_backend,id-c5e84362-6075-425b-bfa3-898abfd9d5a0,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCEPHFS.test_update_access[api_with_backend,id-02939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCIFS.test_delete_access_rule_metadata[api_with_backend,id-abb17315-6510-4b6e-ae6c-dd99a6088954,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCIFS.test_delete_access[api_with_backend,id-8665d1b1-de4c-42d4-93ff-8dc6d2b73a2d,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCIFS.test_get_access[api_with_backend,id-de643909-88a2-470b-8a14-0417696ec451,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCIFS.test_grant_access_rule[api_with_backend,id-61cf6f6c-5d7c-48d7-9d5a-e6ea288afdbc,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCIFS.test_list_access[api_with_backend,id-7c6c4262-5095-4cd7-9d9c-8064009a9055,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCIFS.test_update_access_rule_metadata[api_with_backend,id-c5e84362-6075-425b-bfa3-898abfd9d5a0,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsCIFS.test_update_access[api_with_backend,id-02939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsNFS.test_delete_access_rule_metadata[api_with_backend,id-abb17315-6510-4b6e-ae6c-dd99a6088954,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsNFS.test_delete_access[api_with_backend,id-8665d1b1-de4c-42d4-93ff-8dc6d2b73a2d,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsNFS.test_get_access[api_with_backend,id-de643909-88a2-470b-8a14-0417696ec451,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsNFS.test_grant_access_rule[api_with_backend,id-61cf6f6c-5d7c-48d7-9d5a-e6ea288afdbc,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsNFS.test_list_access[api_with_backend,id-7c6c4262-5095-4cd7-9d9c-8064009a9055,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsNFS.test_update_access_rule_metadata[api_with_backend,id-c5e84362-6075-425b-bfa3-898abfd9d5a0,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectMemberTestsNFS.test_update_access[api_with_backend,id-02939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCEPHFS.test_delete_access_rule_metadata[api_with_backend,id-5faf0e0b-b246-4392-901d-9e7d628f0d6e,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCEPHFS.test_delete_access[api_with_backend,id-7a702c74-8d31-49e3-859a-cc8a78d7915e,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCEPHFS.test_get_access[api_with_backend,id-0eec0f05-f2f3-4500-9d9e-1b77ebc476c2,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCEPHFS.test_grant_access_rule[api_with_backend,id-ace870f9-af91-4259-8760-dc7d7107b7ff,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCEPHFS.test_list_access[api_with_backend,id-9ddc26b6-f8bf-45d9-a2c6-a9eec9bfb8d2,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCEPHFS.test_update_access_rule_metadata[api_with_backend,id-a61d7f06-6f0e-4da3-b11d-1c3a0b5bd416,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCEPHFS.test_update_access[api_with_backend,id-03939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCIFS.test_delete_access_rule_metadata[api_with_backend,id-5faf0e0b-b246-4392-901d-9e7d628f0d6e,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCIFS.test_delete_access[api_with_backend,id-7a702c74-8d31-49e3-859a-cc8a78d7915e,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCIFS.test_get_access[api_with_backend,id-0eec0f05-f2f3-4500-9d9e-1b77ebc476c2,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCIFS.test_grant_access_rule[api_with_backend,id-ace870f9-af91-4259-8760-dc7d7107b7ff,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCIFS.test_list_access[api_with_backend,id-9ddc26b6-f8bf-45d9-a2c6-a9eec9bfb8d2,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCIFS.test_update_access_rule_metadata[api_with_backend,id-a61d7f06-6f0e-4da3-b11d-1c3a0b5bd416,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsCIFS.test_update_access[api_with_backend,id-03939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsNFS.test_delete_access_rule_metadata[api_with_backend,id-5faf0e0b-b246-4392-901d-9e7d628f0d6e,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsNFS.test_delete_access[api_with_backend,id-7a702c74-8d31-49e3-859a-cc8a78d7915e,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsNFS.test_get_access[api_with_backend,id-0eec0f05-f2f3-4500-9d9e-1b77ebc476c2,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsNFS.test_grant_access_rule[api_with_backend,id-ace870f9-af91-4259-8760-dc7d7107b7ff,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsNFS.test_list_access[api_with_backend,id-9ddc26b6-f8bf-45d9-a2c6-a9eec9bfb8d2,positive] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsNFS.test_update_access_rule_metadata[api_with_backend,id-a61d7f06-6f0e-4da3-b11d-1c3a0b5bd416,negative] +manila_tempest_tests.tests.rbac.test_rules.TestProjectReaderTestsNFS.test_update_access[api_with_backend,id-03939b69-ef9b-75cf-abf7-5171fec7c397,positive] +manila_tempest_tests.tests.rbac.test_scheduler_stats.ProjectAdminTests.test_list_storage_pools[api,id-1ec4d0f5-0d60-4bbc-88a4-57fa92f6f62f,positive] +manila_tempest_tests.tests.rbac.test_scheduler_stats.ProjectMemberTests.test_list_storage_pools[api,id-905aa5ea-eff9-4022-be41-df7a8593809d,negative] +manila_tempest_tests.tests.rbac.test_scheduler_stats.ProjectReaderTests.test_list_storage_pools[api,id-faab12f9-ff51-458d-af47-362d872761e9,negative] +manila_tempest_tests.tests.rbac.test_services.TestProjectAdminTests.test_list_services[api,id-08ec3a0b-6e4a-4cbf-bd15-3f48f8ddf71f,positive] +manila_tempest_tests.tests.rbac.test_services.TestProjectMemberTests.test_list_services[api,id-7431dca6-9b03-48d3-b97c-41f72f7ed0a3,negative] +manila_tempest_tests.tests.rbac.test_services.TestProjectReaderTests.test_list_services[api,id-eca71619-d563-4d15-9e49-b661e6da46c0,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_add_share_group_type_access[api,id-89876c46-1167-450d-8b98-746d97fff388,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_create_share_group_type_extra_specs[api,id-80eb22cb-846a-4b51-a71d-ceef0b804901,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_create_share_group_type[api,id-9ea9954a-ae09-4d02-a082-9a72b80009fc,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_delete_share_group_type_extra_spec[api,id-743c18dc-8c3a-4934-9ef8-8b342daffe7c,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_delete_share_group_type[api,id-c23da121-8b1f-4443-80cc-11881745a1c3,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_get_share_group_type[api,id-fcad2b86-ca43-42b0-82bd-37e6f760e4d2,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_list_share_group_type_access[api,id-64a7be53-d1af-40c3-950b-743a2704ac97,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_list_share_group_types[api,id-7871b1b5-610a-425c-9363-d0bcf2beff72,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_remove_share_group_type_access[api,id-fc67ba18-78e9-4c02-9b37-2bd49c8a4470,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectAdminTests.test_update_share_group_type_extra_spec[api,id-fe29877d-2226-42ca-b492-4be0dacd6eaf,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_add_share_group_type_access[api,id-85b69e26-9c67-45c4-80e0-2ce212977c2a,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_create_share_group_type_extra_specs[api,id-2fa2f953-f068-4c60-ab52-eeb1bd69a7a4,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_create_share_group_type[api,id-f9e6f2fd-7c1a-4eee-817c-bf1988904515,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_delete_share_group_type_extra_spec[api,id-95aa1fbf-3eaf-4d65-96b5-4554b0fb0937,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_delete_share_group_type[api,id-8f45798f-717d-41b0-acba-a80dd647cddf,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_get_share_group_type[api,id-8eaf4a99-9706-41c9-8b12-40856d0900f4,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_list_share_group_type_access[api,id-027314a9-6b14-4ae9-83f2-471e84ccaa01,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_list_share_group_types[api,id-d2dd61f4-c763-49f9-9c93-8b587879f554,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_remove_share_group_type_access[api,id-d8518122-dabd-4d2d-8b6a-4eed975e19ee,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectMemberTests.test_update_share_group_type_extra_spec[api,id-30c73c94-b7bc-4e1f-8172-192335997879,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_add_share_group_type_access[api,id-93bba264-104c-48af-af91-5a79ec3e695e,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_create_share_group_type_extra_specs[api,id-caa6d960-4f34-4dff-9cc0-9a0cde44c2ae,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_create_share_group_type[api,id-8c47bbe9-f3f1-419e-b00b-97c9c942a48a,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_delete_share_group_type_extra_spec[api,id-003a0eee-0075-47b0-ba8a-8e57e958c1d3,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_delete_share_group_type[api,id-e853fd60-8906-4e38-b0b4-ec5723696518,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_get_share_group_type[api,id-fe3b28a3-6980-4782-8eaa-518bbd3913d1,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_list_share_group_type_access[api,id-176b280c-7a46-43ae-8164-3bdd945d2dd4,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_list_share_group_types[api,id-47f92f0b-424e-4685-a742-8a4e00cc6901,positive] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_remove_share_group_type_access[api,id-00caa020-3805-4322-aac1-86d4552268a2,negative] +manila_tempest_tests.tests.rbac.test_share_group_types.ProjectReaderTests.test_update_share_group_type_extra_spec[api,id-2782c329-2447-49f7-95e7-0c4c766cfda3,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCEPHFS.test_create_share_group[api_with_backend,id-d060996e-c5f2-4dff-820b-6892a096a425,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCEPHFS.test_delete_share_group[api_with_backend,id-ea6cbb78-057e-4fbc-86bf-125b033cb76f,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCEPHFS.test_force_delete_share_group[api_with_backend,id-2cb00ffb-47e3-495e-853c-007752c9e679,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCEPHFS.test_get_share_group[api_with_backend,id-0de993c5-8389-4997-8f7f-345e27f563f1,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCEPHFS.test_list_share_groups[api_with_backend,id-3b277a44-dcae-46da-a58c-f5281d8abc84,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCEPHFS.test_reset_share_group[api_with_backend,id-069bc68e-6411-44b8-abe9-399885f0eee5,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCEPHFS.test_update_share_group[api_with_backend,id-1bab40d5-bdba-4a23-9300-807fe513bf15,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCIFS.test_create_share_group[api_with_backend,id-d060996e-c5f2-4dff-820b-6892a096a425,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCIFS.test_delete_share_group[api_with_backend,id-ea6cbb78-057e-4fbc-86bf-125b033cb76f,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCIFS.test_force_delete_share_group[api_with_backend,id-2cb00ffb-47e3-495e-853c-007752c9e679,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCIFS.test_get_share_group[api_with_backend,id-0de993c5-8389-4997-8f7f-345e27f563f1,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCIFS.test_list_share_groups[api_with_backend,id-3b277a44-dcae-46da-a58c-f5281d8abc84,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCIFS.test_reset_share_group[api_with_backend,id-069bc68e-6411-44b8-abe9-399885f0eee5,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsCIFS.test_update_share_group[api_with_backend,id-1bab40d5-bdba-4a23-9300-807fe513bf15,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsNFS.test_create_share_group[api_with_backend,id-d060996e-c5f2-4dff-820b-6892a096a425,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsNFS.test_delete_share_group[api_with_backend,id-ea6cbb78-057e-4fbc-86bf-125b033cb76f,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsNFS.test_force_delete_share_group[api_with_backend,id-2cb00ffb-47e3-495e-853c-007752c9e679,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsNFS.test_get_share_group[api_with_backend,id-0de993c5-8389-4997-8f7f-345e27f563f1,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsNFS.test_list_share_groups[api_with_backend,id-3b277a44-dcae-46da-a58c-f5281d8abc84,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsNFS.test_reset_share_group[api_with_backend,id-069bc68e-6411-44b8-abe9-399885f0eee5,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectAdminTestsNFS.test_update_share_group[api_with_backend,id-1bab40d5-bdba-4a23-9300-807fe513bf15,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCEPHFS.test_create_share_group[api_with_backend,id-ebad2242-1fb5-4d99-9a5a-281c1944e03d,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCEPHFS.test_delete_share_group[api_with_backend,id-f5c243e4-5128-4a1c-9a15-8c9f0a44437e,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCEPHFS.test_force_delete_share_group[api_with_backend,id-36a58d50-1257-479f-80a2-f9b7a00814e2,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCEPHFS.test_get_share_group[api_with_backend,id-a29e1a68-220e-40fc-98ea-9092fd256d07,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCEPHFS.test_list_share_groups[api_with_backend,id-d9c04932-c47e-46e0-bfcf-79c2af32c4c7,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCEPHFS.test_reset_share_group[api_with_backend,id-2108c4cd-74e0-467f-823a-e44cf8686afa,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCEPHFS.test_update_share_group[api_with_backend,id-cf9e34b6-6c04-4920-a811-2dbcf07ba14e,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCIFS.test_create_share_group[api_with_backend,id-ebad2242-1fb5-4d99-9a5a-281c1944e03d,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCIFS.test_delete_share_group[api_with_backend,id-f5c243e4-5128-4a1c-9a15-8c9f0a44437e,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCIFS.test_force_delete_share_group[api_with_backend,id-36a58d50-1257-479f-80a2-f9b7a00814e2,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCIFS.test_get_share_group[api_with_backend,id-a29e1a68-220e-40fc-98ea-9092fd256d07,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCIFS.test_list_share_groups[api_with_backend,id-d9c04932-c47e-46e0-bfcf-79c2af32c4c7,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCIFS.test_reset_share_group[api_with_backend,id-2108c4cd-74e0-467f-823a-e44cf8686afa,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsCIFS.test_update_share_group[api_with_backend,id-cf9e34b6-6c04-4920-a811-2dbcf07ba14e,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsNFS.test_create_share_group[api_with_backend,id-ebad2242-1fb5-4d99-9a5a-281c1944e03d,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsNFS.test_delete_share_group[api_with_backend,id-f5c243e4-5128-4a1c-9a15-8c9f0a44437e,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsNFS.test_force_delete_share_group[api_with_backend,id-36a58d50-1257-479f-80a2-f9b7a00814e2,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsNFS.test_get_share_group[api_with_backend,id-a29e1a68-220e-40fc-98ea-9092fd256d07,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsNFS.test_list_share_groups[api_with_backend,id-d9c04932-c47e-46e0-bfcf-79c2af32c4c7,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsNFS.test_reset_share_group[api_with_backend,id-2108c4cd-74e0-467f-823a-e44cf8686afa,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectMemberTestsNFS.test_update_share_group[api_with_backend,id-cf9e34b6-6c04-4920-a811-2dbcf07ba14e,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCEPHFS.test_create_share_group[api_with_backend,id-526dcd91-e789-48f8-b209-c384d77e5803,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCEPHFS.test_delete_share_group[api_with_backend,id-fdf4d49e-a576-441f-9a3c-e2d58c0d8679,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCEPHFS.test_force_delete_share_group[api_with_backend,id-eddca093-e3a1-4a79-a8c7-8fd04c77b02f,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCEPHFS.test_get_share_group[api_with_backend,id-ec0ecbb0-5d45-4624-bb26-8b2e140e2ea9,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCEPHFS.test_list_share_groups[api_with_backend,id-4ac87837-5bdf-4253-ab50-dd6efdcea285,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCEPHFS.test_reset_share_group[api_with_backend,id-37f23531-69b5-418d-bd91-7913341586ec,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCEPHFS.test_update_share_group[api_with_backend,id-4530c19d-0aa5-402e-ac83-a3f2333f6c71,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCIFS.test_create_share_group[api_with_backend,id-526dcd91-e789-48f8-b209-c384d77e5803,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCIFS.test_delete_share_group[api_with_backend,id-fdf4d49e-a576-441f-9a3c-e2d58c0d8679,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCIFS.test_force_delete_share_group[api_with_backend,id-eddca093-e3a1-4a79-a8c7-8fd04c77b02f,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCIFS.test_get_share_group[api_with_backend,id-ec0ecbb0-5d45-4624-bb26-8b2e140e2ea9,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCIFS.test_list_share_groups[api_with_backend,id-4ac87837-5bdf-4253-ab50-dd6efdcea285,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCIFS.test_reset_share_group[api_with_backend,id-37f23531-69b5-418d-bd91-7913341586ec,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsCIFS.test_update_share_group[api_with_backend,id-4530c19d-0aa5-402e-ac83-a3f2333f6c71,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsNFS.test_create_share_group[api_with_backend,id-526dcd91-e789-48f8-b209-c384d77e5803,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsNFS.test_delete_share_group[api_with_backend,id-fdf4d49e-a576-441f-9a3c-e2d58c0d8679,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsNFS.test_force_delete_share_group[api_with_backend,id-eddca093-e3a1-4a79-a8c7-8fd04c77b02f,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsNFS.test_get_share_group[api_with_backend,id-ec0ecbb0-5d45-4624-bb26-8b2e140e2ea9,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsNFS.test_list_share_groups[api_with_backend,id-4ac87837-5bdf-4253-ab50-dd6efdcea285,positive] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsNFS.test_reset_share_group[api_with_backend,id-37f23531-69b5-418d-bd91-7913341586ec,negative] +manila_tempest_tests.tests.rbac.test_share_groups.TestProjectReaderTestsNFS.test_update_share_group[api_with_backend,id-4530c19d-0aa5-402e-ac83-a3f2333f6c71,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectAdminTestsCEPHFS.test_share_instance_list_export_locations[api_with_backend,id-f3218212-d70b-4a3d-bc05-8905a4f14279,positive] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectAdminTestsCEPHFS.test_share_instance_show_export_location[api_with_backend,id-c58c74cf-4fcd-4404-a2f9-1a9e6b4443c4,positive] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectAdminTestsCIFS.test_share_instance_list_export_locations[api_with_backend,id-f3218212-d70b-4a3d-bc05-8905a4f14279,positive] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectAdminTestsCIFS.test_share_instance_show_export_location[api_with_backend,id-c58c74cf-4fcd-4404-a2f9-1a9e6b4443c4,positive] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectAdminTestsNFS.test_share_instance_list_export_locations[api_with_backend,id-f3218212-d70b-4a3d-bc05-8905a4f14279,positive] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectAdminTestsNFS.test_share_instance_show_export_location[api_with_backend,id-c58c74cf-4fcd-4404-a2f9-1a9e6b4443c4,positive] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectMemberTestsCEPHFS.test_share_instance_list_export_locations[api_with_backend,id-27d495dd-b52d-417d-bfbf-9bb700e85f4d,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectMemberTestsCEPHFS.test_share_instance_show_export_location[api_with_backend,id-8de74960-b9cf-4ee6-81dc-fbb5dbb291dd,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectMemberTestsCIFS.test_share_instance_list_export_locations[api_with_backend,id-27d495dd-b52d-417d-bfbf-9bb700e85f4d,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectMemberTestsCIFS.test_share_instance_show_export_location[api_with_backend,id-8de74960-b9cf-4ee6-81dc-fbb5dbb291dd,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectMemberTestsNFS.test_share_instance_list_export_locations[api_with_backend,id-27d495dd-b52d-417d-bfbf-9bb700e85f4d,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectMemberTestsNFS.test_share_instance_show_export_location[api_with_backend,id-8de74960-b9cf-4ee6-81dc-fbb5dbb291dd,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectReaderTestsCEPHFS.test_share_instance_list_export_locations[api_with_backend,id-9db505c1-45d3-4d82-8879-38c4861e4fb3,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectReaderTestsCEPHFS.test_share_instance_show_export_location[api_with_backend,id-09009203-ff20-4914-8764-7865839e29b2,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectReaderTestsCIFS.test_share_instance_list_export_locations[api_with_backend,id-9db505c1-45d3-4d82-8879-38c4861e4fb3,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectReaderTestsCIFS.test_share_instance_show_export_location[api_with_backend,id-09009203-ff20-4914-8764-7865839e29b2,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectReaderTestsNFS.test_share_instance_list_export_locations[api_with_backend,id-9db505c1-45d3-4d82-8879-38c4861e4fb3,negative] +manila_tempest_tests.tests.rbac.test_share_instance_export_locations.TestProjectReaderTestsNFS.test_share_instance_show_export_location[api_with_backend,id-09009203-ff20-4914-8764-7865839e29b2,negative] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectAdminTestsCEPHFS.test_manage_share[api_with_backend,id-8a21f805-2d45-4b0c-8ec5-3f45337bbf66,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectAdminTestsCEPHFS.test_unmanage_share[api_with_backend,id-be5b836d-d6cc-40a5-acf4-e5f249035383,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectAdminTestsCIFS.test_manage_share[api_with_backend,id-8a21f805-2d45-4b0c-8ec5-3f45337bbf66,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectAdminTestsCIFS.test_unmanage_share[api_with_backend,id-be5b836d-d6cc-40a5-acf4-e5f249035383,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectAdminTestsNFS.test_manage_share[api_with_backend,id-8a21f805-2d45-4b0c-8ec5-3f45337bbf66,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectAdminTestsNFS.test_unmanage_share[api_with_backend,id-be5b836d-d6cc-40a5-acf4-e5f249035383,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectMemberTestsCEPHFS.test_manage_share[api_with_backend,id-46f884b2-531d-41c0-8455-8874629b3ea3,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectMemberTestsCEPHFS.test_unmanage_share[api_with_backend,id-9dc2b1a5-8195-46b8-a28a-9710be352f18,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectMemberTestsCIFS.test_manage_share[api_with_backend,id-46f884b2-531d-41c0-8455-8874629b3ea3,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectMemberTestsCIFS.test_unmanage_share[api_with_backend,id-9dc2b1a5-8195-46b8-a28a-9710be352f18,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectMemberTestsNFS.test_manage_share[api_with_backend,id-46f884b2-531d-41c0-8455-8874629b3ea3,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectMemberTestsNFS.test_unmanage_share[api_with_backend,id-9dc2b1a5-8195-46b8-a28a-9710be352f18,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectReaderTestsCEPHFS.test_manage_share[api_with_backend,id-cec85349-b7e3-440e-bbbc-3bb5999b119a,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectReaderTestsCEPHFS.test_unmanage_share[api_with_backend,id-a524620c-90b6-496c-8418-c469e711a607,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectReaderTestsCIFS.test_manage_share[api_with_backend,id-cec85349-b7e3-440e-bbbc-3bb5999b119a,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectReaderTestsCIFS.test_unmanage_share[api_with_backend,id-a524620c-90b6-496c-8418-c469e711a607,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectReaderTestsNFS.test_manage_share[api_with_backend,id-cec85349-b7e3-440e-bbbc-3bb5999b119a,positive] +manila_tempest_tests.tests.rbac.test_share_manage.TestProjectReaderTestsNFS.test_unmanage_share[api_with_backend,id-a524620c-90b6-496c-8418-c469e711a607,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectAdminTests.test_create_share_network[api,id-358dd850-cd81-4b81-aefa-3dfcb7aa4551,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectAdminTests.test_delete_share_network[api,id-6c403ed6-b810-4794-8e9b-d57f173443a2,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectAdminTests.test_list_share_network[api,id-deb20301-9d7c-4c08-b1f0-fc2c403ea708,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectAdminTests.test_show_share_network[api,id-43a3be84-d08b-4f17-89cf-02abda6df580,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectAdminTests.test_update_share_network[api,id-abd2443d-3490-462a-8e51-73b6a8f48795,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectMemberTests.test_create_share_network[api,id-d051c749-3d1c-4485-86c5-6eb860b49cad,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectMemberTests.test_delete_share_network[api,id-717977ab-f077-411a-9bdc-06c8ec9d4f8c,negative] +manila_tempest_tests.tests.rbac.test_share_network.ProjectMemberTests.test_list_share_network[api,id-ac33cd51-1efe-4aaf-99ab-b510b7551571,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectMemberTests.test_show_share_network[api,id-dc3f8f95-f8c5-4030-93dd-e4c56e40b477,negative] +manila_tempest_tests.tests.rbac.test_share_network.ProjectMemberTests.test_update_share_network[api,id-d1fce94c-b163-452d-bf79-13b6edf47e30,negative] +manila_tempest_tests.tests.rbac.test_share_network.ProjectReaderTests.test_create_share_network[api,id-73dd9f09-7106-4fd5-a484-0eb986002e3b,negative] +manila_tempest_tests.tests.rbac.test_share_network.ProjectReaderTests.test_delete_share_network[api,id-f8f26bce-ff82-4472-a8dd-0f46c1757386,negative] +manila_tempest_tests.tests.rbac.test_share_network.ProjectReaderTests.test_list_share_network[api,id-841e9e69-2a22-4572-9147-b233c8a842bc,positive] +manila_tempest_tests.tests.rbac.test_share_network.ProjectReaderTests.test_show_share_network[api,id-c98893c8-cdc6-42af-a842-1ee9466904ae,negative] +manila_tempest_tests.tests.rbac.test_share_network.ProjectReaderTests.test_update_share_network[api,id-67b745cd-e669-4872-bbb7-9307960fbd77,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectAdminTests.test_create_share_type_extra_specs[api,id-d51817f4-b186-4eca-8779-f3c36dcd98e7,positive] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectAdminTests.test_delete_share_type_extra_spec[api,id-81d59322-8ec1-4f32-a50d-2fedd1cca655,positive] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectAdminTests.test_get_share_type_extra_specs[api,id-d4deeb0b-8765-4487-9f1f-9f10088934dd,positive] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectAdminTests.test_update_share_type_extra_spec[api,id-bb27641b-5249-4343-bb08-5a123f31b9f1,positive] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectMemberTests.test_create_share_type_extra_specs[api,id-446c1e7c-5ca2-46f5-b2f3-6417152e5bf8,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectMemberTests.test_delete_share_type_extra_spec[api,id-0bee97ab-b406-481c-9b8b-4813fb9f49dc,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectMemberTests.test_get_share_type_extra_specs[api,id-aabdf0c1-8b68-4c40-a542-cc1dbd039279,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectMemberTests.test_update_share_type_extra_spec[api,id-3629f91c-ad21-4321-acd9-7fca92a4721a,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectReaderTests.test_create_share_type_extra_specs[api,id-da80b823-9a96-45c3-8f86-e9f7fc5ad2c6,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectReaderTests.test_delete_share_type_extra_spec[api,id-46fc8e62-b987-444f-ab9f-cd86a8960156,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectReaderTests.test_get_share_type_extra_specs[api,id-78989220-22dc-46d2-b83b-63f070988eed,negative] +manila_tempest_tests.tests.rbac.test_share_type_extra_specs.ProjectReaderTests.test_update_share_type_extra_spec[api,id-c77173d4-b90c-4edf-a9b6-a26c81aaec42,negative] +manila_tempest_tests.tests.rbac.test_share_types.ProjectAdminTests.test_create_share_type[api,id-b24bf137-352a-4ebd-b736-27518d32c1bd,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectAdminTests.test_get_share_type[api,id-741d69f3-b3fe-49cf-9e33-6b0696b353ec,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectAdminTests.test_list_share_type[api,id-3f811ac6-a345-424f-863a-1a7a49ba0a32,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectAdminTests.test_update_share_type[api,id-3bb9aaab-3c17-45be-a9b1-dd8b6942cb59,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectMemberTests.test_create_share_type[api,id-270761cf-07b4-4fc7-96b5-4deb205adce3,negative] +manila_tempest_tests.tests.rbac.test_share_types.ProjectMemberTests.test_get_share_type[api,id-d3f53218-d92f-489d-8e2e-985178e7fd02,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectMemberTests.test_list_share_type[api,id-757c7ccd-e14e-4c1a-9172-998ae5eed1b8,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectMemberTests.test_update_share_type[api,id-5210170c-b749-4645-a86f-7347c3ba3e99,negative] +manila_tempest_tests.tests.rbac.test_share_types.ProjectReaderTests.test_create_share_type[api,id-f4c352c4-c12b-4722-9fe7-9a2ec639ee63,negative] +manila_tempest_tests.tests.rbac.test_share_types.ProjectReaderTests.test_get_share_type[api,id-e9d9f244-7778-443b-aadc-bac9f2b687b7,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectReaderTests.test_list_share_type[api,id-cf0e97f1-4853-4cf2-9e9a-041c6e57bab5,positive] +manila_tempest_tests.tests.rbac.test_share_types.ProjectReaderTests.test_update_share_type[api,id-338d579b-ff91-4a30-af53-d0b317919efb,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_create_share[api_with_backend,id-34b84af3-a9ea-4c19-8414-e4e44648099c,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_delete_share_metadata[api_with_backend,id-4cd807d6-bac4-4d0f-a207-c84dfe77f032,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_delete_share[api_with_backend,id-44f2eae6-44d4-4962-a94a-d2717b74728f,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_extend_share[api_with_backend,id-2cfa04e5-16cc-43e4-b892-c1a11b0a2f2d,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_force_delete_share[api_with_backend,id-2e915a27-488d-4e33-b2f8-37758ef11653,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_get_share_metadata[api_with_backend,id-2d91e97e-d0e5-4112-8b22-60cd4659586c,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_get_share[api_with_backend,id-14a52454-cba0-4973-926a-28e924ae2e63,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_list_share[api_with_backend,id-5f8c06e6-5b80-45f8-aefb-1b55617d1bd1,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_reset_share[api_with_backend,id-44fb7049-8fc0-4584-9ff1-7527395d2ec5,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_set_share_metadata[api_with_backend,id-d6014579-d772-441a-a9b1-01b1e87caeaa,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_shrink_share[api_with_backend,id-cc49ae58-6696-4030-a029-a66bae2efa96,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCEPHFS.test_update_share[api_with_backend,id-5c2bda4c-0179-4af9-b18c-430a7d31f962,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_create_share[api_with_backend,id-34b84af3-a9ea-4c19-8414-e4e44648099c,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_delete_share_metadata[api_with_backend,id-4cd807d6-bac4-4d0f-a207-c84dfe77f032,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_delete_share[api_with_backend,id-44f2eae6-44d4-4962-a94a-d2717b74728f,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_extend_share[api_with_backend,id-2cfa04e5-16cc-43e4-b892-c1a11b0a2f2d,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_force_delete_share[api_with_backend,id-2e915a27-488d-4e33-b2f8-37758ef11653,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_get_share_metadata[api_with_backend,id-2d91e97e-d0e5-4112-8b22-60cd4659586c,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_get_share[api_with_backend,id-14a52454-cba0-4973-926a-28e924ae2e63,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_list_share[api_with_backend,id-5f8c06e6-5b80-45f8-aefb-1b55617d1bd1,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_reset_share[api_with_backend,id-44fb7049-8fc0-4584-9ff1-7527395d2ec5,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_set_share_metadata[api_with_backend,id-d6014579-d772-441a-a9b1-01b1e87caeaa,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_shrink_share[api_with_backend,id-cc49ae58-6696-4030-a029-a66bae2efa96,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsCIFS.test_update_share[api_with_backend,id-5c2bda4c-0179-4af9-b18c-430a7d31f962,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_create_share[api_with_backend,id-34b84af3-a9ea-4c19-8414-e4e44648099c,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_delete_share_metadata[api_with_backend,id-4cd807d6-bac4-4d0f-a207-c84dfe77f032,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_delete_share[api_with_backend,id-44f2eae6-44d4-4962-a94a-d2717b74728f,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_extend_share[api_with_backend,id-2cfa04e5-16cc-43e4-b892-c1a11b0a2f2d,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_force_delete_share[api_with_backend,id-2e915a27-488d-4e33-b2f8-37758ef11653,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_get_share_metadata[api_with_backend,id-2d91e97e-d0e5-4112-8b22-60cd4659586c,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_get_share[api_with_backend,id-14a52454-cba0-4973-926a-28e924ae2e63,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_list_share[api_with_backend,id-5f8c06e6-5b80-45f8-aefb-1b55617d1bd1,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_reset_share[api_with_backend,id-44fb7049-8fc0-4584-9ff1-7527395d2ec5,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_set_share_metadata[api_with_backend,id-d6014579-d772-441a-a9b1-01b1e87caeaa,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_shrink_share[api_with_backend,id-cc49ae58-6696-4030-a029-a66bae2efa96,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectAdminTestsNFS.test_update_share[api_with_backend,id-5c2bda4c-0179-4af9-b18c-430a7d31f962,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_create_share[api_with_backend,id-7a6eef6b-bf8e-4cb3-a39c-6dc7fbe115ab,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_delete_share_metadata[api_with_backend,id-bea5518a-338e-494d-9034-1d03658ed58b,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_delete_share[api_with_backend,id-6c546ed7-ebfd-4ac5-a626-d333a25a9e66,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_extend_share[api_with_backend,id-c09e6a72-5b99-4be6-8ffe-8ecaad0be990,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_force_delete_share[api_with_backend,id-2349d2b0-6314-4018-85e5-696f8d1ca94a,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_get_share_metadata[api_with_backend,id-a69a2b85-3374-4621-83a9-89937ddb520b,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_get_share[api_with_backend,id-75b9fd40-ae63-4caf-9c93-0fe24b2ce904,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_list_share[api_with_backend,id-92fd157a-f357-4a08-9fc6-9e77a55b89a8,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_reset_share[api_with_backend,id-483cbaef-a53d-433a-9259-f2ecc209f405,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_set_share_metadata[api_with_backend,id-f1c03630-987c-4f19-938d-4a0ef6529177,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_shrink_share[api_with_backend,id-56a07567-d0a9-460a-9267-fcd82306a371,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCEPHFS.test_update_share[api_with_backend,id-20d6360d-5cea-4305-be36-7e1429007598,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_create_share[api_with_backend,id-7a6eef6b-bf8e-4cb3-a39c-6dc7fbe115ab,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_delete_share_metadata[api_with_backend,id-bea5518a-338e-494d-9034-1d03658ed58b,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_delete_share[api_with_backend,id-6c546ed7-ebfd-4ac5-a626-d333a25a9e66,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_extend_share[api_with_backend,id-c09e6a72-5b99-4be6-8ffe-8ecaad0be990,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_force_delete_share[api_with_backend,id-2349d2b0-6314-4018-85e5-696f8d1ca94a,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_get_share_metadata[api_with_backend,id-a69a2b85-3374-4621-83a9-89937ddb520b,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_get_share[api_with_backend,id-75b9fd40-ae63-4caf-9c93-0fe24b2ce904,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_list_share[api_with_backend,id-92fd157a-f357-4a08-9fc6-9e77a55b89a8,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_reset_share[api_with_backend,id-483cbaef-a53d-433a-9259-f2ecc209f405,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_set_share_metadata[api_with_backend,id-f1c03630-987c-4f19-938d-4a0ef6529177,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_shrink_share[api_with_backend,id-56a07567-d0a9-460a-9267-fcd82306a371,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsCIFS.test_update_share[api_with_backend,id-20d6360d-5cea-4305-be36-7e1429007598,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_create_share[api_with_backend,id-7a6eef6b-bf8e-4cb3-a39c-6dc7fbe115ab,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_delete_share_metadata[api_with_backend,id-bea5518a-338e-494d-9034-1d03658ed58b,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_delete_share[api_with_backend,id-6c546ed7-ebfd-4ac5-a626-d333a25a9e66,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_extend_share[api_with_backend,id-c09e6a72-5b99-4be6-8ffe-8ecaad0be990,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_force_delete_share[api_with_backend,id-2349d2b0-6314-4018-85e5-696f8d1ca94a,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_get_share_metadata[api_with_backend,id-a69a2b85-3374-4621-83a9-89937ddb520b,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_get_share[api_with_backend,id-75b9fd40-ae63-4caf-9c93-0fe24b2ce904,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_list_share[api_with_backend,id-92fd157a-f357-4a08-9fc6-9e77a55b89a8,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_reset_share[api_with_backend,id-483cbaef-a53d-433a-9259-f2ecc209f405,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_set_share_metadata[api_with_backend,id-f1c03630-987c-4f19-938d-4a0ef6529177,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_shrink_share[api_with_backend,id-56a07567-d0a9-460a-9267-fcd82306a371,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectMemberTestsNFS.test_update_share[api_with_backend,id-20d6360d-5cea-4305-be36-7e1429007598,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_create_share[api_with_backend,id-350ba4c9-def9-4865-824a-de1ddff5dcf9,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_delete_share_metadata[api_with_backend,id-55486589-a4ef-44f2-b489-96bc29dcd243,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_delete_share[api_with_backend,id-eb92b142-fd8d-47e3-99fe-944cce747ad7,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_extend_share[api_with_backend,id-0b57aedb-6b68-498f-814e-173c47e6c307,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_force_delete_share[api_with_backend,id-cb040955-5897-409f-aea0-84b6ae16b77e,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_get_share_metadata[api_with_backend,id-28cacc77-556f-4707-ba2b-5ef3e56d6ef9,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_get_share[api_with_backend,id-dc439eaf-c885-4002-be8f-4c488beeca81,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_list_share[api_with_backend,id-1fbb1078-4386-4b52-aa88-e6be4a286791,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_reset_share[api_with_backend,id-e5ae5b56-38c0-44ec-b8e0-4bc2a5c1d28a,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_set_share_metadata[api_with_backend,id-3def3f4e-33fc-4726-8818-6cffbc2cab51,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_shrink_share[api_with_backend,id-f85818b1-b93a-4b89-8aa4-b099e582be7c,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCEPHFS.test_update_share[api_with_backend,id-3184269a-11ca-4484-8a4d-b855a6e1800f,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_create_share[api_with_backend,id-350ba4c9-def9-4865-824a-de1ddff5dcf9,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_delete_share_metadata[api_with_backend,id-55486589-a4ef-44f2-b489-96bc29dcd243,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_delete_share[api_with_backend,id-eb92b142-fd8d-47e3-99fe-944cce747ad7,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_extend_share[api_with_backend,id-0b57aedb-6b68-498f-814e-173c47e6c307,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_force_delete_share[api_with_backend,id-cb040955-5897-409f-aea0-84b6ae16b77e,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_get_share_metadata[api_with_backend,id-28cacc77-556f-4707-ba2b-5ef3e56d6ef9,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_get_share[api_with_backend,id-dc439eaf-c885-4002-be8f-4c488beeca81,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_list_share[api_with_backend,id-1fbb1078-4386-4b52-aa88-e6be4a286791,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_reset_share[api_with_backend,id-e5ae5b56-38c0-44ec-b8e0-4bc2a5c1d28a,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_set_share_metadata[api_with_backend,id-3def3f4e-33fc-4726-8818-6cffbc2cab51,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_shrink_share[api_with_backend,id-f85818b1-b93a-4b89-8aa4-b099e582be7c,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsCIFS.test_update_share[api_with_backend,id-3184269a-11ca-4484-8a4d-b855a6e1800f,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_create_share[api_with_backend,id-350ba4c9-def9-4865-824a-de1ddff5dcf9,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_delete_share_metadata[api_with_backend,id-55486589-a4ef-44f2-b489-96bc29dcd243,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_delete_share[api_with_backend,id-eb92b142-fd8d-47e3-99fe-944cce747ad7,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_extend_share[api_with_backend,id-0b57aedb-6b68-498f-814e-173c47e6c307,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_force_delete_share[api_with_backend,id-cb040955-5897-409f-aea0-84b6ae16b77e,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_get_share_metadata[api_with_backend,id-28cacc77-556f-4707-ba2b-5ef3e56d6ef9,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_get_share[api_with_backend,id-dc439eaf-c885-4002-be8f-4c488beeca81,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_list_share[api_with_backend,id-1fbb1078-4386-4b52-aa88-e6be4a286791,positive] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_reset_share[api_with_backend,id-e5ae5b56-38c0-44ec-b8e0-4bc2a5c1d28a,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_set_share_metadata[api_with_backend,id-3def3f4e-33fc-4726-8818-6cffbc2cab51,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_shrink_share[api_with_backend,id-f85818b1-b93a-4b89-8aa4-b099e582be7c,negative] +manila_tempest_tests.tests.rbac.test_shares.TestProjectReaderTestsNFS.test_update_share[api_with_backend,id-3184269a-11ca-4484-8a4d-b855a6e1800f,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_create_snapshot[api_with_backend,id-2b90d3e9-ec71-468a-86e9-e8955139ad48,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_delete_snapshot_metadata[api_with_backend,id-7490eb85-fcdc-45ae-89ba-14cf34c58b3b,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_delete_snapshot[api_with_backend,id-6de91ee0-d27e-409a-957b-75489d4e7291,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_force_delete_snapshot[api_with_backend,id-3ac10dfb-3445-4052-855a-a17056d16a9c,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_get_snapshot_metadata[api_with_backend,id-235b3fe2-55bf-4a43-b703-ca413ce76a41,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_get_snapshot[api_with_backend,id-e55b1a01-0fcb-42aa-8cc4-b041fc75f1e4,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_list_snapshot[api_with_backend,id-3b209017-f5ad-4daa-8932-582a75975bbe,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_rename_snapshot[api_with_backend,id-513c8fef-9597-4e6c-a811-fb89b456d457,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_reset_snapshot[api_with_backend,id-a5e99bfb-8767-4680-9e39-bde767e4b8f8,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCEPHFS.test_set_snapshot_metadata[api_with_backend,id-8c0858d5-5670-4c21-ab1f-7d74a7f8cb6d,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_create_snapshot[api_with_backend,id-2b90d3e9-ec71-468a-86e9-e8955139ad48,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_delete_snapshot_metadata[api_with_backend,id-7490eb85-fcdc-45ae-89ba-14cf34c58b3b,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_delete_snapshot[api_with_backend,id-6de91ee0-d27e-409a-957b-75489d4e7291,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_force_delete_snapshot[api_with_backend,id-3ac10dfb-3445-4052-855a-a17056d16a9c,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_get_snapshot_metadata[api_with_backend,id-235b3fe2-55bf-4a43-b703-ca413ce76a41,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_get_snapshot[api_with_backend,id-e55b1a01-0fcb-42aa-8cc4-b041fc75f1e4,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_list_snapshot[api_with_backend,id-3b209017-f5ad-4daa-8932-582a75975bbe,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_rename_snapshot[api_with_backend,id-513c8fef-9597-4e6c-a811-fb89b456d457,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_reset_snapshot[api_with_backend,id-a5e99bfb-8767-4680-9e39-bde767e4b8f8,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsCIFS.test_set_snapshot_metadata[api_with_backend,id-8c0858d5-5670-4c21-ab1f-7d74a7f8cb6d,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_create_snapshot[api_with_backend,id-2b90d3e9-ec71-468a-86e9-e8955139ad48,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_delete_snapshot_metadata[api_with_backend,id-7490eb85-fcdc-45ae-89ba-14cf34c58b3b,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_delete_snapshot[api_with_backend,id-6de91ee0-d27e-409a-957b-75489d4e7291,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_force_delete_snapshot[api_with_backend,id-3ac10dfb-3445-4052-855a-a17056d16a9c,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_get_snapshot_metadata[api_with_backend,id-235b3fe2-55bf-4a43-b703-ca413ce76a41,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_get_snapshot[api_with_backend,id-e55b1a01-0fcb-42aa-8cc4-b041fc75f1e4,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_list_snapshot[api_with_backend,id-3b209017-f5ad-4daa-8932-582a75975bbe,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_rename_snapshot[api_with_backend,id-513c8fef-9597-4e6c-a811-fb89b456d457,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_reset_snapshot[api_with_backend,id-a5e99bfb-8767-4680-9e39-bde767e4b8f8,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectAdminTestsNFS.test_set_snapshot_metadata[api_with_backend,id-8c0858d5-5670-4c21-ab1f-7d74a7f8cb6d,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_create_snapshot[api_with_backend,id-d880b3f0-9027-4141-b28a-13e797919af7,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_delete_snapshot_metadata[api_with_backend,id-aa3ecadd-b87f-4bcd-ab2c-6a19d27c0adb,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_delete_snapshot[api_with_backend,id-e3fdd270-971f-4478-9e64-9bd11166bab6,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_force_delete_snapshot[api_with_backend,id-a93d6946-1d86-40a1-af01-90e843f8f575,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_get_snapshot_metadata[api_with_backend,id-9a3fc032-eb0c-49e9-8026-d26b05520d95,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_get_snapshot[api_with_backend,id-4ba65029-5c8b-4e96-940a-094d9f662cf6,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_list_snapshot[api_with_backend,id-0dcc1f68-86e2-432e-ad50-51c3cb78b986,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_rename_snapshot[api_with_backend,id-6da7bf79-25ab-4475-a5e0-1046781e9bc7,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_reset_snapshot[api_with_backend,id-22ba2e2e-6788-4075-9e92-af140d3b1238,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCEPHFS.test_set_snapshot_metadata[api_with_backend,id-a2849a7a-66ae-4cf9-9bac-6420dddd8f03,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_create_snapshot[api_with_backend,id-d880b3f0-9027-4141-b28a-13e797919af7,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_delete_snapshot_metadata[api_with_backend,id-aa3ecadd-b87f-4bcd-ab2c-6a19d27c0adb,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_delete_snapshot[api_with_backend,id-e3fdd270-971f-4478-9e64-9bd11166bab6,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_force_delete_snapshot[api_with_backend,id-a93d6946-1d86-40a1-af01-90e843f8f575,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_get_snapshot_metadata[api_with_backend,id-9a3fc032-eb0c-49e9-8026-d26b05520d95,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_get_snapshot[api_with_backend,id-4ba65029-5c8b-4e96-940a-094d9f662cf6,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_list_snapshot[api_with_backend,id-0dcc1f68-86e2-432e-ad50-51c3cb78b986,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_rename_snapshot[api_with_backend,id-6da7bf79-25ab-4475-a5e0-1046781e9bc7,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_reset_snapshot[api_with_backend,id-22ba2e2e-6788-4075-9e92-af140d3b1238,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsCIFS.test_set_snapshot_metadata[api_with_backend,id-a2849a7a-66ae-4cf9-9bac-6420dddd8f03,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_create_snapshot[api_with_backend,id-d880b3f0-9027-4141-b28a-13e797919af7,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_delete_snapshot_metadata[api_with_backend,id-aa3ecadd-b87f-4bcd-ab2c-6a19d27c0adb,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_delete_snapshot[api_with_backend,id-e3fdd270-971f-4478-9e64-9bd11166bab6,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_force_delete_snapshot[api_with_backend,id-a93d6946-1d86-40a1-af01-90e843f8f575,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_get_snapshot_metadata[api_with_backend,id-9a3fc032-eb0c-49e9-8026-d26b05520d95,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_get_snapshot[api_with_backend,id-4ba65029-5c8b-4e96-940a-094d9f662cf6,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_list_snapshot[api_with_backend,id-0dcc1f68-86e2-432e-ad50-51c3cb78b986,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_rename_snapshot[api_with_backend,id-6da7bf79-25ab-4475-a5e0-1046781e9bc7,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_reset_snapshot[api_with_backend,id-22ba2e2e-6788-4075-9e92-af140d3b1238,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectMemberTestsNFS.test_set_snapshot_metadata[api_with_backend,id-a2849a7a-66ae-4cf9-9bac-6420dddd8f03,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_create_snapshot[api_with_backend,id-17a80156-8cd6-420e-8ffe-97103edef4c3,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_delete_snapshot_metadata[api_with_backend,id-e1c0251b-b8f4-439f-b697-ea3fc024c2ff,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_delete_snapshot[api_with_backend,id-b0ca5483-ebdb-484c-a975-525e4d7deca2,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_force_delete_snapshot[api_with_backend,id-ed0af390-e3d0-432b-9147-c0d569181b92,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_get_snapshot_metadata[api_with_backend,id-e5471262-fb4f-4a80-91c1-cc925b94f2c1,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_get_snapshot[api_with_backend,id-46a09178-0264-4f56-9a5f-9a0583e72e4d,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_list_snapshot[api_with_backend,id-fef4285a-a489-4fec-97af-763c2e33282e,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_rename_snapshot[api_with_backend,id-21db863f-c2a4-4d07-b435-2a000255ea3b,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_reset_snapshot[api_with_backend,id-b8c9c9a4-3b2a-4b1c-80d8-2ec87d708111,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCEPHFS.test_set_snapshot_metadata[api_with_backend,id-c9177029-3161-4b5b-b7cb-76f8259a459a,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_create_snapshot[api_with_backend,id-17a80156-8cd6-420e-8ffe-97103edef4c3,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_delete_snapshot_metadata[api_with_backend,id-e1c0251b-b8f4-439f-b697-ea3fc024c2ff,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_delete_snapshot[api_with_backend,id-b0ca5483-ebdb-484c-a975-525e4d7deca2,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_force_delete_snapshot[api_with_backend,id-ed0af390-e3d0-432b-9147-c0d569181b92,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_get_snapshot_metadata[api_with_backend,id-e5471262-fb4f-4a80-91c1-cc925b94f2c1,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_get_snapshot[api_with_backend,id-46a09178-0264-4f56-9a5f-9a0583e72e4d,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_list_snapshot[api_with_backend,id-fef4285a-a489-4fec-97af-763c2e33282e,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_rename_snapshot[api_with_backend,id-21db863f-c2a4-4d07-b435-2a000255ea3b,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_reset_snapshot[api_with_backend,id-b8c9c9a4-3b2a-4b1c-80d8-2ec87d708111,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsCIFS.test_set_snapshot_metadata[api_with_backend,id-c9177029-3161-4b5b-b7cb-76f8259a459a,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_create_snapshot[api_with_backend,id-17a80156-8cd6-420e-8ffe-97103edef4c3,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_delete_snapshot_metadata[api_with_backend,id-e1c0251b-b8f4-439f-b697-ea3fc024c2ff,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_delete_snapshot[api_with_backend,id-b0ca5483-ebdb-484c-a975-525e4d7deca2,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_force_delete_snapshot[api_with_backend,id-ed0af390-e3d0-432b-9147-c0d569181b92,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_get_snapshot_metadata[api_with_backend,id-e5471262-fb4f-4a80-91c1-cc925b94f2c1,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_get_snapshot[api_with_backend,id-46a09178-0264-4f56-9a5f-9a0583e72e4d,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_list_snapshot[api_with_backend,id-fef4285a-a489-4fec-97af-763c2e33282e,positive] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_rename_snapshot[api_with_backend,id-21db863f-c2a4-4d07-b435-2a000255ea3b,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_reset_snapshot[api_with_backend,id-b8c9c9a4-3b2a-4b1c-80d8-2ec87d708111,negative] +manila_tempest_tests.tests.rbac.test_snapshots.TestProjectReaderTestsNFS.test_set_snapshot_metadata[api_with_backend,id-c9177029-3161-4b5b-b7cb-76f8259a459a,negative] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectAdminTests.test_delete_message[api,id-b91c355b-a5f8-47aa-8ab4-00a350f8ac7f,positive] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectAdminTests.test_list_messages[api,id-2067d8ba-953d-4035-b65d-6001b3d4ea8f,positive] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectAdminTests.test_show_message[api,id-ec46f10e-c768-4df5-b75a-0ce3e22d8038,positive] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectMemberTests.test_delete_message[api,id-5821a3a9-6194-414a-9668-0d933a0d4fb0,positive] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectMemberTests.test_list_messages[api,id-1fd0f86d-cb1e-4694-a54e-4b7774c7c652,positive] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectMemberTests.test_show_message[api,id-283d33be-727b-4180-a503-95d31cc99a79,positive] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectReaderTests.test_delete_message[api,id-a03695c7-e05a-4c89-9a04-7d94a8dd2419,negative] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectReaderTests.test_list_messages[api,id-ab3b8812-47df-4472-a410-7f84d52999f3,positive] +manila_tempest_tests.tests.rbac.test_user_messages.ProjectReaderTests.test_show_message[api,id-f0603a61-b620-4f89-afc5-006d1195fa7f,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_migration_files_1_True[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_migration_files_2_False[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_mount_share_one_vm_with_ceph_fuse_client[backend,id-9fb12879-45b3-4042-acac-82be338dbde1,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_mount_share_one_vm[backend,id-825be71c-cf14-4884-a0ad-cf47d511df9a,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_read_mountable_snapshot[backend,id-c98e6876-3a4f-40e8-8b4f-023c94c242c3,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_read_write_two_vms_with_ceph_fuse_client[backend,id-c247f51f-0ffc-4a4f-894c-781647619faf,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_read_write_two_vms[backend,id-5e184576-c7d1-4c16-9b7c-bc9bcd65ba58,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_write_data_to_share_created_from_snapshot_with_ceph_fuse_client[backend,id-5bd64c46-05f4-4891-a08f-e146d1a76437,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_write_data_to_share_created_from_snapshot[backend,id-87b803bf-679a-492b-a538-af4c9ff013c8,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_write_with_ro_access_with_ceph_fuse_client[backend,id-a2a70b94-f5fc-438a-9dfa-53aa60ee3949,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestBaseShareBasicOpsScenarioCEPHFS.test_write_with_ro_access[backend,id-7cc61131-90e1-42fb-9f07-d3786efb338f,negative] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsCIFS.test_migration_files_1_True[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsCIFS.test_migration_files_2_False[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsCIFS.test_mount_share_one_vm[backend,id-825be71c-cf14-4884-a0ad-cf47d511df9a,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsCIFS.test_read_mountable_snapshot[backend,id-a691332b-dd7a-4041-9bbd-3893e168aefa,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsCIFS.test_read_write_two_vms[backend,id-5e184576-c7d1-4c16-9b7c-bc9bcd65ba58,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsCIFS.test_write_data_to_share_created_from_snapshot[backend,id-8c936c3e-4793-49d2-8409-4038f03e7012,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsCIFS.test_write_with_ro_access[backend,id-4344a47a-d316-496b-97a4-12a59297950a,negative] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_migration_files_1_True[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_migration_files_2_False[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_mount_share_one_vm_1_4[backend,id-4bad2073-a19b-4851-8cbe-75b20ade5cdb,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_mount_share_one_vm[backend,id-825be71c-cf14-4884-a0ad-cf47d511df9a,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_read_mountable_snapshot[backend,id-c98e6876-3a4f-40e8-8b4f-023c94c242c3,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_read_write_two_vms[backend,id-5e184576-c7d1-4c16-9b7c-bc9bcd65ba58,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_write_data_to_share_created_from_snapshot[backend,id-87b803bf-679a-492b-a538-af4c9ff013c8,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFS.test_write_with_ro_access[backend,id-7cc61131-90e1-42fb-9f07-d3786efb338f,negative] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_migration_files_1_True[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_migration_files_2_False[backend,id-15d42949-545e-4ad8-b06e-bb2556c54375,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_mount_share_one_vm_1_4[backend,id-4bad2073-a19b-4851-8cbe-75b20ade5cdb,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_mount_share_one_vm[backend,id-825be71c-cf14-4884-a0ad-cf47d511df9a,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_read_mountable_snapshot[backend,id-c98e6876-3a4f-40e8-8b4f-023c94c242c3,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_read_write_two_vms[backend,id-5e184576-c7d1-4c16-9b7c-bc9bcd65ba58,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_write_data_to_share_created_from_snapshot[backend,id-87b803bf-679a-492b-a538-af4c9ff013c8,positive] +manila_tempest_tests.tests.scenario.test_share_basic_ops.TestShareBasicOpsNFSIPv6.test_write_with_ro_access[backend,id-7cc61131-90e1-42fb-9f07-d3786efb338f,negative] +manila_tempest_tests.tests.scenario.test_share_extend.TestBaseShareExtendScenarioCEPHFS.test_create_extend_and_write_with_ceph_fuse_client[backend,id-9ca1e4a9-23e3-4da6-a63e-46e7919335e0,positive] +manila_tempest_tests.tests.scenario.test_share_extend.TestBaseShareExtendScenarioCEPHFS.test_create_extend_and_write[backend,id-e1c0d614-c8f2-43cf-9c49-25808b07ba4a,positive] +manila_tempest_tests.tests.scenario.test_share_extend.TestShareExtendCIFS.test_create_extend_and_write[backend,id-4a9f5cf9-990d-4a35-9ac7-87ef593a09e3,positive] +manila_tempest_tests.tests.scenario.test_share_extend.TestShareExtendNFS.test_create_extend_and_write[backend,id-e1c0d614-c8f2-43cf-9c49-25808b07ba4a,positive] +manila_tempest_tests.tests.scenario.test_share_extend.TestShareExtendNFSIPv6.test_create_extend_and_write[backend,id-e1c0d614-c8f2-43cf-9c49-25808b07ba4a,positive] +manila_tempest_tests.tests.scenario.test_share_manage_unmanage.ShareManageUnmanageCIFS.test_create_manage_and_write[backend,id-165f22b4-9417-45ee-8e1d-2b8669ac2f00,positive] +manila_tempest_tests.tests.scenario.test_share_manage_unmanage.ShareManageUnmanageNFS.test_create_manage_and_write[backend,id-165f22b4-9417-45ee-8e1d-2b8669ac2f00,positive] +manila_tempest_tests.tests.scenario.test_share_manage_unmanage.ShareManageUnmanageNFSIPv6.test_create_manage_and_write[backend,id-165f22b4-9417-45ee-8e1d-2b8669ac2f00,positive] +manila_tempest_tests.tests.scenario.test_share_shrink.TestBaseShareShrinkScenarioCEPHFS.test_create_shrink_and_write_with_ceph_fuse_client[backend,id-7fb324ed-7479-4bd9-b022-b3739dee9bcb,positive] +manila_tempest_tests.tests.scenario.test_share_shrink.TestBaseShareShrinkScenarioCEPHFS.test_create_shrink_and_write[backend,id-ed0f9c0c-5302-4cc9-9f5d-f7641cc3b83b,positive] +manila_tempest_tests.tests.scenario.test_share_shrink.TestShareShrinkCIFS.test_create_shrink_and_write[backend,id-ed0f9c0c-5302-4cc9-9f5d-f7641cc3b83b,positive] +manila_tempest_tests.tests.scenario.test_share_shrink.TestShareShrinkNFS.test_create_shrink_and_write[backend,id-ed0f9c0c-5302-4cc9-9f5d-f7641cc3b83b,positive] +manila_tempest_tests.tests.scenario.test_share_shrink.TestShareShrinkNFSIPv6.test_create_shrink_and_write[backend,id-ed0f9c0c-5302-4cc9-9f5d-f7641cc3b83b,positive] diff --git a/.automation.conf/tempest/load-lists/neutron b/.automation.conf/tempest/load-lists/neutron new file mode 100644 index 000000000..bd6be93ee --- /dev/null +++ b/.automation.conf/tempest/load-lists/neutron @@ -0,0 +1,1005 @@ +neutron_tempest_plugin.api.admin.test_agent_availability_zone.AgentAvailabilityZoneTestCase.test_agents_availability_zone[id-3ffa661e-cfcc-417d-8b63-1c5ec4a22e54] +neutron_tempest_plugin.api.admin.test_agent_management.AgentManagementTestJSON.test_delete_agent_negative[id-b33af888-b6ac-4e68-a0ca-0444c2696cf9] +neutron_tempest_plugin.api.admin.test_agent_management.AgentManagementTestJSON.test_list_agent[id-9c80f04d-11f3-44a4-8738-ed2f879b0ff4] +neutron_tempest_plugin.api.admin.test_agent_management.AgentManagementTestJSON.test_list_agents_non_admin[id-e335be47-b9a1-46fd-be30-0874c0b751e6] +neutron_tempest_plugin.api.admin.test_agent_management.AgentManagementTestJSON.test_show_agent[id-869bc8e8-0fda-4a30-9b71-f8a7cf58ca9f] +neutron_tempest_plugin.api.admin.test_agent_management.AgentManagementTestJSON.test_update_agent_description[id-68a94a14-1243-46e6-83bf-157627e31556] +neutron_tempest_plugin.api.admin.test_agent_management.AgentManagementTestJSON.test_update_agent_status[id-371dfc5b-55b9-4cb5-ac82-c40eadaac941] +neutron_tempest_plugin.api.admin.test_default_security_group_rules.DefaultSecurityGroupRuleTest.test_create_duplicate_default_security_group_rule_different_templates[id-6c5a2f41-5899-47f4-9daf-4f8ddbbd3ad5] +neutron_tempest_plugin.api.admin.test_default_security_group_rules.DefaultSecurityGroupRuleTest.test_create_same_default_security_group_rule_for_different_templates[id-e4696607-1a13-48eb-8912-ee1e742d9471] +neutron_tempest_plugin.api.admin.test_default_security_group_rules.DefaultSecurityGroupRuleTest.test_default_security_group_rule_lifecycle[id-df98f969-ff2d-4597-9765-f5d4f81f775f] +neutron_tempest_plugin.api.admin.test_default_security_group_rules.DefaultSecurityGroupRuleTest.test_legacy_default_sg_rules_created_by_default[id-2f3d3070-e9fa-4127-a33f-f1532fd89108] +neutron_tempest_plugin.api.admin.test_default_security_group_rules.DefaultSecurityGroupRuleTest.test_security_group_rules_created_from_default_sg_rules_template[id-29feedb1-6f04-4a1f-a778-2fae2c7b7dc8] +neutron_tempest_plugin.api.admin.test_extension_driver_port_security_admin.PortSecurityAdminTests.test_create_port_security_false_on_shared_network[id-d39a96e2-2dea-4feb-8093-e7ac991ce6f8,negative] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_delete_policies_while_tenant_attached_to_net[id-d54decee-4203-4ced-91a2-ea42ca63e154] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_external_conversion_on_one_policy_delete[id-fb938b8e-d1d2-4bf8-823a-1b3c79c98a25] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_external_conversion_on_policy_create[id-a5539002-5bdb-48b5-b124-e9eedd5975e6] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_external_conversion_on_policy_delete[id-beddbe9d-304c-4577-8bbe-9b61d0a449b4] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_external_network_on_shared_policy_delete[id-046aff1a-3962-4e2b-9a3b-93a24f255fd0] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_external_update_policy_from_wildcard_to_specific_tenant[id-a5539002-5bdb-48b5-b124-abcd12347865] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_policy_allows_tenant_to_allocate_floatingip[id-01364c50-bfb6-46c4-b44c-edc4564d61cf] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_policy_allows_tenant_to_attach_ext_gw[id-476be1e0-f72e-47dc-9a14-4435926bbe82] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_regular_client_blocked_from_creating_external_wild_policies[id-eff9443a-2d04-48ee-840e-d955ac564bcd] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_regular_client_shares_with_another[id-afd8f1b7-a81e-4629-bca8-a367b3a144bb] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_wildcard_policy_created_from_external_network_api[id-a2e19f06-48a9-4e4c-b717-08cb2008707d] +neutron_tempest_plugin.api.admin.test_external_network_extension.ExternalNetworksRBACTestJSON.test_wildcard_policy_delete_blocked_on_default_ext[id-7041cec7-d8fe-4c78-9b04-b51b2fd49dc9] +neutron_tempest_plugin.api.admin.test_floating_ips_admin_actions.FloatingIPAdminTestJSON.test_associate_floating_ip_with_port_from_another_project[id-11116ee9-4e99-5b15-b8e1-aa7df92ca589,negative] +neutron_tempest_plugin.api.admin.test_floating_ips_admin_actions.FloatingIPAdminTestJSON.test_create_floatingip_with_specified_ip_address[id-332a8ae4-402e-4b98-bb6f-532e5a87b8e0] +neutron_tempest_plugin.api.admin.test_l3_agent_scheduler.L3AgentSchedulerTestJSON.test_add_list_remove_router_on_l3_agent[id-9464e5e7-8625-49c3-8fd1-89c52be59d66] +neutron_tempest_plugin.api.admin.test_l3_agent_scheduler.L3AgentSchedulerTestJSON.test_list_routers_on_l3_agent[id-b7ce6e89-e837-4ded-9b78-9ed3c9c6a45a] +neutron_tempest_plugin.api.admin.test_logging_negative.LoggingNegativeTestJSON.test_create_log_with_invalid_resource_type[id-5fc61e24-cad5-4d86-a2d4-f40c0fa0a54c,negative] +neutron_tempest_plugin.api.admin.test_logging_negative.LoggingNegativeTestJSON.test_create_log_with_nonexistent_port[id-7ed63170-0748-44b7-b0a0-64bfd9390dac,negative] +neutron_tempest_plugin.api.admin.test_logging_negative.LoggingNegativeTestJSON.test_create_log_with_nonexistent_sg[id-89194c6b-8f47-400b-979b-072b1c1f767b,negative] +neutron_tempest_plugin.api.admin.test_logging.LoggingTestJSON.test_list_supported_logging_types[id-1af6cdab-0eb0-4e13-8027-d89cf1c7a87a] +neutron_tempest_plugin.api.admin.test_logging.LoggingTestJSON.test_log_deleted_with_corresponding_security_group[id-1ab4eb2a-76f5-45b9-816b-1aa497a71eea] +neutron_tempest_plugin.api.admin.test_logging.LoggingTestJSON.test_log_lifecycle[id-8d2e1ba5-455b-4519-a88e-e587002faba6] +neutron_tempest_plugin.api.admin.test_network_segment_range.NetworkSegmentRangeTestJson.test_create_network_with_default_network_segment_range[id-2129a26b-a97b-43d6-b0b2-04253c6046f8] +neutron_tempest_plugin.api.admin.test_network_segment_range.NetworkSegmentRangeTestJson.test_create_network_with_tenant_specific_network_segment_range[id-0019de49-c0ea-4554-af57-18ad4ae30195] +neutron_tempest_plugin.api.admin.test_network_segment_range.NetworkSegmentRangeTestJson.test_create_update_delete_network_segment_range[id-44a1ece1-d85d-4253-92f8-4efc318a6d8e] +neutron_tempest_plugin.api.admin.test_network_segment_range.NetworkSegmentRangeTestJson.test_delete_network_segment_range_failed_with_segment_referenced[id-42959544-9956-4b0c-aec6-d56533323924] +neutron_tempest_plugin.api.admin.test_network_segment_range.NetworkSegmentRangeTestJson.test_list_network_segment_ranges[id-17139cc1-4826-4bf9-9c39-85b74894d938] +neutron_tempest_plugin.api.admin.test_network_segment_range.NetworkSegmentRangeTestJson.test_show_network_segment_range[id-54fa26c9-37b5-4df4-a934-a705f29920fc] +neutron_tempest_plugin.api.admin.test_network_segment_range.NetworkSegmentRangeTestJson.test_update_network_segment_range_failed_with_existing_range_impacted[id-5e118fef-a139-4886-8250-07e73d2cbe7a] +neutron_tempest_plugin.api.admin.test_networks.NetworksTestAdmin.test_create_network_with_project_and_other_tenant[id-08b92179-669d-45ee-8233-ef6611190809] +neutron_tempest_plugin.api.admin.test_networks.NetworksTestAdmin.test_create_network_with_project_and_tenant[id-8d21aaca-4364-4eb9-8b79-44b4fff6373b] +neutron_tempest_plugin.api.admin.test_networks.NetworksTestAdmin.test_create_network_with_project[id-d3c76044-d067-4cb0-ae47-8cdd875c7f67] +neutron_tempest_plugin.api.admin.test_networks.NetworksTestAdmin.test_create_tenant_network_vxlan[id-571d0dde-0f84-11e7-b565-fa163e4fa634] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesAdmin.test_port_create_with_trusted_attr_set[id-4d75cc60-99d0-4949-b3ce-5826b81aa0a9] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesAdmin.test_port_set_trusted_attr[id-26c15e2a-55b2-410f-8ed3-84db9406ff3f] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesAdmin.test_regenerate_mac_address[id-dfe8cc79-18d9-4ae8-acef-3ec6bb719cc2] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesAdmin.test_update_mac_address[id-dfe8cc79-18d9-4ae8-acef-3ec6bb719bb1] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesResourceRequest.test_port_resource_request_empty[id-0eeb6ffa-9a7a-40b5-83dd-dbdcd67e2e64] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesResourceRequest.test_port_resource_request_inherited_policy[id-b6c34ae4-44c8-47f0-86de-7ef9866fa000] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesResourceRequest.test_port_resource_request_min_bw_and_min_pps[id-5ae93aa0-408a-11ec-bbca-17b1a60f3438] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesResourceRequest.test_port_resource_request_no_provider_net_conflict[id-7261391f-64cc-45a6-a1e3-435694c54bf5] +neutron_tempest_plugin.api.admin.test_ports.PortTestCasesResourceRequest.test_port_resource_request[id-ebb86dc4-716c-4558-8516-6dfc4a67601f] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_floatingip_when_quotas_is_full[id-d00fe5bb-9db8-4e1a-9c31-490f52897e6f,negative] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_network_when_quotas_is_full[id-952f9b24-9156-4bdc-90f3-682a3d4302f0,negative] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_port_when_quotas_is_full[id-fe20d9f9-346c-4a20-bbfa-d9ca390f4dc6,negative] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_router_when_quotas_is_full[id-bb1e9c3c-7e6f-41f1-b579-63dbc655ecb7,negative] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_security_group_rule_when_quotas_is_full[id-b7143480-6118-4ed4-be38-1b6f15f30d05,negative] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_security_group_when_quotas_is_full[id-5c924ff7-b7a9-474f-92a3-dbe0f976ec13,negative] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_create_subnet_when_quotas_is_full[id-0b7f99e3-9f77-45ce-9a89-b39a184de618,negative] +neutron_tempest_plugin.api.admin.test_quotas_negative.QuotasAdminNegativeTestJSON.test_set_network_quota_lower_than_networks_amount[id-9f676a6e-d729-428b-adcd-4de2867c50e6,negative] +neutron_tempest_plugin.api.admin.test_quotas.QuotasTest.test_detail_quotas[gate,id-e974b5ba-090a-452c-a578-f9710151d9fc] +neutron_tempest_plugin.api.admin.test_quotas.QuotasTest.test_quotas_force_false[gate,id-43d01327-d8be-4773-a8f0-1d2e9664fda2] +neutron_tempest_plugin.api.admin.test_quotas.QuotasTest.test_quotas[gate,id-2390f766-836d-40ef-9aeb-e810d78207fb] +neutron_tempest_plugin.api.admin.test_routers_flavors.RoutersFlavorTestCase.test_badrequest_on_requesting_flags_and_flavor[id-83939cf7-5070-41bc-9a3e-cd9f22df2186] +neutron_tempest_plugin.api.admin.test_routers_flavors.RoutersFlavorTestCase.test_create_router_with_flavor[id-a4d01977-e968-4983-b4d9-824ea6c33f4b] +neutron_tempest_plugin.api.admin.test_routers_flavors.RoutersFlavorTestCase.test_delete_router_flavor_in_use[id-30e73858-a0fc-409c-a2e0-e9cd2826f6a2] +neutron_tempest_plugin.api.admin.test_routers_ha.RoutersTestHA.test_delete_ha_router_keeps_ha_network_segment_data[id-0d8c0c8f-3809-4acc-a2c8-e0941333ff6c] +neutron_tempest_plugin.api.admin.test_routers_ha.RoutersTestHA.test_ha_router_creation[id-8abc177d-14f1-4018-9f01-589b299cbee1] +neutron_tempest_plugin.api.admin.test_routers_ha.RoutersTestHA.test_legacy_router_creation[id-97b5f7ef-2192-4fa3-901e-979cd5c1097a] +neutron_tempest_plugin.api.admin.test_routers_ha.RoutersTestHA.test_legacy_router_update_to_ha[id-5a6bfe82-5b23-45a4-b027-5160997d4753] +neutron_tempest_plugin.api.admin.test_security_groups.SecGroupAdminTest.test_security_group_recreated_on_port_update[id-44f1e1c4-af10-4aa0-972f-87c1c8fa25cc] +neutron_tempest_plugin.api.admin.test_shared_network_extension.AllowedAddressPairSharedNetworkTest.test_create_with_address_pair_blocked_on_other_network[id-86c3529b-1231-40de-803c-ffffffff1fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.AllowedAddressPairSharedNetworkTest.test_update_with_address_pair_blocked_on_other_network[id-86c3529b-1231-40de-803c-ffffffff2fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_create_rbac_policy_with_target_tenant_none[id-86c3529b-1231-40de-803c-bfffffff1eee,smoke] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_create_rbac_policy_with_target_tenant_too_long_id[id-86c3529b-1231-40de-803c-bfffffff1fff,smoke] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_delete_self_share_rule[id-34d627da-a732-68c0-2e1a-bc4a19246698] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_duplicate_policy_error[id-86c3529b-1231-40de-803c-affefefef321] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_filter_fields[id-bf5052b8-b11e-407c-8e43-113447404d3e] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_filter_policies[id-e7bcb1ea-4877-4266-87bb-76f68b421f31] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_filtering_works_with_rbac_records_present[id-86c3529b-1231-40de-803c-aeeeeeee7fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_network_only_visible_to_policy_target[id-86c3529b-1231-40de-803c-afffffff1fff,smoke] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_policy_show[id-86c3529b-1231-40de-803c-afffffff5fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_policy_target_update[id-86c3529b-1231-40de-803c-afffffff2eee] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_port_presence_prevents_network_rbac_policy_deletion[id-86c3529b-1231-40de-803c-afffffff3fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_rbac_bumps_network_revision[id-86c3529b-1231-40de-1234-89664291a4cb] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_rbac_policy_quota[id-c5f8f785-ce8d-4430-af7e-a236205862fb] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_regular_client_blocked_from_sharing_anothers_network[id-86c3529b-1231-40de-803c-afffffff6fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_regular_client_blocked_from_sharing_with_wildcard[id-86c3529b-1231-40de-803c-afffffff7fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_regular_client_shares_to_another_regular_client[id-86c3529b-1231-40de-803c-afffffff4fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_subnet_on_network_only_visible_to_policy_target[id-86c3529b-1231-40de-803c-afffffff2fff] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_tenant_can_delete_port_on_own_network[id-86c3529b-1231-40de-803c-beefbeefbeef] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_tenant_cant_delete_other_tenants_ports[id-f7539232-389a-4e9c-9e37-e42a129eb541] +neutron_tempest_plugin.api.admin.test_shared_network_extension.RBACSharedNetworksTest.test_update_self_share_rule[id-34d627da-869f-68c0-2e1a-bc4a19246698] +neutron_tempest_plugin.api.admin.test_shared_network_extension.SharedNetworksTest.test_create_bulk_shared_network[id-3e39c4a6-9caf-4710-88f1-d20073c6dd76] +neutron_tempest_plugin.api.admin.test_shared_network_extension.SharedNetworksTest.test_create_port_shared_network_as_non_admin_tenant[id-9c31fabb-0181-464f-9ace-95144fe9ca77] +neutron_tempest_plugin.api.admin.test_shared_network_extension.SharedNetworksTest.test_create_update_shared_network[id-6661d219-b96d-4597-ad10-55766ce4abf7] +neutron_tempest_plugin.api.admin.test_shared_network_extension.SharedNetworksTest.test_filtering_shared_networks[id-6661d219-b96d-4597-ad10-55766123421a] +neutron_tempest_plugin.api.admin.test_shared_network_extension.SharedNetworksTest.test_filtering_shared_subnets[id-6661d219-b96d-4597-ad10-51672353421a] +neutron_tempest_plugin.api.admin.test_shared_network_extension.SharedNetworksTest.test_list_shared_networks[id-a064a9fd-e02f-474a-8159-f828cd636a28] +neutron_tempest_plugin.api.admin.test_shared_network_extension.SharedNetworksTest.test_show_shared_networks_attribute[id-e03c92a2-638d-4bfa-b50a-b1f66f087e58] +neutron_tempest_plugin.api.admin.test_tag.TagFilterFloatingIpTestJSON.test_filter_floatingip_tags[id-01f00afc-dbec-432a-bfee-2a1f0510e7a8,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterNetworkTestJSON.test_filter_network_tags[id-a66b5cca-7db2-40f5-a33d-8ac9f864e53e,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterPortTestJSON.test_filter_port_tags[id-09c036b8-c8d0-4bee-b776-7f4601512898,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterQosPolicyTestJSON.test_filter_qos_policy_tags[id-c2f9a6ae-2529-4cb9-a44b-b16f8ba27832,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterRouterTestJSON.test_filter_router_tags[id-cdd3f3ea-073d-4435-a6cb-826a4064193d,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterSecGroupTestJSON.test_filter_security_group_tags[id-d4d1d681-0116-4800-9725-16cb88f8171a,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterSubnetpoolTestJSON.test_filter_subnetpool_tags[id-16ae7ad2-55c2-4821-9195-bfd04ab245b7,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterSubnetTestJSON.test_filter_subnet_tags[id-dd8f9ba7-bcf6-496f-bead-714bd3daac10,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFilterTrunkTestJSON.test_filter_trunk_tags[id-3fb3ca3a-8e3a-4565-ba73-16413d445e25,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagFloatingIpTestJSON.test_floatingip_tags[id-53f6c2bf-e272-4e9e-b9a9-b165eb7be807,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagNetworkTestJSON.test_network_tags[id-5621062d-fbfb-4437-9d69-138c78ea4188,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagPortTestJSON.test_port_tags[id-c7c44f2c-edb0-4ebd-a386-d37cec155c34,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagQosPolicyTestJSON.test_qos_policy_tags[id-e9bac15e-c8bc-4317-8295-4bf1d8d522b8,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagRouterTestJSON.test_router_tags[id-b898ff92-dc33-4232-8ab9-2c6158c80d28,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagSecGroupTestJSON.test_security_group_tags[id-0f1a78eb-c5be-42cf-919d-2ce3621a51c2,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagSubnetPoolTestJSON.test_subnetpool_tags[id-bdc1c24b-c0b5-4835-953c-8f67dc11edfe,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagSubnetTestJSON.test_subnet_tags[id-2805aabf-a94c-4e70-a0b2-9814f06beb03,smoke] +neutron_tempest_plugin.api.admin.test_tag.TagTrunkTestJSON.test_trunk_tags[id-4c63708b-c4c3-407c-8101-7a9593882f5f,smoke] +neutron_tempest_plugin.api.admin.test_tag.UpdateTagsTest.test_update_tags_affects_only_updated_resource[id-74c56fb1-a3b1-4a62-a8d2-d04dca6bd4cd,smoke] +neutron_tempest_plugin.api.test_address_groups.AddressGroupTest.test_add_wrong_address_to_address_group[id-feec6747-b4b8-49e3-8cff-817d3f097f2c] +neutron_tempest_plugin.api.test_address_groups.AddressGroupTest.test_address_group_create_with_wrong_address[id-8a42029a-40eb-4b44-a7cf-38500046f9b8] +neutron_tempest_plugin.api.test_address_groups.AddressGroupTest.test_address_group_lifecycle[id-496fef1b-22ce-483b-ab93-d28bf46954b0] +neutron_tempest_plugin.api.test_address_groups.AddressGroupTest.test_edit_addresses_in_address_group[id-27c03921-bb12-4b9a-b32e-7083bc90ff1f] +neutron_tempest_plugin.api.test_address_groups.AddressGroupTest.test_remove_wrong_address_from_address_group[id-74f6fd4c-257b-4725-887b-470e96960e24] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_filter_fields[id-55a9fbb6-3333-48e8-90e4-11ab2a49567b] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_filter_rbac_policies[id-774fc038-486c-4507-ab04-c5aac0fca5ab] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_policy_target_update[id-95f59a88-c47e-4dd9-a231-85f1782753a7] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_rbac_policy_show[id-20b2706b-1cea-4724-ab72-d7452ecb1fc4] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_regular_client_blocked_from_sharing_anothers_policy[id-a0f3a01a-e2c7-47d6-9385-0cd7a7f0c996] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_regular_client_blocked_from_updating_shared_address_group[id-f39e32d9-4733-48ec-b550-07f0ec4998a9] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_regular_client_shares_to_another_regular_client[id-c89db8d4-0b52-4072-ac7e-672860491843] +neutron_tempest_plugin.api.test_address_groups.RbacSharedAddressGroupTest.test_secgrprule_presence_prevents_policy_rbac_policy_deletion[id-35a214c9-5c99-468f-9242-34d0529cabfa] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_delete_address_scope_associated_with_subnetpool[id-1e471e5c-6f9c-437a-9257-fd9bc4b6f0fb,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_delete_non_existent_address_scope[id-5c25dc6a-1e92-467a-9cc7-cda74b6003db,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_get_non_existent_address_scope[id-9b6dd7ad-cabb-4f55-bd5e-e61176ef41f6,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_tenant_create_shared_address_scope[id-9c92ec34-0c50-4104-aa47-9ce98d5088df,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_tenant_get_not_shared_admin_address_scope[id-ef213552-f2da-487d-bf4a-e1705d115ff1,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_tenant_update_address_scope_shared_false[id-a859ef2f-9c76-4e2e-ba0f-e0339a489e8c,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_tenant_update_address_scope_shared_true[id-a857b61e-bf53-4fab-b21a-b0daaf81b5bd,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_update_non_existent_address_scope[id-47c25dc5-e886-4a84-88c3-ac5031969661,negative] +neutron_tempest_plugin.api.test_address_scopes_negative.AddressScopeTestNegative.test_update_shared_address_scope_to_unshare[id-702d0515-82cb-4207-b0d9-703336e54665,negative] +neutron_tempest_plugin.api.test_address_scopes.AddressScopeTest.test_admin_create_shared_address_scope[id-5a06c287-8036-4d04-9d78-def8e06d43df] +neutron_tempest_plugin.api.test_address_scopes.AddressScopeTest.test_admin_update_shared_address_scope[id-e9e1ccdd-9ccd-4076-9503-71820529508b] +neutron_tempest_plugin.api.test_address_scopes.AddressScopeTest.test_delete_address_scope[id-22b3b600-72a8-4b60-bc94-0f29dd6271df] +neutron_tempest_plugin.api.test_address_scopes.AddressScopeTest.test_show_address_scope_project_id[id-bbd57364-6d57-48e4-b0f1-8b9a998f5e06] +neutron_tempest_plugin.api.test_address_scopes.AddressScopeTest.test_show_address_scope[id-85e0326b-4c75-4b92-bd6e-7c7de6aaf05c] +neutron_tempest_plugin.api.test_address_scopes.AddressScopeTest.test_tenant_create_list_address_scope[id-045f9294-8b1a-4848-b6a8-edf1b41e9d06] +neutron_tempest_plugin.api.test_address_scopes.AddressScopeTest.test_tenant_update_address_scope[id-85a259b2-ace6-4e32-9657-a9a392b452aa] +neutron_tempest_plugin.api.test_address_scopes.RbacAddressScopeTest.test_filter_fields[id-051248e7-d66f-4c69-9022-2b73ee5b9e73] +neutron_tempest_plugin.api.test_address_scopes.RbacAddressScopeTest.test_filter_rbac_policies[id-88852ba0-8546-4ce7-8f79-4a9c840c881d] +neutron_tempest_plugin.api.test_address_scopes.RbacAddressScopeTest.test_policy_target_update[id-038e999b-cd4b-4021-a9ff-ebb734f6e056] +neutron_tempest_plugin.api.test_address_scopes.RbacAddressScopeTest.test_rbac_policy_show[id-19cbd62e-c6c3-4495-98b9-b9c6c6c9c127] +neutron_tempest_plugin.api.test_address_scopes.RbacAddressScopeTest.test_regular_client_blocked_from_sharing_anothers_policy[id-222a638d-819e-41a7-a3fe-550265c06e79] +neutron_tempest_plugin.api.test_address_scopes.RbacAddressScopeTest.test_regular_client_shares_to_another_regular_client[id-57da6ba2-6329-49c8-974c-9858fe187136] +neutron_tempest_plugin.api.test_address_scopes.RbacAddressScopeTest.test_subnet_pool_presence_prevents_rbac_policy_deletion[id-798ac6c6-96cc-49ce-ba5c-c6eced7a09d3] +neutron_tempest_plugin.api.test_auto_allocated_topology.TestAutoAllocatedTopology.test_check_requirements_for_auto_allocate_net_topology[id-10e4ecef-4309-4bda-9c4d-192e1b5a5831] +neutron_tempest_plugin.api.test_auto_allocated_topology.TestAutoAllocatedTopology.test_delete_allocated_net_topology_as_tenant[id-aabc0b02-cee4-11e5-9f3c-091127605a2b] +neutron_tempest_plugin.api.test_auto_allocated_topology.TestAutoAllocatedTopology.test_get_allocated_net_topology_as_tenant[id-64bc0b02-cee4-11e5-9f3c-080027605a2b] +neutron_tempest_plugin.api.test_availability_zones.ListAvailableZonesTest.test_list_available_zones[id-5a8a8a1a-c265-11e8-a611-080027758b73] +neutron_tempest_plugin.api.test_conntrack_helper.ConntrackHelperTestJSON.test_2_conntrack_helpers_to_same_router[id-134469d9-fb25-4165-adc8-f4747f07caf1] +neutron_tempest_plugin.api.test_conntrack_helper.ConntrackHelperTestJSON.test_conntrack_helper_info_in_router_details[id-0a6ae20c-3f66-423e-93c6-cfedd1c93b8d] +neutron_tempest_plugin.api.test_conntrack_helper.ConntrackHelperTestJSON.test_create_list_update_show_delete_conntrack_helper[id-6361c80e-902d-4c2a-88b4-ea8066507eee] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_allowed_address_pairs[id-ed93e453-3f8d-495e-8e7e-b0e268c2ebd9,negative] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_create_port_sec_with_security_group[id-fe7c27b9-f320-4daf-b977-b1547c43daf6] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_delete_with_port_sec[id-2df6114b-b8c3-48a1-96e8-47f08159d35c] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_port_sec_default_value[id-7c338ddf-e64e-4118-bd33-e49a1f2f1495] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_port_sec_specific_value_1[id-e60eafd2-31de-4c38-8106-55447d033b57] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_port_sec_specific_value_2[id-e60eafd2-31de-4c38-8106-55447d033b57] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_port_sec_update_pass[id-05642059-1bfc-4581-9bc9-aaa5db08dd60] +neutron_tempest_plugin.api.test_extension_driver_port_security.PortSecTest.test_port_sec_update_port_failed[id-ff11226c-a5ff-4ad4-8480-0840e36e47a9,negative] +neutron_tempest_plugin.api.test_extensions.ExtensionsTest.test_list_extensions_includes_all[id-c7597fac-2404-45b1-beb4-523c8b1d4604] +neutron_tempest_plugin.api.test_extensions.ExtensionsTest.test_list_extensions_pagination[id-19db409e-a23f-445d-8bc8-ca3d64c84706] +neutron_tempest_plugin.api.test_extensions.ExtensionsTest.test_list_extensions_project_id[id-155b7bc2-e358-4dd8-bf3e-1774c084567f] +neutron_tempest_plugin.api.test_extensions.ExtensionsTest.test_list_extensions_sorting[id-262420b7-a4bb-4a3e-b4b5-e73bad18df8c] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsIpV6TestJSON.test_create_update_delete_flavor[id-136bcf09-00af-4da7-9b7f-174735d4aebd] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsIpV6TestJSON.test_create_update_delete_service_profile[id-b12a9487-b6a2-4cff-a69a-fe2a0b64fae6] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsIpV6TestJSON.test_list_flavors[id-eb3dd12e-6dfd-45f4-8393-46e0fa19860e] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsIpV6TestJSON.test_list_service_profiles[id-e2fb2f8c-45bf-429a-9f17-171c70444612] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsIpV6TestJSON.test_show_flavor[id-362f9658-164b-44dd-8356-151bc9b7be72] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsIpV6TestJSON.test_show_service_profile[id-30abb445-0eea-472e-bd02-8649f54a5968] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsJson.test_create_update_delete_flavor[id-136bcf09-00af-4da7-9b7f-174735d4aebd] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsJson.test_create_update_delete_service_profile[id-b12a9487-b6a2-4cff-a69a-fe2a0b64fae6] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsJson.test_list_flavors[id-eb3dd12e-6dfd-45f4-8393-46e0fa19860e] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsJson.test_list_service_profiles[id-e2fb2f8c-45bf-429a-9f17-171c70444612] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsJson.test_show_flavor[id-362f9658-164b-44dd-8356-151bc9b7be72] +neutron_tempest_plugin.api.test_flavors_extensions.TestFlavorsJson.test_show_service_profile[id-30abb445-0eea-472e-bd02-8649f54a5968] +neutron_tempest_plugin.api.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_associate_floatingip_with_port_with_floatingip[id-0b5b8797-6de7-4191-905c-a48b888eb429,negative] +neutron_tempest_plugin.api.test_floating_ips.FloatingIPPoolTestJSON.test_create_floatingip_from_specific_pool[id-6c438332-4554-461c-9668-512ae09bf952] +neutron_tempest_plugin.api.test_floating_ips.FloatingIPTestJSON.test_blank_update_clears_association[id-f6a0fb6c-cb64-4b81-b0d5-f41d8f69d22d] +neutron_tempest_plugin.api.test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_description[id-c72c1c0c-2193-4aca-eeee-b1442641ffff] +neutron_tempest_plugin.api.test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_port_details[id-cecae820-ebaa-4f96-b386-6a9fbf25c552] +neutron_tempest_plugin.api.test_floating_ips.FloatingIPTestJSON.test_floatingip_update_extra_attributes_port_id_not_changed[id-fd7161e1-2167-4686-a6ff-0f3df08001bb] +neutron_tempest_plugin.api.test_local_ip.LocalIPAssociationTestJSON.test_local_ip_association_lifecycle[id-602d2874-49be-4c72-8799-b20c95853b6b] +neutron_tempest_plugin.api.test_local_ip.LocalIPAssociationTestJSON.test_local_ip_association_with_two_ips_on_port[id-5d26edab-78d2-4cbd-9d0b-3c0b19f0f52d] +neutron_tempest_plugin.api.test_local_ip.LocalIPTestJSON.test_create_local_ip_with_network[id-e32df8ac-4e29-4adf-8057-46ae8684eff2] +neutron_tempest_plugin.api.test_local_ip.LocalIPTestJSON.test_local_ip_lifecycle[id-369257b0-521d-43f5-9482-50e18e87a472] +neutron_tempest_plugin.api.test_metering_extensions.MeteringIpV6TestJSON.test_create_delete_metering_label_rule_with_filters[id-f4d547cd-3aee-408f-bf36-454f8825e045] +neutron_tempest_plugin.api.test_metering_extensions.MeteringIpV6TestJSON.test_create_delete_metering_label_with_filters[id-ec8e15ff-95d0-433b-b8a6-b466bddb1e50] +neutron_tempest_plugin.api.test_metering_extensions.MeteringIpV6TestJSON.test_create_delete_metering_label_with_name_max_length[id-46608f8d-2e27-4eb6-a0b4-dbe405144c4d] +neutron_tempest_plugin.api.test_metering_extensions.MeteringIpV6TestJSON.test_list_metering_label_rules[id-cc832399-6681-493b-9d79-0202831a1281] +neutron_tempest_plugin.api.test_metering_extensions.MeteringIpV6TestJSON.test_list_metering_labels[id-05d7c750-6d26-44d6-82f3-c9dd1f81f358] +neutron_tempest_plugin.api.test_metering_extensions.MeteringIpV6TestJSON.test_show_metering_label_rule[id-b7354489-96ea-41f3-9452-bace120fb4a7] +neutron_tempest_plugin.api.test_metering_extensions.MeteringIpV6TestJSON.test_show_metering_label[id-cfc500d9-9de6-4847-8803-62889c097d45] +neutron_tempest_plugin.api.test_metering_extensions.MeteringTestJSON.test_create_delete_metering_label_rule_with_filters[id-f4d547cd-3aee-408f-bf36-454f8825e045] +neutron_tempest_plugin.api.test_metering_extensions.MeteringTestJSON.test_create_delete_metering_label_with_filters[id-ec8e15ff-95d0-433b-b8a6-b466bddb1e50] +neutron_tempest_plugin.api.test_metering_extensions.MeteringTestJSON.test_create_delete_metering_label_with_name_max_length[id-46608f8d-2e27-4eb6-a0b4-dbe405144c4d] +neutron_tempest_plugin.api.test_metering_extensions.MeteringTestJSON.test_list_metering_label_rules[id-cc832399-6681-493b-9d79-0202831a1281] +neutron_tempest_plugin.api.test_metering_extensions.MeteringTestJSON.test_list_metering_labels[id-05d7c750-6d26-44d6-82f3-c9dd1f81f358] +neutron_tempest_plugin.api.test_metering_extensions.MeteringTestJSON.test_show_metering_label_rule[id-b7354489-96ea-41f3-9452-bace120fb4a7] +neutron_tempest_plugin.api.test_metering_extensions.MeteringTestJSON.test_show_metering_label[id-cfc500d9-9de6-4847-8803-62889c097d45] +neutron_tempest_plugin.api.test_metering_negative.MeteringNegativeTestJSON.test_create_metering_label_with_too_long_name[id-8b3f7c84-9d37-4771-8681-bfd2c07f3c2d,negative] +neutron_tempest_plugin.api.test_ndp_proxy_negative.NDPProxyNegativeTestJSON.test_create_ndp_proxy_with_different_address_scope[id-a0d93fd6-1219-4b05-9db8-bdf02846c447,negative] +neutron_tempest_plugin.api.test_ndp_proxy_negative.NDPProxyNegativeTestJSON.test_create_ndp_proxy_with_subnet_not_connect_router[id-26e534a0-3e47-4894-8cb5-20a078ce76a9,negative] +neutron_tempest_plugin.api.test_ndp_proxy_negative.NDPProxyNegativeTestJSON.test_create_ndp_proxy_without_ipv6_address[id-f9a4e56d-3836-40cd-8c05-585b3f1e034a,negative] +neutron_tempest_plugin.api.test_ndp_proxy_negative.NDPProxyNegativeTestJSON.test_enable_ndp_proxy_and_unset_gateway[id-e035b3af-ebf9-466d-9ef5-a73b063a1f56,negative] +neutron_tempest_plugin.api.test_ndp_proxy_negative.NDPProxyNegativeTestJSON.test_enable_ndp_proxy_without_address_scope[id-194b5ee7-4c59-4643-aabf-80a125c3f688,negative] +neutron_tempest_plugin.api.test_ndp_proxy_negative.NDPProxyNegativeTestJSON.test_enable_ndp_proxy_without_external_gw[id-a0897204-bb85-41cc-a5fd-5d0ab8116a07,negative] +neutron_tempest_plugin.api.test_ndp_proxy.NDPProxyTestJSON.test_ndp_proxy_lifecycle[id-481bc712-d504-4128-bffb-62d98b88886b] +neutron_tempest_plugin.api.test_network_ip_availability_negative.NetworksIpAvailabilityNegativeTest.test_network_availability_nonexistent_network_id[id-3b8693eb-6c57-4ea1-ab84-3730c9ee9c84,negative] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv4Test.test_list_ip_availability_after_port_delete[id-9f11254d-757b-492e-b14b-f52144e4ee7b] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv4Test.test_list_ip_availability_after_subnet_and_ports[id-3aecd3b2-16ed-4b87-a54a-91d7b3c2986b] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv4Test.test_list_ip_availability_before_subnet[id-0f33cc8c-1bf6-47d1-9ce1-010618240599] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv4Test.test_show_ip_availability_after_port_delete[id-a4d1e291-c152-4d62-9316-8c9bf1c6aee2] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv4Test.test_show_ip_availability_after_subnet_and_ports_create[id-da1fbed5-b4a9-45b3-bdcb-b1660710d565] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv6Test.test_list_ip_availability_after_port_delete[id-9f11254d-757b-492e-b14b-f52144e4ee7b] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv6Test.test_list_ip_availability_after_subnet_and_ports[id-3aecd3b2-16ed-4b87-a54a-91d7b3c2986b] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv6Test.test_list_ip_availability_before_subnet[id-0f33cc8c-1bf6-47d1-9ce1-010618240599] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv6Test.test_list_ipv6_ip_availability_after_subnet_and_ports[id-0d5a03f2-fdb7-4ec3-b746-734c51d74b69] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv6Test.test_show_ip_availability_after_port_delete[id-a4d1e291-c152-4d62-9316-8c9bf1c6aee2] +neutron_tempest_plugin.api.test_network_ip_availability.NetworksIpAvailabilityIPv6Test.test_show_ip_availability_after_subnet_and_ports_create[id-da1fbed5-b4a9-45b3-bdcb-b1660710d565] +neutron_tempest_plugin.api.test_networks_negative.NetworksNegativeTest.test_delete_network_in_use[id-9f80f25b-5d1b-4f26-9f6b-774b9b270819,negative] +neutron_tempest_plugin.api.test_networks_negative.NetworksNegativeTest.test_update_network_mtu[id-9f80f25b-5d1b-4f26-9f6b-774b9b270820,negative] +neutron_tempest_plugin.api.test_networks.NetworksMtuTestJSON.test_create_network_custom_mtu[id-c79dbf94-ee26-420f-a56f-382aaccb1a41] +neutron_tempest_plugin.api.test_networks.NetworksMtuTestJSON.test_update_network_custom_mtu[id-2d35d49d-9d16-465c-92c7-4768eb717688] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_no_pagination_limit_0[id-f1867fc5-e1d6-431f-bc9f-8b882e43a7f9] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_pagination_page_reverse_asc[id-79a52810-2156-4ab6-b577-9e46e58d4b58] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_pagination_page_reverse_desc[id-36a4671f-a542-442f-bc44-a8873ee778d1] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_pagination_page_reverse_with_href_links[id-13eb066c-aa90-406d-b4c3-39595bf8f910] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_pagination_with_href_links[id-8a9c89df-0ee7-4c0d-8f1d-ec8f27cf362f] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_pagination_with_marker[id-b7e153d2-37c3-48d4-8390-ec13498fee3d] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_pagination[id-71389852-f57b-49f2-b109-77b705e9e8af] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_sorts_asc[id-de27d34a-bd9d-4516-83d6-81ef723f7d0d] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_sorts_desc[id-e767a160-59f9-4c4b-8dc1-72124a68640a] +neutron_tempest_plugin.api.test_networks.NetworksSearchCriteriaTest.test_list_validation_filters[id-3574ec9b-a8b8-43e3-9c11-98f5875df6a9] +neutron_tempest_plugin.api.test_networks.NetworksTestJSON.test_create_network_with_project[id-0cc0552f-afaf-4231-b7a7-c2a1774616da] +neutron_tempest_plugin.api.test_networks.NetworksTestJSON.test_create_update_network_dns_domain[id-94e2a44c-3367-4253-8c2a-22deaf59e96c] +neutron_tempest_plugin.api.test_networks.NetworksTestJSON.test_list_networks_fields_keystone_v3[id-a23186b9-aa6f-4b08-b877-35ca3b9cd54c] +neutron_tempest_plugin.api.test_networks.NetworksTestJSON.test_show_network_fields_keystone_v3[id-26f2b7a5-2cd1-4f3a-b11f-ad259b099b11] +neutron_tempest_plugin.api.test_networks.NetworksTestJSON.test_show_network[id-2bf13842-c93f-4a69-83ed-717d2ec3b44e] +neutron_tempest_plugin.api.test_port_forwarding_negative.PortForwardingNegativeTestJSON.test_mapping_different_external_ports_to_the_same_destination[id-0c229a4c-9f28-11ea-bb37-0242ac130002,negative] +neutron_tempest_plugin.api.test_port_forwarding_negative.PortForwardingNegativeTestJSON.test_mapping_same_fip_and_external_port_to_different_dest[id-63c0d406-99d5-11ea-bb37-0242ac130002,negative] +neutron_tempest_plugin.api.test_port_forwarding_negative.PortForwardingNegativeTestJSON.test_out_of_range_ports[id-e9d3ffb6-e5bf-421d-acaa-ee6010dfbf14,negative] +neutron_tempest_plugin.api.test_port_forwardings.PortForwardingTestJSON.test_associate_2_port_forwardings_to_floating_ip[id-8202cded-7e82-4420-9585-c091105404f6] +neutron_tempest_plugin.api.test_port_forwardings.PortForwardingTestJSON.test_associate_port_forwarding_to_2_fixed_ips[id-a7e6cc48-8a9b-49be-82fb-cef6f5c29381] +neutron_tempest_plugin.api.test_port_forwardings.PortForwardingTestJSON.test_associate_port_forwarding_to_port_with_fip[id-4ca72d40-93e4-485f-a876-76caf33c1fe6] +neutron_tempest_plugin.api.test_port_forwardings.PortForwardingTestJSON.test_associate_port_forwarding_to_used_floating_ip[id-6a34e811-66d1-4f63-aa4d-9013f15deb62] +neutron_tempest_plugin.api.test_port_forwardings.PortForwardingTestJSON.test_port_forwarding_info_in_fip_details[id-aa842070-39ef-4b09-9df9-e723934f96f8] +neutron_tempest_plugin.api.test_port_forwardings.PortForwardingTestJSON.test_port_forwarding_life_cycle[id-829a446e-46bc-41ce-b442-6e428aeb3c19] +neutron_tempest_plugin.api.test_ports_negative.PortsNegativeTestJSON.test_add_port_with_illegal_ip[id-9b0a4152-9aa4-4169-9b2c-579609e2fb4a,negative] +neutron_tempest_plugin.api.test_ports_negative.PortsNegativeTestJSON.test_add_port_with_nonexist_network_id[id-7cf473ae-7ec8-4834-ae17-9ef6ec6b8a32,negative] +neutron_tempest_plugin.api.test_ports_negative.PortsNegativeTestJSON.test_add_port_with_nonexist_security_groups_id[id-cad2d349-25fa-490e-9675-cd2ea24164bc,negative] +neutron_tempest_plugin.api.test_ports_negative.PortsNegativeTestJSON.test_add_port_with_nonexist_tenant_id[id-5b69a905-3a84-43a4-807a-1a67ab85caeb,negative] +neutron_tempest_plugin.api.test_ports_negative.PortsNegativeTestJSON.test_add_port_with_too_long_description[id-e10da38c-1071-49c9-95c2-0c451e18ae31,negative] +neutron_tempest_plugin.api.test_ports_negative.PortsNegativeTestJSON.test_add_port_with_too_long_name[id-0cbd256a-a6d4-4afa-a039-44cc13704bab,negative] +neutron_tempest_plugin.api.test_ports.PortsIpv6TestJSON.test_add_ipv6_ips_to_port[id-b85879fb-4852-4b99-aa32-3f8a7a6a3f01] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_no_pagination_limit_0[id-3afe7024-77ab-4cfe-824b-0b2bf4217727] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_pagination_page_reverse_asc[id-b8857391-dc44-40cc-89b7-2800402e03ce] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_pagination_page_reverse_desc[id-4e51e9c9-ceae-4ec0-afd4-147569247699] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_pagination_page_reverse_with_href_links[id-74293e59-d794-4a93-be09-38667199ef68] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_pagination_with_href_links[id-fcd02a7a-f07e-4d5e-b0ca-b58e48927a9b] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_pagination_with_marker[id-f4723b8e-8186-4b9a-bf9e-57519967e048] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_pagination[id-a202fdc8-6616-45df-b6a0-463932de6f94] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_sorts_asc[id-9ab73df4-960a-4ae3-87d3-60992b8d3e2d] +neutron_tempest_plugin.api.test_ports.PortsSearchCriteriaTest.test_list_sorts_desc[id-b426671d-7270-430f-82ff-8f33eec93010] +neutron_tempest_plugin.api.test_ports.PortsTaggingOnCreationTestJSON.test_tagging_ports_during_bulk_creation_no_tags[id-33eda785-a08a-44a0-1bbb-fb50a2f1cd78] +neutron_tempest_plugin.api.test_ports.PortsTaggingOnCreationTestJSON.test_tagging_ports_during_bulk_creation[id-5cf26014-fdd3-4a6d-b94d-a05f0c55da89] +neutron_tempest_plugin.api.test_ports.PortsTaggingOnCreationTestJSON.test_tagging_ports_during_creation[id-6baa43bf-88fb-8bca-6051-97ea1a5e8f4f] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_add_ips_to_port[id-1d6d8683-8691-43c6-a7ba-c69723258726] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_change_dhcp_flag_then_create_port[id-c72c1c0c-2193-4aca-bbb4-b1442640c123] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_create_update_port_description[id-c72c1c0c-2193-4aca-bbb4-b1442640bbbb] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_create_update_port_security[id-3ae162e8-ff00-490c-a423-6a88e48f1ed6] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_create_update_port_with_dns_domain[id-dfe8cc79-18d9-4ae8-acef-3ec6bb719aa7] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_create_update_port_with_dns_name[id-539fbefe-fb36-48aa-9a53-8c5fbd44e492] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_create_update_port_with_no_dns_name[id-435e89df-a8bb-4b41-801a-9f20d362d777] +neutron_tempest_plugin.api.test_ports.PortsTestJSON.test_port_shut_down[id-12a3f5f1-4d9c-4fe1-9e7b-08c75fe77790] +neutron_tempest_plugin.api.test_qos_negative.QosBandwidthLimitRuleNegativeTestJSON.test_rule_create_rule_non_existent_policy[id-1b592566-745f-4e15-a439-073afe341244,negative] +neutron_tempest_plugin.api.test_qos_negative.QosBandwidthLimitRuleNegativeTestJSON.test_rule_update_rule_nonexistent_policy[id-e9ce8042-c828-4cb9-b1f1-85bd35e6553a,negative] +neutron_tempest_plugin.api.test_qos_negative.QosBandwidthLimitRuleNegativeTestJSON.test_rule_update_rule_nonexistent_rule[id-a2c72066-0c32-4f28-be7f-78fa721588b6,negative] +neutron_tempest_plugin.api.test_qos_negative.QosDscpRuleNegativeTestJSON.test_rule_create_rule_non_existent_policy[id-07d17f09-3dc4-4c24-9bb1-49081a153c5a,negative] +neutron_tempest_plugin.api.test_qos_negative.QosDscpRuleNegativeTestJSON.test_rule_update_rule_nonexistent_policy[id-d47d5fbe-3e98-476f-b2fd-97818175dea5,negative] +neutron_tempest_plugin.api.test_qos_negative.QosDscpRuleNegativeTestJSON.test_rule_update_rule_nonexistent_rule[id-9c0bd085-5a7a-496f-a984-50dc631a64f2,negative] +neutron_tempest_plugin.api.test_qos_negative.QosMinimumBandwidthRuleNegativeTestJSON.test_rule_create_rule_non_existent_policy[id-5a714a4a-bfbc-4cf9-b0c0-13fd185204f7,negative] +neutron_tempest_plugin.api.test_qos_negative.QosMinimumBandwidthRuleNegativeTestJSON.test_rule_update_rule_nonexistent_policy[id-08b8455b-4d4f-4119-bad3-9357085c3a80,negative] +neutron_tempest_plugin.api.test_qos_negative.QosMinimumBandwidthRuleNegativeTestJSON.test_rule_update_rule_nonexistent_rule[id-8470cbe0-8ca5-46ab-9c66-7cf69301b121,negative] +neutron_tempest_plugin.api.test_qos_negative.QosMinimumPpsRuleNegativeTestJSON.test_rule_create_rule_non_existent_policy[id-de4f5540-3e10-11ec-9700-4bf3629b843e,negative] +neutron_tempest_plugin.api.test_qos_negative.QosMinimumPpsRuleNegativeTestJSON.test_rule_update_rule_nonexistent_policy[id-ddd16824-3e10-11ec-928d-5b1ef3fb9f43,negative] +neutron_tempest_plugin.api.test_qos_negative.QosMinimumPpsRuleNegativeTestJSON.test_rule_update_rule_nonexistent_rule[id-deb914ee-3e10-11ec-b3dc-03e52f9269c9,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_add_policy_with_too_long_description[id-b9dce444-d3b3-11e5-950a-54ee747c99db,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_add_policy_with_too_long_name[id-b9dce555-d3b3-11e5-950a-54ee757c77da,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_add_policy_with_too_long_tenant_id[id-b9dce444-d3b3-11e5-950a-54ee757c77dc,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_delete_non_existent_qos_policy[id-09e435b7-44d3-4f9d-8aa8-c295d46b5866,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_get_non_existent_qos_policy[id-88b54ab0-804b-446c-bc19-8e54222d70ef,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_update_non_existent_qos_policy[id-21050859-1284-4bf5-b05a-13846f83988f,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_update_policy_with_too_long_description[id-925c7eaf-474b-4a02-a4ba-76a9f82bc45a,negative] +neutron_tempest_plugin.api.test_qos_negative.QosNegativeTestJSON.test_update_policy_with_too_long_name[id-0e85f3e4-7a93-4187-b847-8f4e835aae1b,negative] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_attach_and_detach_a_policy_by_a_tenant[id-d911707e-fa2c-11e9-9553-5076af30bbf5] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_get_rules_by_policy[id-ce0bd0c2-54d9-4e29-85f1-cfb36ac3ebe2] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_create_fail_for_the_same_type[id-8a59b00b-ab01-4787-92f8-93a5cdf5e378] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_create_forbidden_for_regular_tenants[id-a4a2e7ad-786f-4927-a85a-e545a93bd274] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_create_rule_nonexistent_policy[id-f211222c-5808-46cb-a961-983bbab6b852] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_create[id-8a59b00b-3e9c-4787-92f8-93a5cdf5e378] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_delete[id-67ee6efd-7b33-4a68-927d-275b4f8ba958] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_update_forbidden_for_regular_tenants_foreign_policy[id-9a607936-4b6f-4c2f-ad21-bd5b3d4fc91f] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_update_forbidden_for_regular_tenants_own_policy[id-1bfc55d9-6fd8-4293-ab3a-b1d69bf7cd2e] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleTestJSON.test_rule_update[id-149a6988-2568-47d2-931e-2dbc858943b3] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_attach_and_detach_a_policy_by_a_tenant[id-d911707e-fa2c-11e9-9553-5076af30bbf5] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_create_policy_with_multiple_rules[id-c8cbe502-0f7e-11ea-8d71-362b9e155667] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_get_rules_by_policy[id-ce0bd0c2-54d9-4e29-85f1-cfb36ac3ebe2] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_create_fail_for_the_same_type[id-8a59b00b-ab01-4787-92f8-93a5cdf5e378] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_create_forbidden_for_regular_tenants[id-a4a2e7ad-786f-4927-a85a-e545a93bd274] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_create_rule_nonexistent_policy[id-f211222c-5808-46cb-a961-983bbab6b852] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_create[id-8a59b00b-3e9c-4787-92f8-93a5cdf5e378] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_delete[id-67ee6efd-7b33-4a68-927d-275b4f8ba958] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_update_1[id-7ca7c83b-0555-4a0f-a422-4338f77f79e5] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_update_2[id-7ca7c83b-0555-4a0f-a422-4338f77f79e5] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_update_forbidden_for_regular_tenants_foreign_policy[id-9a607936-4b6f-4c2f-ad21-bd5b3d4fc91f] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_update_forbidden_for_regular_tenants_own_policy[id-1bfc55d9-6fd8-4293-ab3a-b1d69bf7cd2e] +neutron_tempest_plugin.api.test_qos.QosBandwidthLimitRuleWithDirectionTestJSON.test_rule_update[id-149a6988-2568-47d2-931e-2dbc858943b3] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_get_rules_by_policy[id-c565131d-4c80-4231-b0f3-9ae2be4de129] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_invalid_rule_create[id-33646b08-4f05-4493-a48a-bde768a18533] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_qos_dscp_create_and_update[id-19ed2286-ccb1-11e9-87d7-525400d6f522] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_rule_create_fail_for_the_same_type[id-08553ffe-030f-4037-b486-7e0b8fb9385a] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_rule_create_forbidden_for_regular_tenants[id-bf6002ea-29de-486f-b65d-08aea6d4c4e2] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_rule_create_rule_nonexistent_policy[id-9cb8ef5c-96fc-4978-9ee0-e3b02bab628a] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_rule_create[id-f5cbaceb-5829-497c-9c60-ad70969e9a08] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_rule_delete[id-74f81904-c35f-48a3-adae-1f5424cb3c18] +neutron_tempest_plugin.api.test_qos.QosDscpMarkingRuleTestJSON.test_rule_update[id-76f632e5-3175-4408-9a32-3625e599c8a2] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_get_rules_by_policy[id-de0bd0c2-54d9-4e29-85f1-cfb36ac3ebe2] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_create_fail_for_missing_min_kbps[id-266d9b87-e51c-48bd-9aa7-8269573621be] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_create_fail_for_the_same_type[id-aa59b00b-ab01-4787-92f8-93a5cdf5e378] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_create_forbidden_for_regular_tenants[id-b4a2e7ad-786f-4927-a85a-e545a93bd274] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_create_pass_for_direction_ingress[id-35baf998-ae65-495c-9902-35a0d11e8936] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_create_rule_nonexistent_policy[id-a211222c-5808-46cb-a961-983bbab6b852] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_create[id-aa59b00b-3e9c-4787-92f8-93a5cdf5e378] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_delete[id-a7ee6efd-7b33-4a68-927d-275b4f8ba958] +neutron_tempest_plugin.api.test_qos.QosMinimumBandwidthRuleTestJSON.test_rule_update[id-a49a6988-2568-47d2-931e-2dbc858943b3] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_get_rules_by_policy[id-1b94f4e2-3d3e-11ec-bb21-6f98e4044b8b] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_create_any_direction_when_egress_direction_exists[id-ceb8e41e-3d72-11ec-a446-d7faae6daec2] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_create_egress_direction_when_any_direction_exists[id-a147a71e-3d7b-11ec-8097-278b1afd5fa2] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_create_fail_for_missing_min_kpps[id-6b656b57-d2bf-47f9-89a9-1baad1bd5418] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_create_fail_for_the_same_type[id-f41213e5-2ab8-4916-b106-38d2cac5e18c] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_create_forbidden_for_regular_tenants[id-1a6b6128-3d3e-11ec-bf49-57b326d417c0] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_create[id-66a5b9b4-d4f9-4af8-b238-9e1881b78487] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_delete[id-c49018b6-d358-49a1-a94b-d53224165045] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_update_direction_conflict[id-a020e186-3d60-11ec-88ca-d7f5eec22764] +neutron_tempest_plugin.api.test_qos.QosMinimumPpsRuleTestJSON.test_rule_update[id-522ed09a-1d7f-4c1b-9195-61f19caf916f] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_no_pagination_limit_0[id-40e09b53-4eb8-4526-9181-d438c8005a20] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_pagination_page_reverse_asc[id-d6a8bacd-d5e8-4ef3-bc55-23ca6998d208] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_pagination_page_reverse_desc[id-0b9aecdc-2b27-421b-b104-53d24e905ae8] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_pagination_page_reverse_with_href_links[id-1a3dc257-dafd-4870-8c71-639ae7ddc6ea] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_pagination_with_href_links[id-3bad0747-8082-46e9-be4d-c428a842db41] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_pagination_with_marker[id-3bd8fb58-c0f8-4954-87fb-f286e1eb096a] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_pagination[id-719e61cc-e33c-4918-aa4d-1a791e6e0e86] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_sorts_asc[id-55fc0103-fdc1-4d34-ab62-c579bb739a91] +neutron_tempest_plugin.api.test_qos.QosSearchCriteriaTest.test_list_sorts_desc[id-13e08ac3-bfed-426b-892a-b3b158560c23] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_create_policy[id-108fbdf7-3463-4e47-9871-d07f3dcf5bbb] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_default_policy_creating_network_with_policy[id-807cce45-38e5-482d-94db-36e1796aba73] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_default_policy_creating_network_without_policy[id-18d94f22-b9d5-4390-af12-d30a0cfc4cd3] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_delete_not_allowed_if_policy_in_use_by_network[id-18163237-8ba9-4db5-9525-bad6d2343c75] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_delete_not_allowed_if_policy_in_use_by_port[id-24153230-84a9-4dd5-9525-bad6d2343c75] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_delete_policy[id-1cb42653-54bd-4a9a-b888-c55e18199201] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_get_policy_that_is_shared[id-fb384bde-a973-41c3-a542-6f77a092155f] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_list_admin_rule_types[id-cf776f77-8d3d-49f2-8572-12d6a1557224] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_list_policy_filter_by_name[id-f8d20e92-f06d-4805-b54f-230f77715815] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_list_policy_sort_by_name[id-dde0b449-a400-4a87-b5a5-4d1c413c917b] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_list_regular_rule_types[id-49c8ea35-83a9-453a-bd23-239cf3b13929] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_association_with_admin_network[id-65b9ef75-1911-406a-bbdb-ca1d68d528b0] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_association_with_network_non_shared_policy[id-1aa55a79-324f-47d9-a076-894a8fc2448b] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_association_with_network_nonexistent_policy[id-9efe63d0-836f-4cc2-b00c-468e63aa614e] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_association_with_port_non_shared_policy[id-f53d961c-9fe5-4422-8b66-7add972c6031] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_association_with_port_nonexistent_policy[id-49e02f5a-e1dd-41d5-9855-cfa37f2d195e] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_association_with_port_shared_policy[id-98fcd95e-84cf-4746-860e-44692e674f2e] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_association_with_tenant_network[id-1738de5d-0476-4163-9022-5e1b548c208e] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_create_forbidden_for_regular_tenants[id-aed8e2a6-22da-421b-89b9-935a2c1a1b50] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_update_association_with_admin_network[id-09a9392c-1359-4cbb-989f-fb768e5834a8] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_update_association_with_port_shared_policy[id-f8163237-fba9-4db5-9526-bad6d2343c76] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_update_forbidden_for_regular_tenants_foreign_policy[id-4ecfd7e7-47b6-4702-be38-be9235901a87] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_update_forbidden_for_regular_tenants_own_policy[id-6e880e0f-bbfc-4e54-87c6-680f90e1b618] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_policy_update[id-8e88a54b-f0b2-4b7d-b061-a15d93c2c7d6] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_qos_policy_delete_with_rules[id-a2a5849b-dd06-4b18-9664-0b6828a1fc27] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_shared_policy_update[id-ee263db4-009a-4641-83e5-d0e83506ba4c] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_show_policy_has_project_id[id-606a48e2-5403-4052-b40f-4d54b855af76] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_show_rule_type_details_as_admin[id-d0a2460b-7325-481f-a531-050bd96ab25e] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_show_rule_type_details_as_user[id-8ececa21-ef97-4904-a152-9f04c90f484d] +neutron_tempest_plugin.api.test_qos.QosTestJSON.test_user_create_port_with_admin_qos_policy[id-06060880-2956-4c16-9a63-f284c3879229] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_filter_fields[id-ba88d0ca-a350-11e5-a06f-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_filter_rbac_policies[id-c7496f86-a350-11e5-b380-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_net_bound_shared_policy_wildcard_and_project_id_wild_remove[id-328b1f70-d424-11e5-a57f-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_net_bound_shared_policy_wildcard_and_projectid_wild_remains[id-1997b00c-0c75-4e43-8ce2-999f9fa555ee] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_network_presence_prevents_policy_rbac_policy_deletion[id-a9b39f46-a350-11e5-97c7-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_policy_sharing_with_wildcard_and_project_id[id-2ace9adc-da6e-11e5-aafe-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_policy_sharing_with_wildcard[id-b9dcf582-d3b3-11e5-950a-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_policy_target_update[id-9f85c76a-a350-11e5-8ae5-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_rbac_policy_show[id-c10d993a-a350-11e5-9c7a-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_regular_client_blocked_from_sharing_anothers_policy[id-cd7d755a-a350-11e5-a344-54ee756c66df] +neutron_tempest_plugin.api.test_qos.RbacSharedQosPoliciesTest.test_regular_client_shares_to_another_regular_client[id-b0fe87e8-a350-11e5-9f08-54ee756c66df] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_portbinding_bumps_revision[id-68d5ac3a-11a1-4847-8e2e-5843c043d89b] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_allowed_address_pairs_bumps_revision[id-db70c285-0365-4fac-9f55-2a0ad8cf55a8] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_dns_domain_bumps_revision[id-40ba648f-f374-4c29-a5b7-489dd5a38a4e] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_external_network_bumps_revision[id-90743b00-b0e2-40e4-9524-1c884fe3ef23] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_extra_dhcp_opt_bumps_revision[id-a21ec3b4-3569-4b77-bf29-4177edaa2df5] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_floatingip_bumps_revision[id-9de71ebc-f5df-4cd0-80bc-60299fce3ce9] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_network_bumps_revision[id-4a26a4be-9c53-483c-bc50-b53f1db10ac6] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_network_constrained_by_revision[id-4a26a4be-9c53-483c-bc50-b11111113333] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_port_bumps_revision[id-cac7ecde-12d5-4331-9a03-420899dea077] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_port_security_bumps_revisions[id-9c83105c-9973-45ff-9ca2-e66d64700abe] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_port_sg_binding_bumps_revision[id-6489632f-8550-4453-a674-c98849742967] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_qos_network_policy_binding_bumps_revision[id-817da343-c6e4-445c-9519-a621f124dfbe] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_qos_port_policy_binding_bumps_revision[id-5af6450a-0f61-49c3-b628-38db77c7b856] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_router_bumps_revision[id-4a37bde9-1975-47e0-9b8c-2c9ca36415b0] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_router_extra_attributes_bumps_revision[id-afb6486c-41b5-483e-a500-3c506f4deb49] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_router_extra_routes_bumps_revision[id-8482324f-cf59-4d73-b98e-d37119255300] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_sg_group_bumps_revision[id-6c256f71-c929-4200-b3dc-4e1843506be5] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_sg_rule_bumps_sg_revision[id-29c7ab2b-d1d8-425d-8cec-fcf632960f22] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_subnet_bumps_network_revision[id-e8c5d7db-2b8d-4567-a326-6e123437c4d1] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_subnet_bumps_revision[id-c1c4fa41-8e89-44d0-9bfc-409f3b66dc57] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_subnet_service_types_bumps_revisions[id-6bd18702-e25a-4b4b-8c0c-680113533511] +neutron_tempest_plugin.api.test_revisions.TestRevisions.test_update_subnetpool_bumps_revision[id-e8c5d7db-2b8d-4615-a476-6e537437c4f2] +neutron_tempest_plugin.api.test_router_interface_fip.RouterInterfaceFip.test_router_interface_fip[id-943ab44d-0ea7-4c6a-bdfd-8ba759622992] +neutron_tempest_plugin.api.test_routers_negative.DvrRoutersNegativeTest.test_router_create_tenant_distributed_returns_forbidden[id-4990b055-8fc7-48ab-bba7-aa28beaad0b9,negative] +neutron_tempest_plugin.api.test_routers_negative.DvrRoutersNegativeTestExtended.test_router_update_centralized_returns_exception[id-c277e945-3b39-442d-b149-e2e8cc6a2b40,negative] +neutron_tempest_plugin.api.test_routers_negative.DvrRoutersNegativeTestExtended.test_router_update_distributed_returns_exception[id-5379fe06-e45e-4a4f-8b4a-9e28a924b451,negative] +neutron_tempest_plugin.api.test_routers_negative.HaRoutersNegativeTest.test_router_create_tenant_ha_returns_forbidden[id-821b85b9-9c51-40f3-831f-bf223a7e0084,negative] +neutron_tempest_plugin.api.test_routers_negative.RoutersNegativePolicyTest.test_add_interface_in_use[id-8010d27e-4ab7-4ea3-98b1-3995b7910efd,negative] +neutron_tempest_plugin.api.test_routers_negative.RoutersNegativePolicyTest.test_add_interface_port_nonexist[id-ed84c800-ee29-4b76-9419-d6d7b143fc47,negative] +neutron_tempest_plugin.api.test_routers_negative.RoutersNegativePolicyTest.test_add_interface_wrong_tenant[id-159f576d-a423-46b5-b501-622694c02f6b,negative] +neutron_tempest_plugin.api.test_routers_negative.RoutersNegativePolicyTest.test_remove_associated_ports[id-dad7a8ba-2726-11eb-82dd-74e5f9e2a801,negative] +neutron_tempest_plugin.api.test_routers.DvrRoutersTest.test_create_distributed_router[id-141297aa-3424-455d-aa8d-f2d95731e00a] +neutron_tempest_plugin.api.test_routers.DvrRoutersTestToCentralized.test_convert_distributed_router_back_to_centralized[id-644d7a4a-01a1-4b68-bb8d-0c0042cb1729] +neutron_tempest_plugin.api.test_routers.DvrRoutersTestUpdateDistributedExtended.test_convert_centralized_router_to_distributed_extended[id-0ffb9973-0c1a-4b76-a1f2-060178057661] +neutron_tempest_plugin.api.test_routers.DvrRoutersTestUpdateDistributedExtended.test_convert_distributed_router_to_centralized_extended[id-e9a8f55b-c535-44b7-8b0a-20af6a7c2921] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_add_external_gateways_compat[id-e49efc57-7b25-43a3-8e55-2d87a3759c57] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_add_external_gateways_one[id-089fa304-3726-4120-9759-668e8ff1114c] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_add_external_gateways[id-60a1e7db-04ef-4a3a-9ff1-01a990d365fd] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_add_remove_external_gateways[id-03ab196a-dac0-4363-93e4-ea799246870b] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_enable_default_route_bfd[id-842f6edb-e072-4805-bf11-04c25420776d] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_enable_default_route_ecmp[id-33e9a156-a83f-435f-90ee-1a49dc9c350d] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_remove_external_gateways_compat[id-2a238eec-d9d5-435a-9013-d6e195ecd5d1] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_create_router_update_external_gateways[id-17e94c9f-c59f-4e50-abd5-d1256460e311] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_update_router_enable_default_route_bfd[id-a22016a6-f118-4eb5-abab-7e241ae01848] +neutron_tempest_plugin.api.test_routers.ExternalGWMultihomingRoutersTest.test_update_router_enable_default_route_ecmp[id-bfbad985-2df2-4cd9-9c32-819b5508c40e] +neutron_tempest_plugin.api.test_routers.HaRoutersTest.test_convert_legacy_router[id-77db8eae-3aa3-4e61-bf2a-e739ce042e53] +neutron_tempest_plugin.api.test_routers.RoutersDeleteTest.test_delete_router[id-dbbc5c74-63c8-11eb-8881-74e5f9e2a801] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_create_router_with_default_snat_value[id-847257cc-6afd-4154-b8fb-af49f5670ce8] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_create_router_with_snat_explicit[id-ea74068d-09e9-4fd7-8995-9b6a1ace920f] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_create_update_router_description[id-c72c1c0c-2193-4aca-eeee-b1442640eeee] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_extra_routes_atomic[id-b29d1698-d603-11e9-9c66-079cc4aec539] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_network_attached_with_two_routers[id-01f185d1-d1a6-4cf9-abf7-e0e1384c169c] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_router_interface_status[id-db3093b1-93b6-4893-be83-c4716c251b3e] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_router_interface_update_and_remove_gateway_ip[id-4f8a2a1e-7fe9-4d99-9bff-5dc0e78b7e06] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_update_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_update_router_reset_gateway_without_snat[id-f2faf994-97f4-410b-a831-9bc977b64374] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_update_router_set_gateway_with_snat_explicit[id-b386c111-3b21-466d-880c-5e72b01e1a33] +neutron_tempest_plugin.api.test_routers.RoutersIpV6Test.test_update_router_set_gateway_without_snat[id-96536bc7-8262-4fb2-9967-5c46940fa279] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_no_pagination_limit_0[id-fb102124-20f8-4cb3-8c81-f16f5e41d192] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_pagination_page_reverse_asc[id-77b9676c-d3cb-43af-a0e8-a5b8c6099e70] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_pagination_page_reverse_desc[id-3133a2c5-1bb9-4fc7-833e-cf9a1d160255] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_pagination_page_reverse_with_href_links[id-8252e2f0-b3da-4738-8e25-f6f8d878a2da] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_pagination_with_href_links[id-40804af8-c25d-45f8-b8a8-b4c70345215d] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_pagination_with_marker[id-a5b83e83-3d98-45bb-a2c7-0ee179ffd42c] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_pagination[id-7f7d40b1-e165-4817-8dc5-02f8e2f0dff3] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_sorts_asc[id-03a69efb-90a7-435b-bb5c-3add3612085a] +neutron_tempest_plugin.api.test_routers.RoutersSearchCriteriaTest.test_list_sorts_desc[id-95913d30-ff41-4b17-9f44-5258c651e78c] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_create_router_with_default_snat_value[id-847257cc-6afd-4154-b8fb-af49f5670ce8] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_create_router_with_snat_explicit[id-ea74068d-09e9-4fd7-8995-9b6a1ace920f] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_create_update_router_description[id-c72c1c0c-2193-4aca-eeee-b1442640eeee] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_extra_routes_atomic[id-b29d1698-d603-11e9-9c66-079cc4aec539] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_network_attached_with_two_routers[id-01f185d1-d1a6-4cf9-abf7-e0e1384c169c] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_router_interface_status[id-db3093b1-93b6-4893-be83-c4716c251b3e] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_router_interface_update_and_remove_gateway_ip[id-4f8a2a1e-7fe9-4d99-9bff-5dc0e78b7e06] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_update_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_update_router_reset_gateway_without_snat[id-f2faf994-97f4-410b-a831-9bc977b64374] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_update_router_set_gateway_with_snat_explicit[id-b386c111-3b21-466d-880c-5e72b01e1a33] +neutron_tempest_plugin.api.test_routers.RoutersTest.test_update_router_set_gateway_without_snat[id-96536bc7-8262-4fb2-9967-5c46940fa279] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_assign_nonexistent_sec_group[id-d5ecb408-eb7e-47c1-a56f-353967dbd1c2,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_assign_sec_group_twice[id-867d67c3-7e26-4288-a27b-e3d0649ee54b,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_with_boolean_type_name[id-b6b79838-7430-4d3f-8e07-51dfb61802c2,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_with_too_long_name[id-594edfa8-9a5b-438e-9344-49aece337d49,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_delete_in_use_sec_group[id-3200b1a8-d73b-48e9-b03f-e891a4abe2d3,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_no_sec_group_changes_after_assignment_failure[id-98ef378d-81a2-43f6-bb6f-735c04cdef91,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_update_default_security_group_name[id-55100aa8-b24f-333c-0bef-64eefd85f15c,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_update_security_group_with_boolean_type_name[id-d9a14917-f66f-4eca-ab72-018563917f1b,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupIPv6Test.test_update_security_group_with_too_long_name[id-c8510dd8-c3a8-4df9-ae44-24354db50960,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupProtocolTest.test_create_security_group_rule_with_ipv6_protocol_integers[id-8aa636bd-7060-4fdf-b722-cdae28e2f1ef,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupProtocolTest.test_create_security_group_rule_with_ipv6_protocol_names[id-cccbb0f3-c273-43ed-b3fc-1efc48833810,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupQuotaTest.test_create_excess_sg[id-63f00cba-fcf5-4000-a3ee-eca58a1795c1,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupQuotaTest.test_sg_quota_incorrect_values[id-90a83445-bbc2-49d8-8c85-a111c08cd7fb,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupRulesQuotaTest.test_sg_creation_with_insufficient_sg_rules_quota[id-8336e6ea-2e0a-4a1a-8673-a6f81b577d57] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_assign_nonexistent_sec_group[id-d5ecb408-eb7e-47c1-a56f-353967dbd1c2,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_assign_sec_group_twice[id-867d67c3-7e26-4288-a27b-e3d0649ee54b,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_with_boolean_type_name[id-b6b79838-7430-4d3f-8e07-51dfb61802c2,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_with_too_long_name[id-594edfa8-9a5b-438e-9344-49aece337d49,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_delete_in_use_sec_group[id-3200b1a8-d73b-48e9-b03f-e891a4abe2d3,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_no_sec_group_changes_after_assignment_failure[id-98ef378d-81a2-43f6-bb6f-735c04cdef91,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_update_default_security_group_name[id-55100aa8-b24f-333c-0bef-64eefd85f15c,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_update_security_group_with_boolean_type_name[id-d9a14917-f66f-4eca-ab72-018563917f1b,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeSecGroupTest.test_update_security_group_with_too_long_name[id-c8510dd8-c3a8-4df9-ae44-24354db50960,negative] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeStatelessSecGroupTest.test_adding_stateful_sg_to_port_with_stateless_sg[id-63374580-3154-410b-ab31-e98a136094f8] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeStatelessSecGroupTest.test_adding_stateless_sg_to_port_with_stateful_sg[id-3854a4c6-4ace-4133-be83-4a2820ede06f] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeStatelessSecGroupTest.test_create_port_with_stateful_and_stateless_sg[id-9e85ce0d-37b2-4044-88a8-09ae965069ba] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeStatelessSecGroupTest.test_update_used_stateful_sg_to_stateless[id-afe4d777-7a98-44ed-a1dc-588861f6daba] +neutron_tempest_plugin.api.test_security_groups_negative.NegativeStatelessSecGroupTest.test_update_used_stateless_sg_to_stateful[id-5e1e3053-16dc-4f0b-a327-ff953f527248] +neutron_tempest_plugin.api.test_security_groups.RbacSharedSecurityGroupTest.test_filter_fields[id-2a9fd480-2a35-11e9-9cb6-acde48001122] +neutron_tempest_plugin.api.test_security_groups.RbacSharedSecurityGroupTest.test_filter_rbac_policies[id-2adf6bd7-2a35-11e9-9c62-acde48001122] +neutron_tempest_plugin.api.test_security_groups.RbacSharedSecurityGroupTest.test_policy_target_update[id-2a41eb8f-2a35-11e9-bae9-acde48001122] +neutron_tempest_plugin.api.test_security_groups.RbacSharedSecurityGroupTest.test_port_presence_prevents_policy_rbac_policy_deletion[id-2a619a8a-2a35-11e9-90d9-acde48001122] +neutron_tempest_plugin.api.test_security_groups.RbacSharedSecurityGroupTest.test_rbac_policy_show[id-2abf8f9e-2a35-11e9-85f7-acde48001122] +neutron_tempest_plugin.api.test_security_groups.RbacSharedSecurityGroupTest.test_regular_client_blocked_from_sharing_anothers_policy[id-2aff3900-2a35-11e9-96b3-acde48001122] +neutron_tempest_plugin.api.test_security_groups.RbacSharedSecurityGroupTest.test_regular_client_shares_to_another_regular_client[id-2a81795c-2a35-11e9-9d86-acde48001122] +neutron_tempest_plugin.api.test_security_groups.SecGroupNormalizedCidrTest.test_normalized_cidr_in_rule[id-f87bb108-205c-4f06-a378-81a26f71b829] +neutron_tempest_plugin.api.test_security_groups.SecGroupQuotaTest.test_create_sg_when_quota_disabled[id-d43cf1a7-aa7e-4c41-9340-627a1a6ab961] +neutron_tempest_plugin.api.test_security_groups.SecGroupQuotaTest.test_max_allowed_sg_amount[id-df7981fb-b83a-4779-b13e-65494ef44a72] +neutron_tempest_plugin.api.test_security_groups.SecGroupQuotaTest.test_sg_quota_decrease_less_than_created[id-ba95676c-8d9a-4482-b4ec-74d51a4602a6] +neutron_tempest_plugin.api.test_security_groups.SecGroupQuotaTest.test_sg_quota_increased[id-623d909c-6ef8-43d6-93ee-97086e2651e8] +neutron_tempest_plugin.api.test_security_groups.SecGroupQuotaTest.test_sg_quota_values[id-1826aa02-090d-4717-b43a-50ee449b02e7] +neutron_tempest_plugin.api.test_security_groups.SecGroupRulesQuotaTest.test_create_sg_rules_when_quota_disabled[id-4459e066-d9c8-4a13-9e98-018f95ce2dbf] +neutron_tempest_plugin.api.test_security_groups.SecGroupRulesQuotaTest.test_sg_rules_quota_decrease_less_than_created[id-3fc39bd6-3132-4e6f-a09c-456fb18d600c] +neutron_tempest_plugin.api.test_security_groups.SecGroupRulesQuotaTest.test_sg_rules_quota_increased[id-77ec038c-5638-11ea-8e2d-0242ac130003] +neutron_tempest_plugin.api.test_security_groups.SecGroupRulesQuotaTest.test_sg_rules_quota_values[id-37508c8d-270b-4b93-8007-72876a1fec38] +neutron_tempest_plugin.api.test_security_groups.SecGroupSearchCriteriaTest.test_list_no_pagination_limit_0[id-5c78bd57-e6e9-4e71-a05c-9c4082a3f139] +neutron_tempest_plugin.api.test_security_groups.SecGroupSearchCriteriaTest.test_list_pagination[id-b9654cdc-80bc-43f8-844e-dfe88fd2f125] +neutron_tempest_plugin.api.test_security_groups.SecGroupSearchCriteriaTest.test_list_sorts_by_name_asc[id-0064aa80-8a29-442d-a8de-9101af8210fa] +neutron_tempest_plugin.api.test_security_groups.SecGroupSearchCriteriaTest.test_list_sorts_by_name_desc[id-35e86832-53cd-4e63-97ec-31a2413da591] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupProtocolIPv6Test.test_security_group_rule_protocol_legacy_icmpv6[id-c7d17b41-3b4e-4add-bb3b-6af59baaaffa] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupProtocolTest.test_security_group_rule_protocol_ints[id-66e47f1f-20b6-4417-8839-3cc671c7afa3] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupProtocolTest.test_security_group_rule_protocol_names[id-282e3681-aa6e-42a7-b05c-c341aa1e3cdf] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupTest.test_create_bulk_sec_groups[id-7c0ecb10-b2db-11e6-9b14-000c29248b0d] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupTest.test_create_list_update_show_delete_security_group[id-bfd128e5-3c92-44b6-9d66-7fe29d22c802] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupTest.test_create_sec_groups_with_the_same_name[id-e93f33d8-57ea-11eb-b69b-74e5f9e2a801] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupTest.test_list_security_group_rules_contains_all_rules[id-b5923b1a-4d33-44e1-af25-088dcb55b02b] +neutron_tempest_plugin.api.test_security_groups.StatefulSecGroupTest.test_show_security_group_contains_all_rules[id-1fff0d57-bb6c-4528-9c1d-2326dce1c087] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupProtocolIPv6Test.test_security_group_rule_protocol_legacy_icmpv6[id-a034814e-0fa5-4437-8e6f-0d2eebd668b3] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupProtocolTest.test_security_group_rule_protocol_ints[id-b0332b5d-6fac-49d5-a79d-ae4fe62600f7] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupProtocolTest.test_security_group_rule_protocol_names[id-3a065cdd-99bd-409f-a08e-385c6674bec2] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupTest.test_create_bulk_sec_groups[id-b33e612e-65f0-467b-9bf2-b5b2ce67f72f] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupTest.test_create_list_update_show_delete_security_group[id-0214d58a-2177-47e1-af83-dcd45c024829] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupTest.test_create_sec_groups_with_the_same_name[id-a6896935-db18-413d-95f5-4f465e0e2209] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupTest.test_list_security_group_rules_contains_all_rules[id-cdf3a63a-08fe-4091-bab4-62180847990f] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupTest.test_show_security_group_contains_all_rules[id-ddbc0e4c-840f-44ab-8718-0b95b7c7b575] +neutron_tempest_plugin.api.test_security_groups.StatelessSecGroupTest.test_stateless_security_group_update[id-0a6c1476-3d1a-11ec-b0ec-0800277ac3d9] +neutron_tempest_plugin.api.test_service_type_management.ServiceTypeManagementTest.test_service_provider_list[id-2cbbeea9-f010-40f6-8df5-4eaa0c918ea6] +neutron_tempest_plugin.api.test_subnetpool_prefix_ops.SubnetPoolPrefixOpsIpv4Test.test_add_overlapping_prefix[id-a36c18fc-10b5-4ebc-ab79-914f826c5bf5] +neutron_tempest_plugin.api.test_subnetpool_prefix_ops.SubnetPoolPrefixOpsIpv4Test.test_add_remove_prefix[id-b1d56d1f-2818-44ee-b6a3-3c1327c25318] +neutron_tempest_plugin.api.test_subnetpool_prefix_ops.SubnetPoolPrefixOpsIpv6Test.test_add_overlapping_prefix[id-a36c18fc-10b5-4ebc-ab79-914f826c5bf5] +neutron_tempest_plugin.api.test_subnetpool_prefix_ops.SubnetPoolPrefixOpsIpv6Test.test_add_remove_prefix[id-b1d56d1f-2818-44ee-b6a3-3c1327c25318] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_create_sp_associate_address_scope_multiple_prefix_intersect[id-83a19a13-5384-42e2-b579-43fc69c80914,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_create_subnet_different_pools_same_network[id-fc011824-153e-4469-97ad-9808eb88cae1,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_create_subnetpool_associate_address_scope_of_other_owner[id-f06d8e7b-908b-4e94-b570-8156be6a4bf1,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_create_subnetpool_associate_address_scope_prefix_intersect[id-2dfb4269-8657-485a-a053-b022e911456e,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_create_subnetpool_associate_invalid_address_scope[id-9589e332-638e-476e-81bd-013d964aa3cb,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_create_subnetpool_associate_non_exist_address_scope[id-3b6c5942-485d-4964-a560-55608af020b5,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_delete_non_existent_subnetpool[id-5e1f2f86-d81a-498c-82ed-32a49f4dc4d3,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_get_non_existent_subnetpool[id-0212a042-603a-4f46-99e0-e37de9374d30,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_default_subnetpool[id-6ae09d8f-95be-40ed-b1cf-8b850d45bab5,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_shared_subnetpool[id-d1143fe2-212b-4e23-a308-d18f7d8d78d6,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_create_subnetpool_associate_shared_address_scope[id-3396ec6c-cb80-4ebe-b897-84e904580bdf,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_get_not_shared_admin_subnetpool[id-dc9336e5-f28f-4658-a0b0-cc79e607007d,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_tenant_update_sp_prefix_associated_with_shared_addr_scope[id-7438e49e-1351-45d8-937b-892059fb97f5,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_non_existent_subnetpool[id-4be84d30-60ca-4bd3-8512-db5b36ce1378,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_subnetpool_associate_address_scope_of_other_owner[id-6d3d9ad5-32d4-4d63-aa00-8c62f73e2881,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_subnetpool_associate_address_scope_wrong_ip_version[id-648fee7d-a909-4ced-bad3-3a169444c0a8,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_subnetpool_multiple_prefix_intersect[id-4d3f8a79-c530-4e59-9acf-6c05968adbfe,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_subnetpool_not_modifiable_shared[id-e6cd6d87-6173-45dd-bf04-c18ea7ec7537,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_subnetpool_prefix_intersect[id-96006292-7214-40e0-a471-153fb76e6b31,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_subnetpool_prefixes_shrink[id-62f7c43b-bff1-4def-8bb7-4754b840aaad,negative] +neutron_tempest_plugin.api.test_subnetpools_negative.SubnetPoolsNegativeTestJSON.test_update_subnetpool_tenant_id[id-2f66dc2f-cc32-4caa-91ec-0c0cd7c46d70,negative] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_cannot_share_if_no_access_to_address_scope[id-f74a71de-9abf-49c6-8199-4ac7f53e383b] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_filter_fields[id-17b2b437-a5fa-4340-ad98-912a986d0d7c] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_filter_rbac_policies[id-1c24c28c-eb1e-466e-af29-255cf127653a] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_policy_target_update[id-71b35ad0-51cd-40da-985d-89a51c95ec6a] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_rbac_policy_show[id-e59e4502-4e6a-4e49-b446-a5d5642bbd69] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_regular_client_blocked_from_sharing_anothers_policy[id-63d9acbe-403c-4e77-9ffd-80e636a4621e] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_regular_client_shares_to_another_regular_client[id-9cf8bba5-0163-4083-9397-678bb9b5f5a2] +neutron_tempest_plugin.api.test_subnetpools.RbacSubnetPoolTest.test_subnet_presence_prevents_rbac_policy_deletion[id-451d9d38-65a0-4916-a805-1460d6a938d1] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_no_pagination_limit_0[id-82a13efc-c18f-4249-b8ec-cec7cf26fbd6] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_pagination_page_reverse_asc[id-1ec1f325-43b0-406e-96ce-20539e38a61d] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_pagination_page_reverse_desc[id-f43a293e-4aaa-48f4-aeaf-de63a676357c] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_pagination_page_reverse_with_href_links[id-73511385-839c-4829-8ac1-b5ad992126c4] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_pagination_with_href_links[id-b3bd9665-2769-4a43-b50c-31b1add12891] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_pagination_with_marker[id-ddb20d14-1952-49b4-a17e-231cc2239a52] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_pagination[id-1291fae7-c196-4372-ad59-ce7988518f7b] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_sorts_asc[id-6e3f842e-6bfb-49cb-82d3-0026be4e8e04] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_sorts_desc[id-f336859b-b868-438c-a6fc-2c06374115f2] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsSearchCriteriaTest.test_list_validation_filters[id-27feb3f8-40f4-4e50-8cd2-7d0096a98682] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_admin_create_shared_subnetpool[id-d70c6c35-913b-4f24-909f-14cd0d29b2d2] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_create_list_subnetpool[id-6e1781ec-b45b-4042-aebe-f485c022996e] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_create_subnet_from_pool_with_default_prefixlen[id-83f76e3a-9c40-40c2-a015-b7c5242178d8] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_create_subnet_from_pool_with_prefixlen[id-1362ed7d-3089-42eb-b3a5-d6cb8398ee77] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_create_subnet_from_pool_with_quota[id-a64af292-ec52-4bde-b654-a6984acaf477] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_create_subnet_from_pool_with_subnet_cidr[id-86b86189-9789-4582-9c3b-7e2bfe5735ee] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_create_subnetpool_associate_address_scope[id-49b44c64-1619-4b29-b527-ffc3c3115dc4] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_create_update_subnetpool_description[id-c72c1c0c-2193-4aca-ddd4-b1442640bbbb] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_get_subnetpool[id-741d08c2-1e3f-42be-99c7-0ea93c5b728c] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_show_subnetpool_has_project_id[id-5bf9f1e2-efc8-4195-acf3-d12b2bd68dd3] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_tenant_create_non_default_subnetpool[id-4c6963c2-f54c-4347-b288-75d18421c4c4] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_tenant_update_subnetpool[id-764f1b93-1c4a-4513-9e7b-6c2fc5e9270c] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_update_subnetpool_associate_address_scope[id-910b6393-db24-4f6f-87dc-b36892ad6c8c] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_update_subnetpool_associate_another_address_scope[id-18302e80-46a3-4563-82ac-ccd1dd57f652] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_update_subnetpool_disassociate_address_scope[id-f8970048-e41b-42d6-934b-a1297b07706a] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_update_subnetpool_prefixes_append[id-4b496082-c992-4319-90be-d4a7ce646290] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTest.test_update_subnetpool_prefixes_extend[id-2cae5d6a-9d32-42d8-8067-f13970ae13bb] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_admin_create_shared_subnetpool[id-d70c6c35-913b-4f24-909f-14cd0d29b2d2] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_dual_stack_subnets_from_subnetpools[id-f62d73dc-cf6f-4879-b94b-dab53982bf3b] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_list_subnetpool[id-6e1781ec-b45b-4042-aebe-f485c022996e] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_subnet_from_pool_with_default_prefixlen[id-83f76e3a-9c40-40c2-a015-b7c5242178d8] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_subnet_from_pool_with_prefixlen[id-1362ed7d-3089-42eb-b3a5-d6cb8398ee77] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_subnet_from_pool_with_quota[id-a64af292-ec52-4bde-b654-a6984acaf477] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_subnet_from_pool_with_subnet_cidr[id-86b86189-9789-4582-9c3b-7e2bfe5735ee] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_subnetpool_associate_address_scope[id-49b44c64-1619-4b29-b527-ffc3c3115dc4] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_create_update_subnetpool_description[id-c72c1c0c-2193-4aca-ddd4-b1442640bbbb] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_get_subnetpool[id-741d08c2-1e3f-42be-99c7-0ea93c5b728c] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_show_subnetpool_has_project_id[id-5bf9f1e2-efc8-4195-acf3-d12b2bd68dd3] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_tenant_create_non_default_subnetpool[id-4c6963c2-f54c-4347-b288-75d18421c4c4] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_tenant_update_subnetpool[id-764f1b93-1c4a-4513-9e7b-6c2fc5e9270c] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_update_subnetpool_associate_address_scope[id-910b6393-db24-4f6f-87dc-b36892ad6c8c] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_update_subnetpool_associate_another_address_scope[id-18302e80-46a3-4563-82ac-ccd1dd57f652] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_update_subnetpool_disassociate_address_scope[id-f8970048-e41b-42d6-934b-a1297b07706a] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_update_subnetpool_prefixes_append[id-4b496082-c992-4319-90be-d4a7ce646290] +neutron_tempest_plugin.api.test_subnetpools.SubnetPoolsTestV6.test_update_subnetpool_prefixes_extend[id-2cae5d6a-9d32-42d8-8067-f13970ae13bb] +neutron_tempest_plugin.api.test_subnets.SubnetServiceTypeTestJSON.test_allocate_ips_are_from_correct_subnet[id-7e0edb66-1bb2-4473-ab83-d039cddced0d] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_no_pagination_limit_0[id-d851937c-9821-4b46-9d18-43e9077ecac0] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_pagination_page_reverse_asc[id-dfaa20ca-6d84-4f26-962f-2fee4d247cd9] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_pagination_page_reverse_desc[id-40552213-3e12-4d6a-86f3-dda92f3de88c] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_pagination_page_reverse_with_href_links[id-3cea9053-a731-4480-93ee-19b2c28a9ce4] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_pagination_with_href_links[id-351183ef-6ed9-4d71-a9f2-a5ac049bd7ea] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_pagination_with_marker[id-2ddd9aa6-de28-410f-9cbc-ce752893c407] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_pagination[id-b93063b3-f713-406e-bf93-e5738e09153c] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_sorts_asc[id-d2d61995-5dd5-4b93-bce7-3edefdb79563] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_sorts_desc[id-c3c6b0af-c4ac-4da0-b568-8d08ae550604] +neutron_tempest_plugin.api.test_subnets.SubnetsSearchCriteriaTest.test_list_validation_filters[id-c0f9280b-9d81-4728-a967-6be22659d4c8] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_create_network_with_timestamp[id-462be770-b310-4df9-9c42-773217e4c8b1] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_create_port_with_timestamp[id-e2450a7b-d84f-4600-a093-45e78597bbac] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_create_subnet_with_timestamp[id-8ee55186-454f-4b97-9f9f-eb2772ee891c] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_create_subnetpool_with_timestamp[id-87a8b196-4b90-44f0-b7f3-d2057d7d658e] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_segment_with_timestamp[id-396a97dc-b66c-4c46-9171-c39eefe6936c] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_show_networks_attribute_with_timestamp[id-2ac50ab2-7ebd-4e27-b3ce-a9e399faaea2] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_show_port_attribute_with_timestamp[id-584c6723-40b6-4f26-81dd-f508f9d9fb51] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_show_subnet_attribute_with_timestamp[id-1836a086-e7cf-4141-bf57-0cfe79e8051e] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_show_subnetpool_attribute_with_timestamp[id-1d3970e6-bcf7-46cd-b7d7-0807759c73b4] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_update_network_with_timestamp[id-4db5417a-e11c-474d-a361-af00ebef57c5] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_update_port_with_timestamp[id-4241e0d3-54b4-46ce-a9a7-093fc764161b] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_update_subnet_with_timestamp[id-a490215a-6f4c-4af9-9a4c-57c41f1c4fa1] +neutron_tempest_plugin.api.test_timestamp.TestTimeStamp.test_update_subnetpool_with_timestamp[id-d48c7578-c3d2-4f9b-a7a1-be2008c770a0] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithL3.test_create_floatingip_with_timestamp[id-8ae55186-464f-4b87-1c9f-eb2765ee81ac] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithL3.test_create_router_with_timestamp[id-433ba770-b310-4da9-5d42-733217a1c7b1] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithL3.test_show_floatingip_attribute_with_timestamp[id-32a6a086-e1ef-413b-b13a-0cfe13ef051e] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithL3.test_show_router_attribute_with_timestamp[id-1ab50ac2-7cbd-4a17-b23e-a9e36cfa4ec2] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithL3.test_update_floatingip_with_timestamp[id-a3ac215a-61ac-13f9-9d3c-57c51f11afa1] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithL3.test_update_router_with_timestamp[id-4a65417a-c11c-4b4d-a351-af01abcf57c6] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithSecurityGroup.test_create_sg_with_timestamp[id-a3150a7b-d31a-423a-abf3-45e71c97cbac] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithSecurityGroup.test_create_sgrule_with_timestamp[id-83e8bd32-43e0-a3f0-1af3-12a5733c653e] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithSecurityGroup.test_show_sg_attribute_with_timestamp[id-521e6723-43d6-12a6-8c3d-f5042ad9fc32] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithSecurityGroup.test_show_sgrule_attribute_with_timestamp[id-143da0e6-ba17-43ad-b3d7-03aa759c3cb4] +neutron_tempest_plugin.api.test_timestamp.TestTimeStampWithSecurityGroup.test_update_sg_with_timestamp[id-432ae0d3-32b4-413e-a9b3-091ac76da31b] +neutron_tempest_plugin.api.test_trunk_details.TestTrunkDetailsJSON.test_port_resource_empty_trunk_details[id-fe6d865f-1d5c-432e-b65d-904157172f24] +neutron_tempest_plugin.api.test_trunk_details.TestTrunkDetailsJSON.test_port_resource_trunk_details_no_subports[id-f0bed24f-d36a-498b-b4e7-0d66e3fb7308] +neutron_tempest_plugin.api.test_trunk_details.TestTrunkDetailsJSON.test_port_resource_trunk_details_with_subport[id-544bcaf2-86fb-4930-93ab-ece1c3cc33df] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_add_subport_duplicate_segmentation_details[id-00cb40bb-1593-44c8-808c-72b47e64252f,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_add_subport_passing_dict[id-4eac8c25-83ee-4051-9620-34774f565730,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_add_subport_port_id_disabled_trunk[id-7f132ccc-1380-42d8-9c44-50411612bd01,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_add_subport_port_id_uses_parent_port_id[id-6f132ccc-1380-42d8-9c44-50411612bd01,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_subport_invalid_inherit_network_segmentation_type[id-40aed9be-e976-47d0-dada-bde2c7e74e57,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_subport_missing_segmentation_id[id-7e0f99ab-fe37-408b-a889-9e44ef300084,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_subport_nonexistent_port_id[id-a5c5200a-72a0-43c5-a11a-52f808490344,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_subport_nonexistent_trunk[id-80deb6a9-da2a-48db-b7fd-bcef5b14edc1,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_trunk_duplicate_subport_segmentation_ids[id-40aed9be-e976-47d0-a555-bde2c7e74e57,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_trunk_nonexistent_port_id[id-1b5cf87a-1d3a-4a94-ba64-647153d54f32,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_trunk_nonexistent_subport_port_id[id-980bca3b-b0be-45ac-8067-b401e445b796,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_trunk_with_subport_missing_port_id[id-a717691c-4e07-4d81-a98d-6f1c18c5d183,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_trunk_with_subport_missing_segmentation_id[id-a315d78b-2f43-4efa-89ae-166044c568aa,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_create_trunk_with_subport_missing_segmentation_type[id-33498618-f75a-4796-8ae6-93d4fd203fa4,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_delete_port_in_use_by_subport[id-343a03d0-4f7c-11e6-97fa-40a8f063c891,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_delete_port_in_use_by_trunk[id-6c9c5126-4f61-11e6-8248-40a8f063c891,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_delete_trunk_disabled_trunk[id-9f132ccc-1380-42d8-9c44-50411612bd01,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_remove_subport_not_found[id-aaca7dd7-96b8-445a-931e-63f0d86d2fe2,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_remove_subport_passing_dict[id-17ca7dd7-96a8-445a-941e-53c0c86c2fe2,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestJSON.test_remove_subport_port_id_disabled_trunk[id-8f132ccc-1380-42d8-9c44-50411612bd01,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestMtusJSON.test_add_subport_with_mtu_greater_than_trunk[id-3b32bf77-8002-403e-ad01-6f4cf018daa5,negative] +neutron_tempest_plugin.api.test_trunk_negative.TrunkTestMtusJSON.test_create_trunk_with_mtu_smaller_than_subport[id-228380ef-1b7a-495e-b759-5b1f08e3e858,negative] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_no_pagination_limit_0[id-eafe7024-77ab-4cfe-824b-0b2bf4217727] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_pagination_page_reverse_asc[id-f8857391-dc44-40cc-89b7-2800402e03ce] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_pagination_page_reverse_desc[id-ae51e9c9-ceae-4ec0-afd4-147569247699] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_pagination_page_reverse_with_href_links[id-b4293e59-d794-4a93-be09-38667199ef68] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_pagination_with_href_links[id-dcd02a7a-f07e-4d5e-b0ca-b58e48927a9b] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_pagination_with_marker[id-c4723b8e-8186-4b9a-bf9e-57519967e048] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_pagination[id-b202fdc8-6616-45df-b6a0-463932de6f94] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_sorts_asc[id-fab73df4-960a-4ae3-87d3-60992b8d3e2d] +neutron_tempest_plugin.api.test_trunk.TrunksSearchCriteriaTest.test_list_sorts_desc[id-a426671d-7270-430f-82ff-8f33eec93010] +neutron_tempest_plugin.api.test_trunk.TrunkTestInheritJSONBase.test_add_subport[id-0f05d98e-41f5-4629-dada-9aee269c9602] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_add_subports[id-bb5fcead-09b5-484a-bbe6-46d1e06d6cc0] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_create_show_delete_trunk[id-7de46c22-e2b6-4959-ac5a-0e624632ab32] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_create_trunk_empty_subports_list[id-e1a6355c-4768-41f3-9bf8-0f1d192bd501] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_create_trunk_subports_not_specified[id-382dfa39-ca03-4bd3-9a1c-91e36d2e3796] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_create_update_trunk_with_description[id-5ff46c22-a2b6-5559-bc5a-0ef2463cab32] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_create_update_trunk[id-4ce46c22-a2b6-4659-bc5a-0ef2463cab32] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_delete_trunk_with_subport_is_allowed[id-ee5fcead-1abf-483a-bce6-43d1e06d6aa0] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_get_subports[id-bb5fcaad-09b5-484a-dde6-4cd1ea6d6ff0] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_list_trunks[id-73365f73-bed6-42cd-960b-ec04e0c99d85] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_remove_subport[id-96eea398-a03c-4c3e-a99e-864392c2ca53] +neutron_tempest_plugin.api.test_trunk.TrunkTestJSON.test_show_trunk_has_project_id[id-8d83a6ca-662d-45b8-8062-d513077296aa] +neutron_tempest_plugin.api.test_trunk.TrunkTestMtusJSON.test_add_subport_with_mtu_equal_to_trunk[id-175b05ae-66ad-44c7-857a-a12d16f1058f] +neutron_tempest_plugin.api.test_trunk.TrunkTestMtusJSON.test_add_subport_with_mtu_greater_than_subport[id-2004c5c6-e557-4c43-8100-c820ad4953e8] +neutron_tempest_plugin.api.test_trunk.TrunkTestMtusJSON.test_create_trunk_with_mtu_equal_to_subport[id-22725101-f4bc-4e00-84ec-4e02cd7e0500] +neutron_tempest_plugin.api.test_trunk.TrunkTestMtusJSON.test_create_trunk_with_mtu_greater_than_subport[id-0f05d98e-41f5-4629-ac29-9aee269c9602] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_associate_disassociate_network[id-51e1b079-aefa-4c37-8b1a-0567b3ef7954] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_associate_disassociate_router[id-de8d94b0-0239-4a48-9574-c3a4a4f7cacb] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_associate_invalid_network[id-f049ce21-d239-47c0-b13f-fb57a2a558ce,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_attach_associated_subnet_to_associated_router[id-4be1f073-fe57-4858-b7b9-9a189e90b770,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_create_bgpvpn_as_non_admin_fail[id-0a911d61-d908-4c21-a11e-e403ac0d8e38,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_create_bgpvpn_with_invalid_routetargets[id-464ca6f9-86e4-4ee3-9c65-f1edae93223d,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_create_bgpvpn[id-4f90deb2-eb8e-4e7d-9d68-c5b5cc657f7e] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_delete_bgpvpn_as_non_admin_fail[id-596abfc2-fd89-491d-863d-25459db1df4b,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_delete_bgpvpn[id-709b23b0-9719-47df-9f53-b0812a5d5a48] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_disassociate_invalid_network[id-078b2660-4adb-4c4c-abf0-b77bf0bface5,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_list_bgpvpn_as_non_owner_fail[id-4875e65d-0b65-40c0-9efd-309420686ab4,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_list_bgpvpn[id-7a7feca2-1c24-4f5d-ad4b-b0e5a712adb1] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_list_netassoc_as_non_owner_fail[id-2cbb10af-bf9c-4b32-b6a6-4066de783758,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_list_show_network_association[id-096281da-356d-4c04-bd55-784a26bb1b0c] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_list_show_router_association[id-3ae91755-b1b6-4c62-a699-a44eeb4ee522] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_show_bgpvpn_as_non_owner_fail[id-b20110bb-393b-4342-8b30-6486cd2b4fc6,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_show_bgpvpn[id-9fa29db8-35d0-4beb-a986-23c369499ab1] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_show_netassoc_as_non_owner_fail[id-57b0da93-8e37-459f-9aaf-f903acc36025,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_update_bgpvpn_invalid_routetargets[id-7d4e9b87-e1ab-47a7-a8d6-9d179365556a,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_update_route_target_non_admin_fail[id-e35eb9be-fe1f-406c-b36b-fc1879328313,negative] +neutron_tempest_plugin.bgpvpn.api.test_bgpvpn.BgpvpnTest.test_update_route_target[id-559013fd-1e34-4fde-9599-f8aafe9ae716] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_basic[compute,id-afdd6cad-871a-4343-b97b-6319c76c815d,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_negative_delete_bgpvpn[compute,id-dc92643f-1b2c-4a3e-bc5e-ea780d721ef7,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_negative_delete_net_association[compute,id-2e6bf893-1410-4ef6-9948-1877f3a8f3d1,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_negative_delete_router_association[compute,id-9ebf4342-4448-4d63-98f9-44d3a606b6cf,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_negative_disjoint_import_export[compute,id-b6d219b2-90bb-431f-a566-bf6a780d1578,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_negative_ping_to_unassociated_net[compute,id-8478074e-22df-4234-b02b-61257b475b18,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_negative_update_to_disjoint_import_export[compute,id-be4471b3-5f57-4022-b719-b45a673a728b,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_negative_update_to_empty_rt_list[compute,id-fb37a546-7263-4ffe-a42c-77eca377ff1a,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_port_association_bgpvpn_route[compute,id-73f629fa-fdae-40fc-8a7e-da3aedcf797a,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_port_association_create_and_delete[compute,id-9c3280b5-0b32-4562-800c-0b50d9d52bfd,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_port_association_create_and_update[compute,id-3b44b2f4-f514-4004-8623-2682bc46bb07,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_port_association_local_pref[compute,id-876b49bc-f34a-451b-ba3c-d74295838130,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_tenant_separation_and_local_connectivity[compute,id-f762e6ac-920e-4d0f-aa67-02bdd4ab8433,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_update_route_targets_and_unassociated_net[compute,id-08d4f40e-3cec-485b-9da2-76e67fbd9881,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_update_route_targets_common_target[compute,id-bf417cad-0bc4-446a-b367-850aa619ca4f,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_update_route_targets_disjoint_targets[compute,id-133497a1-2788-40f7-be01-b3b64b5ef8cd,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_update_rt_and_keep_local_connectivity_variant1[compute,id-c8bfd695-f731-47a6-86e3-3dfa492e08e0,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_update_rt_and_keep_local_connectivity_variant2[compute,id-758a8731-5070-4b1e-9a66-d6ff05bb5be1,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_variant1[compute,id-8a5a6fac-313c-464b-9c5c-29d4e1c0a51e,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_variant2[compute,id-e7468636-0816-4092-82ca-3590680ed00b,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_variant3[compute,id-7c66aa31-fb3a-4e15-8808-46eb361f153a,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_variant4[compute,id-973ab26d-c7d8-4a32-9aa9-2d7e6f406135,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_variant5[compute,id-2ac0696b-e828-4299-9e94-5f9c4988d961,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_bgpvpn_variant6[compute,id-9081338e-a52e-46bb-a40e-bda24ec4b1bd,network] +neutron_tempest_plugin.bgpvpn.scenario.test_bgpvpn_basic.TestBGPVPNBasic.test_port_association_many_bgpvpn_routes[compute,id-d92a8a18-c4d0-40d5-8592-713d7dae7d25,network] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_create_show_delete_firewall_group[id-48dfcd75-3924-479d-bb65-b3ed33397663] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_create_update_delete_firewall_policy[id-6ef9bd02-7349-4d61-8d1f-80479f64d904] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_create_update_delete_firewall_rule[id-ffc009fa-cd17-4029-8025-c4b81a7dd923] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_default_fwg_created_on_list_firewall_groups[id-fd24db16-c8cb-4cb4-ba60-b0cd18a66b83] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_error_on_create_firewall_group_name_default[id-a1f524d8-5336-4769-aa7b-0830bb4df6c8] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_list_firewall_policies[id-f6b83902-746f-4e74-9403-2ec9899583a3] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_list_firewall_rules[id-ddccfa87-4af7-48a6-9e50-0bd0ad1348cb] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_show_firewall_policy[id-164381de-61f4-483f-9a5a-48105b8e70e2] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_show_firewall_rule[id-76b07afc-444e-4bb9-abec-9b8c5f994dcd] +neutron_tempest_plugin.fwaas.api.test_fwaasv2_extensions.FWaaSv2ExtensionTestJSON.test_update_firewall_group[id-e021baab-d4f7-4bad-b382-bde4946e0e0b] +neutron_tempest_plugin.fwaas.scenario.test_fwaas_v2.TestFWaaS_v2.test_icmp_reachability_scenarios[id-77fdf3ea-82c1-453d-bfec-f7efe335625d] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions_negative.BgpSpeakerTestJSONNegative.test_create_bgp_peer_non_admin[id-33f7aaf0-9786-478b-b2d1-a51086a50eb4,negative] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions_negative.BgpSpeakerTestJSONNegative.test_create_bgp_speaker_illegal_local_asn[id-75e9ee2f-6efd-4320-bff7-ae24741c8b06,negative] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions_negative.BgpSpeakerTestJSONNegative.test_create_bgp_speaker_non_admin[id-6742ec2e-382a-4453-8791-13a19b47cd13,negative] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions_negative.BgpSpeakerTestJSONNegative.test_get_advertised_routes_tenant_networks[id-9cc33701-51e5-421f-a5d5-fd7b330e550f] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions_negative.BgpSpeakerTestJSONNegative.test_update_bgp_speaker_local_asn[id-39435932-0266-4358-899b-0e9b1e53c3e9,negative] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_add_bgp_peer[id-6ade0319-1ee2-493c-ac4b-5eb230ff3a77] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_add_gateway_network[id-23c8eb37-d10d-4f43-b2e7-6542cb6a4405] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_create_bgp_peer[id-81d9dc45-19f8-4c6e-88b8-401d965cd1b0] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_delete_bgp_speaker[id-df259771-7104-4ffa-b77f-bd183600d7f9] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_get_advertised_routes_floating_ips[id-cae9cdb1-ad65-423c-9604-d4cd0073616e] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_get_advertised_routes_null_address_scope[id-5bef22ad-5e70-4f7b-937a-dc1944642996] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_get_advertised_routes_tenant_networks[id-c9ad566e-fe8f-4559-8303-bbad9062a30c] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_remove_bgp_peer[id-f9737708-1d79-440b-8350-779f97d882ee] +neutron_tempest_plugin.neutron_dynamic_routing.api.test_bgp_speaker_extensions.BgpSpeakerTestJSON.test_remove_gateway_network[id-6cfc7137-0d99-4a3d-826c-9d1a3a1767b0] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_4byte_asn.BgpSpeaker4byteASNTest.test_check_advertised_multiple_tenant_network[id-c3158328-2f69-4aa2-b1b7-5a06ab58afaf] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_4byte_asn.BgpSpeaker4byteASNTest.test_check_advertised_tenant_network_with_multiple_peers[id-c72411c8-ea79-495d-bdbd-a10159642676] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_4byte_asn.BgpSpeaker4byteASNTest.test_check_advertised_tenant_network[id-73466aa5-7d1d-4f9f-8fb4-4100fad2dffe] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_4byte_asn.BgpSpeaker4byteASNTest.test_check_neighbor_established_with_multiple_peers[id-212a3d82-ac50-43dc-b657-030b1133643e] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_4byte_asn.BgpSpeaker4byteASNTest.test_check_neighbor_established[id-9f18c931-a59e-4a27-939b-21124995ffe2] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_basic.BgpSpeakerBasicTest.test_remove_add_speaker_agent[id-aa6c565c-ded3-413b-8dc9-3928b3b0e38f] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_basic.BgpSpeakerBasicTest.test_schedule_added_speaker[id-cc615252-c6cb-4d75-a70e-608fb2c3736a] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.basic.test_basic.BgpSpeakerBasicTest.test_unschedule_deleted_speaker[id-ce98c33c-0ffa-49ae-b365-da836406793b] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv4.test_ipv4.BgpSpeakerIpv4Test.test_check_advertised_multiple_tenant_network[id-a5c238de-b750-499c-aaa2-b44a057e9ed3] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv4.test_ipv4.BgpSpeakerIpv4Test.test_check_advertised_tenant_network_with_multiple_peers[id-91971dfb-c129-4744-9fbb-ac4f9e8d56c0] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv4.test_ipv4.BgpSpeakerIpv4Test.test_check_advertised_tenant_network[id-f32245fc-aeab-4244-acfa-3af9dd662e8d] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv4.test_ipv4.BgpSpeakerIpv4Test.test_check_neighbor_established_with_multiple_peers[id-e4961cc1-0c47-4081-a896-caaa9342ca75] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv4.test_ipv4.BgpSpeakerIpv4Test.test_check_neighbor_established[id-7f2acbc2-ff88-4a63-aa02-a2f9feb3f5b0] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv6.test_ipv6.BgpSpeakerIpv6Test.test_check_advertised_multiple_tenant_network[id-aca5d678-c249-4de5-921b-6b6ba621e4f7] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv6.test_ipv6.BgpSpeakerIpv6Test.test_check_advertised_tenant_network_with_multiple_peers[id-be710ec1-a338-44c9-8b89-31c3532aae65] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv6.test_ipv6.BgpSpeakerIpv6Test.test_check_advertised_tenant_network[id-6a3483fc-8c8a-4387-bda6-c7061410e04b] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv6.test_ipv6.BgpSpeakerIpv6Test.test_check_neighbor_established_with_multiple_peers[id-f81012f3-2f7e-4b3c-8c1d-b1778146d712] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.ipv6.test_ipv6.BgpSpeakerIpv6Test.test_check_neighbor_established[id-5194a8e2-95bd-49f0-872d-1e3e875ede32] +neutron_tempest_plugin.neutron_dynamic_routing.scenario.test_simple_bgp.Bgp.test_bgp[id-c1208ce2-c55f-4424-9035-25de83161d6f] +neutron_tempest_plugin.scenario.admin.test_floatingip.FloatingIpTestCasesAdmin.test_two_vms_fips[id-6bba729b-3fb6-494b-9e1e-82bbd89a1045] +neutron_tempest_plugin.scenario.test_basic.NetworkBasicTest.test_basic_instance[id-de07fe0a-e955-449e-b48b-8641c14cd52e] +neutron_tempest_plugin.scenario.test_basic.NetworkBasicTest.test_ping_global_ip_from_vm_with_fip[id-49609189-3a0e-43c7-832e-a7e114aad1c9] +neutron_tempest_plugin.scenario.test_connectivity.NetworkConnectivityTest.test_connectivity_dvr_and_no_dvr_routers_in_same_subnet[id-69d3650a-5c32-40bc-ae56-5c4c849ddd37] +neutron_tempest_plugin.scenario.test_connectivity.NetworkConnectivityTest.test_connectivity_router_east_west_traffic[id-b72c3b77-3396-4144-b05d-9cd3c0099893] +neutron_tempest_plugin.scenario.test_connectivity.NetworkConnectivityTest.test_connectivity_through_2_routers[id-8944b90d-1766-4669-bd8a-672b5d106bb7] +neutron_tempest_plugin.scenario.test_dhcp.DHCPPortUpdateTest.test_modify_dhcp_port_ip_address[id-8171cc68-9dbb-46ca-b065-17b5b2e26094] +neutron_tempest_plugin.scenario.test_dhcp.DHCPTest.test_extra_dhcp_opts[id-58f7c094-1980-4e03-b0d3-6c4dd27217b1] +neutron_tempest_plugin.scenario.test_dns_integration.DNSIntegrationAdminTests.test_fip_admin_delete[id-d44cd5b8-ac67-4965-96ff-cb77ab6aea8b] +neutron_tempest_plugin.scenario.test_dns_integration.DNSIntegrationAdminTests.test_port_on_special_network[id-fa6477ce-a12b-41da-b671-5a3bbdafab07] +neutron_tempest_plugin.scenario.test_dns_integration.DNSIntegrationDomainPerProjectTests.test_fip_admin_delete[id-ac89db9b-5ca4-43bd-85ba-40fbeb47e208] +neutron_tempest_plugin.scenario.test_dns_integration.DNSIntegrationDomainPerProjectTests.test_port_with_dns_name[id-43a67509-3161-4125-8f2c-0d4a67599721] +neutron_tempest_plugin.scenario.test_dns_integration.DNSIntegrationExtraTests.test_port_with_publishing_subnet[id-e10e0e5d-69ac-4172-b39f-27ab344b7f99] +neutron_tempest_plugin.scenario.test_dns_integration.DNSIntegrationTests.test_fip[id-a8f2fade-8d5c-40f9-80f0-3de4b8d91985] +neutron_tempest_plugin.scenario.test_dns_integration.DNSIntegrationTests.test_server_with_fip[id-850ee378-4b5a-4f71-960e-0e7b12e03a34] +neutron_tempest_plugin.scenario.test_dvr.NetworkDvrTest.test_vm_reachable_through_compute[id-3d73ec1a-2ec6-45a9-b0f8-04a283d9d344] +neutron_tempest_plugin.scenario.test_fip64.Fip64.test_fip64[id-63f7da91-c7dd-449b-b50b-1c56853ce0ef] +neutron_tempest_plugin.scenario.test_floatingip.DefaultSnatToExternal.test_nested_snat_external_ip[id-b911b124-b6cb-449d-83d9-b34f3665741d] +neutron_tempest_plugin.scenario.test_floatingip.DefaultSnatToExternal.test_snat_external_ip[id-3d73ea1a-27c6-45a9-b0f8-04a283d9d764] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpMultipleRoutersTest.test_reuse_ip_address_with_other_fip_on_other_router[id-b0382ab3-3c86-4415-84e3-649a8b040dab] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIPPortDetailsTest.test_floatingip_port_details[id-a663aeee-dd81-492b-a207-354fd6284dbe] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIPQosTest.test_qos[id-5eb48aea-eaba-4c20-8a6f-7740070a0aa3] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSameNetwork.test_east_west_1[id-05c4e3b3-7319-4052-90ad-e8916436c23b] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSameNetwork.test_east_west_2[id-05c4e3b3-7319-4052-90ad-e8916436c23b] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSameNetwork.test_east_west_3[id-05c4e3b3-7319-4052-90ad-e8916436c23b] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSameNetwork.test_east_west_4[id-05c4e3b3-7319-4052-90ad-e8916436c23b] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSeparateNetwork.test_east_west_1[id-f18f0090-3289-4783-b956-a0f8ac511e8b] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSeparateNetwork.test_east_west_2[id-f18f0090-3289-4783-b956-a0f8ac511e8b] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSeparateNetwork.test_east_west_3[id-f18f0090-3289-4783-b956-a0f8ac511e8b] +neutron_tempest_plugin.scenario.test_floatingip.FloatingIpSeparateNetwork.test_east_west_4[id-f18f0090-3289-4783-b956-a0f8ac511e8b] +neutron_tempest_plugin.scenario.test_floatingip.TestFloatingIPUpdate.test_floating_ip_update[id-1bdd849b-03dd-4b8f-994f-457cf8a36f93] +neutron_tempest_plugin.scenario.test_internal_dns.InternalDNSTest.test_create_and_update_port_with_dns_name[id-db5e612f-f17f-4974-b5f1-9fe89f4a6fc9] +neutron_tempest_plugin.scenario.test_internal_dns.InternalDNSTest.test_dns_domain_and_name[id-988347de-07af-471a-abfa-65aea9f452a6] +neutron_tempest_plugin.scenario.test_ipv6.IPv6Test.test_ipv6_hotplug_dhcpv6stateless[id-9aaedbc4-986d-42d5-9177-3e721728e7e0] +neutron_tempest_plugin.scenario.test_ipv6.IPv6Test.test_ipv6_hotplug_slaac[id-b13e5408-5250-4a42-8e46-6996ce613e91] +neutron_tempest_plugin.scenario.test_local_ip.LocalIPTest.test_local_ip_connectivity[id-3aa4b288-011a-4aa2-9024-19ad2ce40bfd] +neutron_tempest_plugin.scenario.test_mac_learning.MacLearningTest.test_mac_learning_vms_on_same_network[id-013686ac-23b1-23e4-8361-10b1c98a2861] +neutron_tempest_plugin.scenario.test_metadata.MetadataTest.test_metadata_ipv6_only_network[id-7542892a-d132-471c-addb-172dcf888ff6] +neutron_tempest_plugin.scenario.test_metadata.MetadataTest.test_metadata_routed[id-e680949a-f1cc-11ea-b49a-cba39bbbe5ad] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromDVR.test_from_dvr_to_dvr_ha[id-25304a51-93a8-4cf3-9523-bce8b4eaecf8] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromDVR.test_from_dvr_to_ha[id-a00d5ad7-8509-4bb0-bdd2-7f1ee052d1cd] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromDVR.test_from_dvr_to_legacy[id-e5cac02c-248d-4aac-bd5e-9d47c5197307] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromDVRHA.test_from_dvr_ha_to_dvr[id-d6bedff1-72be-4a9a-8ea2-dc037cd838e0] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromDVRHA.test_from_dvr_ha_to_ha[id-55957267-4e84-4314-a2f7-7cd36a2df04b] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromDVRHA.test_from_dvr_ha_to_legacy[id-1be9b2e2-379c-40a4-a269-6687b81df691] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromHA.test_from_ha_to_dvr_ha[id-e4149576-248b-43fa-9d0b-a5c2f51967ce] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromHA.test_from_ha_to_dvr[id-42260eea-5d56-4d30-b62a-a62694dfe4d5] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromHA.test_from_ha_to_legacy[id-b4e68ac0-3b76-4306-ae8a-51cf4d363b22] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromLegacy.test_from_legacy_to_dvr_ha[id-fe169f2c-6ed3-4eb0-8afe-2d540c4b49e2] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromLegacy.test_from_legacy_to_dvr[id-23724222-483a-4129-bc15-7a9278f3828b] +neutron_tempest_plugin.scenario.test_migration.NetworkMigrationFromLegacy.test_from_legacy_to_ha[id-09d85102-994f-4ff9-bf3e-17051145ca12] +neutron_tempest_plugin.scenario.test_mtu.NetworkMtuTest.test_connectivity_min_max_mtu[id-3d73ec1a-2ec6-45a9-b0f8-04a273d9d344] +neutron_tempest_plugin.scenario.test_mtu.NetworkWritableMtuTest.test_connectivity_min_max_mtu[id-bc470200-d8f4-4f07-b294-1b4cbaaa35b9] +neutron_tempest_plugin.scenario.test_multicast.MulticastTestIPv4.test_multicast_between_vms_on_same_network[id-113486fc-24c9-4be4-8361-03b1c9892867] +neutron_tempest_plugin.scenario.test_multiple_gws.NetworkMultipleGWTest.test_create_router_multiple_gw_bfd_and_ecmp[id-5117587d-9633-48b7-aa8f-ec9d59a601a5] +neutron_tempest_plugin.scenario.test_multiple_gws.NetworkMultipleGWTest.test_create_router_single_gw_bfd[id-9baa05e6-ba10-4850-93e3-695f4d97b8f8] +neutron_tempest_plugin.scenario.test_multiple_gws.NetworkMultipleGWTest.test_update_router_single_gw_bfd[id-75202251-c384-4962-8685-60cf2c530906] +neutron_tempest_plugin.scenario.test_port_forwardings.PortForwardingTestJSON.test_port_forwarding_editing_and_deleting_tcp_rule[id-aa19d46c-a4a6-11ea-bb37-0242ac130002] +neutron_tempest_plugin.scenario.test_port_forwardings.PortForwardingTestJSON.test_port_forwarding_editing_and_deleting_udp_rule[id-6d05b1b2-6109-4c30-b402-1503f4634acb] +neutron_tempest_plugin.scenario.test_port_forwardings.PortForwardingTestJSON.test_port_forwarding_to_2_fixed_ips[id-5971881d-06a0-459e-b636-ce5d1929e2d4] +neutron_tempest_plugin.scenario.test_port_forwardings.PortForwardingTestJSON.test_port_forwarding_to_2_servers[id-ab40fc48-ca8d-41a0-b2a3-f6679c847bfe] +neutron_tempest_plugin.scenario.test_ports.PortsTest.test_port_with_fixed_ip[id-62e32802-1d21-11eb-b322-74e5f9e2a801] +neutron_tempest_plugin.scenario.test_ports.PortsTest.test_previously_used_port[id-5500797e-b8c2-4e07-a5e0-89fa4e814965] +neutron_tempest_plugin.scenario.test_portsecurity.PortSecurityTest.test_port_security_removed_added_stateful_sg[id-61ab176e-d48b-42b7-b38a-1ba571ecc033] +neutron_tempest_plugin.scenario.test_portsecurity.PortSecurityTest.test_port_security_removed_added_stateless_sg[id-2f4005e1-cee1-40e5-adbd-b3e3cf218065] +neutron_tempest_plugin.scenario.test_qos.QoSTest.test_attach_previously_used_port_to_new_instance[id-66e5673e-0522-11ea-8d71-362b9e155667] +neutron_tempest_plugin.scenario.test_qos.QoSTest.test_create_instance_using_network_with_existing_policy[id-4eee64da-5646-11ea-82b4-0242ac130003] +neutron_tempest_plugin.scenario.test_qos.QoSTest.test_qos_basic_and_update[id-00682a0c-b72e-11e8-b81e-8c16450ea513] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_connectivity_between_vms_using_different_sec_groups[id-4a724164-bbc0-4029-a844-644ece66c026] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_default_sec_grp_scenarios[id-3d73ec1a-2ec6-45a9-b0f8-04a283d9d764] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_established_tcp_session_after_re_attachinging_sg[id-01f0ddca-b049-47eb-befd-82acb502c9ec] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_intra_sg_isolation[id-f07d0159-8f9e-4faa-87f5-a869ab0ad490] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_ip_prefix_negative[id-a01cd2ef-3cfc-4614-8aac-9d1333ea21dd,negative] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_ip_prefix[id-3d73ec1a-2ec6-45a9-b0f8-04a283d9d664] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_multiple_ports_portrange_remote[id-f07d0159-8f9e-4faa-87f5-a869ab0ad489] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_multiple_ports_secgroup_inheritance[id-f07d0159-8f9e-4faa-87f5-a869ab0ad488] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_overlapping_sec_grp_rules[id-cd66b826-d86c-4fb4-ab37-17c8391753cb] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_protocol_number_rule[id-3d73ec1a-2ec6-45a9-b0f8-04a283d9d864] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_remote_group_and_remote_address_group[id-678dd4c0-2953-4626-b89c-8e7e4110ec4b] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_remote_group[id-7ed39b86-006d-40fb-887a-ae46693dabc9] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_remove_sec_grp_from_active_vm[id-96dcd5ff-9d45-4e0d-bea0-0b438cbd388f] +neutron_tempest_plugin.scenario.test_security_groups.StatefulNetworkSecGroupTest.test_two_sec_groups[id-3d73ec1a-2ec6-45a9-b0f8-04a283d9d964] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_connectivity_between_vms_using_different_sec_groups[id-7ede9ab5-a615-46c5-9dea-cf2aa1ea43cb] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_default_sec_grp_scenarios[id-9e193e3f-56f2-4f4e-886c-988a147958ef] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_fragmented_traffic_is_accepted[id-14c4af2c-8077-4756-a6e3-6bebd642ed92] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_ip_prefix_negative[id-1ad469c4-0d8f-42ae-8ec3-46cc424565c4,negative] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_ip_prefix[id-07985496-58da-4c1f-a6ef-2fdd88128a81] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_multiple_ports_portrange_remote[id-4043ca0a-eabb-4198-be53-3d3051cc0804] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_multiple_ports_secgroup_inheritance[id-4f1eb6db-ae7f-4f26-b371-cbd8363f9b0b] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_overlapping_sec_grp_rules[id-bfe25138-ceac-4944-849a-b9b90aff100f] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_packets_of_any_connection_state_can_reach_dest[id-c3bb8073-97a2-4bea-a6fb-0a9d2e4df13f] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_protocol_number_rule[id-afae8654-a389-4887-b21d-7f07ec350177] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_reattach_sg_with_changed_mode[id-8d4753cc-cd7a-48a0-8ece-e11efce2af10] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_remote_group_and_remote_address_group[id-9fae530d-2711-4c61-a4a5-8efe6e58ab14] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_remote_group[id-fa1e93bf-67c5-4590-9962-38ee1f43a46a] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_remove_sec_grp_from_active_vm[id-e4340e47-39cd-49ed-967c-fc2c40b47c5a] +neutron_tempest_plugin.scenario.test_security_groups.StatelessNetworkSecGroupIPv4Test.test_two_sec_groups[id-b51cc0eb-8f9a-49e7-96ab-61cd31243b67] +neutron_tempest_plugin.scenario.test_security_groups.StatelessSecGroupDualStackDHCPv6StatelessTest.test_default_sec_grp_scenarios[id-c61c127c-e08f-4ddf-87a3-58b3c86e5476] +neutron_tempest_plugin.scenario.test_security_groups.StatelessSecGroupDualStackSlaacTest.test_default_sec_grp_scenarios[id-e7d64384-ea6a-40aa-b454-854f0990153c] +neutron_tempest_plugin.scenario.test_trunk.TrunkTest.test_parent_port_connectivity_after_trunk_deleted_lb[id-d61cbdf6-1896-491c-b4b4-871caf7fbffe] +neutron_tempest_plugin.scenario.test_trunk.TrunkTest.test_subport_connectivity_soft_reboot[id-a8a02c9b-b453-49b5-89a2-cce7da6680fb] +neutron_tempest_plugin.scenario.test_trunk.TrunkTest.test_subport_connectivity[id-a8a02c9b-b453-49b5-89a2-cce7da66bbcb] +neutron_tempest_plugin.scenario.test_trunk.TrunkTest.test_trunk_subport_lifecycle[id-bb13fe28-f152-4000-8131-37890a40c79e] +neutron_tempest_plugin.scenario.test_trunk.TrunkTest.test_trunk_vm_migration[id-ecd7de30-1c90-4280-b97c-1bed776d5d07,slow] +neutron_tempest_plugin.scenario.test_vlan_transparency.VlanTransparencyTest.test_vlan_transparent_allowed_address_pairs[id-2dd03b4f-9c20-4cda-8c6a-40fa453ec69a] +neutron_tempest_plugin.scenario.test_vlan_transparency.VlanTransparencyTest.test_vlan_transparent_port_sec_disabled[id-a2694e3a-6d4d-4a23-9fcc-c3ed3ef37b16] +neutron_tempest_plugin.sfc.tests.api.test_flowclassifier_extensions.FlowClassifierExtensionTestJSON.test_list_flowclassifier_with_logical_destination_port[id-b2ed2a37-fc64-4be5-819b-9cf2a13db70b] +neutron_tempest_plugin.sfc.tests.api.test_flowclassifier_extensions.FlowClassifierExtensionTestJSON.test_list_flowclassifier[id-1b84cf01-9c09-4ce7-bc72-b15e39076468] +neutron_tempest_plugin.sfc.tests.api.test_flowclassifier_extensions.FlowClassifierExtensionTestJSON.test_show_flowclassifier[id-3ff8c08e-26ff-4034-ae48-810ed213a998] +neutron_tempest_plugin.sfc.tests.api.test_flowclassifier_extensions.FlowClassifierExtensionTestJSON.test_update_flowclassifier[id-563564f7-7077-4f5e-8cdc-51f37ae5a2b9] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_chain_empty_flow_classifiers[id-d93d7ec3-f12e-4fad-b82b-759d358ff044] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_chain_flowclassifiers_symmetric[id-81f0faba-49ae-435a-8454-566c1e0a929e] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_chain_multi_flowclassifiers[id-0c5ac396-6027-4bd1-af21-79fda6df9b77] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_chain_multi_port_pair_groups[id-3f82c78f-e119-449f-bf6c-a964db45be3a] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_chain_port_pair_group_symmetric[id-144629ec-7538-4595-93ea-89e28ba50724] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_pair_different_ingress_egress[id-1a6067bf-b967-42a7-8b62-158a9ec185b4] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_pair_group_empty_port_pairs[id-4fff9a4a-a98a-42bd-b3f4-483b93e6f297] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_create_port_pair_group_multi_port_pairs[id-1a1c98a0-ff54-4647-a798-011e902825fa] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_list_port_chain[id-83cfceba-f9d9-41e2-b27f-f919d8ff83a9] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_list_port_pair_group[id-e7d432c4-a7b4-444b-88cc-f420c5c1c29e] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_list_port_pair[id-264cc4b8-aa17-4cea-88bf-26400e9751d9] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_show_port_chain[id-0433ca11-dbc9-448d-8433-0df252e3d0cd] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_show_port_pair_group[id-f12faa84-8dcb-4fbb-b03a-9ab05040a350] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_show_port_pair[id-83018ad7-3666-4396-bf3a-288a2b6a0e7c] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_update_port_chain[id-4ad641d3-823f-4b25-9438-68970593253d] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_update_port_pair_group[id-8991c2ef-71ba-4033-9037-5c8bf52a0c88] +neutron_tempest_plugin.sfc.tests.api.test_sfc_extensions.SfcExtensionTestJSON.test_update_port_pair[id-69d21fa4-bdd5-4142-b1cc-6578037f605a] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_multi_port_chain[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e22ab,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain_multi_flow_classifiers_symmetric[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21b1,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain_multi_flow_classifiers[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21a9,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain_multi_port_pair_groups_symmetric[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21b0,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain_multi_port_pair_groups[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21ab,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain_multi_port_pairs_symmetric[compute,id-f970f6b3-6541-47ac-a9ea-f869be1e21ad,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain_multi_port_pairs[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21aa,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain_symmetric[compute,id-35927961-1904-4a6b-9d08-ad819f1cf812,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_create_port_chain[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21a8,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_multi_port_chains_update_port_pair_group_replace_port_pairs[compute,id-69f8f427-a3a1-4f8f-814f-87e6faea9d54,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_chain_add_flow_classifiers[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21ac,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_chain_add_port_pair_groups[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21af,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_chain_remove_flow_classifiers[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21ad,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_chain_remove_port_pair_groups[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21bf,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_chain_replace_flow_classifiers[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21ae,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_chain_replace_port_pair_groups_flow_classifiers[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21bc,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_chain_replace_port_pair_groups[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21be,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_pair_group_add_port_pairs[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21bb,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_pair_group_remove_port_pairs[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21ba,network] +neutron_tempest_plugin.sfc.tests.scenario.test_sfc.TestSfc.test_update_port_pair_group_replace_port_pairs[compute,id-f970f6b3-6541-47ac-a9ea-f769be1e21b9,network] +neutron_tempest_plugin.tap_as_a_service.api.test_taas.TaaSExtensionTestJSON.test_create_and_update_tap_flow[id-bb4d5482-37fc-46b5-85a5-5867e9adbfae] +neutron_tempest_plugin.tap_as_a_service.api.test_taas.TaaSExtensionTestJSON.test_create_and_update_tap_service[id-687089b8-b045-496d-86bf-030b380039d1] +neutron_tempest_plugin.tap_as_a_service.api.test_taas.TaaSExtensionTestJSON.test_create_tap_service_and_flow_vlan_filter[id-897a0aaf-1b55-4ea8-9d9f-1bc0fd09cb60] +neutron_tempest_plugin.tap_as_a_service.api.test_taas.TaaSExtensionTestJSON.test_create_tap_service_and_flow[id-b993c14e-797a-4c91-b4da-8cb1a450aa2f] +neutron_tempest_plugin.tap_as_a_service.api.test_taas.TaaSExtensionTestJSON.test_delete_tap_resources_after_tf_port_delete[id-9ba4edfd-4002-4c44-b02b-6c4f71b40a92] +neutron_tempest_plugin.tap_as_a_service.api.test_taas.TaaSExtensionTestJSON.test_delete_tap_resources_after_ts_port_delete[id-d7a2115d-16b4-41cf-95a6-dcebc3682b24] +neutron_tempest_plugin.tap_as_a_service.scenario.test_taas.TestTaaS.test_tap_flow_data_mirroring[id-40903cbd-0e3c-464d-b311-dc77d3894e65,network,slow] +neutron_tempest_plugin.tap_as_a_service.scenario.test_traffic_impact.TestTaaSTrafficScenarios.test_taas_forwarded_traffic_negative[compute,id-6c54d9c5-075a-4a1f-bbe6-12c3c9abf1e2,network,slow] +neutron_tempest_plugin.tap_as_a_service.scenario.test_traffic_impact.TestTaaSTrafficScenarios.test_taas_forwarded_traffic_positive[compute,id-fcb15ca3-ef61-11e9-9792-f45c89c47e11,network,slow] +neutron_tempest_plugin.tap_as_a_service.scenario.test_traffic_impact.TestTaaSTrafficScenarios.test_taas_forwarded_traffic_provider_net_negative[compute,id-6c54d9c5-075a-4a1f-bbe6-12c3c9abf1e3,network,slow] +neutron_tempest_plugin.tap_as_a_service.scenario.test_traffic_impact.TestTaaSTrafficScenarios.test_taas_forwarded_traffic_provider_net_positive[compute,id-fcb15ca3-ef61-11e9-9792-f45c89c47e12,network,slow] +neutron_tempest_plugin.tap_as_a_service.scenario.test_traffic_impact.TestTaaSTrafficScenarios.test_taas_network_connectivity[compute,id-e3c52e91-7abf-4dfd-8687-f7c071cdd333,network,slow] +neutron_tempest_plugin.tap_as_a_service.scenario.test_traffic_impact.TestTaaSTrafficScenarios.test_taas_provider_network_connectivity[compute,id-ff414b7d-e81c-47f2-b6c8-53bc2f1e9b00,network,slow] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_admin_create_endpoint_group_for_tenant[id-69ee29e9-f72c-4d84-b6ab-c3f5576440fc,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_admin_create_ike_policy_for_tenant[id-8f33c292-558d-4fdb-b32c-ab1677e8bdc8,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_admin_create_ipsec_policy_for_tenant[id-74dcf2d3-a40e-4a6c-a25a-747d764bee81,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_admin_create_vpn_service_for_tenant[id-016b1861-fe55-4184-ba3c-e049ebbeb570,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_connection_with_cidr_and_endpoint_group[id-68e17a9b-5b33-4d3e-90f2-01a6728d2c26,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_connection_with_mix_ip_endpoint_group[id-427bbc1b-4040-42e4-b661-6395a0bd8762,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_connection_with_subnet_and_local_endpoint_group[id-92cd1221-3c7c-47ea-adad-8d8c2fc0f247,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_connection_with_subnet_and_remote_endpoint_group[id-eac56769-ec2d-4817-ba45-ca101d676bfc,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_delete_vpn_connection_with_ep_group[id-386f703e-445e-4208-abd2-df66066fd876,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_delete_vpn_connection_with_legacy_mode[id-92cfbb15-4299-4fe9-a4e1-f11167e3b057,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_update_delete_endpoint_group[id-e0cb784d-ebd9-4ad1-84e1-0cba915784d0,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_update_delete_ike_policy[id-d9ecb858-7136-4ef5-abba-325c84c57d78,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_update_delete_ipsec_policy[id-e3952941-9b9e-48fc-9f36-fc41707a16b4,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_update_delete_vpn_service[id-8780a28a-deb2-40b0-8433-66f37005c058,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpn_connection_missing_peer_cidr[id-e9977b38-9cd8-4aa6-8bd5-ba15f41c368c,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpn_connection_small_dpd[id-d9e1af94-1fbf-4183-b857-82328d4f4b97,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpn_connection_small_MTU[id-7678798a-fc20-46cd-ad78-b6b3d599de18,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpn_connection_with_missing_local_endpoint_group[id-b5dd8d1e-587b-4e26-84f0-dcb814c65e57,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpn_connection_with_missing_remote_endpoint_group[id-d101a6a7-67d3-4418-9d9e-65c4507a9148,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpn_connection_wrong_peer_cidr[id-ea6ae270-9ea4-4446-9022-b7ef7d986762,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpn_service_subnet_not_on_router[id-06a268fa-c53b-4cd1-9350-b83892e4a394,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpnservice_long_description[id-6e88b4fb-de5e-464e-97db-a04c68070589,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_create_vpnservice_long_name[id-95a4a55a-3a10-4f53-9e8a-45be359a245e,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_fail_create_endpoint_group_when_provide_subnet_id_with_cidr[id-670844d2-f7b0-466c-8fd7-7d2ad6e832ee,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_fail_create_endpoint_group_when_wrong_type[id-0ef1b2b0-9b4b-49f1-9473-cb4a0b625543,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_fail_create_endpoint_group_with_mixed_IP_version[id-d7516513-f2a2-42bd-8cea-baba73b93a22,negative,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_list_ike_policies[id-b6a665cf-d3df-417f-9477-bd0ef2c56c36,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_list_ipsec_policies[id-c3f7a459-3ccb-42b3-8fab-5ad7a5a791b1,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_list_vpn_connections[id-fdb5a215-0671-449b-91e4-dd58fb223947,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_list_vpn_services[id-2997641b-3f2f-4fdf-9af6-bfb9997ecd3b,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_show_endpoint_group[id-7a584d9c-0f64-4d39-a66c-261bfa46e369,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_show_ike_policy[id-5efd625b-41f2-4b80-90e0-81786f3e3309,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_show_ipsec_policy[id-86d68b8d-5935-46db-b096-235f70825678,smoke] +neutron_tempest_plugin.vpnaas.api.test_vpnaas.VPNaaSTestJSON.test_show_vpn_service[id-631d33ec-8d34-49e1-9e53-576959ea2c57,smoke] +neutron_tempest_plugin.vpnaas.scenario.test_vpnaas.Vpnaas4in4.test_vpnaas[id-aa932ab2-63aa-49cf-a2a0-8ae71ac2bc24] +neutron_tempest_plugin.vpnaas.scenario.test_vpnaas.Vpnaas4in6.test_vpnaas_4in6[id-2d5f18dc-6186-4deb-842b-051325bd0466] +neutron_tempest_plugin.vpnaas.scenario.test_vpnaas.Vpnaas6in4.test_vpnaas_6in4[id-10febf33-c5b7-48af-aa13-94b4fb585a55] +neutron_tempest_plugin.vpnaas.scenario.test_vpnaas.Vpnaas6in6.test_vpnaas_6in6[id-8b503ffc-aeb0-4938-8dba-73c7323e276d] diff --git a/.automation.conf/tempest/load-lists/octavia b/.automation.conf/tempest/load-lists/octavia new file mode 100644 index 000000000..65fd12663 --- /dev/null +++ b/.automation.conf/tempest/load-lists/octavia @@ -0,0 +1,67 @@ +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_round_robin_listener_CRUD[id-4a874014-b7d1-49a4-ac9a-2400b3434700] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_least_connections_listener_CRUD[id-ecdd65b0-cf8f-48ee-972b-2f09425472f1] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_source_ip_port_listener_CRUD[id-807a421e-5e99-4556-b0eb-512d39b25eac] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_source_ip_listener_CRUD[id-7830aba8-12ca-40d9-9d9b-a63f7a43b287] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_round_robin_listener_CRUD[id-2b888812-d916-44f0-b620-8d83dbb45975] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_source_ip_listener_CRUD[id-b2ae8604-7a4f-477c-9658-fac27734671a] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_least_connections_listener_CRUD[id-27a2ba7d-6147-46e4-886a-47c1ba63bf89] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_round_robin_listener_CRUD[id-dd913f74-c6a6-4998-9bed-095babb9cb47] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_least_connections_listener_CRUD[id-0681b2ac-8301-4e6c-bf29-b35244864af3] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_source_ip_port_listener_CRUD[id-6211f8ad-622d-404d-b199-8c2eb55ab340] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_source_ip_port_listener_CRUD[id-3f9a2de9-5012-437d-a907-a25e1f68ccfb] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_source_ip_listener_CRUD[id-0ad3fdee-e8c2-4c44-9690-b8a838fbc7a5] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_TCP_healthmonitor_CRUD[id-b4cbe603-0a14-4778-b38c-f330053c86b6] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_TCP_healthmonitor_CRUD[id-cacec696-10f4-430d-bc9e-2c5f235a3324] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_UDP_healthmonitor_CRUD[id-cc4abf84-361b-409b-b859-9a860d539deb] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_HTTP_healthmonitor_CRUD[id-a51e09aa-6e44-4c67-a9e4-df70d0e08f96] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_HTTP_healthmonitor_CRUD[id-20a2905f-2b53-4395-9a7f-1ded67ef4408] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_UDP_healthmonitor_CRUD[id-265d7359-f0a5-4083-92a8-07cb1787fe36] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_PING_healthmonitor_CRUD[id-de01b73d-dba0-4426-9e20-9be3a34cfc44] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_TLS_healthmonitor_CRUD[id-0de0e021-fd3c-4f7c-b959-67d758394fd2] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_PING_healthmonitor_CRUD[id-74611ffb-45f8-4cf5-a28c-7cc37879a27b] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_TLS_healthmonitor_CRUD[id-57714d4c-d584-4345-9ceb-becc3ae37b7f] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_TCP_healthmonitor_CRUD[id-141a121a-8918-4f9c-a070-eaf8ec29008d] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_HTTPS_healthmonitor_CRUD[id-fef9eabc-9d1e-4ad2-ae3e-05afc8c84c48] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_HTTP_healthmonitor_CRUD[id-4c2058f9-b8e2-4a5b-a2f3-3bd58a29f63b] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_PING_healthmonitor_CRUD[id-f9458ffd-5af7-402b-9c15-c061bf2eb9ba] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_HTTPS_healthmonitor_CRUD[id-08681eac-e907-4f71-8799-4b8fdf23914a] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_UDP_healthmonitor_CRUD[id-3c79750a-aba6-4838-acbe-bc937ccf2118] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_TLS_healthmonitor_CRUD[id-6becafb2-1e15-4977-bb29-b08f5728d028] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_HTTPS_healthmonitor_CRUD[id-8a8cc776-b68f-4761-9bf9-cae566cdc155] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_TCP_healthmonitor_CRUD[id-94be34b1-4dc6-492b-a777-0587626a785f] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_UDP_healthmonitor_CRUD[id-fe43ee90-093d-4175-837e-92f803958ef1] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_HTTPS_healthmonitor_CRUD[id-e188daac-6db9-4dc2-8ecb-b47932e1984a] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_TLS_healthmonitor_CRUD[id-de80d87a-5479-41c6-8c6b-518cc64ec62d] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_HTTP_healthmonitor_CRUD[id-d5e0d1b6-7cce-4592-abce-0ac6bee18818] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_PING_healthmonitor_CRUD[id-296a445c-5cc8-47a7-ae26-8d548f9712c3] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_healthmonitor_udp_traffic[id-80b86513-1a76-4e42-91c9-cb23c879e536] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_SIP_listener_with_allowed_cidrs[id-d198ddc5-1bcb-4310-a1b0-fa1a6328c4e9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_SI_listener_with_allowed_cidrs[id-fb5f35c1-08c9-43f7-8ed1-0395a3ef4735] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_insert_headers[id-d3a28e76-76bc-11eb-a7c3-74e5f9e2a801] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_port_tcp_traffic[id-60108f30-d870-487c-ab96-8d8a9b587b94] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_least_connections_tcp_traffic[id-e1056709-6a1a-4a15-80c2-5cbb8279f924] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_least_connections_udp_traffic[id-b5285410-507c-4629-90d4-6161540033d9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_SIP_listener_with_allowed_cidrs[id-bbb09dbb-2aad-4281-9383-4bb4ad420ee1] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_port_http_traffic[id-a446585b-5651-40ce-a4db-cb2ab4d37c03] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_http_traffic[id-881cc3e9-a011-4043-b0e3-a6185f736053] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_LC_listener_with_allowed_cidrs[id-04399db0-04f0-4cb5-bb27-a12bf18bfe08] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_SI_listener_with_allowed_cidrs[id-4f73bac5-2c98-45f9-8976-724c99e39979] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_healthmonitor_http_traffic[id-a16f8eb4-a77c-4b0e-8b1b-91c237039713] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_basic_http_traffic[id-6751135d-e15a-4e22-89f4-bfcc3408d424,slow,smoke] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_healthmonitor_tcp_traffic[id-22f00c34-343b-4aa9-90be-4567ecf85772] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_and_udp_traffic_on_same_port[id-2b05229c-0254-11eb-8610-74e5f9e2a801] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_basic_udp_traffic[id-1e459663-2315-4067-bb47-c8a72f4928f0] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tenant_flow_log[id-571dddd9-f5bd-404e-a799-9df7ac9e2fa9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_udp_traffic[id-be9e6ef2-7840-47d7-9315-cdb1e897b202] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_LC_listener_with_allowed_cidrs[id-7456b558-9add-4e0e-988e-06803f8047f7] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_LC_listener_with_allowed_cidrs[id-3d8d95b6-55e8-4bb9-b474-4ac35abaff22] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_basic_tcp_traffic[id-332a08e0-eff1-4c19-b46c-bf87148a6d84] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_RR_listener_with_allowed_cidrs[id-8bca1325-f894-494d-95c6-3ea4c3df6a0b] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_RR_listener_with_allowed_cidrs[id-13b0f2de-9934-457b-8be0-f1bffc6915a0] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_least_connections_http_traffic[id-a58063fb-b9e8-4cfc-8a8c-7b2e9e884e7a] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_l7policies_and_l7rules[id-3558186d-6dcd-4d9d-b7f7-adc190b66149] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_port_udp_traffic[id-a67dfa58-6953-4a0f-8a65-3f153b254c98] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_SIP_listener_with_allowed_cidrs[id-70290a9d-0065-42ad-bb46-884a535d2da2] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_RR_listener_with_allowed_cidrs[id-93675cc3-e765-464b-9563-e0848dc75330] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_SI_listener_with_allowed_cidrs[id-c0904c88-2479-42e2-974f-55041f30e6c5] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_tcp_traffic[id-4568db0e-4243-4191-a822-9d327a55fa64] diff --git a/.automation.conf/tempest/load-lists/octavia-full b/.automation.conf/tempest/load-lists/octavia-full new file mode 100644 index 000000000..20146ff6a --- /dev/null +++ b/.automation.conf/tempest/load-lists/octavia-full @@ -0,0 +1,820 @@ +octavia_tempest_plugin.tests.act_stdby_scenario.v2.test_active_standby_iptables.ActiveStandbyIptablesScenarioTest.test_active_standby_vrrp_failover[id-deab2b3f-62c7-4a05-9e92-aa45a04773fd] +octavia_tempest_plugin.tests.act_stdby_scenario.v2.test_active_standby.ActiveStandbyScenarioTest.test_active_standby_vrrp_failover[id-e591fa7a-0eee-485a-8ca0-5cf1a556bdf0] +octavia_tempest_plugin.tests.api.v2.test_amphora.AmphoraAPITest.test_admin_log[id-4e3c6fcb-5f83-4da1-8296-56f209eae30d] +octavia_tempest_plugin.tests.api.v2.test_amphora.AmphoraAPITest.test_amphora_failover[id-fb772680-b2ba-4fc3-989b-95ad8492ccaf] +octavia_tempest_plugin.tests.api.v2.test_amphora.AmphoraAPITest.test_amphora_list_and_show[id-a0e9ff99-2c4f-45d5-81c9-78d3107c236f] +octavia_tempest_plugin.tests.api.v2.test_amphora.AmphoraAPITest.test_amphora_update[id-b7fc231b-dcfa-47a5-99f3-ec5ddcc48f30] +octavia_tempest_plugin.tests.api.v2.test_availability_zone_capabilities.AvailabilityZoneCapabilitiesAPITest.test_availability_zone_capabilities_list[id-cb3e4c59-4114-420b-9837-2666d4d5fef4] +octavia_tempest_plugin.tests.api.v2.test_availability_zone_profile.AvailabilityZoneProfileAPITest.test_availability_zone_profile_create[id-e512b580-ef32-44c3-bbd2-efdc27ba2ea6] +octavia_tempest_plugin.tests.api.v2.test_availability_zone_profile.AvailabilityZoneProfileAPITest.test_availability_zone_profile_delete[id-371cee1d-3404-4744-b5c5-8a3d37aa8425] +octavia_tempest_plugin.tests.api.v2.test_availability_zone_profile.AvailabilityZoneProfileAPITest.test_availability_zone_profile_list[id-ef7d1c45-e312-46ce-8dcb-f2fe26295658] +octavia_tempest_plugin.tests.api.v2.test_availability_zone_profile.AvailabilityZoneProfileAPITest.test_availability_zone_profile_show[id-379d92dc-7f6d-4674-ae6f-b3aa2120c677] +octavia_tempest_plugin.tests.api.v2.test_availability_zone_profile.AvailabilityZoneProfileAPITest.test_availability_zone_profile_update[id-7121d4c0-f751-4b4e-a4c1-ab06c27a54a4] +octavia_tempest_plugin.tests.api.v2.test_availability_zone.AvailabilityZoneAPITest.test_availability_zone_create[id-3899ef15-37c3-48a3-807f-8bb10bd295f0] +octavia_tempest_plugin.tests.api.v2.test_availability_zone.AvailabilityZoneAPITest.test_availability_zone_delete[id-11585b33-2689-4693-be3b-26b210bb7fc5] +octavia_tempest_plugin.tests.api.v2.test_availability_zone.AvailabilityZoneAPITest.test_availability_zone_list[id-bba84c0c-2832-4c4c-90ff-d28acfe4ae36] +octavia_tempest_plugin.tests.api.v2.test_availability_zone.AvailabilityZoneAPITest.test_availability_zone_show[id-4fa77f96-ba75-4255-bef8-6710cd7cb762] +octavia_tempest_plugin.tests.api.v2.test_availability_zone.AvailabilityZoneAPITest.test_availability_zone_update[id-9c466b9f-b70a-456d-9172-eb79b7820c7f] +octavia_tempest_plugin.tests.api.v2.test_flavor_capabilities.FlavorCapabilitiesAPITest.test_flavor_capabilities_list[id-df837ee3-ca4b-4a4d-a7a3-27fa57cf3a33] +octavia_tempest_plugin.tests.api.v2.test_flavor_profile.FlavorProfileAPITest.test_flavor_profile_create[id-d0e3a08e-d58a-4460-83ed-34307ca04cde] +octavia_tempest_plugin.tests.api.v2.test_flavor_profile.FlavorProfileAPITest.test_flavor_profile_delete[id-4c2eaacf-c2c8-422a-b7dc-a30ceba6bcd4] +octavia_tempest_plugin.tests.api.v2.test_flavor_profile.FlavorProfileAPITest.test_flavor_profile_list[id-c4e17fdf-849a-4132-93ae-dfca21ce4444] +octavia_tempest_plugin.tests.api.v2.test_flavor_profile.FlavorProfileAPITest.test_flavor_profile_show[id-a2c2ff9a-fce1-42fd-8cfd-56dea31610f6] +octavia_tempest_plugin.tests.api.v2.test_flavor_profile.FlavorProfileAPITest.test_flavor_profile_update[id-32a2e285-8dfc-485f-a450-a4d450d3c3ec] +octavia_tempest_plugin.tests.api.v2.test_flavor.FlavorAPITest.test_flavor_create[id-7e8f39ce-53e0-4364-8778-6da9b9a59e5a] +octavia_tempest_plugin.tests.api.v2.test_flavor.FlavorAPITest.test_flavor_delete[id-dfe9173a-26f3-4bba-9e69-d2b817ff2b86] +octavia_tempest_plugin.tests.api.v2.test_flavor.FlavorAPITest.test_flavor_list[id-3ef040ee-fe7e-457b-a56f-8b152f7afa3b] +octavia_tempest_plugin.tests.api.v2.test_flavor.FlavorAPITest.test_flavor_show[id-7492a862-4011-4924-8e81-70763f479cf8] +octavia_tempest_plugin.tests.api.v2.test_flavor.FlavorAPITest.test_flavor_update[id-3d9e2820-a68e-4db9-bf94-53cbcff2dc15] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTP_healthmonitor_create[id-bc3fc817-3368-4e1e-bb6d-52c4de3fb10c] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTP_healthmonitor_delete[id-76b3d116-0190-4de8-a58e-8e450a46a621] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTP_healthmonitor_list[id-9e4b1298-b6a0-46c7-b8e4-afcd31f904d3] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTP_healthmonitor_show[id-358afb0b-6259-46be-a0b3-b11e5e202624] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTP_healthmonitor_update[id-2417164b-ec03-4488-afd2-60b096dc0077] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTPS_healthmonitor_create[id-18fcd6b8-9fc3-4858-83f6-d8800052b655] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTPS_healthmonitor_delete[id-8e277e7f-49ea-4fcf-98e6-12566cc33846] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTPS_healthmonitor_list[id-7488b1e1-12b5-4a42-9d78-9e08060ec7b1] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTPS_healthmonitor_show[id-c2476eea-8ae1-40ed-be55-0125f9399bd4] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_HTTPS_healthmonitor_update[id-5d88aa1e-2db9-43f8-bb9b-4673c2060835] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_PING_healthmonitor_create[id-5b0446f2-374e-4e74-9865-72e16a19c587] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_PING_healthmonitor_delete[id-c6646a25-b46d-4541-82de-75ee2beef052] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_PING_healthmonitor_list[id-1f93483c-29ae-4ac1-a588-9ce9bd837232] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_PING_healthmonitor_show[id-1489251c-0704-4735-bf62-801b5277c5c9] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_PING_healthmonitor_update[id-61908142-9768-44aa-9a4b-b3904560a0dc] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TCP_healthmonitor_create[id-46c4e159-84d2-4876-9298-ac85561d3bd0] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TCP_healthmonitor_delete[id-5cfacffe-63f2-4aa3-856a-9fa3dafa2d33] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TCP_healthmonitor_list[id-49ccd27c-3a4c-41e5-bcb0-d4f03ecc3e79] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TCP_healthmonitor_show[id-c761f90e-7b9c-400b-b540-e7c14f65d0a8] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TCP_healthmonitor_update[id-649bdfa3-1009-4f88-bc92-c3e3141c493e] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TLS_healthmonitor_create[id-a15f7319-2d3b-4ec8-9d70-e77a55045145] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TLS_healthmonitor_delete[id-a4055e48-7740-4ff5-b6e2-9e69b1d40dce] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TLS_healthmonitor_list[id-60a254dc-4764-45f2-a183-8102063462e0] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TLS_healthmonitor_show[id-8ed512d7-9fd7-4932-bf5f-090498b384bb] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_TLS_healthmonitor_update[id-365fffd3-0817-4907-aab1-7da60736ba60] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_UDP_healthmonitor_create[id-f609b2c2-391d-4bc9-9793-9a4bc30ab00b] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_UDP_healthmonitor_delete[id-369bd443-54ec-4071-a279-5ac1ed38c52d] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_UDP_healthmonitor_list[id-6637d37d-76aa-455a-ba73-8f1a12edcedd] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_UDP_healthmonitor_show[id-a3a9be7e-83d6-42cb-b603-f14a464b8268] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_LC_UDP_healthmonitor_update[id-df29f696-a321-4626-acb2-6f66105e1661] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTP_healthmonitor_create[id-4f20473f-ab02-4426-8d15-cf34b3c72558] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTP_healthmonitor_delete[id-a7bab4ac-340c-4776-ab9d-9fcb66869432] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTP_healthmonitor_list[id-c9a9f20c-3680-4ae8-b657-33c687258fea] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTP_healthmonitor_show[id-284e8d3b-7b2d-4697-9e41-580b3423c0b4] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTP_healthmonitor_update[id-fa584b2c-f179-4c4e-ad2e-ff51fd1c5973] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTPS_healthmonitor_create[id-abfff805-5491-4aef-9952-45b553cbebbb] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTPS_healthmonitor_delete[id-2991664a-9138-4c10-8e30-2cb6a82bb5b4] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTPS_healthmonitor_list[id-93c0d03a-eb32-457b-a5af-96c03a891c06] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTPS_healthmonitor_show[id-eeb4967b-ce46-4717-a750-3e740223a804] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_HTTPS_healthmonitor_update[id-b927836a-2770-46ff-92de-3031c5240da6] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_PING_healthmonitor_create[id-ac64228e-dc4c-4116-b610-5783a85a87f1] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_PING_healthmonitor_delete[id-31f7c2c2-9174-4538-8dce-35128bc47ce7] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_PING_healthmonitor_list[id-2ef2944f-dd56-40a5-9100-4e1b86c623af] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_PING_healthmonitor_show[id-799ada1b-c082-42c5-b6ea-477f10fc88ce] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_PING_healthmonitor_update[id-cdd559b1-5c7b-492f-9c8d-c1da6e8d7b3b] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TCP_healthmonitor_create[id-e3ac7193-1dc9-41df-a4e2-7d40ca70a678] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TCP_healthmonitor_delete[id-56e0cd28-3e74-498c-b55b-21078a758d1f] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TCP_healthmonitor_list[id-5912534f-20b3-45b7-9907-9247bf05cd13] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TCP_healthmonitor_show[id-d2eae7f4-43b4-4696-93ed-a30f95c978fe] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TCP_healthmonitor_update[id-1b9c310e-cffe-4f6a-b1af-021f751fc2a9] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TLS_healthmonitor_create[id-54952a9a-b3ef-4b36-a586-9adcd63dfc49] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TLS_healthmonitor_delete[id-e22a02e2-411a-46d5-9a3a-20ff37cbc835] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TLS_healthmonitor_list[id-5d169fe7-16f3-4f70-8b1e-72aeeec4fd61] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TLS_healthmonitor_show[id-5ce6eecc-d425-47cd-809f-aab5c56e1a9d] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_TLS_healthmonitor_update[id-4b3c9a67-f884-43a3-8f42-bac68be7060b] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_UDP_healthmonitor_create[id-9af20b3c-fc42-4365-a4e9-cecbdddf90c0] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_UDP_healthmonitor_delete[id-9221a59f-3f0f-41e8-b65c-cdbcca1a2eca] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_UDP_healthmonitor_list[id-3f81050e-4218-46fa-8d85-09807b8cdded] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_UDP_healthmonitor_show[id-a6a46e6a-a063-46bf-972c-86d0305fb766] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_RR_UDP_healthmonitor_update[id-805e2976-962a-4bb0-a9cc-97270a42c376] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTP_healthmonitor_create[id-35b076a4-dfb1-4557-9eac-a33982f73856] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTP_healthmonitor_delete[id-717e447f-d5c8-485a-923b-da83e560273b] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTP_healthmonitor_list[id-2df98839-7b2a-46c4-9da7-34e3d1c33851] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTP_healthmonitor_show[id-54d693ba-1ba4-4388-b020-c29dc3184522] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTP_healthmonitor_update[id-654e4ce3-b579-4595-b1a1-6762f64b2408] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTPS_healthmonitor_create[id-965804cb-d6a1-4fdd-99dc-948829e0c046] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTPS_healthmonitor_delete[id-2e177a95-4ec8-4847-bd35-706b6452406a] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTPS_healthmonitor_list[id-1ce28897-917c-4e7a-89bb-cc2aabd67037] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTPS_healthmonitor_show[id-2638718e-b345-4868-b527-9bed575e27d6] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_HTTPS_healthmonitor_update[id-879ef60a-4621-45aa-a520-b57da3b1fddc] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_PING_healthmonitor_create[id-eb86eab1-4cdf-40fc-b633-679d7fb64806] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_PING_healthmonitor_delete[id-17127700-dc5f-4546-a6e6-c0b851704836] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_PING_healthmonitor_list[id-e8418eaa-73a5-4d56-8ca5-314dd2141dc9] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_PING_healthmonitor_show[id-124a5ca1-5eae-4ed0-8528-7a499e9ad7a2] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_PING_healthmonitor_update[id-66e83157-53c3-4eac-a7f0-e3dc4f51de06] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TCP_healthmonitor_create[id-6bdab976-b6ad-4c83-87ab-0d184e80eb2c] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TCP_healthmonitor_delete[id-0b1699c2-ff3e-47b6-a1ad-7128465d1233] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TCP_healthmonitor_list[id-25203919-a039-43a4-84db-15279cbd2ec7] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TCP_healthmonitor_show[id-098ae671-1791-455a-a5a8-ada8c592a2dd] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TCP_healthmonitor_update[id-ce475c4c-d01a-4cde-be71-555c84f2b8da] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TLS_healthmonitor_create[id-4f8111dd-4f03-4740-ae4b-b13b731f45a0] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TLS_healthmonitor_delete[id-4ad99064-9015-40b3-8d5b-6cc99e2cc8b9] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TLS_healthmonitor_list[id-4f84c05e-d4a7-4998-98cd-bc74024309f4] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TLS_healthmonitor_show[id-eef8f780-b557-447f-9f61-b1f3e6daec77] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_TLS_healthmonitor_update[id-b292478f-5c26-462e-b222-103be3b115d3] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_UDP_healthmonitor_create[id-d98365ca-56ba-450d-955f-d1c06c329960] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_UDP_healthmonitor_delete[id-a17825ff-e774-4230-9c53-b53bfc355d61] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_UDP_healthmonitor_list[id-4cb10c86-a875-4a9e-be8f-c0afc8aa5633] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_UDP_healthmonitor_show[id-9b6d9c45-4696-4f6a-8816-594b03e3ee5b] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SI_UDP_healthmonitor_update[id-49ccc552-752b-4f84-9900-65908cb13add] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTP_healthmonitor_create[id-acab47f5-3006-4e84-a55f-e9dfe33113d2] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTP_healthmonitor_delete[id-142022cc-9be3-4695-9acf-a7576e4b3268] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTP_healthmonitor_list[id-45bdd757-3132-4ede-8584-c46bc2f8f19e] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTP_healthmonitor_show[id-5653ea25-c7e0-4c72-8b2a-19dd97dd5a69] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTP_healthmonitor_update[id-d141d8d1-fa12-49cb-9d6d-413998aa2dc5] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTPS_healthmonitor_create[id-ca42a541-3280-4496-8604-9ce64e1992d6] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTPS_healthmonitor_delete[id-dfd994b2-511b-433f-95f7-0786f1857090] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTPS_healthmonitor_list[id-300db789-b231-45be-863d-f4d6116660d3] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTPS_healthmonitor_show[id-fff4472d-b4de-4b8b-9748-476ffc7c8e13] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_HTTPS_healthmonitor_update[id-9a1bbbbb-c640-48cb-bd1a-e3d3fd2602af] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_PING_healthmonitor_create[id-78c8e5fc-3ba0-44d0-ac4a-93a90fb59c3f] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_PING_healthmonitor_delete[id-0d08ffc0-6e6a-470d-abed-5c101a828401] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_PING_healthmonitor_list[id-a1d534ba-ada9-4d7c-8e17-6e520a27c110] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_PING_healthmonitor_show[id-1af98ebe-3f3c-4e5f-8f72-ecbd9b25c69f] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_PING_healthmonitor_update[id-3f573e31-49b9-45d0-bb00-8483f48ae422] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TCP_healthmonitor_create[id-d5b50306-a3bd-4293-96ed-17a2897d57cc] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TCP_healthmonitor_delete[id-ccf4a10c-2e72-4dbf-bc2f-134156eac3e2] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TCP_healthmonitor_list[id-12c29b89-bbbc-46b0-89c5-beb42fc52181] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TCP_healthmonitor_show[id-40169a7d-84ac-4362-b8d7-64b9b807ce7e] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TCP_healthmonitor_update[id-1a6922fd-9e8c-4836-9a6a-087f09249a49] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TLS_healthmonitor_create[id-204556df-185e-4305-b1b7-e8d345d645e4] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TLS_healthmonitor_delete[id-aaacc11e-98cd-4322-a7db-7c720eafd2b2] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TLS_healthmonitor_list[id-a3b01b12-f4cb-4b2a-9f62-af24834ce19b] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TLS_healthmonitor_show[id-15146c2e-e1c1-48ac-a7d8-3a1b4de590b2] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_TLS_healthmonitor_update[id-e9468e81-dbff-4e88-8d4b-e2a54835c2d8] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_UDP_healthmonitor_create[id-2de53a00-c631-4985-abc1-89070ac2515f] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_UDP_healthmonitor_delete[id-559eccf4-eb7c-4d23-9dc8-741cc1601fc7] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_UDP_healthmonitor_list[id-571e139b-08ae-4e8c-b25c-0e0bb9d198af] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_UDP_healthmonitor_show[id-008042da-3734-4bbb-b8b2-f4ad9e2dab21] +octavia_tempest_plugin.tests.api.v2.test_healthmonitor.HealthMonitorAPITest.test_SIP_UDP_healthmonitor_update[id-64379cb2-b789-4fe6-ae3a-e62b907c6365] +octavia_tempest_plugin.tests.api.v2.test_l7policy.L7PolicyAPITest.test_l7policy_create_redirect_pool[id-eba4ddc2-758b-4dd5-bd28-c1b41d6575ca] +octavia_tempest_plugin.tests.api.v2.test_l7policy.L7PolicyAPITest.test_l7policy_create_redirect_url[id-2b529135-71bc-46f7-912f-74d238d67190] +octavia_tempest_plugin.tests.api.v2.test_l7policy.L7PolicyAPITest.test_l7policy_create_reject[id-aa9b0d50-0d16-4365-85eb-846b17eb8398] +octavia_tempest_plugin.tests.api.v2.test_l7policy.L7PolicyAPITest.test_l7policy_delete[id-7925eb4b-94b6-4c28-98c2-fd0b4f0976cc] +octavia_tempest_plugin.tests.api.v2.test_l7policy.L7PolicyAPITest.test_l7policy_list[id-42fa14ba-caf1-465e-ab36-27e7501f95ef] +octavia_tempest_plugin.tests.api.v2.test_l7policy.L7PolicyAPITest.test_l7policy_show[id-baaa8104-a037-4976-b908-82a0b3e08129] +octavia_tempest_plugin.tests.api.v2.test_l7policy.L7PolicyAPITest.test_l7policy_update[id-08f73b22-550b-4e5a-b3d6-2ec03251ca13] +octavia_tempest_plugin.tests.api.v2.test_l7rule.L7RuleAPITest.test_l7rule_create[id-55ac1337-189d-40a6-b614-47d7a8e991f6] +octavia_tempest_plugin.tests.api.v2.test_l7rule.L7RuleAPITest.test_l7rule_delete[id-8e15d68d-70e7-4cf3-82bc-9604384654a0] +octavia_tempest_plugin.tests.api.v2.test_l7rule.L7RuleAPITest.test_l7rule_list[id-69095254-f106-4fb6-9f54-7a78cc14fb51] +octavia_tempest_plugin.tests.api.v2.test_l7rule.L7RuleAPITest.test_l7rule_show[id-b80b34c3-09fc-467b-8027-7350adb17070] +octavia_tempest_plugin.tests.api.v2.test_l7rule.L7RuleAPITest.test_l7rule_update[id-f8cee23b-89b6-4f3a-a842-1463daf42cf7] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_listener_create[id-88d0ec83-7b08-48d9-96e2-0df1d2f8cd98] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_listener_delete[id-16f11c82-f069-4592-8954-81b35a98e3b7] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_listener_list[id-78ba6eb0-178c-477e-9156-b6775ca7b271] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_listener_show_stats[id-6f14a6c1-945e-43bc-8215-410c8a5edb25] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_listener_show[id-6e299eae-6907-4dfc-89c2-e57709d25d3d] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_listener_update[id-aaae0298-5778-4c7e-a27a-01549a71b319] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_udp_sctp_https_listener_create_on_same_port[id-930338b8-3029-48a6-89b2-8b062060fe61] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_udp_sctp_tcp_listener_create_on_same_port[id-cceac303-4db5-4d5a-9f6e-ff33780a5f29] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_http_udp_sctp_terminated_https_listener_create_on_same_port[id-128dabd0-3a9b-4c11-9ef5-8d189a290f17] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_https_listener_create[id-2cc89237-fc6b-434d-b38e-b3309823e71f] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_https_listener_delete[id-769526a0-df71-47cd-996e-46484de32223] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_https_listener_list[id-61b7c643-f5fa-4471-8f9e-2e0ccdaf5ac7] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_https_listener_show_stats[id-f8a43c27-f0a0-496d-a287-1958f337ac04] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_https_listener_show[id-aa838646-435f-4a20-8442-519a7a138e7e] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_https_listener_update[id-9679b061-2b2c-469f-abd9-26ed140ef001] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_prometheus_listener_create[id-1a6ba0d0-f309-4088-a686-dda0e9ab7e43] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_prometheus_listener_delete[id-322a6372-6b56-4a3c-87e3-dd82074bc83e] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_prometheus_listener_list[id-5473e071-8277-4ac5-9277-01ecaf46e274] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_prometheus_listener_show[id-b851b754-4333-4115-9063-a9fce44c2e46] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_prometheus_listener_update[id-cbba6bf8-9184-4da5-95e9-5efe1f89ddf0] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_sctp_listener_create[id-d6d36c32-27ff-4977-9d21-fd71a14e3b20] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_sctp_listener_delete[id-0de6f1ad-58ae-4b31-86b6-b440fce70244] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_sctp_listener_list[id-0abc3998-aacd-4edd-88f5-c5c35557646f] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_sctp_listener_show_stats[id-7f6d3906-529c-4b99-8376-b836059df220] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_sctp_listener_show[id-10992529-1d0a-47a3-855c-3dbcd868db4e] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_sctp_listener_update[id-c590b485-4e08-4e49-b384-2282b3f6f1b9] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_listener_create[id-45580065-5653-436b-aaff-dc465fa0a542] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_listener_delete[id-f5ca019d-2b33-48f9-9c2d-2ec169b423ca] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_listener_list[id-1cd476e2-7788-415e-bcaf-c377acfc9794] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_listener_show_stats[id-8a999856-f448-498c-b891-21af449b5208] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_listener_show[id-1fcbbee2-b697-4890-b6bf-d308ac1c94cd] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_listener_update[id-8d933121-db03-4ccc-8b77-4e879064a9ba] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_udp_sctp_http_listener_create_on_same_port[id-01a21892-008a-4327-b4fd-fbf194ecb1a5] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_udp_sctp_https_listener_create_on_same_port[id-5da764a4-c03a-46ed-848b-98b9d9fa9089] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_tcp_udp_sctp_terminated_https_listener_create_on_same_port[id-21da2598-c79e-4548-8fe0-b47749027010] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_terminated_https_listener_create[id-df9861c5-4a2a-4122-8d8f-5556156e343e] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_terminated_https_listener_delete[id-ef357dcc-c9a0-40fe-a15c-b368f15d7187] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_terminated_https_listener_list[id-aed69f58-fe69-401d-bf07-37b0d6d8437f] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_terminated_https_listener_show_stats[id-c39c996f-9633-4d81-a5f1-e94643f0c650] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_terminated_https_listener_show[id-2c2e7146-0efc-44b6-8401-f1c69c2422fe] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_terminated_https_listener_update[id-2ae08e10-fbf8-46d8-a073-15f90454d718] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_udp_listener_create[id-7b53f336-47bc-45ae-bbd7-4342ef0673fc] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_udp_listener_delete[id-86bd9717-e3e9-41e3-86c4-888c64455926] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_udp_listener_list[id-c08fb77e-b317-4d6f-b430-91f5b27ebac6] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_udp_listener_show_stats[id-a4c1f199-923b-41e4-a134-c91e590e20c4] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_udp_listener_show[id-1dea3a6b-c95b-4e91-b591-1aa9cbcd0d1d] +octavia_tempest_plugin.tests.api.v2.test_listener.ListenerAPITest.test_udp_listener_update[id-fd02dbfd-39ce-41c2-b181-54fc7ad91707] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_delete_cascade[id-abd784e3-485f-442a-85da-d91365c6b5dd] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_delete[id-643ef031-c800-45f2-b229-3c8f8b37c829] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_failover[id-fc2e07a6-9776-4559-90c9-141170d4c397] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_ipv4_create[id-61c6343c-a5d2-4b9f-8c7d-34ea83f0596b] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_ipv6_create[id-fc9996de-4f55-4fc4-b8ef-a4b9170c7078] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_list[id-6546ef3c-c0e2-46af-b892-f795f4d01119] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_show_stats[id-105afcba-4dd6-46d6-8fa4-bd7330aa1259] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_show_status[id-60acc1b0-fa46-41f8-b526-c81ae2f42c30] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_show[id-826ae612-8717-4c64-a8a7-cb9570a85870] +octavia_tempest_plugin.tests.api.v2.test_load_balancer.LoadBalancerAPITest.test_load_balancer_update[id-b75a4d15-49d2-4149-a745-635eed1aacc3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTP_LC_backup_member_list[id-b982188a-d55f-438a-a1b2-224f0ec8ff12] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTP_LC_member_list[id-fcc5c6cd-d1c2-4a49-8d26-2268608e59a6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTP_RR_member_list[id-9ce7ad78-915b-42ce-b0d8-44d88a929f3d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTP_SI_member_list[id-0d142f26-c9e4-45bf-8cd7-1f5659301047] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTP_SIP_member_list[id-2f9d0974-2e55-49c1-b83a-8bdd6dfdb46c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTPS_LC_member_list[id-33450ca2-db09-451a-bd46-6f260bf520f5] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTPS_RR_member_list[id-8f6362a1-d98b-4696-b88d-41e1eb4a9f70] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTPS_SI_member_list[id-a2be8c21-c7b1-4c1d-ab39-43042bf75a19] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_HTTPS_SIP_member_list[id-d31c5b8b-7ec1-4e78-a821-30e9a1e05139] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_LC_alt_monitor_member_create[id-0684575a-0970-4fa8-8006-10c2b39c5f2b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_LC_backup_member_create[id-c1e029b0-b6d6-4fa6-8ccb-5c3f3aa293b0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_LC_member_create[id-10641ec2-981e-4092-a0d0-89a434506eef] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_RR_alt_monitor_member_create[id-0623aa1f-753d-44e7-afa1-017d274eace7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_RR_member_create[id-96b709fa-dca3-4780-8de7-fb168d455d76] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_SI_alt_monitor_member_create[id-0d9a8b32-0c13-49ea-8dd3-a124ec4ac6f9] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_SI_member_create[id-57c52d0c-0a62-4988-a02e-2f9f8b440d08] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_SIP_alt_monitor_member_create[id-a2dbf216-a974-45e1-822d-859f76c89ed6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTP_SIP_member_create[id-926eeed9-ecf4-4d22-9417-ef7a7e0a7788] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_LC_alt_monitor_member_create[id-dce70b40-502b-4b1c-8592-180817324ea0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_LC_member_create[id-46555ea6-12a8-4961-b105-bffdead7abcd] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_RR_alt_monitor_member_create[id-5354ac6c-653d-43ce-8096-1f9de961de73] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_RR_member_create[id-e5c8503a-4bc5-43ad-b0da-3e5c1ef719f7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_SI_alt_monitor_member_create[id-df427d31-9843-4840-9137-6b88c633d329] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_SI_member_create[id-e64c28f6-09f5-4fd8-a59e-bcf90975581a] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_SIP_alt_monitor_member_create[id-55da07bc-bf2c-4924-aba3-a03456843e14] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_HTTPS_SIP_member_create[id-6773a8bd-1c51-4040-84ba-1aa2b6c4280d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_LC_alt_monitor_member_create[id-c1a5e297-f38e-4fc8-92a4-4177a37c4794] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_LC_member_create[id-dfe24159-96a4-4496-888e-e74acd9d390d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_RR_alt_monitor_member_create[id-b2c8632b-f833-4844-9af3-ffee655be6bf] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_RR_member_create[id-e565de98-88e2-4529-9730-a66073e31480] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_SI_alt_monitor_member_create[id-5c32e2fd-9148-466c-b788-e11d7a48483b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_SI_member_create[id-76435fb2-dcb3-4be2-ada9-2dbc375c100b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_SIP_alt_monitor_member_create[id-ab462d7c-069d-4b55-b6a7-dd199bde65b3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_PROXY_SIP_member_create[id-b1f6f779-2535-4e47-add2-24561545ba59] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_LC_alt_monitor_member_create[id-2716d05d-6b04-405e-bda9-e79c778eb6dd] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_LC_member_create[id-14ee6e7d-c434-4e2e-947b-1a37d5ffa3bd] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_RR_alt_monitor_member_create[id-9306b599-c8e2-4ce9-b789-9e32d42406c4] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_RR_member_create[id-9bbfec96-a7e5-414d-96d1-710e468b8700] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_SI_alt_monitor_member_create[id-62db0223-1e44-4d6c-8499-9f72c86d30e3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_SI_member_create[id-2da54523-cefc-4a44-ab07-c33ffe891bf0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_SIP_alt_monitor_member_create[id-73673efc-5b70-4394-b831-1d59fe283e7d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_TCP_SIP_member_create[id-0a08da1e-84f5-4068-84ec-1312b6b8bee3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_LC_alt_monitor_member_create[id-fb9d0708-e320-45d7-be30-f6e7ea45c644] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_LC_member_create[id-5d36d4a4-3b9c-4d54-af61-5f80080bb040] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_RR_alt_monitor_member_create[id-ccced84a-994d-4d30-a07a-30fa83e4dde2] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_RR_member_create[id-bc0802dd-633f-42d4-8c6a-b4c70af29870] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_SI_alt_monitor_member_create[id-9a298318-89a5-416f-b027-af5eda94f813] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_SI_member_create[id-b0455c5e-3702-41d7-8069-6ce55563767c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_SIP_alt_monitor_member_create[id-caf68a97-1911-466c-b392-50b946e2395c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv4_UDP_SIP_member_create[id-bb750dc5-73a8-4722-bf3b-cdafaefe7914] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_LC_alt_monitor_member_create[id-cd894c4f-2256-405f-aa6e-2f77973c749a] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_LC_backup_member_create[id-b1994c5d-74b8-44be-b9e5-5e18e9219b61] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_LC_member_create[id-3bbb8554-f757-4673-92e3-8593eef83f19] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_RR_alt_monitor_member_create[id-141944cc-5e2c-4e83-88f8-f61a6797c9b7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_RR_member_create[id-883db951-adb1-4e05-8369-99f38fde6b3c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_SI_alt_monitor_member_create[id-6759674b-dca0-4a48-b166-3f87dc1cc727] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_SI_member_create[id-683d9ef4-6af3-48e2-aba4-9f404d493467] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_SIP_alt_monitor_member_create[id-d816d324-2434-4812-9b3e-a3f0d4949008] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTP_SIP_member_create[id-d4cfe315-b6d6-4940-8ff6-5f5252028eec] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_LC_alt_monitor_member_create[id-6056724b-d046-497a-ae31-c02af67d4fbb] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_LC_member_create[id-354e44d3-db08-4ba9-8e3e-8c3210542a86] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_RR_alt_monitor_member_create[id-364ba4b9-825a-4f92-9bf2-8d76bcba0288] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_RR_member_create[id-70ba1deb-d644-437f-af80-4299461b20af] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_SI_alt_monitor_member_create[id-3d308996-3522-4c91-9bfd-48fedc2ed2f2] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_SI_member_create[id-dc26ce07-d580-4a55-b7cd-1b4f09c13572] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_SIP_alt_monitor_member_create[id-05684ab0-dff3-41aa-8b42-7f95fd6aa4ab] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_HTTPS_SIP_member_create[id-ed42872c-1ffc-4210-9f69-5f7eb8ec732f] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_LC_alt_monitor_member_create[id-0f8b924e-dd0b-44f9-92b6-8f3dfb0a720c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_LC_member_create[id-d1efbfab-b674-4b78-8014-7ecf7ab464ac] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_RR_alt_monitor_member_create[id-06facdb2-5b7e-4e8b-810d-8f829c619a6d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_RR_member_create[id-4ec5a74b-06bd-4005-8fcc-25d1bced4807] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_SI_alt_monitor_member_create[id-56944f91-bf4b-4e9a-9b05-6207e8184c75] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_SI_member_create[id-e8f4eb38-8e8b-485a-b70a-b1679ad58b66] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_SIP_alt_monitor_member_create[id-ab87132f-5a0e-40a1-9498-9883780d31a9] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_PROXY_SIP_member_create[id-227d7f40-a224-4e67-8844-2d28abc5171e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_LC_alt_monitor_member_create[id-699f1c0d-65ae-40d7-9abd-2cef0a1560b9] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_LC_member_create[id-595255f9-f595-43e7-a398-80dd76719aa8] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_RR_alt_monitor_member_create[id-22600457-ffe5-44a0-90b0-da4f48051023] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_RR_member_create[id-57e6f861-5a55-43a5-9cae-a966bd2a48eb] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_SI_alt_monitor_member_create[id-fc0e77b1-e115-4ec7-80e3-c00d79932549] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_SI_member_create[id-66ddafd2-ace3-43ea-b78b-78b6b0a4d9eb] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_SIP_alt_monitor_member_create[id-b8394de8-a898-4cab-aa0c-f3168d702ee0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_TCP_SIP_member_create[id-7a86e00b-90bf-4fd3-8636-ae7264929106] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_LC_alt_monitor_member_create[id-0c2c2d5f-9602-4602-82e7-94a1393c295d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_LC_member_create[id-d41f5b46-ba06-42bf-a320-0fda106a7543] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_RR_alt_monitor_member_create[id-4cddcf8a-566e-4a5a-bf81-99026b17f676] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_RR_member_create[id-26547c9a-6bbc-429a-9436-e94f2930b9e1] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_SI_alt_monitor_member_create[id-ccfc1283-9e8e-4aa5-a5d3-1d18d57bec65] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_SI_member_create[id-b62c8562-fdbb-4989-a5ae-d9e1c1b76cd5] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_SIP_alt_monitor_member_create[id-5abdfbcd-d1cd-4e6a-b98f-79afea442ad8] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_ipv6_UDP_SIP_member_create[id-63f558b4-d2f8-4e4c-828b-3651e50844b7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_PROXY_LC_member_list[id-c17241d4-5cff-41e2-9742-047647d61546] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_PROXY_RR_member_list[id-ce4109f8-3bad-4965-95ec-7170519e4a3f] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_PROXY_SI_member_list[id-1ab3978b-8a37-45d3-8e2c-aab4c2187d43] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_PROXY_SIP_member_list[id-15f8690b-f345-413c-9b4e-af39d546fbec] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_TCP_LC_member_list[id-d1300e9a-64ba-4d02-baf6-2523439b80d7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_TCP_RR_member_list[id-5c35df5d-8951-4506-905b-502f623cc9e4] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_TCP_SI_member_list[id-2da08931-bc4c-4339-b16a-43d40ca7734d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_TCP_SIP_member_list[id-db5769ce-f4b0-4a0f-92a7-4eeed66b6730] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_UDP_LC_member_list[id-dffc1dfb-7506-4f81-b1e5-5835b9690079] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_UDP_RR_member_list[id-c7de7d31-2910-4864-84df-61a883e916fb] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_UDP_SI_member_list[id-ea12a6bc-6267-4790-b2b3-cbd6a146533b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest1.test_UDP_SIP_member_list[id-638811fa-26ce-44f3-8ac7-29cf1ef41838] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_alt_monitor_member_batch_update[id-c4331afe-c129-44e8-8388-fcbbd28cf783] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_alt_monitor_member_show[id-2674b363-7922-494a-b121-cf415dbbb716] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_alt_monitor_member_update[id-65680d48-1d49-4959-a7d1-677797e54f6b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_backup_member_delete[id-eab8f0dc-0959-4b50-aea2-2f2319305d15] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_backup_member_show[id-048f4b15-1cb4-49ac-82d6-b2ac7fe9d03b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_backup_member_update[id-592c19c3-1e0d-4d6d-b2ff-0d39d8654c99] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_member_batch_update[id-8104628d-6f30-4037-ae65-c6f6c1b3af42] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_member_delete[id-8b6574a3-17e8-4950-b24e-66d0c28960d3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_member_show[id-a99da0e8-0595-49a5-a788-efc37fad2dc2] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_LC_member_update[id-52124440-c95c-48fb-af26-70377bcba7d6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_RR_alt_monitor_member_batch_update[id-83e0a9f2-491f-46a8-b3ce-6969d70a4e9f] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_RR_alt_monitor_member_show[id-7674ae04-7e92-44ef-9adf-40718d7ec705] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_RR_alt_monitor_member_update[id-c07572b8-e853-48f3-a8ea-37fc293a4724] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_RR_member_batch_update[id-5f412e52-8ee0-4ee5-8b0e-0e8fc68279a6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_RR_member_delete[id-f129ba5e-a16e-4178-924f-6a9c5b8b1589] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_RR_member_show[id-2c4a29f4-be25-416c-9546-9585298cfe4c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_RR_member_update[id-f83993ce-b053-42ff-9022-612ed67e8db6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SI_alt_monitor_member_batch_update[id-6c1fe175-8e99-4adf-934d-bee79c89fa02] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SI_alt_monitor_member_show[id-6c88a35e-b7c1-4b14-bdae-1a710890555a] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SI_alt_monitor_member_update[id-af40e333-caed-4808-a46c-05c977f3cebc] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SI_member_batch_update[id-546ac0c3-4025-4c88-8276-1c05e7198e82] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SI_member_delete[id-87d0eac7-e391-4633-88cb-e691eeeab4fc] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SI_member_show[id-266b60e2-7c3a-4edb-950b-66d57aa64b80] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SI_member_update[id-f63a9deb-4a45-42c4-9aeb-f7c304ecbc16] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SIP_alt_monitor_member_batch_update[id-44a2508d-ecea-4f92-ba66-a64d6d7f12da] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SIP_alt_monitor_member_show[id-7c4fcb3e-a994-4d39-97cc-929c022c001e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SIP_alt_monitor_member_update[id-5215ecc4-fd47-451a-b073-399bad8b522c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SIP_member_batch_update[id-617028e2-89fb-4e7e-ba62-1a8a7af697ca] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SIP_member_delete[id-1c9f9dc5-4ba3-44cd-a840-fd0629abfddd] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SIP_member_show[id-704d04c3-e639-4dee-b55d-09ebf55f8a0d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTP_SIP_member_update[id-7c89fb05-d949-4c0f-8c61-7e55e494c76f] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_LC_alt_monitor_member_batch_update[id-dd4d0a10-0473-47a2-8ec5-815fbdf0c5ee] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_LC_alt_monitor_member_show[id-c3db94b3-a38c-4a0a-8c53-85888c2e1876] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_LC_alt_monitor_member_update[id-caf95728-5e9c-4295-bd4a-a15263ba5714] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_LC_member_batch_update[id-5ed41de6-8994-4ba4-8107-29eab89fab1e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_LC_member_delete[id-a122557b-4824-4a4f-87f0-6ba5c9ca1e32] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_LC_member_show[id-5844580e-6d01-42dc-b951-d995c9612167] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_LC_member_update[id-d1d98798-20cb-4290-818c-e814911d25e5] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_RR_alt_monitor_member_batch_update[id-7bed4b1c-f862-45bf-ae30-3b4ad0b48870] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_RR_alt_monitor_member_show[id-a06a137d-f6d1-44a6-978b-22fe8e23752c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_RR_alt_monitor_member_update[id-e419d49f-22e4-4331-985e-3a1cc8d0b6b0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_RR_member_batch_update[id-ee622c92-a4b4-41a6-96e3-b3b2429276a2] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_RR_member_delete[id-f961105a-9874-4765-b457-3de9f342e226] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_RR_member_show[id-bfe7dfea-878e-4e7c-afd8-9860d7282930] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_RR_member_update[id-42ac5187-799a-4714-972c-fea6e1c6a7b2] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SI_alt_monitor_member_batch_update[id-59303a97-cc97-441e-b1a6-395271ec2287] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SI_alt_monitor_member_show[id-c2c8ba90-6ade-4fd3-bf12-e15627983917] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SI_alt_monitor_member_update[id-27c17512-51b4-49ae-ac92-3e141599cdda] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SI_member_batch_update[id-3e6c76e0-3630-45f0-a674-5d79b662812b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SI_member_delete[id-a919aa78-5221-4321-aa26-fcd3432d843c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SI_member_show[id-4af933ae-5c6d-4849-af85-e06f7d5a661c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SI_member_update[id-cc67064f-becc-4e31-b9e5-b3ea7e78a187] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SIP_alt_monitor_member_batch_update[id-c1c8930d-0436-4075-b47a-f3bd263ab8a8] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SIP_alt_monitor_member_show[id-69de0c60-3e4f-40cf-9bf7-d2b1e6c83715] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SIP_alt_monitor_member_update[id-edff98be-6208-4f1c-9cd3-376b7ac47f80] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SIP_member_batch_update[id-794137e5-28c1-4b0e-bc2f-fc030d03a689] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SIP_member_delete[id-93ef7ac1-da00-420d-a367-22e86d968e1c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SIP_member_show[id-2d970b1c-c157-4974-b605-b8e08d97e874] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_HTTPS_SIP_member_update[id-a1f214fe-6c09-4298-b03e-7069b615dec2] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_LC_alt_monitor_member_batch_update[id-de033003-1dcb-4896-ad5d-9e68e31addf0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_LC_alt_monitor_member_show[id-ebc52abf-9133-4922-902c-505f890bb44e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_LC_alt_monitor_member_update[id-b22288fa-4e25-4779-bd78-6b4802926457] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_LC_member_batch_update[id-7ae0bd6f-d04c-4d53-bb7e-fef0680726db] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_LC_member_delete[id-069b8558-138f-4d6c-a3ec-9e803d5e2a14] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_LC_member_show[id-43f90043-65d4-483a-99ab-564f25acc0d7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_LC_member_update[id-5ee3df40-381a-4497-9e31-df82d8c2e514] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_RR_alt_monitor_member_batch_update[id-098f73c5-e3c1-4dfa-bd7f-c87df90743e6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_RR_alt_monitor_member_show[id-007c4f0c-8192-4806-9c25-c2f27ea4ba57] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_RR_alt_monitor_member_update[id-43d30b1d-0f99-4b46-ad37-542e899ceae7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_RR_member_batch_update[id-82325d1a-ad01-471e-bfb3-b75ca86ae8eb] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_RR_member_delete[id-6bca4760-bfb4-4cee-b77f-a77abec3e38e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_RR_member_show[id-0d6d2875-d1b3-4508-8e17-1c656a5f31ec] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_RR_member_update[id-d5cf47a6-a3bb-4238-a5cf-a74b122edce4] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SI_alt_monitor_member_batch_update[id-f55d8bff-ac68-4e3b-8f05-a9bb69bb0881] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SI_alt_monitor_member_show[id-383148a5-a9ec-413a-a44c-85c1bbb39729] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SI_alt_monitor_member_update[id-2061a0cf-49e0-49b8-af4d-f197cf84ef11] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SI_member_batch_update[id-963d7e71-d8b7-4257-9b01-f1d7ab57cbc8] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SI_member_delete[id-92368eef-d9ce-47d9-b3f2-7624601010a0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SI_member_show[id-518c4d3f-2b5a-4f8a-9c5e-fad15127502e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SI_member_update[id-5946b163-8684-402a-b228-c0648a3e0734] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SIP_alt_monitor_member_batch_update[id-4de187b6-4948-4394-af6b-8828e96d8f3e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SIP_alt_monitor_member_show[id-28e1e3e7-454b-409d-84c3-1826f82ca9dd] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SIP_alt_monitor_member_update[id-56a21d8e-825a-4780-a073-41061a0d55ca] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SIP_member_batch_update[id-1ab05cf2-265e-4291-b17c-19caa0a1b6ff] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SIP_member_delete[id-e83b9389-768f-4bcf-a650-17af01243d2b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SIP_member_show[id-189ec327-b1c5-47a8-a843-10963cba0a9c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_PROXY_SIP_member_update[id-25a24e27-218b-4dcd-99aa-e9ca9f8163e5] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_LC_alt_monitor_member_batch_update[id-69bd512a-d561-43b1-9a4a-ea7134ee8f9e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_LC_alt_monitor_member_show[id-ecec1c11-2c2c-408c-9b4e-01620266dab6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_LC_alt_monitor_member_update[id-b693b5ba-d8e7-4b89-ad6c-41b56cf258f7] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_LC_member_batch_update[id-1d79ab5a-1110-43f1-bfc3-1cb4e2ab5011] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_LC_member_delete[id-9b2a08cf-c9ae-4f8a-a15c-2acab09a7613] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_LC_member_show[id-baa9b4ec-1ac5-431f-bae0-f2ef68d1c81a] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_LC_member_update[id-0df9463f-de6b-43c1-934f-6523873f3530] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_RR_alt_monitor_member_batch_update[id-e952a399-c817-461e-9e37-fdf7e7b34983] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_RR_alt_monitor_member_show[id-9c968920-1fcc-4a71-8dc9-fdf2ff59af7c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_RR_alt_monitor_member_update[id-5a05f8b0-52ca-4ed7-a1a7-c62aee16c960] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_RR_member_batch_update[id-176cd46e-f5b1-47d1-9403-a0246272eea4] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_RR_member_delete[id-f0f5a651-f7f0-40d7-a051-32da07c28252] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_RR_member_show[id-1acd8a34-dd13-411e-bdf3-414b3fcc569d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_RR_member_update[id-46e5d4e8-0ecc-40a7-87bd-f9ccdfc9a2d3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SI_alt_monitor_member_batch_update[id-fee72f7a-928d-477f-b09b-5a866be717a3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SI_alt_monitor_member_show[id-9b95b5f2-8823-4019-be86-311a1bde5b20] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SI_alt_monitor_member_update[id-7a6314dd-83c5-41ee-92f6-e18409ac213d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SI_member_batch_update[id-71d20f78-ffe3-49a3-b0c6-38cd5804f255] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SI_member_delete[id-1631c730-f34a-4ae7-91eb-5f5b5052cb55] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SI_member_show[id-d8576e58-e8ff-491d-beee-b7c439d2c41c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SI_member_update[id-93818084-a9fb-480d-a7e1-04066ee0e393] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SIP_alt_monitor_member_batch_update[id-52328f7a-bec4-4f23-9293-f5f1283c0af9] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SIP_alt_monitor_member_show[id-6953a9a4-5fac-4470-bfda-4fafbd67288b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SIP_alt_monitor_member_update[id-d6625773-2665-423d-8500-cf9b1b38b53e] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SIP_member_batch_update[id-1d171080-c7e5-4ee0-83d4-51bb1655cb21] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SIP_member_delete[id-eca3d41d-21bd-4547-b8b8-8f87867eb4ad] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SIP_member_show[id-6d8d546f-8c41-49b9-bd9d-8f8ea3975816] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_TCP_SIP_member_update[id-87a3e3aa-580a-41ca-bc15-8cb2995c9125] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_LC_alt_monitor_member_batch_update[id-f9a125e0-84c4-4f7e-8a82-fe84ca3175e5] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_LC_alt_monitor_member_show[id-3a291344-0a88-46fc-9eca-c2c6b9048076] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_LC_alt_monitor_member_update[id-60baa2d7-927a-4b58-80b9-a2e5196985ee] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_LC_member_batch_update[id-b9fadfe2-c3f2-48a4-97a4-04c58c40df87] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_LC_member_delete[id-5824438d-cda2-4cea-a7d0-e7f5e5a11cac] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_LC_member_show[id-e53b2d6a-ad3f-46be-b899-56324874ad24] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_LC_member_update[id-b036a40f-d220-4be6-abc9-8ca8e01b96c3] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_RR_alt_monitor_member_batch_update[id-fdf2423c-c312-466a-b021-130a52b5be35] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_RR_alt_monitor_member_show[id-2d154e0c-4955-4b00-92d5-e9df7b2fbf63] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_RR_alt_monitor_member_update[id-7609cd2f-32ac-4488-869a-7e14827df6ef] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_RR_member_batch_update[id-dec6b06d-6a69-48d9-b7a9-67eb287fe95a] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_RR_member_delete[id-03424069-302d-4020-996c-0a346a97c847] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_RR_member_show[id-06de8b15-caf3-4a75-b278-cdfe6208c8db] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_RR_member_update[id-171ae461-af7d-4fe1-961d-78376fcc2b3f] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SI_alt_monitor_member_batch_update[id-04259015-b6d7-411b-8c19-f21e05994b7c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SI_alt_monitor_member_show[id-896acc77-3b73-4565-ad87-9467218b143b] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SI_alt_monitor_member_update[id-523cf4e8-c071-4778-bc89-367a0b8469e6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SI_member_batch_update[id-086b407e-3ace-47a8-94e4-cf563674ceb6] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SI_member_delete[id-8e05deee-f385-44d8-a112-2649aeea6006] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SI_member_show[id-ffc64ff1-ec8c-4201-a295-a179adc0c7e0] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SI_member_update[id-9f655415-13d6-4ceb-9ea6-9a32baf0e093] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SIP_alt_monitor_member_batch_update[id-e4a357a3-1d07-46f4-a8ff-67a279783b24] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SIP_alt_monitor_member_show[id-9c0d4668-5a0e-41b3-b3b4-3d0372fe28af] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SIP_alt_monitor_member_update[id-796a2972-38e6-41fc-a885-6316195acd70] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SIP_member_batch_update[id-00b3ebda-c28c-471b-bbf8-01de6567b4b5] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SIP_member_delete[id-23788358-ac5f-46c4-922a-164e6a13fe0d] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SIP_member_show[id-a03d02d0-830c-4aad-a10b-96c47974483c] +octavia_tempest_plugin.tests.api.v2.test_member.MemberAPITest2.test_UDP_SIP_member_update[id-2f4efa91-e61d-4dd6-8006-ebfdb00c1246] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_LC_pool_delete[id-008088c8-696e-47ba-bc18-75827fe5956b] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_LC_pool_list[id-4b4c8021-f4dd-4826-b825-7e3dc0beaba4] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_LC_pool_show[id-416c72c6-ef63-4e70-b27e-3ed95b93c02d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_LC_pool_standalone_create[id-29f1a69d-6a0d-4a85-b178-f50f5b4bdfbc] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_LC_pool_update[id-d73755fe-ba3a-4248-9543-8e167a5aa7f4] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_LC_pool_with_listener_create[id-5086402a-2339-4238-bddb-d30508e6cc53] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_app_cookie_pool_delete[id-f9aa5a8c-4e2a-4029-8581-2980f1d111cf] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_app_cookie_pool_show[id-6fa12ae6-a61a-43d0-85d7-5367811c9c5a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_app_cookie_pool_update[id-8677a512-77e1-4af3-96f7-8a3d66725e08] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_app_cookie_pool_with_listener_create[id-c8b84032-1c20-4d85-9db2-2fe5b9eff37a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_http_cookie_pool_delete[id-12a31fb5-85fc-4ec8-9475-079dc06f358b] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_http_cookie_pool_show[id-4a1b6e2c-c216-4589-9ab6-2cd63217f06a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_http_cookie_pool_update[id-4d3b3a4a-a652-4dca-9a49-b14471ce5309] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_http_cookie_pool_with_listener_create[id-0296cccb-83be-425c-ac6a-828774734d5a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_pool_delete[id-35ed3800-7a4a-47a6-9b94-c1033fff1112] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_pool_list[id-6959a32e-fb34-4f3e-be68-8880c6450016] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_pool_show[id-b7932438-1aea-4175-a50c-984fee1c0cad] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_pool_standalone_create[id-7587fe48-87ba-4538-9f03-190911f100ff] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_pool_update[id-7bd0a6bf-57b4-46a6-83ef-f9991896658a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_pool_with_listener_create[id-c9c0df79-f07e-428c-ae57-b9d4078eec79] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_source_IP_pool_delete[id-07528fe6-12a6-4fca-8819-9980e9d3db84] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_source_IP_pool_show[id-373f1c80-e51e-4260-b8d8-f6aeb512f81c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_source_IP_pool_update[id-2e7bbf67-ed32-4a3c-b5ae-1aff8b07bacc] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_RR_source_IP_pool_with_listener_create[id-882263e6-d50f-47b4-9083-f76c2b92eef0] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SI_pool_delete[id-06bec76b-8fbf-4be8-9350-92590ac48606] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SI_pool_list[id-b2cb9879-c1b3-491a-bd20-773bc57625b0] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SI_pool_show[id-af1c1b10-a6ac-4f28-82ba-2c0770903a5c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SI_pool_standalone_create[id-a8b1b41c-5c3c-4c17-a2d4-b7c344520e3d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SI_pool_update[id-42a742d2-ef9c-47fd-8585-5588bb867431] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SI_pool_with_listener_create[id-0136341c-4622-4f65-a59d-b9983331d627] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SIP_pool_delete[id-9b40351e-1140-4b98-974c-46bd1a19763d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SIP_pool_list[id-871b5a7f-c8f3-4d05-9533-f9498e2465fa] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SIP_pool_show[id-59964666-3dfe-4bad-81e0-bc5a4809c10c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SIP_pool_standalone_create[id-265ba978-a528-429c-9ef7-c36373ee2225] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SIP_pool_update[id-7c770af5-782e-453c-bd2e-41ec90b37907] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTP_SIP_pool_with_listener_create[id-a07e2ff0-90f3-43d3-a7ec-5ca93b7f29bf] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_LC_pool_delete[id-c4b2dad0-378f-4f85-a1f3-597de609b0f3] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_LC_pool_list[id-8ace3b65-7183-4b55-837d-3e7f438ea079] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_LC_pool_show[id-82568809-bdb9-444b-9790-128d0c328d72] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_LC_pool_standalone_create[id-eb84fabc-68e6-44f7-955d-4919f045cd08] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_LC_pool_update[id-ca641999-7817-4f8f-a58b-2ccd7a5dca97] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_LC_pool_with_listener_create[id-664fc4d3-70d9-41c5-b3f8-c006726062ae] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_RR_pool_delete[id-c305126b-3ead-4ea8-a886-77d355c0d4a2] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_RR_pool_list[id-7e01fa71-34fd-42e6-9db8-4b2a57cda38d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_RR_pool_show[id-a1bffe2f-ce20-4d79-a168-bc930de5edcb] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_RR_pool_standalone_create[id-cc940a8b-b21c-46f5-9976-d2c8dd73b626] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_RR_pool_update[id-ac097c2e-4f79-4714-8de4-517598d37919] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_RR_pool_with_listener_create[id-10df2793-63b2-42a3-a5d0-9241a9d700a3] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SI_pool_delete[id-4b21149e-64f8-4e5f-8f71-020abbd0d0eb] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SI_pool_list[id-25d4b0d2-ab46-40ad-afec-1b0afa88a559] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SI_pool_show[id-64434a6d-222e-4056-bcc0-335ebe4f03ee] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SI_pool_standalone_create[id-d69c5e0b-43b5-4afe-a94a-1a4f93e44a93] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SI_pool_update[id-3b4e6462-4e0c-4c05-bc30-d6f86f67bb60] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SI_pool_with_listener_create[id-04cc43f2-9eab-4552-b8c1-cea9e1325696] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SIP_pool_delete[id-391a4ed7-be97-4231-8198-5c2802bc6e30] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SIP_pool_list[id-250e6bf6-5017-47c9-ae12-1e64515d3bfd] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SIP_pool_show[id-9f8f855c-cd09-4a74-b5f2-c5c13b59422e] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SIP_pool_standalone_create[id-7da08af5-e225-46df-b0b4-a5f1834a5377] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SIP_pool_update[id-72cee49d-36d5-4b54-8883-8fe7be9fd0f0] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_HTTPS_SIP_pool_with_listener_create[id-3d52a82e-e488-445a-8599-87e9bb7153eb] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_LC_pool_delete[id-d906c63e-6090-422c-9627-e5a971e1665c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_LC_pool_list[id-f3f3565e-a6b3-4541-9fb3-d9900231771b] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_LC_pool_show[id-93e4bc67-ce12-43fb-a50d-97de47c3a63f] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_LC_pool_standalone_create[id-08e35f78-a85d-48d2-8ac3-14c5e68b64f7] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_LC_pool_update[id-67a2cbab-f4fc-41c8-93e8-97ddba39c1ab] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_LC_pool_with_listener_create[id-044f460b-47ec-4e97-96be-c7ab812bfa16] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_RR_pool_delete[id-729be91c-82c5-4b4e-9feb-08a1c786488b] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_RR_pool_list[id-fb267dae-b4e3-4858-a85e-72ecb1d91eff] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_RR_pool_show[id-4cf9fa5c-d8e0-4253-8b79-2eb59e066772] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_RR_pool_standalone_create[id-4d6c124e-73ea-4b32-bd1c-3ff7be2c4e55] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_RR_pool_update[id-4392bc21-c18e-4e25-bb7e-2c9e3777d784] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_RR_pool_with_listener_create[id-60406086-b0a9-4f55-8f64-df161981443c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SI_pool_delete[id-277fba8f-d72b-47f0-9723-5e013f53fb7a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SI_pool_list[id-42a3e3e3-ad71-418e-a262-628a213a7b03] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SI_pool_show[id-adbb7f3b-7a37-4a8e-a2b4-c3f827dad0ba] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SI_pool_standalone_create[id-cc42779d-71f8-4a7c-8217-02127be344ce] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SI_pool_update[id-d2cb7a0a-8268-46bc-a519-08474c42c4ca] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SI_pool_with_listener_create[id-c845d8d3-30d7-42c3-8943-9a4582c62e2d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SIP_pool_delete[id-1f6b6b9c-96c6-420b-bc51-8568c081a1ee] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SIP_pool_list[id-69d3f811-6ce8-403e-bae9-745d51cb268a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SIP_pool_show[id-1194bd18-20bc-43e7-b588-9f78a72e0021] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SIP_pool_standalone_create[id-ad4cb862-fa37-4874-99c1-511cdcd86f91] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SIP_pool_update[id-f269293c-b6fa-4fe2-82a6-57fc8ef89260] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_PROXY_SIP_pool_with_listener_create[id-bcf76f26-e801-4ab4-b338-b210457d592e] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_LC_pool_delete[id-3bfd2d96-813b-48af-86e0-97361873a68a] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_LC_pool_list[id-f3a74c0c-3083-44a5-9938-a245176babcd] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_LC_pool_show[id-ee910c1e-1704-4b41-99c1-0c1f904e577d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_LC_pool_standalone_create[id-8f552da6-38f8-44b8-b69b-072cc1e232a6] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_LC_pool_update[id-c9f1ed23-f6d4-4d44-9d22-bdc1fbe5854d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_LC_pool_with_listener_create[id-728b974f-ff59-479b-ada5-de280bbaaf02] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_RR_pool_delete[id-a455dea2-19ce-435c-90ae-e143fe84245e] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_RR_pool_list[id-1a0b616f-ee77-4ac1-bb5f-300c2a10a7f2] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_RR_pool_show[id-e6e91bb3-76b3-4ff8-ad60-d20ac1e64381] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_RR_pool_standalone_create[id-41a5f22e-80e8-4d85-bfd6-2726846ed2ce] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_RR_pool_update[id-f6a5970d-2f27-419b-a0ee-7a420ee7b396] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_RR_pool_with_listener_create[id-b76432ae-1aa2-4048-b326-1cbda28415ac] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SI_pool_delete[id-8411295f-aec0-40ab-a25d-a4677c711d98] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SI_pool_list[id-ec0fda75-f2d7-4fa6-ba91-c5eb9a7e9874] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SI_pool_show[id-55f62d1a-33b0-4263-84af-672a30ee52bd] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SI_pool_standalone_create[id-869b4208-5821-44f5-acf6-4f087c4dd79c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SI_pool_update[id-84e63663-0bf7-45bc-a4d9-b3bbd664fd8c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SI_pool_with_listener_create[id-4b0be593-b2e4-4704-a347-c36dae76aaad] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SIP_pool_delete[id-fe54f133-865a-4613-9cf0-0469c780f53e] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SIP_pool_list[id-8ca217d0-705f-4a7a-87c2-752bb1ee88f1] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SIP_pool_show[id-6003cbe8-73a5-416a-9be0-7aa5699dc157] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SIP_pool_standalone_create[id-42382080-7fd5-46d7-afd7-d47c880f0397] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SIP_pool_update[id-f5445a86-247f-4075-8b71-e4e5415d0bed] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_TCP_SIP_pool_with_listener_create[id-53687dd3-e076-4e93-b917-93c76a160444] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_LC_pool_delete[id-ea58bede-1934-480b-86fc-665b872fc946] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_LC_pool_list[id-ed26899d-a590-46fc-bf70-27c5a9c59cbf] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_LC_pool_show[id-5f4339f6-0387-44f4-a5f9-e385d44b5ee2] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_LC_pool_standalone_create[id-0cb032d9-e092-476e-9aaf-463eea58fc16] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_LC_pool_update[id-8e6b8802-c4a7-43eb-a4e8-9f6bf7899a7d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_LC_pool_with_listener_create[id-9bf3bb05-ee36-47f3-b669-78f06a94035d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_pool_delete[id-63559694-6b38-4bad-9f10-3675131b28c0] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_pool_list[id-6fb6a6e3-ff65-4f2c-8876-2997d3903cfe] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_pool_show[id-7c4a1c90-7fc2-42ee-ad78-d7c75b5a56d2] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_pool_standalone_create[id-c09be35d-8a8b-4abd-8752-2cb4d7d7fab2] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_pool_update[id-a1cded67-9fd6-4155-8761-ce165d518b47] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_pool_with_listener_create[id-4fb59ed9-5c44-437e-a5f9-bb01b9ba6a72] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_source_IP_pool_delete[id-cc69c0d0-9191-4faf-a154-e33df880f44e] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_source_IP_pool_show[id-bd732c36-bdaa-4591-bf4e-28268874d22c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_source_IP_pool_update[id-28b90650-a612-4b10-981f-d3dd6a366e4f] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_RR_source_ip_pool_with_listener_create[id-1d5eed30-86bf-4bf4-87d0-22adee3defa1] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SI_pool_delete[id-33322f21-12cc-4f2e-b406-0c11b05a1c6e] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SI_pool_list[id-0bb02800-d7c9-4916-a532-ac1ac7b945d0] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SI_pool_show[id-5b961eee-183d-435c-bdf5-b83ca68c4944] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SI_pool_standalone_create[id-47af95cb-685a-48de-9d17-66108cdfd3fa] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SI_pool_update[id-cdb230b9-996a-4933-a7a2-a7b09465c18c] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SI_pool_with_listener_create[id-0115ab0c-e8fd-434a-9448-7fba55a8f27d] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SIP_pool_delete[id-7cac0520-c7ea-49cb-8b38-0e309af2ea53] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SIP_pool_list[id-bdc7df5f-ffdb-48c8-823e-a3b5d76868a0] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SIP_pool_show[id-1400529e-3a0c-4bac-b6ed-669fdd723956] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SIP_pool_standalone_create[id-02d40127-d46e-4aba-8428-96f6deff3554] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SIP_pool_update[id-d9e1aeee-21c3-4b0f-9685-834768597607] +octavia_tempest_plugin.tests.api.v2.test_pool.PoolAPITest.test_UDP_SIP_pool_with_listener_create[id-5f1acd3e-305d-40d5-81e9-fe6250411d49] +octavia_tempest_plugin.tests.api.v2.test_provider.ProviderAPITest.test_provider_list[id-8b94e0cc-a24d-4c29-bc8e-53f58214dc67] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_alpn_fallback_tls_traffic[id-ee0d15a3-05b7-498d-9b2f-280d4896e597] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_alpn_proto_not_supported_tls_traffic[id-56f4274a-ebd9-42f7-b897-baebc4b8eb5b] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_alpn_tls_traffic[id-19bade6f-302f-45dc-b316-553f1dfff49c] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_basic_h2_listener_http_listener_pool_reencryption[id-d3e4c5fe-1726-49e4-b0b0-7a5a47749fc9] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_basic_tls_SNI_multi_listener_traffic[id-bfac9bf4-8cd0-4519-8d99-5ad0c75abf5c] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_basic_tls_SNI_traffic[id-08405802-4411-4454-b008-8607408f424a] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_basic_tls_traffic[id-887ece26-0f7b-4933-89ab-5bb00b106ee0] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_http_1_1_tls_hsts_traffic[id-7436c6b7-44be-4544-a40b-31d2b7b2ad0b] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_http_1_1_tls_traffic[id-a0dff0f2-d53e-497c-9ded-dca64e82991f] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_http_2_tls_traffic[id-9965828d-24af-4fa0-91ae-21c6bc47ab4c] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_mixed_http_https_traffic[id-dcf11f78-7af3-4832-b716-9a01648f439c] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_pool_reencryption_client_authentication[id-11b67c96-a553-4b47-9fc6-4c3d7a2a10ce] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_pool_reencryption[id-ee0faf71-d11e-4323-8673-e5e15779749b] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_tls_client_auth_mandatory[id-af6bb7d2-acbb-4f6e-861f-39a2a3f02331] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_tls_client_auth_optional[id-42d696bf-e7f5-44f0-9331-4a5e01d69ef3] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_tls_multi_listener_client_auth[id-13271ce6-f9f7-4017-a017-c2fc390b9438] +octavia_tempest_plugin.tests.barbican_scenario.v2.test_tls_barbican.TLSWithBarbicanTest.test_tls_prometheus_client_auth_mandatory[id-7d9dcae6-3e2c-4eae-9bfb-1ef0d00aa530] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_HTTP_healthmonitor_CRUD[id-4c2058f9-b8e2-4a5b-a2f3-3bd58a29f63b] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_HTTPS_healthmonitor_CRUD[id-08681eac-e907-4f71-8799-4b8fdf23914a] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_PING_healthmonitor_CRUD[id-74611ffb-45f8-4cf5-a28c-7cc37879a27b] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_TCP_healthmonitor_CRUD[id-cacec696-10f4-430d-bc9e-2c5f235a3324] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_TLS_healthmonitor_CRUD[id-6becafb2-1e15-4977-bb29-b08f5728d028] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_LC_UDP_healthmonitor_CRUD[id-fe43ee90-093d-4175-837e-92f803958ef1] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_HTTP_healthmonitor_CRUD[id-a51e09aa-6e44-4c67-a9e4-df70d0e08f96] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_HTTPS_healthmonitor_CRUD[id-fef9eabc-9d1e-4ad2-ae3e-05afc8c84c48] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_PING_healthmonitor_CRUD[id-de01b73d-dba0-4426-9e20-9be3a34cfc44] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_TCP_healthmonitor_CRUD[id-141a121a-8918-4f9c-a070-eaf8ec29008d] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_TLS_healthmonitor_CRUD[id-de80d87a-5479-41c6-8c6b-518cc64ec62d] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_RR_UDP_healthmonitor_CRUD[id-265d7359-f0a5-4083-92a8-07cb1787fe36] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_HTTP_healthmonitor_CRUD[id-20a2905f-2b53-4395-9a7f-1ded67ef4408] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_HTTPS_healthmonitor_CRUD[id-8a8cc776-b68f-4761-9bf9-cae566cdc155] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_PING_healthmonitor_CRUD[id-296a445c-5cc8-47a7-ae26-8d548f9712c3] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_TCP_healthmonitor_CRUD[id-94be34b1-4dc6-492b-a777-0587626a785f] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_TLS_healthmonitor_CRUD[id-0de0e021-fd3c-4f7c-b959-67d758394fd2] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SI_UDP_healthmonitor_CRUD[id-3c79750a-aba6-4838-acbe-bc937ccf2118] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_HTTP_healthmonitor_CRUD[id-d5e0d1b6-7cce-4592-abce-0ac6bee18818] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_HTTPS_healthmonitor_CRUD[id-e188daac-6db9-4dc2-8ecb-b47932e1984a] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_PING_healthmonitor_CRUD[id-f9458ffd-5af7-402b-9c15-c061bf2eb9ba] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_TCP_healthmonitor_CRUD[id-b4cbe603-0a14-4778-b38c-f330053c86b6] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_TLS_healthmonitor_CRUD[id-57714d4c-d584-4345-9ceb-becc3ae37b7f] +octavia_tempest_plugin.tests.scenario.v2.test_healthmonitor.HealthMonitorScenarioTest.test_SIP_UDP_healthmonitor_CRUD[id-cc4abf84-361b-409b-b859-9a860d539deb] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_http_ipv6_vip_ipv6_members_traffic[id-dd75f41a-5b29-47ad-963d-3434f1056ca3] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_http_ipv6_vip_mixed_ipv4_ipv6_members_traffic[id-219ac17d-c5c1-4e7e-a9d5-0764d7ce7746] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_http_LC_listener_with_allowed_cidrs[id-9bead31b-0760-4c8f-b70a-f758fc5edd6a] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_http_RR_listener_with_allowed_cidrs[id-84b23f68-4bc3-49e5-8372-60c25fe69613] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_http_SI_listener_with_allowed_cidrs[id-d1256195-3d85-4ffd-bda3-1c0ab78b8ce1] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_http_SIP_listener_with_allowed_cidrs[id-c11768f1-19b4-48cc-99a5-0737379b1957] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_tcp_LC_listener_with_allowed_cidrs[id-843a13f7-e00f-4151-8817-b5395eb69b52] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_tcp_RR_listener_with_allowed_cidrs[id-52c07510-5755-44a3-9231-64c9cbb4bbd4] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_tcp_SI_listener_with_allowed_cidrs[id-bf8504b6-b95a-4f8a-9032-ab432db46eec] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_tcp_SIP_listener_with_allowed_cidrs[id-fcfe2ab1-2c36-4793-a926-1fec589a9a2a] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_udp_LC_listener_with_allowed_cidrs[id-cc0d55b1-87e8-4a87-bf50-66299947a469] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_udp_RR_listener_with_allowed_cidrs[id-df0417d9-dc72-4bb5-b3ce-1e2558a3c4a9] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_udp_SI_listener_with_allowed_cidrs[id-ce75bf28-5288-4821-a603-460e602de8b9] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_udp_SIP_listener_with_allowed_cidrs[id-80f31bc1-819e-4d9e-8820-bf3e28600540] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_vip_ipv6_members_udp_traffic[id-26317013-a9b5-4a00-a993-d4c55b764e40] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_ipv6_vip_mixed_ipv4_ipv6_members_udp_traffic[id-c468434d-bc84-4bfa-825f-d4761daa0d76] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_tcp_ipv6_vip_ipv6_members_traffic[id-9bb93619-14cb-45d9-ad60-2f80c201486a] +octavia_tempest_plugin.tests.scenario.v2.test_ipv6_traffic_ops.IPv6TrafficOperationsScenarioTest.test_tcp_ipv6_vip_mixed_ipv4_ipv6_members_traffic[id-a4e8d5d1-03d5-4252-9300-e89b9b2bdafc] +octavia_tempest_plugin.tests.scenario.v2.test_l7policy.L7PolicyScenarioTest.test_l7policy_CRUD[id-ffd598d9-d8cd-4586-a749-cde4897e64dd] +octavia_tempest_plugin.tests.scenario.v2.test_l7rule.L7RuleScenarioTest.test_l7rule_CRUD[id-a1c268b9-5304-48c7-9a34-0ef0e8e9307e] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_least_connections_listener_CRUD[id-ecdd65b0-cf8f-48ee-972b-2f09425472f1] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_round_robin_listener_CRUD[id-4a874014-b7d1-49a4-ac9a-2400b3434700] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_source_ip_listener_CRUD[id-b2ae8604-7a4f-477c-9658-fac27734671a] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_http_source_ip_port_listener_CRUD[id-807a421e-5e99-4556-b0eb-512d39b25eac] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_least_connections_listener_CRUD[id-0681b2ac-8301-4e6c-bf29-b35244864af3] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_round_robin_listener_CRUD[id-2b888812-d916-44f0-b620-8d83dbb45975] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_source_ip_listener_CRUD[id-0ad3fdee-e8c2-4c44-9690-b8a838fbc7a5] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_tcp_source_ip_port_listener_CRUD[id-6211f8ad-622d-404d-b199-8c2eb55ab340] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_least_connections_listener_CRUD[id-27a2ba7d-6147-46e4-886a-47c1ba63bf89] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_round_robin_listener_CRUD[id-dd913f74-c6a6-4998-9bed-095babb9cb47] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_source_ip_listener_CRUD[id-7830aba8-12ca-40d9-9d9b-a63f7a43b287] +octavia_tempest_plugin.tests.scenario.v2.test_listener.ListenerScenarioTest.test_udp_source_ip_port_listener_CRUD[id-3f9a2de9-5012-437d-a907-a25e1f68ccfb] +octavia_tempest_plugin.tests.scenario.v2.test_load_balancer.LoadBalancerScenarioTest.test_load_balancer_ipv4_CRUD_with_compute_flavor[id-c9d8b6dd-ef29-40d8-b329-86d31857df3f] +octavia_tempest_plugin.tests.scenario.v2.test_load_balancer.LoadBalancerScenarioTest.test_load_balancer_ipv4_CRUD[id-a5e2e120-4f7e-4c8b-8aac-cf09cb56711c] +octavia_tempest_plugin.tests.scenario.v2.test_load_balancer.LoadBalancerScenarioTest.test_load_balancer_ipv6_CRUD_with_compute_flavor[id-2f1c2bdc-0df9-4c1e-be83-910fcd5af8f2] +octavia_tempest_plugin.tests.scenario.v2.test_load_balancer.LoadBalancerScenarioTest.test_load_balancer_ipv6_CRUD[id-86ffecc4-dce8-46f9-936e-8a4c6bcf3959] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_LC_alt_monitor_member_crud[id-eab4eb32-b26f-4fe1-a606-1574b5b6182c] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_LC_member_crud[id-33abafca-ce57-479e-8480-843ef412d6a6] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_RR_alt_monitor_member_crud[id-a0f02494-ffb3-47be-8670-f56c0df9ec94] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_RR_member_crud[id-15c8c3e3-569c-4029-95df-a9f72049e267] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_SI_alt_monitor_member_crud[id-3910a7ec-63c5-4152-9fe1-ce21d3e1cdca] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_SI_member_crud[id-07d1e571-d12c-4e04-90d1-8f4f42610df3] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_SIP_alt_monitor_member_crud[id-7470bea5-9ea0-4e04-a82f-a0bed202b97d] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTP_SIP_member_crud[id-ab8f46fe-0c84-4755-a9a2-80cc1fbdea18] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_LC_alt_monitor_member_crud[id-6f8fce94-b2aa-4497-b80f-74293d977d25] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_LC_member_crud[id-80355701-bc68-4cba-a9b3-4f35fc192b6a] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_RR_alt_monitor_member_crud[id-6c3e5bd7-4573-4f6d-ac64-31b238c9ea51] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_RR_member_crud[id-57aee0db-3295-42b7-a7d3-aae942a6cb41] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_SI_alt_monitor_member_crud[id-d59ea523-8dac-4e19-8df4-a7076a17296c] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_SI_member_crud[id-32b0b541-29dc-464b-91c1-115413539de7] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_SIP_alt_monitor_member_crud[id-761e1acd-3f4c-4e02-89e1-f89adfe2e3f9] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_HTTPS_SIP_member_crud[id-e59e9a7d-b6e7-43e9-b9d5-0717f113d769] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTP_LC_member_create[id-f9bc8ef1-cf21-41e5-819d-7561173e5286] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTP_RR_member_create[id-b8afb91d-9b85-4569-85c7-03453df8990b] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTP_SI_member_create[id-cc7f9272-84a6-436c-a529-171b67a45b62] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTP_SIP_member_create[id-b3dc557a-88ec-4bc6-84fd-c3aaab5d5920] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTPS_LC_member_create[id-e63c89a7-30a3-4eff-8ff5-dd62a5ecec0f] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTPS_RR_member_create[id-a64dc345-4afe-4a2c-8a6a-178dd5a94670] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTPS_SI_member_create[id-704a10ed-d52d-4c75-9445-9ef98f7f540f] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_HTTPS_SIP_member_create[id-d6f3908d-470a-4939-b407-c6d6324c06b6] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_PROXY_LC_member_create[id-efaa9ed0-c261-4184-9693-0020965606a8] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_PROXY_RR_member_create[id-909aebf2-f9e4-4b96-943e-c02b8a415cd2] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_PROXY_SI_member_create[id-4c516b5b-eb7b-4a4c-9a73-fba823332e25] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_PROXY_SIP_member_create[id-ab745620-bf92-49e1-ac35-e42f266a7612] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_TCP_LC_member_create[id-f4ac056c-2cb8-457f-b1b1-9b49226f9b9f] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_TCP_RR_member_create[id-407ff3d4-f0a2-4d27-be69-3f2ec039a6a0] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_TCP_SI_member_create[id-61973bc8-8bc4-4aec-bf57-b37583887544] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_TCP_SIP_member_create[id-c7ffbd6e-5d9f-45e8-a5d0-2d26ea6b0ed0] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_UDP_LC_member_create[id-90e22b80-d52b-4af2-9c4d-9be44eed9575] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_UDP_RR_member_create[id-e74b28cf-ab1a-423b-a1c5-d940e3c0a5ab] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_UDP_SI_member_create[id-ddab1836-ba9f-42e5-9630-1572d4a63501] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_mixed_UDP_SIP_member_create[id-aa6b282c-d1c2-4a39-b085-33c224d4faff] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_LC_alt_monitor_member_crud[id-3ea2aad1-5650-4ec6-8394-501de33cce70] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_LC_member_crud[id-0e45c423-db43-4fee-8442-d9daabe6b2aa] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_RR_alt_monitor_member_crud[id-534fbc38-1c70-4c67-8f89-74a6905b1c98] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_RR_member_crud[id-e0ad1fa0-1fdb-472d-9d69-8968631c9239] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_SI_alt_monitor_member_crud[id-5d3879a6-d103-4800-bca4-1ef18ecbee68] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_SI_member_crud[id-12348506-1cfc-4d62-9cc2-d380776a9154] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_SIP_alt_monitor_member_crud[id-ad43bc3f-2664-42c4-999f-9763facb8d15] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_PROXY_SIP_member_crud[id-ba7b0c73-df44-4a1a-a610-a107daabc36d] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_LC_alt_monitor_member_crud[id-06b95367-dc81-41e5-9a53-981833fb2979] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_LC_member_crud[id-9b2e7e2d-776b-419c-9717-ab4fef9cd5ca] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_RR_alt_monitor_member_crud[id-673425e0-2a57-4c92-a416-7b4e0824708f] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_RR_member_crud[id-c4c72e4b-5abe-41df-9f1d-6a8a27c75a80] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_SI_alt_monitor_member_crud[id-4f1661e5-1dff-4910-9ecd-96327ea3e873] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_SI_member_crud[id-efb158e2-de75-4d8b-8566-a0fa5fd75173] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_SIP_alt_monitor_member_crud[id-5872f1de-1a33-4c20-bc02-7d058e3c3b55] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_TCP_SIP_member_crud[id-3341d05c-c199-496f-ac40-6248818ce831] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_LC_alt_monitor_member_crud[id-0b4ec248-c6a0-4d29-b77e-189453ec0535] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_LC_member_crud[id-c70bd8c6-0f6a-4ee7-840f-a3355aefd471] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_RR_alt_monitor_member_crud[id-94829e1e-506e-4f3c-ab04-4e338787ccfd] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_RR_member_crud[id-f08c9efc-b69c-4c0f-a731-74ec8c17fc91] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_SI_alt_monitor_member_crud[id-16b84495-e8f8-4e7b-b242-43a6e00fb8ad] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_SI_member_crud[id-0984583b-daaf-4509-bf1f-ff3acf33836b] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_SIP_alt_monitor_member_crud[id-5f40b080-0f2c-4791-a509-da7cfe9eace4] +octavia_tempest_plugin.tests.scenario.v2.test_member.MemberScenarioTest.test_UDP_SIP_member_crud[id-9550835b-c9ef-44e3-8087-151c25a95168] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_LC_pool_standalone_CRUD[id-f30bd185-ca13-45c1-8a2f-f4179e7f0c3a] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_LC_pool_with_listener_CRUD[id-d8c428b0-dee4-4374-8286-31e52aeb7fe5] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_RR_app_cookie_pool_with_listener_CRUD[id-d6b8119b-40e9-487d-a037-9972a1e688e8] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_RR_http_cookie_pool_with_listener_CRUD[id-a67f2276-6469-48d4-bf7e-ddf6d8694dba] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_RR_pool_standalone_CRUD[id-dfa120bf-81b9-4f22-bb5e-7df660c18173] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_RR_pool_with_listener_CRUD[id-087da8ab-79c7-48ba-871c-5769185cea3e] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_RR_source_IP_pool_with_listener_CRUD[id-c248e3d8-43d9-4fd4-93af-845747c9b939] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_SI_pool_standalone_CRUD[id-4ef47185-ef22-4396-8c9c-b98b9b476605] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_SI_pool_with_listener_CRUD[id-13a5caba-42a5-4b8c-a389-74d630a91687] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_SIP_pool_standalone_CRUD[id-fee61d34-e272-42f5-92e2-69b515c6cded] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTP_SIP_pool_with_listener_CRUD[id-d99948da-649d-493c-a74d-72e532df0605] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_LC_pool_standalone_CRUD[id-82d8e035-4068-4bad-a87b-e4907bf6d464] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_LC_pool_with_listener_CRUD[id-726beb03-de8c-43cd-ba5f-e7d6faf627a3] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_RR_pool_standalone_CRUD[id-6179a5d1-6425-4144-a437-b0d260b7b883] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_RR_pool_with_listener_CRUD[id-1b4585b4-c521-48e8-a69a-8a1d729a2949] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_SI_pool_standalone_CRUD[id-5ff7732a-7481-4c03-8efc-5ee794feb11a] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_SI_pool_with_listener_CRUD[id-30f3d93c-cc22-4821-8805-d5c41023eccd] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_SIP_pool_standalone_CRUD[id-b3c68f89-634e-4279-9546-9f2d2eac4bfa] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_HTTPS_SIP_pool_with_listener_CRUD[id-4327f636-50c3-411c-b90e-0b907bdaffc5] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_LC_pool_standalone_CRUD[id-b3cef24e-343a-4e77-833b-422158d54673] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_LC_pool_with_listener_CRUD[id-f1edfb45-a9d3-4150-8bc9-4fc3427c6346] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_RR_pool_standalone_CRUD[id-6b9f4f01-cb78-409a-b9fe-cbbeb27d0c5f] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_RR_pool_with_listener_CRUD[id-0228ea63-dff5-4dfb-b48a-193e8509caa8] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_SI_pool_standalone_CRUD[id-7cbb01b8-196b-4ac3-9fec-a41abf867850] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_SI_pool_with_listener_CRUD[id-29327103-4949-4a77-a748-87ab725237b7] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_SIP_pool_standalone_CRUD[id-95a93e91-6ac0-40d5-999c-84a8b68c14f4] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_PROXY_SIP_pool_with_listener_CRUD[id-13893ac9-150f-4605-be68-6bdf65e2bb12] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_LC_pool_standalone_CRUD[id-d6d067c3-ec63-4b5d-a364-acc7493ae3b8] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_LC_pool_with_listener_CRUD[id-a159c345-9463-4c01-b571-086c789bd7d5] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_RR_pool_standalone_CRUD[id-71088923-cfdf-4821-a6a8-c7c9045b624d] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_RR_pool_with_listener_CRUD[id-4b663772-5c6b-49a3-b592-49d91bd71ff1] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_SI_pool_standalone_CRUD[id-6a4dd425-d7d9-40dd-b451-feb4b3c551cc] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_SI_pool_with_listener_CRUD[id-4391d6a5-bb1c-4ff0-9f74-7b8c43a0b150] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_SIP_pool_standalone_CRUD[id-d045ea39-b6dd-4171-bb90-2b9970e25303] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_TCP_SIP_pool_with_listener_CRUD[id-ec22ab54-8e0a-4472-8f70-78c34f28dc36] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_LC_pool_standalone_CRUD[id-6fea6a39-19eb-4a0e-b507-82ecc57c1dc5] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_LC_pool_with_listener_CRUD[id-6ce12d8c-ad59-4e48-8de1-d26926735457] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_RR_pool_standalone_CRUD[id-45aefaa0-c909-4861-91c6-517ea10285a5] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_RR_pool_with_listener_CRUD[id-cff21560-52be-439f-a41f-789d365db567] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_RR_source_ip_pool_with_listener_CRUD[id-dc7f0ed5-f94c-4498-9dca-5dbc08e7162f] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_SI_pool_standalone_CRUD[id-211a688c-f495-4f32-a297-c64d240b5de0] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_SI_pool_with_listener_CRUD[id-b19f1285-dbf2-4ac9-9199-3c3693148133] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_SIP_pool_standalone_CRUD[id-0e0f0299-8c5e-4d7c-a99e-85db43b45446] +octavia_tempest_plugin.tests.scenario.v2.test_pool.PoolScenarioTest.test_UDP_SIP_pool_with_listener_CRUD[id-66d50010-13ca-4588-ae36-61bb783d556e] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_backup_member[id-b2d5cefe-eac0-4eb3-b7c2-54f22578def9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_basic_http_traffic[id-6751135d-e15a-4e22-89f4-bfcc3408d424,slow,smoke] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_basic_tcp_traffic[id-332a08e0-eff1-4c19-b46c-bf87148a6d84] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_basic_udp_traffic[id-1e459663-2315-4067-bb47-c8a72f4928f0] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_healthmonitor_http_traffic[id-a16f8eb4-a77c-4b0e-8b1b-91c237039713] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_healthmonitor_tcp_traffic[id-22f00c34-343b-4aa9-90be-4567ecf85772] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_healthmonitor_udp_traffic[id-80b86513-1a76-4e42-91c9-cb23c879e536] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_hm_op_status_changed_as_expected_on_update[id-cd5aeefa-0e16-11eb-b8dc-74e5f9e2a801,slow] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_l7policies_and_l7rules[id-3558186d-6dcd-4d9d-b7f7-adc190b66149] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_LC_listener_with_allowed_cidrs[id-04399db0-04f0-4cb5-bb27-a12bf18bfe08] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_RR_listener_with_allowed_cidrs[id-13b0f2de-9934-457b-8be0-f1bffc6915a0] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_SI_listener_with_allowed_cidrs[id-fb5f35c1-08c9-43f7-8ed1-0395a3ef4735] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_http_SIP_listener_with_allowed_cidrs[id-d198ddc5-1bcb-4310-a1b0-fa1a6328c4e9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_insert_headers[id-d3a28e76-76bc-11eb-a7c3-74e5f9e2a801] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_least_connections_http_traffic[id-a58063fb-b9e8-4cfc-8a8c-7b2e9e884e7a] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_least_connections_tcp_traffic[id-e1056709-6a1a-4a15-80c2-5cbb8279f924] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_least_connections_udp_traffic[id-b5285410-507c-4629-90d4-6161540033d9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_mixed_ipv4_ipv6_members_http_traffic[id-20b6b671-0101-4bed-a249-9af6ee3aa6d9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_mixed_ipv4_ipv6_members_tcp_traffic[id-c442ae84-0abc-4470-8c7e-14a07e92a6fa] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_mixed_ipv4_ipv6_members_udp_traffic[id-56823616-34e1-4e17-beb9-15dd6b1593af] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_prometheus_listener_metrics_page[id-05e99fb3-2b37-478e-889b-77f1c731a471] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_http_traffic[id-881cc3e9-a011-4043-b0e3-a6185f736053] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_port_http_traffic[id-a446585b-5651-40ce-a4db-cb2ab4d37c03] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_port_tcp_traffic[id-60108f30-d870-487c-ab96-8d8a9b587b94] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_port_udp_traffic[id-a67dfa58-6953-4a0f-8a65-3f153b254c98] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_tcp_traffic[id-4568db0e-4243-4191-a822-9d327a55fa64] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_source_ip_udp_traffic[id-be9e6ef2-7840-47d7-9315-cdb1e897b202] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_and_udp_traffic_on_same_port[id-2b05229c-0254-11eb-8610-74e5f9e2a801] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_LC_listener_with_allowed_cidrs[id-3d8d95b6-55e8-4bb9-b474-4ac35abaff22] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_RR_listener_with_allowed_cidrs[id-8bca1325-f894-494d-95c6-3ea4c3df6a0b] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_SI_listener_with_allowed_cidrs[id-c0904c88-2479-42e2-974f-55041f30e6c5] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tcp_SIP_listener_with_allowed_cidrs[id-bbb09dbb-2aad-4281-9383-4bb4ad420ee1] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_tenant_flow_log[id-571dddd9-f5bd-404e-a799-9df7ac9e2fa9] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_LC_listener_with_allowed_cidrs[id-7456b558-9add-4e0e-988e-06803f8047f7] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_RR_listener_with_allowed_cidrs[id-93675cc3-e765-464b-9563-e0848dc75330] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_SI_listener_with_allowed_cidrs[id-4f73bac5-2c98-45f9-8976-724c99e39979] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_SIP_listener_with_allowed_cidrs[id-70290a9d-0065-42ad-bb46-884a535d2da2] +octavia_tempest_plugin.tests.scenario.v2.test_traffic_ops.TrafficOperationsScenarioTest.test_udp_update_pool_healthmonitor_listener[id-c79f2cd0-0324-11eb-bc8e-74e5f9e2a801] +octavia_tempest_plugin.tests.spare_pool_scenario.v2.test_spare_pool.SparePoolTest.test_health_manager_failover_to_spare_amp[id-2ba3a2c2-de9d-4556-9535-cbe9209b4eaa] diff --git a/.automation.conf/tempest/load-lists/platform.2022.11-test-list.txt b/.automation.conf/tempest/load-lists/platform.2022.11-test-list.txt new file mode 100644 index 000000000..4eca58790 --- /dev/null +++ b/.automation.conf/tempest/load-lists/platform.2022.11-test-list.txt @@ -0,0 +1,432 @@ +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors[id-e36c0eaa-dff5-4082-ad1f-3f9a80aa3f59] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_with_detail[id-6e85fde4-b3cd-4137-ab72-ed5f418e8c24] +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image[id-3731d080-d4c5-4872-b41a-64d0d0021314] +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name[id-3b7c6fe4-dfe7-477c-9243-b06359db51e6] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypairsv22_create_list_show_with_type[id-89d59d43-f735-441a-abcf-0601727f47b6] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_list_security_groups_by_server[id-79517d60-535a-438f-af3d-e6feab1cbea7] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_security_group_create_get_delete[id-ecc0da4a-2117-48af-91af-993cca39a615] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_security_groups_create_list_delete[id-eb2b087d-633d-4d0d-a7bd-9e6ba35b32de] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_server_security_groups[id-fe4abc0d-83f5-4c50-ad11-57a1127297a2] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_update_security_groups[id-7d4e1d3c-3209-4d6d-b020-986304ebad1f] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_nonexistent_security_group[id-6727c00b-214c-4f9e-9a52-017ac3e98411] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_security_group_without_passing_id[id-1438f330-8fa4-4aeb-8a94-37c250106d7f] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_the_default_security_group[id-36a1629f-c6da-4a26-b8b8-55e7e5d5cd58] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_get_nonexistent_group[id-673eaec1-9b3e-48ed-bdf1-2786c1b9661c] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_non_existent_security_group[id-27edee9c-873d-4da6-a68a-3c256efebe8f] +tempest.api.compute.servers.test_availability_zone.AZV2TestJSON.test_get_availability_zone_list_with_non_admin_user[id-a8333aa2-205c-449f-a828-d38c2489bf25] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_active_server[id-925fdfb4-5b13-47ea-ac8a-c36ae6fddb05] +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action[id-aacc71ca-1d70-4aa5-bbf6-0ff71470e43c] +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions[id-77ca5cc5-9990-45e0-ab98-1de8fead201a] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor[id-80c574cc-0925-44ba-8602-299028357dd9] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image[id-b3304c3b-97df-46d2-8cd3-e2b6659724e7] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name[id-f9eb2b70-735f-416c-b260-9914ac6181e4] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status[id-de2612ab-b7dd-4044-b0b1-d2539601911f] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results[id-67aec2d0-35fe-4503-9f92-f13272b867ed] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_active_status[id-ca78e20e-fddb-4ce6-b7f7-bcbf8605e66e] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor[id-573637f5-7325-47bb-9144-3476d0416908] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image[id-05e8a8e7-9659-459a-989d-92c2f501f4ba] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit[id-614cdfc1-d557-4bac-915b-3e67b48eee76] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name[id-9b067a7b-7fee-4f6a-b29c-be43fe18fc5a] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_status[id-ca78e20e-fddb-4ce6-b7f7-bcbf8605e66e] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex[id-a905e287-c35e-42f2-b132-d02b09f3654a] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard[id-e9f624ee-92af-4562-8bec-437945a18dcb] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date[id-74745ad8-b346-45b5-b9b8-509d7447fc1f] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date[id-87d12517-e20a-4c9c-97b6-dd1628d6d6c9] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count[id-d47c17fb-eebd-4287-8e95-f20a7e627b18] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value[id-62610dd9-4713-4ee0-8beb-fd2c1aa7f950] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string[id-679bc053-5e70-4514-9800-3dfab1a380a6] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor[id-5913660b-223b-44d4-a651-a0fbfd44ca75] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image[id-ff01387d-c7ad-47b4-ae9e-64fa214638fe] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name[id-e2c77c4a-000a-4af3-a0bd-629a328bde7c] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted[id-93055106-2d34-46fe-af68-d9ddbf7ee570] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing[id-fcdf192d-0f74-4d89-911f-1ec002b822c4] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server[id-24a26f1a-1ddc-4eea-b0d7-a90cc874ad8f] +tempest.api.compute.servers.test_multiple_create.MultipleCreateTestJSON.test_multiple_create[id-61e03386-89c3-449c-9bb1-a06f423fd9d1] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server[id-80a8094c-211e-440a-ab88-9e59d556c7ee] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server[id-aaa6cdf3-55a7-461a-add9-1c8596b9a07c] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server[id-af8eafd4-38a7-4a4b-bdbc-75145a580560] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_check_tag_existence[id-81279a66-61c3-4759-b830-a2dbe64cbe08] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_create_delete_tag[id-8d95abe2-c658-4c42-9a44-c0258500306b] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_delete_all_tags[id-a63b2a74-e918-4b7c-bcab-10c855f3a57e] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_update_all_tags[id-a2c1af8c-127d-417d-974b-8115f7e3d831] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password[id-b92d5ec7-b1dd-44a2-87e4-45e888c46ef0] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair[id-f9e15296-d7f9-4e62-b53f-a04e89160833] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name[id-8fea6be7-065e-47cf-89b8-496e6f96c699] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address[id-89b90870-bc13-4b73-96af-f9d4f2b70077] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name[id-5e6ccff8-349d-4852-a8b3-055df7988dd2] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name[id-fd57f159-68d6-4c2a-902b-03070828a87e] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit[id-7fc74810-0bd2-4cd7-8244-4f33a9db865a] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256[id-c3e0fb12-07fc-4d76-a22e-37409887afe8] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor[id-18f5227f-d155-4429-807c-ccb103887537] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image[id-fcba1052-0a50-4cf3-b1ac-fae241edf02f] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid[id-4e72dc2d-44c5-4336-9667-f7972e95c402] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit[id-f4d7279b-5fd2-4bf2-9ba4-ae35df0d18c5] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id[id-75f79124-277c-45e6-a373-a1d6803f4cc4] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server[id-3436b02f-1b1e-4f03-881e-c6a602327439] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address[id-5226dd80-1e9c-4d8a-b5f9-b26ca4763fd0] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server[id-d4c023a0-9c55-4747-9dd5-413b820143c7] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server[id-98fa0458-1485-440f-873b-fe7f0d714930] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server[id-d86141a7-906e-4731-b187-d64a2ea61422] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_reboot_deleted_server[id-98fa0458-1485-440f-873b-fe7f0d714930] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank[id-dbbfd247-c40c-449e-8f6c-d2aa7c7da7cf] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server[id-a31460a9-49e1-42aa-82ee-06e0bb7c2d03] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server[id-aa8eed43-e2cb-4ebf-930b-da14f6a21d81] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256[id-5c8e244c-dada-4590-9944-749c455b431f] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name[id-38204696-17c6-44da-9590-40f87fb5a899] +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas[id-9bfecac7-b966-4f47-913f-1a9e2c12134a] +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas[id-f1ef0a97-dbbb-4cca-adc5-c9fbc4f76107] +tempest.api.compute.test_versions.TestVersions.test_list_api_versions[id-6c0a0990-43b6-4529-9b61-5fd8daf7c55c] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume[id-52e9045a-e90d-4c0d-9087-79d657faffff] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments[id-7fa563fe-f0f7-43eb-9e22-a1ece036b513] +tempest.api.identity.v3.TestApiDiscovery.test_api_media_types[id-657c1970-4722-4189-8831-7325f3bc4265] +tempest.api.identity.v3.TestApiDiscovery.test_api_version_resources[id-b9232f5e-d9e5-4d97-b96c-28d3db4de1bd] +tempest.api.identity.v3.TestApiDiscovery.test_api_version_statuses[id-8879a470-abfb-47bb-bb8d-5a7fd279ad1e] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types[id-657c1970-4722-4189-8831-7325f3bc4265] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources[id-b9232f5e-d9e5-4d97-b96c-28d3db4de1bd] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses[id-8879a470-abfb-47bb-bb8d-5a7fd279ad1e] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_list_api_versions[id-721f480f-35b6-46c7-846e-047e6acea0dc] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential[id-8080c75c-eddc-4786-941a-c2da7039ae61] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_expires[id-852daf0c-42b5-4239-8466-d193d0543ed3] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_list_application_credentials[id-ff0cd457-6224-46e7-b79e-0ada4964a8a6] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_query_application_credentials[id-9bb5e5cc-5250-493a-8869-8b665f6aa5f6] +tempest.api.identity.v3.test_catalog.IdentityCatalogTest.test_catalog_standardization[id-56b57ced-22b8-4127-9b8a-565dfb0207e2] +tempest.api.identity.v3.test_domains.DefaultDomainTestJSON.test_default_domain_exists[id-17a5de24-e6a0-4e4a-a9ee-d85b6e5612b5] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token[id-6f8e4436-fc96-4282-8122-e41df57197a9] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_token_auth_creation_existence_deletion[id-0f9f5a5f-d5cd-4a86-8a5b-c5ded151f212] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_validate_token[id-a9512ac3-3909-48a4-b395-11f438e16260] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_password_history_check_self_service_api[id-941784ee-5342-4571-959b-b80dd2cea516] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_user_account_lockout[id-a7ad8bbf-2cff-4520-8c1d-96332e151658] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image[id-f848bb94-1c6e-45a4-8726-39e3a5b23535] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_register_upload_get_image_file[id-139b765e-7f3d-4b3d-8b37-3ca3876ee318] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image[id-f66891a7-a35c-41a8-b590-a065c2a1caa6] +tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema[id-622b925c-479f-4736-860d-adeaf13bc371] +tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema[id-25c8d7b2-df21-460f-87ac-93130bcdc684] +tempest.api.image.v2.test_images.ListImagesTest.test_index_no_params[id-1e341d7a-90a9-494c-b143-2cdf2aeb6aee] +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format[id-9959ca1d-1aa7-4b7a-a1ea-0fff0499b37e] +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format[id-4a4735a7-f22f-49b6-b0d9-66e1ef7453eb] +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_limit[id-e914a891-3cc8-4b40-ad32-e0a39ffbddbb] +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_min_max_size[id-4ad8c157-971a-4ba8-aa84-ed61154b1e7f] +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_size[id-cf1b9a48-8340-480e-af7b-fe7e17690876] +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_status[id-7fc9e369-0f58-4d05-9aa5-0969e2d59d15] +tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_visibility[id-7a95bb92-d99e-4b12-9718-7bc6ab73e6d2] +tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params[id-1e341d7a-90a9-494c-b143-2cdf2aeb6aee] +tempest.api.image.v2.test_images.ListUserImagesTest.test_get_image_schema[id-622b925c-479f-4736-860d-adeaf13bc371] +tempest.api.image.v2.test_images.ListUserImagesTest.test_get_images_schema[id-25c8d7b2-df21-460f-87ac-93130bcdc684] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_container_format[id-9959ca1d-1aa7-4b7a-a1ea-0fff0499b37e] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_disk_format[id-4a4735a7-f22f-49b6-b0d9-66e1ef7453eb] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_limit[id-e914a891-3cc8-4b40-ad32-e0a39ffbddbb] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_min_max_size[id-4ad8c157-971a-4ba8-aa84-ed61154b1e7f] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_size[id-cf1b9a48-8340-480e-af7b-fe7e17690876] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_status[id-7fc9e369-0f58-4d05-9aa5-0969e2d59d15] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_visibility[id-7a95bb92-d99e-4b12-9718-7bc6ab73e6d2] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_no_params[id-1e341d7a-90a9-494c-b143-2cdf2aeb6aee] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id[id-32248db1-ab88-4821-9604-c7c369f1f88c] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image[id-6fe40f1c-57bd-4918-89cc-8500f850f3de] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image[id-e57fc127-7ba0-4693-92d7-1d8a05ebcba9] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id[id-ef45000d-0a72-4781-866d-4cb7bf2562ad] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image[id-668743d5-08ad-4480-b2b8-15da34f81d9f] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_container_format[id-292bd310-369b-41c7-a7a3-10276ef76753] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_disk_format[id-70c6040c-5a97-4111-9e13-e73665264ce1] +tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image[id-10407036-6059-4f95-a2cd-cbbbee7ed329] +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag[id-39c023a2-325a-433a-9eea-649bf1414b19] +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image[id-8cd30f82-6f9a-4c6e-8034-c1b51fba43d9] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful[id-4ab211a0-276f-4552-9070-51e27f58fecf] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips[id-51a5e97f-f02e-4e4e-9a17-a69811d300e3] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips_duplicate[id-57b8302b-cba9-4fbb-8835-9168df029051] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips_outrange[id-98244d88-d990-4570-91d4-6b25d70d08af] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_router[id-e98f65db-68f4-4330-9fea-abd8c5192d4d] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_64_subnets[id-4256c61d-c538-41ea-9147-3c450c36669e] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_invalid_options[id-81f18ef6-95b5-4584-9966-10d480b7496a] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_eui64[id-e5517e62-6f16-430d-a672-f80875493d4c] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_no_ra[id-ae2f4a5d-03ff-4c42-a3b0-ce2fcb7ea832] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_no_ra_no_dhcp[id-21635b6f-165a-4d42-bf49-7d195e47342f] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_two_subnets[id-4544adf7-bb5f-4bdc-b769-b3e77026cef2] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_floating_ip_specifying_a_fixed_ip_address[id-36de4bd0-f09c-43e3-a8e1-1decc1ffd3a5] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_with_port_multiple_ip_address[id-45c4c683-ea97-41ef-9c51-5e9802f2f3d7] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_floating_ip_delete_port[id-e1f6bffd-442f-4668-b30e-df13f2705e77] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_floating_ip_update_different_router[id-1bb2f731-fe5a-4b8c-8409-799ade1bed4d] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_associate_floatingip_port_ext_net_unreachable[id-6b3b8797-6d43-4191-985c-c48b773eb429] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_create_floatingip_in_private_network[id-50b9aeb4-9f0b-48ee-aa31-fa955a48ff54] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_create_floatingip_with_port_ext_net_unreachable[id-22996ea8-4a81-4b27-b6e1-fa5df92fa5e8] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes[id-a4d9ec4c-0306-4111-a75c-db01a709030b] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools[id-bec949c4-3147-4ba6-af5f-cd2306118404] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled[id-94ce038d-ff0a-4a4c-a56b-09da3ca0b55d] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw[id-9393b468-186d-496d-aa36-732348cd76e7] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools[id-8217a149-0c6c-4cfb-93db-0486f707d13f] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers[id-d830de0a-be47-468f-8f02-1fd996118289] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway[id-d2d596e2-8e76-47a9-ac51-d4648009f4d3] +tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet[id-0e269138-0da6-4efc-a46d-578161e7b221] +tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet[id-f04f61a9-b7f3-4194-90b2-9bcf660d1bfe] +tempest.api.network.test_networks.NetworksTest.test_external_network_visibility[id-af774677-42a9-4e4b-bb58-16fe6a5bc1ec] +tempest.api.network.test_networks.NetworksTest.test_list_networks[id-f7ffdeda-e200-4a7a-bcbe-05716e86bf43] +tempest.api.network.test_networks.NetworksTest.test_list_networks_fields[id-6ae6d24f-9194-4869-9c85-c313cb20e080] +tempest.api.network.test_networks.NetworksTest.test_list_subnets[id-db68ba48-f4ea-49e9-81d1-e367f6d0b20a] +tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields[id-842589e3-9663-46b0-85e4-7f01273b0412] +tempest.api.network.test_networks.NetworksTest.test_show_network[id-2bf13842-c93f-4a69-83ed-717d2ec3b44e] +tempest.api.network.test_networks.NetworksTest.test_show_network_fields[id-867819bb-c4b6-45f7-acf9-90edcf70aa5e] +tempest.api.network.test_networks.NetworksTest.test_show_subnet[id-bd635d81-6030-4dd1-b3b9-31ba0cfdf6cc] +tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields[id-270fff0b-8bfc-411f-a184-1e8fd35286f0] +tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp[id-3d3852eb-3009-49ec-97ac-5ce83b73010a] +tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_all_attributes[id-a4d9ec4c-0306-4111-a75c-db01a709030b] +tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_allocation_pools[id-bec949c4-3147-4ba6-af5f-cd2306118404] +tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_dhcp_enabled[id-94ce038d-ff0a-4a4c-a56b-09da3ca0b55d] +tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_gw[id-9393b468-186d-496d-aa36-732348cd76e7] +tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_gw_and_allocation_pools[id-8217a149-0c6c-4cfb-93db-0486f707d13f] +tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_with_host_routes_and_dns_nameservers[id-d830de0a-be47-468f-8f02-1fd996118289] +tempest.api.network.test_networks.NetworksTestJSON.test_create_delete_subnet_without_gateway[id-d2d596e2-8e76-47a9-ac51-d4648009f4d3] +tempest.api.network.test_networks.NetworksTestJSON.test_create_update_delete_network_subnet[id-0e269138-0da6-4efc-a46d-578161e7b221] +tempest.api.network.test_networks.NetworksTestJSON.test_delete_network_with_subnet[id-f04f61a9-b7f3-4194-90b2-9bcf660d1bfe] +tempest.api.network.test_networks.NetworksTestJSON.test_external_network_visibility[id-af774677-42a9-4e4b-bb58-16fe6a5bc1ec] +tempest.api.network.test_networks.NetworksTestJSON.test_list_networks[id-f7ffdeda-e200-4a7a-bcbe-05716e86bf43] +tempest.api.network.test_networks.NetworksTestJSON.test_list_networks_fields[id-6ae6d24f-9194-4869-9c85-c313cb20e080] +tempest.api.network.test_networks.NetworksTestJSON.test_list_subnets[id-db68ba48-f4ea-49e9-81d1-e367f6d0b20a] +tempest.api.network.test_networks.NetworksTestJSON.test_list_subnets_fields[id-842589e3-9663-46b0-85e4-7f01273b0412] +tempest.api.network.test_networks.NetworksTestJSON.test_show_network[id-2bf13842-c93f-4a69-83ed-717d2ec3b44e] +tempest.api.network.test_networks.NetworksTestJSON.test_show_network_fields[id-867819bb-c4b6-45f7-acf9-90edcf70aa5e] +tempest.api.network.test_networks.NetworksTestJSON.test_show_subnet[id-bd635d81-6030-4dd1-b3b9-31ba0cfdf6cc] +tempest.api.network.test_networks.NetworksTestJSON.test_show_subnet_fields[id-270fff0b-8bfc-411f-a184-1e8fd35286f0] +tempest.api.network.test_networks.NetworksTestJSON.test_update_subnet_gw_dns_host_routes_dhcp[id-3d3852eb-3009-49ec-97ac-5ce83b73010a] +tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port[id-67f1b811-f8db-43e2-86bd-72c074d4a42c] +tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools[id-0435f278-40ae-48cb-a404-b8a087bc09b1] +tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port[id-c72c1c0c-2193-4aca-aaa4-b1442640f51c] +tempest.api.network.test_ports.PortsTestJSON.test_list_ports[id-cf95b358-3e92-4a29-a148-52445e1ac50e] +tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields[id-ff7f117f-f034-4e0e-abff-ccef05c454b4] +tempest.api.network.test_ports.PortsTestJSON.test_port_list_filter_by_router_id[id-5ad01ed0-0e6e-4c5d-8194-232801b15c72] +tempest.api.network.test_ports.PortsTestJSON.test_show_port[id-c9a685bd-e83f-499c-939f-9f7863ca259f] +tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields[id-45fcdaf2-dab0-4c13-ac6c-fcddfb579dbd] +tempest.api.network.test_routers.RoutersTest.test_add_multiple_router_interfaces[id-802c73c9-c937-4cef-824b-2191e24a6aab] +tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_port_id[id-2b7d2f37-6748-4d78-92e5-1d590234f0d5] +tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_subnet_id[id-b42e6e39-2e37-49cc-a6f4-8467e940900a] +tempest.api.network.test_routers.RoutersTest.test_create_show_list_update_delete_router[id-f64403e2-8483-4b34-8ccd-b09a87bcc68c] +tempest.api.network.test_routers.RoutersTest.test_router_interface_port_update_with_fixed_ip[id-96522edf-b4b5-45d9-8443-fa11c26e6eff] +tempest.api.network.test_routers.RoutersTest.test_update_delete_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +tempest.api.network.test_routers.RoutersTest.test_update_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +tempest.api.network.test_routers.RoutersTest.test_update_router_admin_state[id-a8902683-c788-4246-95c7-ad9c6d63a4d9] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_add_router_interfaces_on_overlapping_subnets_returns_400[id-957751a3-3c68-4fa2-93b6-eb52ea10db6e] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_delete_non_existent_router_returns_404[id-c7edc5ad-d09d-41e6-a344-5c0c31e2e3e4] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_add_gateway_invalid_network_returns_404[id-37a94fc0-a834-45b9-bd23-9a81d2fd1e22] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_add_gateway_net_not_external_returns_400[id-11836a18-0b15-4327-a50b-f0d9dc66bddd] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_remove_interface_in_use_returns_409[id-04df80f9-224d-47f5-837a-bf23e33d1c20] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_show_non_existent_router_returns_404[id-c2a70d72-8826-43a7-8208-0209e6360c47] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_update_non_existent_router_returns_404[id-b23d1569-8b0c-4169-8d4b-6abd34fad5c7] +tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group[id-bfd128e5-3c92-44b6-9d66-7fe29d22c802] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args[id-87dfbcf9-1849-43ea-b1e4-efa3eeae9f71] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code[id-c9463db8-b44d-4f52-b6c0-8dbda99f26ce] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value[id-0a307599-6655-4220-bebc-fd70c64f2290] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id[id-c2ed2deb-7a0c-44d8-8b4c-a5825b5c310b] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix[id-16459776-5da2-4634-bce4-4b55ee3ec188] +tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule[id-cfb99e0e-7410-4a3d-8a0c-959a63ee77e9] +tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups[id-e30abd17-fef9-4739-8617-dc26da88e686] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails[id-2323061e-9fbf-4eb0-b547-7e8fafc90849] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails[id-8fde898f-ce88-493b-adc9-4e4692879fc5] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype[id-5666968c-fff3-40d6-9efc-df1c8bd01abb] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol[id-981bdc22-ce48-41ed-900a-73148b583958] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix[id-5f8daf69-3c5f-4aaa-88c9-db1d66f68679] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports[id-0d9c7791-f2ad-4e2f-ac73-abf2373b0d2d] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid[id-4bf786fd-2f02-443c-9716-5b98e159a49a] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group[id-be308db6-a7cf-4d5c-9baf-71bafd73f35e] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group[id-1f1bb89d-5664-4956-9fcd-83ee0fa603df] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group[id-424fd5c3-9ddc-486a-b45f-39bf0c820fc6] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule[id-4c094c09-000b-4e41-8100-9617600c02a6] +tempest.api.network.test_subnetpools_extensions.SubnetPoolsTestJSON.test_create_list_show_update_delete_subnetpools[id-62595970-ab1c-4b7f-8fcc-fddfe55e9811] +tempest.api.network.test_versions.NetworksApiDiscovery.test_api_version_resources[id-cac8a836-c2e0-4304-b556-cd299c7281d1] +tempest.api.object_storage.test_account_bulk.BulkTest.test_bulk_delete[id-c075e682-0d2a-43b2-808d-4116200d736d] +tempest.api.object_storage.test_account_bulk.BulkTest.test_extract_archive[id-a407de51-1983-47cc-9f14-47c2b059413c] +tempest.api.object_storage.test_account_quotas.AccountQuotasTest.test_upload_valid_object[id-a22ef352-a342-4587-8f47-3bbdb5b039c4] +tempest.api.object_storage.test_account_quotas_negative.AccountQuotasNegativeTest.test_user_modify_quota[id-d1dc5076-555e-4e6d-9697-28f1fe976324] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers[id-3499406a-ae53-4f8c-b43a-133d4dc6fe3f] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_end_marker[id-5ca164e4-7bde-43fa-bafb-913b53b9e786] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_format_json[id-1c7efa35-e8a2-4b0b-b5ff-862c7fd83704] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit[id-5cfa4ab2-4373-48dd-a41f-a532b12b08b2] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_end_marker[id-888a3f0e-7214-4806-8e50-5e0c9a69bb5e] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_marker[id-f7064ae8-dbcc-48da-b594-82feef6ea5af] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_marker_and_end_marker[id-8cf98d9c-e3a0-4e44-971b-c87656fdddbd] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_marker[id-638f876d-6a43-482a-bbb3-0840bca101c6] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_marker_and_end_marker[id-ac8502c2-d4e4-4f68-85a6-40befea2ef5e] +tempest.api.object_storage.test_account_services.AccountTest.test_list_extensions[id-6eb04a6a-4860-4e31-ba91-ea3347d76b58] +tempest.api.object_storage.test_account_services.AccountTest.test_list_extensions[id-6eb04a6a-4860-4e31-ba91-ea3347d76b58] +tempest.api.object_storage.test_account_services.AccountTest.test_list_no_account_metadata[id-b904c2e3-24c2-4dba-ad7d-04e90a761be5] +tempest.api.object_storage.test_account_services.AccountTest.test_list_no_containers[id-884ec421-fbad-4fcc-916b-0580f2699565] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_create_metadata_key[id-64fd53f3-adbd-4639-af54-436e4982dbfb] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_delete_metadata[id-9f60348d-c46f-4465-ae06-d51dbd470953] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_delete_metadata_key[id-d4d884d3-4696-4b85-bc98-4f57c4dd2bf1] +tempest.api.object_storage.test_container_acl.ObjectTestACLs.test_read_object_with_rights[id-a3270f3f-7640-4944-8448-c7ea783ea5b6] +tempest.api.object_storage.test_container_acl.ObjectTestACLs.test_write_object_with_rights[id-aa58bfa5-40d9-4bc3-82b4-d07f4a9e392a] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_large_object[id-22eeeb2b-3668-4160-baef-44790f65a5a0] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_too_many_objects[id-3a387039-697a-44fc-a9c0-935de31f426b] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_valid_object[id-9a0fb034-86af-4df0-86fa-f8bd7db21ae0] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container[id-92139d73-7819-4db1-85f8-3f2f22a8d91f] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_overwrite[id-49f866ed-d6af-4395-93e7-4187eb56d322] +tempest.api.object_storage.test_container_services.ContainerTest.test_delete_container[id-95d3a249-b702-4082-a2c4-14bb860cf06a] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents[id-312ff6bd-5290-497f-bda1-7c5fec6697ab] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_delimiter[id-fe323a32-57b9-4704-a996-2e68f83b09bc] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_end_marker[id-55b4fa5c-e12e-4ca9-8fcf-a79afe118522] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_format_json[id-196f5034-6ab0-4032-9da9-a937bbb9fba9] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_limit[id-297ec38b-2b61-4ff4-bcd1-7fa055e97b61] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_marker[id-c31ddc63-2a58-4f6b-b25c-94d2937e6867] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_no_object[id-4646ac2d-9bfb-4c7d-a3c5-0f527402b3df] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_path[id-58ca6cc9-6af0-408d-aaec-2a6a7b2f0df9] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_prefix[id-77e742c7-caf2-4ec9-8aa4-f7d509a3344c] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_metadata[id-96e68f0e-19ec-4aa2-86f3-adc6a45e14dd] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_no_container_metadata[id-a2faf936-6b13-4f8d-92a2-c2278355821e] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_and_delete_metadata[id-cf19bc0b-7e16-4a5a-aaed-cb0c2fe8deef] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_metadata[id-2ae5f295-4bf1-4e04-bfad-21e54b62cec5] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_metadata_key[id-31f40a5f-6a52-4314-8794-cd89baed3040] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_delete_metadata[id-3a5ce7d4-6e4b-47d0-9d87-7cd42c325094] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_delete_metadata_key[id-a2e36378-6f1f-43f4-840a-ffd9cfd61914] +tempest.api.object_storage.test_object_expiry.ObjectExpiryTest.test_get_object_after_expiry_time[id-fb024a42-37f3-4ba5-9684-4f40a7910b41] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_2d_way[id-06f90388-2d0e-40aa-934c-e9a8833e958a] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_across_containers[id-aa467252-44f3-472a-b5ae-5b57c3c9c147] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_in_same_container[id-1a9ab572-1b66-4981-8c21-416e2a5e6011] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_to_itself[id-2248abba-415d-410b-9c30-22dff9cd6e67] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object[id-5b4ce26f-3545-46c9-a2ba-5754358a4c62] +tempest.api.object_storage.test_object_services.ObjectTest.test_delete_object[id-17738d45-03bd-4d45-9e0b-7b2f58f98687] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object[id-02610ba7-86b7-4272-9ed8-aa8d417cb3cd] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_if_different[id-50d01f12-526f-4360-9ac2-75dd508d7b68] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_x_object_manifest[id-e3e6a64a-9f50-4955-b987-6ce6767c97fb] +tempest.api.object_storage.test_object_services.ObjectTest.test_list_object_metadata_with_x_object_manifest[id-23a3674c-d6de-46c3-86af-ff92bfc8a3da] +tempest.api.object_storage.test_object_services.ObjectTest.test_object_upload_in_segments[id-e3e6a64a-9f50-4955-b987-6ce6767c97fb] +tempest.api.object_storage.test_object_services.ObjectTest.test_object_upload_in_segments[id-e3e6a64a-9f50-4955-b987-6ce6767c97fb] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_delete_large_object[id-87b6dfa1-abe9-404d-8bf0-6c3751e6aa77] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_list_large_object_metadata[id-e69ad766-e1aa-44a2-bdd2-bf62c09c1456] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_retrieve_large_object[id-49bc49bc-dd1b-4c0f-904e-d9f10b830ee8] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_upload_manifest[id-2c3f24a6-36e8-4711-9aa2-800ee1fc7b5b] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_get_object_using_temp_url[id-f91c96d4-1230-4bba-8eb9-84476d18d991] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_put_object_using_temp_url[id-9b08dade-3571-4152-8a4f-a4f2a873a735] +tempest.api.object_storage.test_object_version.ContainerTest.test_versioned_container[id-a151e158-dcbf-4a1f-a1e7-46cd65895a6f] +tempest.api.volume.test_availability_zone.AvailabilityZoneTestJSON.test_get_availability_zone_list[id-01f1ae88-eba9-4c6b-a011-6f7ace06b725] +tempest.api.volume.test_availability_zone.AvailabilityZoneV2TestJSON.test_get_availability_zone_list[id-01f1ae88-eba9-4c6b-a011-6f7ace06b725] +tempest.api.volume.test_extensions.ExtensionsTestJSON.test_list_extensions[id-94607eb0-43a5-47ca-82aa-736b41bd2e2c] +tempest.api.volume.test_extensions.ExtensionsV2TestJSON.test_list_extensions[id-94607eb0-43a5-47ca-82aa-736b41bd2e2c] +tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_crud_snapshot_metadata[id-a2f20f99-e363-4584-be97-bc33afb1a56c] +tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_update_show_snapshot_metadata_item[id-e8ff85c5-8f97-477f-806a-3ac364a949ed] +tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_update_snapshot_metadata_item[id-e8ff85c5-8f97-477f-806a-3ac364a949ed] +tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_create_get_delete_snapshot_metadata[id-a2f20f99-e363-4584-be97-bc33afb1a56c] +tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_crud_snapshot_metadata[id-a2f20f99-e363-4584-be97-bc33afb1a56c] +tempest.api.volume.test_snapshot_metadata.SnapshotV2MetadataTestJSON.test_update_snapshot_metadata_item[id-e8ff85c5-8f97-477f-806a-3ac364a949ed] +tempest.api.volume.test_versions.VersionsTest.test_list_versions[id-77838fc4-b49b-4c64-9533-166762517369] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_create_get_delete_volume_metadata[id-6f5b125b-f664-44bf-910f-751591fe5769] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_crud_volume_metadata[id-6f5b125b-f664-44bf-910f-751591fe5769] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_update_show_volume_metadata_item[id-862261c5-8df4-475a-8c21-946e50e36a20] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_update_volume_metadata_item[id-862261c5-8df4-475a-8c21-946e50e36a20] +tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_create_get_delete_volume_metadata[id-6f5b125b-f664-44bf-910f-751591fe5769] +tempest.api.volume.test_volume_metadata.VolumesV2MetadataTest.test_update_volume_metadata_item[id-862261c5-8df4-475a-8c21-946e50e36a20] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_bootable[id-63e21b4c-0a0c-41f6-bfc3-7c2816815599] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_readonly_update[id-fff74e1e-5bd3-4b33-9ea9-24c103bc3f59] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_upload[id-d8f1ca95-3d5b-44a3-b8ca-909691c9532d] +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_bootable[id-63e21b4c-0a0c-41f6-bfc3-7c2816815599] +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_readonly_update[id-fff74e1e-5bd3-4b33-9ea9-24c103bc3f59] +tempest.api.volume.test_volumes_actions.VolumesV2ActionsTest.test_volume_upload[id-d8f1ca95-3d5b-44a3-b8ca-909691c9532d] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete[id-27fb0e9f-fb64-41dd-8bdb-1ffa762f0d51] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_as_clone[id-3f591b4a-7dc6-444c-bd51-77469506b3a1] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_from_image[id-54a01030-c7fc-447c-86ee-c1182beae638] +tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete[id-27fb0e9f-fb64-41dd-8bdb-1ffa762f0d51] +tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_as_clone[id-3f591b4a-7dc6-444c-bd51-77469506b3a1] +tempest.api.volume.test_volumes_get.VolumesV2GetTest.test_volume_create_get_update_delete_from_image[id-54a01030-c7fc-447c-86ee-c1182beae638] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list[id-0b6ddd39-b948-471f-8038-4787978747c4] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_by_name[id-a28e8da4-0b56-472f-87a8-0f4d3f819c02] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_by_name[id-2de3a6d4-12aa-403b-a8f2-fdeb42a89623] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_pagination[id-e9138a2c-f67b-4796-8efa-635c196d01de] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_with_multiple_params[id-2a7064eb-b9c3-429b-b888-33928fc5edd3] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_pagination[id-af55e775-8e4b-4feb-8719-215c43b0238c] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_param_display_name_and_status[id-777c87c1-2fc4-4883-8b8e-5c0b951d1ec8] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_display_name_and_status[id-856ab8ca-6009-4c37-b691-be1065528ad4] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_metadata[id-1ca92d3c-4a8e-4b43-93f5-e4c7fb3b291d] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_details[id-adcbb5a7-5ad8-4b61-bd10-5380e111a877] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_param_metadata[id-b5ebea1b-0603-40a0-bb41-15fcd0a53214] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_availability_zone[id-c0cfa863-3020-40d7-b587-e35f597d5d87] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_status[id-39654e13-734c-4dab-95ce-7613bf8407ce] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_availability_zone[id-e1b80d13-94f0-4ba2-a40e-386af29f8db1] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_status[id-2943f712-71ec-482a-bf49-d5ca06216b9f] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list[id-0b6ddd39-b948-471f-8038-4787978747c4] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_by_name[id-a28e8da4-0b56-472f-87a8-0f4d3f819c02] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_by_name[id-2de3a6d4-12aa-403b-a8f2-fdeb42a89623] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_param_display_name_and_status[id-777c87c1-2fc4-4883-8b8e-5c0b951d1ec8] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_display_name_and_status[id-856ab8ca-6009-4c37-b691-be1065528ad4] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_detail_param_metadata[id-1ca92d3c-4a8e-4b43-93f5-e4c7fb3b291d] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_details[id-adcbb5a7-5ad8-4b61-bd10-5380e111a877] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_with_param_metadata[id-b5ebea1b-0603-40a0-bb41-15fcd0a53214] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_availability_zone[id-c0cfa863-3020-40d7-b587-e35f597d5d87] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_by_status[id-39654e13-734c-4dab-95ce-7613bf8407ce] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_availability_zone[id-e1b80d13-94f0-4ba2-a40e-386af29f8db1] +tempest.api.volume.test_volumes_list.VolumesV2ListTestJSON.test_volumes_list_details_by_status[id-2943f712-71ec-482a-bf49-d5ca06216b9f] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_invalid_size[id-1ed83a8a-682d-4dfb-a30e-ee63ffd6c049] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_snapshot_id[id-0c36f6ae-4604-4017-b0a9-34fdc63096f9] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_source_volid[id-47c73e08-4be8-45bb-bfdf-0c4e79b88344] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_volume_type[id-10254ed8-3849-454e-862e-3ab8e6aa01d2] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_negative[id-8b472729-9eba-446e-a83b-916bdb34bef7] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_zero[id-41331caa-eaf4-4001-869d-bc18c1869360] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_without_passing_size[id-9387686f-334f-4d31-a439-33494b9e2683] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_invalid_volume_id[id-1f035827-7c32-4019-9240-b4ec2dbd9dfd] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_volume_without_passing_volume_id[id-441a1550-5d44-4b30-af0f-a6d402f52026] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_invalid_volume_id[id-30799cfd-7ee4-446c-b66c-45b383ed211b] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_volume_without_passing_volume_id[id-c6c3db06-29ad-4e91-beb0-2ab195fe49e3] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_invalid_status[id-ba94b27b-be3f-496c-a00e-0283b373fa75] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_nonexistent_name[id-9ca17820-a0e7-4cbd-a7fa-f4468735e359] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_invalid_status[id-143b279b-7522-466b-81be-34a87d564a7c] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_nonexistent_name[id-0f4aa809-8c7b-418f-8fb3-84c7a5dfc52f] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_empty_volume_id[id-72aeca85-57a5-4c1f-9057-f320f9ea575b] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_invalid_volume_id[id-e66e40d6-65e6-4e75-bdc7-636792fa152d] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_nonexistent_volume_id[id-0186422c-999a-480e-a026-6a665744c30c] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_delete_nonexistent_volume_id[id-555efa6e-efcd-44ef-8a3b-4a7ca4837a29] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_get_nonexistent_volume_id[id-f131c586-9448-44a4-a8b0-54ca838aa43e] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_invalid_size[id-1ed83a8a-682d-4dfb-a30e-ee63ffd6c049] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_snapshot_id[id-0c36f6ae-4604-4017-b0a9-34fdc63096f9] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_source_volid[id-47c73e08-4be8-45bb-bfdf-0c4e79b88344] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_nonexistent_volume_type[id-10254ed8-3849-454e-862e-3ab8e6aa01d2] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_out_passing_size[id-9387686f-334f-4d31-a439-33494b9e2683] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_negative[id-8b472729-9eba-446e-a83b-916bdb34bef7] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_with_size_zero[id-41331caa-eaf4-4001-869d-bc18c1869360] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_create_volume_without_passing_size[id-9387686f-334f-4d31-a439-33494b9e2683] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_invalid_volume_id[id-1f035827-7c32-4019-9240-b4ec2dbd9dfd] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_delete_volume_without_passing_volume_id[id-441a1550-5d44-4b30-af0f-a6d402f52026] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_invalid_volume_id[id-30799cfd-7ee4-446c-b66c-45b383ed211b] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_get_volume_without_passing_volume_id[id-c6c3db06-29ad-4e91-beb0-2ab195fe49e3] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_invalid_status[id-ba94b27b-be3f-496c-a00e-0283b373fa75] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_detail_with_nonexistent_name[id-9ca17820-a0e7-4cbd-a7fa-f4468735e359] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_invalid_status[id-143b279b-7522-466b-81be-34a87d564a7c] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_list_volumes_with_nonexistent_name[id-0f4aa809-8c7b-418f-8fb3-84c7a5dfc52f] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_empty_volume_id[id-72aeca85-57a5-4c1f-9057-f320f9ea575b] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_invalid_volume_id[id-e66e40d6-65e6-4e75-bdc7-636792fa152d] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_update_volume_with_nonexistent_volume_id[id-0186422c-999a-480e-a026-6a665744c30c] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_delete_nonexistent_volume_id[id-555efa6e-efcd-44ef-8a3b-4a7ca4837a29] +tempest.api.volume.test_volumes_negative.VolumesV2NegativeTest.test_volume_get_nonexistent_volume_id[id-f131c586-9448-44a4-a8b0-54ca838aa43e] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_get_list_update_delete[id-2a8abbe4-d871-46db-b049-c41f5af8216e] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_volume_from_snapshot[id-677863d1-3142-456d-b6ac-9924f667a7f4] +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshot_create_get_list_update_delete[id-2a8abbe4-d871-46db-b049-c41f5af8216e] +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_details_with_params[id-220a1022-1fcd-4a74-a7bd-6b859156cda2] +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_snapshots_list_with_params[id-59f41f43-aebf-48a9-ab5d-d76340fab32b] +tempest.api.volume.test_volumes_snapshots.VolumesV2SnapshotTestJSON.test_volume_from_snapshot[id-677863d1-3142-456d-b6ac-9924f667a7f4] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit[id-db4d8e0a-7a2e-41cc-a712-961f6844e896] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit_equals_infinite[id-a1427f61-420e-48a5-b6e3-0b394fa95400] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit_equals_zero[id-e3b44b7f-ae87-45b5-8a8c-66110eb24d0a] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_marker[id-05489dde-44bc-4961-a1f5-3ce7ee7824f7] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_offset[id-ca96d551-17c6-4e11-b0e8-52d3bb8a63c7] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_created_at_asc[id-4052c3a0-2415-440a-a8cc-305a875331b0] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_created_at_desc[id-dcbbe24a-f3c0-4ec8-9274-55d48db8d1cf] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_id_asc[id-c5513ada-64c1-4d28-83b9-af3307ec1388] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_id_desc[id-8a7fe058-0b41-402a-8afd-2dbc5a4a718b] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_name_asc[id-d58b5fed-0c37-42d3-8c5d-39014ac13c00] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_name_desc[id-96ba6f4d-1f18-47e1-b4bc-76edc6c21250] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_details_with_params[id-220a1022-1fcd-4a74-a7bd-6b859156cda2] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_with_params[id-59f41f43-aebf-48a9-ab5d-d76340fab32b] +tempest.api.volume.test_volumes_snapshots_list.VolumesV2SnapshotListTestJSON.test_snapshots_list_details_with_params[id-220a1022-1fcd-4a74-a7bd-6b859156cda2] +tempest.api.volume.test_volumes_snapshots_list.VolumesV2SnapshotListTestJSON.test_snapshots_list_with_params[id-59f41f43-aebf-48a9-ab5d-d76340fab32b] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id[id-e3e466af-70ab-4f4b-a967-ab04e3532ea7] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id[id-bb9da53e-d335-4309-9c15-7e76fd5e4d6d] +tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id[id-e3e466af-70ab-4f4b-a967-ab04e3532ea7] +tempest.api.volume.test_volumes_snapshots_negative.VolumesV2SnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id[id-bb9da53e-d335-4309-9c15-7e76fd5e4d6d] +tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_pagination[id-e9138a2c-f67b-4796-8efa-635c196d01de] +tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_details_with_multiple_params[id-2a7064eb-b9c3-429b-b888-33928fc5edd3] +tempest.api.volume.v2.test_volumes_list.VolumesV2ListTestJSON.test_volume_list_pagination[id-af55e775-8e4b-4feb-8719-215c43b0238c] diff --git a/.automation.conf/tempest/load-lists/tempest-full b/.automation.conf/tempest/load-lists/tempest-full new file mode 100644 index 000000000..c063d803e --- /dev/null +++ b/.automation.conf/tempest/load-lists/tempest-full @@ -0,0 +1,1601 @@ +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_upload_manifest[id-2c3f24a6-36e8-4711-9aa2-800ee1fc7b5b] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_reboot[id-db22b618-f157-4566-a317-1b6d467a8094,negative] +tempest.api.network.test_networks.NetworksTest.test_list_networks_fields[id-6ae6d24f-9194-4869-9c85-c313cb20e080] +tempest.api.compute.admin.test_flavors_extra_specs.FlavorsExtraSpecsTestJSON.test_flavor_set_get_update_show_unset_keys[id-0b2f9d4b-1ca2-4b99-bb40-165d4bb94208] +tempest.api.compute.servers.test_servers.ServerShowV247Test.test_show_server[id-88b0bdb2-494c-11e7-a919-92ebcb67fe33] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_for_non_existent_tenant[id-b2285aaa-9e76-4704-93a9-7a8acd0a6c8f,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_name_asc[id-d58b5fed-0c37-42d3-8c5d-39014ac13c00] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_user_update[id-b537d090-afb9-4519-b95d-270b0708e87e] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_update_quota_normal_user[id-733abfe8-166e-47bb-8363-23dbd7ff3476,negative] +tempest.api.image.v2.test_images.ListSharedImagesTest.test_list_images_param_member_status[id-3fa50be4-8e38-4c02-a8db-7811bb780122] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_snapshot_volume_backed_multiattach[id-885ac48a-2d7a-40c5-ae8b-1993882d724c,image] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_dhcpv6_stateless[compute,id-cf1c4425-766b-45b8-be35-e2959728eb00,network,slow] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_update_non_existent_port[id-cf8eef21-4351-4f53-adcd-cc5cb1e76b92,negative] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_with_repeated_name[id-969b10c7-3d77-4e1b-a4f2-2d265980f7e5,negative] +tempest.api.volume.admin.test_volume_type_access.VolumeTypesAccessTest.test_volume_type_access_list[id-5220eb28-a435-43ce-baaf-ed46f0e95159] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_update_user_password[id-1aeb25ac-6ec5-4d8b-97cb-7ac3567a989f] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_list_roles[id-f5654bcc-08c4-4f71-88fe-05d64e06de94] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_remote_group_id[id-c2ed2deb-7a0c-44d8-8b4c-a5825b5c310b] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_list_all_implied_roles[id-3748c316-c18f-4b08-997b-c60567bc6235] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_remove_host[id-c8e85064-e79b-4906-9931-c11c24294d02] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_get_details_as_user[id-557cad12-34c9-4ff4-95f0-22f0dfbaf7dc,negative] +tempest.api.network.test_routers.RoutersIpV6Test.test_update_delete_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +tempest.api.image.v2.test_images.ImportImagesTest.test_remote_delete[id-44d60544-1524-42f7-8899-315301105dd8] +tempest.scenario.test_shelve_instance.TestShelveInstance.test_cold_migrate_unshelved_instance[compute,id-1295fd9e-193a-4cf8-b211-55358e021bae,image,network,slow] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_associate_disassociate_qos[id-1dd93c76-6420-485d-a771-874044c416ac] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_remove_metadata_value[id-8a21ebad-a5c7-4e29-b428-384edc8cd156] +tempest.api.network.admin.test_routers_dvr.RoutersTestDVR.test_centralized_router_update_to_dvr[id-acd43596-c1fb-439d-ada8-31ad48ae3c2e] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_get_users_by_unauthorized_user[id-284192ce-fb7c-4909-a63b-9a502e0ddd11,negative] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_container_format[id-9959ca1d-1aa7-4b7a-a1ea-0fff0499b37e] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_subnets_fields[id-842589e3-9663-46b0-85e4-7f01273b0412] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_metadata_value_exceeds_max_length[id-81e36922-326b-4b7c-8155-3bbceecd7a82,negative] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_list_ec2_credentials[id-897813f0-160c-4fdc-aabc-24ee635ce4a9] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful[id-4ab211a0-276f-4552-9070-51e27f58fecf] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_update_all_quota_resources_for_tenant[id-55fbe2bf-21a9-435b-bbd2-4162b0ed799a] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_delete_non_empty_container[id-42da116e-1e8c-4c96-9e06-2f13884ed2b1,negative] +tempest.api.object_storage.test_object_temp_url_negative.ObjectTempUrlNegativeTest.test_get_object_after_expiration_time[id-5a583aca-c804-41ba-9d9a-e7be132bdf0b,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_range[id-05a1890e-7db9-4a6c-90a8-ce998a2bddfa] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_validate_token[id-a9512ac3-3909-48a4-b395-11f438e16260] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_update_delete_port[id-c72c1c0c-2193-4aca-aaa4-b1442640f51c,smoke] +tempest.api.compute.servers.test_instance_actions_negative.InstanceActionsNegativeTestJSON.test_get_instance_action_invalid_request[id-0269f40a-6f18-456c-b336-c03623c897f1,negative] +tempest.api.network.test_ports.PortsTestJSON.test_list_ports_fields[id-ff7f117f-f034-4e0e-abff-ccef05c454b4] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_slaac[compute,id-9178ad42-10e4-47e9-8987-e02b170cc5cd,network,slow] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_metadata_items_limit[id-d8c0a210-a5c3-4664-be04-69d96746b547,negative] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_create_delete_metering_label_rule_with_filters[id-f4d547cd-3aee-408f-bf36-454f8825e045] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_paused[id-1e107f21-61b2-4988-8f22-b196e938ab88] +tempest.api.volume.test_versions.VersionsTest.test_show_version[id-7f755ae2-caa9-4049-988c-331d8f7a579f] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_get_updated_quotas[id-ce9e0815-8091-4abd-8345-7fe5b85faa1d] +tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_crud_snapshot_metadata[id-a2f20f99-e363-4584-be97-bc33afb1a56c] +tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port[id-48037ff2-e889-4c3b-b86a-8e3f34d2d060,smoke] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_end_marker[id-5ca164e4-7bde-43fa-bafb-913b53b9e786] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_create_domain_config_and_show_config_groups_and_options[id-9e3ff13c-f597-4f01-9377-d6c06c2a1477] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_show_port_binding_ext_attr[id-b54ac0ff-35fc-4c79-9ca3-c7dbd4ea4f13] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_metadata[id-2ae5f295-4bf1-4e04-bfad-21e54b62cec5] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_create[id-850795d7-d4d3-4e55-b527-a774c0123d3a,smoke] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_assignments_for_implied_roles_create_delete[id-c8828027-df48-4021-95df-b65b92c7429e] +tempest.api.volume.admin.test_encrypted_volumes_extend.EncryptedVolumesExtendAttachedTest.test_extend_attached_encrypted_volume_luksv1[compute,id-e93243ec-7c37-4b5b-a099-ebf052c13216] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_empty_public_key[id-dade320e-69ca-42a9-ba4a-345300f127e0,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_container_format[id-2143655d-96d9-4bec-9188-8674206b4b3b] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_delete_users_by_unauthorized_user[id-d45195d5-33ed-41b9-a452-7d0d6a00f6e9,negative] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_create_domain_with_name_length_over_64[id-37b1bbf2-d664-4785-9a11-333438586eae,negative] +tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_list_services[id-e55908e8-360e-439e-8719-c3230a3e179e] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents[id-312ff6bd-5290-497f-bda1-7c5fec6697ab,smoke] +tempest.api.compute.admin.test_simple_tenant_usage_negative.TenantUsagesNegativeTestJSON.test_get_usage_tenant_with_empty_tenant_id[id-8b21e135-d94b-4991-b6e9-87059609c8ed,negative] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_remove_user_role[id-f0b9292c-d3ba-4082-aa6c-440489beef69] +tempest.api.network.test_routers.RoutersIpV6Test.test_create_show_list_update_delete_router[id-f64403e2-8483-4b34-8ccd-b09a87bcc68c,smoke] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_with_detail_param_offset_and_limit[id-06b6abc4-3f10-48e9-a7a1-3facc98f03e5] +tempest.api.object_storage.test_object_version.ContainerTest.test_versioned_container[id-a151e158-dcbf-4a1f-a1e7-46cd65895a6f] +tempest.api.compute.admin.test_auto_allocate_network.AutoAllocateNetworkTest.test_server_multi_create_auto_allocate[id-2e6cf129-9e28-4e8a-aaaa-045ea826b2a6] +tempest.api.object_storage.test_account_quotas_negative.AccountQuotasNegativeTest.test_user_modify_quota[id-d1dc5076-555e-4e6d-9697-28f1fe976324,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_resize[compute,id-719eb59d-2f42-4b66-b8b1-bb1254473967,network,slow] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_list_metering_labels[id-e2fb2f8c-45bf-429a-9f17-171c70444612] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_implied_roles_create_check_show_delete[id-c90c316c-d706-4728-bcba-eb1912081b69] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_read_object_without_rights[id-9ed01334-01e9-41ea-87ea-e6f465582823,negative] +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_delete_image[id-3731d080-d4c5-4872-b41a-64d0d0021314] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_volume_delete_nonexistent_volume_id[id-54a34226-d910-4b00-9ef8-8683e6c55846,negative] +tempest.api.volume.admin.test_volume_quota_classes.VolumeQuotaClassesTest.test_update_default_quota[id-a7644c63-2669-467a-b00e-452dd5c5397b] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_update_show_volume_metadata_item[id-862261c5-8df4-475a-8c21-946e50e36a20] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_delete_non_existent_router_returns_404[id-c7edc5ad-d09d-41e6-a344-5c0c31e2e3e4,negative] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_reset_snapshot_status[id-3e13ca2f-48ea-49f3-ae1a-488e9180d535] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_delete_with_invalid_id[id-c74f4bf1-4708-4ff2-95a0-f49eaca951bd,negative] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_remote_image[id-69da74d9-68a9-404b-9664-ff7164ccb0f5] +tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_subnet_id[id-b42e6e39-2e37-49cc-a6f4-8467e940900a,smoke] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_disable_service_with_invalid_binary[id-c571f179-c6e6-4c50-a0ab-368b628a8ac1,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_network_fields[id-867819bb-c4b6-45f7-acf9-90edcf70aa5e] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_delete_non_existent_user[id-7cc82f7e-9998-4f89-abae-23df36495867,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_objects_schema[id-dee4a891-b38b-3bf0-a3b2-e03ee67b3a3c] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_delete_object_without_write_rights[id-b4e366f8-f185-47ab-b789-df4416f9ecdb,negative] +tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network[id-d4f9024d-1e28-4fc1-a6b1-25dbc6fa11e2,smoke] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_get_object_using_temp_url_with_inline_query_parameter[id-9d9cfd90-708b-465d-802c-e4a8090b823d] +tempest.api.volume.test_volumes_clone.VolumesCloneTest.test_create_from_bootable_volume[id-cbbcd7c6-5a6c-481a-97ac-ca55ab715d16,image] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers[id-3499406a-ae53-4f8c-b43a-133d4dc6fe3f,smoke] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_verify_entry_in_list[id-68089c38-04b1-4758-bdf0-cf0daec4defd] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_port_list_filter_by_ip_substr[id-79895408-85d5-460d-94e7-9531c5fd9123] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsTest.test_group_snapshot_create_show_list_delete[id-1298e537-f1f0-47a3-a1dd-8adec8168897] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_metadata_empty_body[id-0f58d402-e34a-481d-8af8-b392b17426d9] +tempest.api.object_storage.test_crossdomain.CrossdomainTest.test_get_crossdomain_policy[id-d1b8b031-b622-4010-82f9-ff78a9e915c7] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_with_existent_aggregate_name[id-9c23a291-b0b1-487b-b464-132e061151b3,negative] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_quota_usage[id-ae8b6091-48ad-4bfa-a188-bbf5cc02115f] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_suspend_non_existent_server[id-d1f032d5-7b6e-48aa-b252-d5f16dd994ca,negative] +tempest.api.identity.v3.test_access_rules.AccessRulesV3Test.test_delete_access_rule[id-278757e9-e193-4bf8-adf2-0b0a229a17d0] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_size_smaller_than_original_size[id-e0c75c74-ee34-41a9-9288-2a2051452854,negative] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts_with_zone[id-5dc06f5b-d887-47a2-bb2a-67762ef3c6de] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_user_account_lockout[id-a7ad8bbf-2cff-4520-8c1d-96332e151658] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_description[id-777b6f14-aca9-4758-9e84-38783cfa58bc,negative] +tempest.api.identity.v2.test_api_discovery.TestApiDiscovery.test_api_version_statuses[id-77fd6be0-8801-48e6-b9bf-38cdd2f253ec,smoke] +tempest.api.compute.admin.test_simple_tenant_usage_negative.TenantUsagesNegativeTestJSON.test_get_usage_tenant_with_invalid_date[id-4079dd2a-9e8d-479f-869d-6fa985ce45b6,negative] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_delete_nonexistent_type_id[id-6b3926d2-7d73-4896-bc3d-e42dfd11a9f6,negative] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_offline_delete_online[compute,id-5210a1de-85a0-11e6-bb21-641c676a5d61] +tempest.api.image.v2.admin.test_images_metadefs_namespace_tags.MetadataNamespaceTagsTest.test_create_update_delete_tag[id-a2a3765e-1a2c-3f6d-a3a7-3cc3466ab875] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_expect_continue[id-84dafe57-9666-4f6d-84c8-0814d37923b8] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_create_port_on_non_existent_network[id-13d3b106-47e6-4b9b-8d53-dae947f092fe,negative] +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_list_instance_actions[id-77ca5cc5-9990-45e0-ab98-1de8fead201a] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_format_json[id-196f5034-6ab0-4032-9da9-a937bbb9fba9] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_security_group_without_passing_id[id-1438f330-8fa4-4aeb-8a94-37c250106d7f,negative] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_delete_deleted_image[id-e57fc127-7ba0-4693-92d7-1d8a05ebcba9,negative] +tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_subnet[id-8936533b-c0aa-4f29-8e53-6cc873aec489,smoke] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_with_default_snat_value[id-847257cc-6afd-4154-b8fb-af49f5670ce8] +tempest.api.volume.admin.test_groups.GroupsV320Test.test_reset_group_status[id-b20c696b-0cbc-49a5-8b3a-b1fb9338f45c] +tempest.api.network.admin.test_quotas.QuotasTest.test_quotas[id-2390f766-836d-40ef-9aeb-e810d78207fb] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_create_project_by_unauthorized_user[id-8fba9de2-3e1f-4e77-812a-60cb68f8df13,negative] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_update_request_without_token[id-7a421573-72c7-4c22-a98e-ce539219c657,negative] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_volume_host_name[id-67ec6902-f91d-4dec-91fa-338523208bbc] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_update_port_with_cidr_address_pair[id-4d6d178f-34f6-4bff-a01c-0a2f8fe909e4] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_list_services[id-e0218299-0a59-4f43-8b2b-f1c035b3d26d] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_name_length_over_255[id-7704b4f3-3b75-4b82-87cc-931d41c8f780,negative] +tempest.api.volume.test_volumes_backup.VolumesBackupsTest.test_backup_create_attached_volume[compute,id-07af8f6d-80af-44c9-a5dc-c8427b1b62e6] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_create_domain_with_disabled_status[id-036df86e-bb5d-42c0-a7c2-66b9db3a6046] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_get_trusts_all[id-4773ebd5-ecbf-4255-b8d8-b63e6f72b65d,smoke] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsTestJSON.test_list_projects_with_parent[id-6edc66f5-2941-4a17-9526-4073311c1fac] +tempest.api.compute.admin.test_fixed_ips.FixedIPsTestJson.test_set_unreserve[id-7476e322-b9ff-4710-bf82-49d51bac6e2e] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_http_image[id-6d0e13a7-515b-460c-b91f-9f4793f09816] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_list_show_update_delete_floating_ip[id-62595970-ab1c-4b7f-8fcc-fddfe55e8718,smoke] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_rebuild_server_with_personality[id-128966d8-71fc-443c-8cab-08e24114ecc9,slow] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.test_create_list_show_delete_interfaces_by_network_port[id-73fe8f02-590d-4bf1-b184-e9ca81065051,network] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_get_details_with_invalid_id[id-3c916244-2c46-49a4-9b55-b20bb0ae512c,negative] +tempest.api.image.v2.admin.test_images_metadefs_namespace_tags.MetadataNamespaceTagsTest.test_create_list_delete_namespace_tags[id-a2a3765e-3a6d-4f6d-a3a7-3cc3476aa876] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata[id-211021f6-21de-4657-a68f-908878cfe251] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_domain_delete_cascades_content[id-d8d318b7-d1b3-4c37-94c5-3c5ba0b121ea] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_greater_than_actual_count[id-d47c17fb-eebd-4287-8e95-f20a7e627b18,negative] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_invalid_options[id-81f18ef6-95b5-4584-9966-10d480b7496a] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_invalid_size[id-1ed83a8a-682d-4dfb-a30e-ee63ffd6c049,negative] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_image_share_reject[id-d9e83e5f-3524-4b38-a900-22abcb26e90e] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway_without_snat[id-96536bc7-8262-4fb2-9967-5c46940fa279] +tempest.api.compute.images.test_list_images.ListImagesTestJSON.test_get_image[id-490d0898-e12a-463f-aef0-c50156b9f789] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_with_multiple_ports[compute,id-756ced7f-6f1a-43e7-a851-2fcfc16f3dd7,network] +tempest.api.volume.test_volumes_extend.VolumesExtendTest.test_volume_extend_when_volume_has_snapshot[id-86be1cba-2640-11e5-9c82-635fb964c912] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts_with_nonexistent_zone[id-c6ddbadb-c94e-4500-b12f-8ffc43843ff8] +tempest.api.compute.admin.test_networks.NetworksTest.test_get_network[id-d206d211-8912-486f-86e2-a9d090d1f416] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_create_delete_metering_label_with_filters[id-ec8e15ff-95d0-433b-b8a6-b466bddb1e50] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_list_all_container_objects_with_nonexistent_container[id-14331d21-1e81-420a-beea-19cb5e5207f5,negative] +tempest.api.volume.test_volumes_backup.VolumesBackupsV39Test.test_update_backup[id-9b374cbc-be5f-4d37-8848-7efb8a873dcc] +tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_two_security_groups_and_extra_attributes[id-edf6766d-3d40-4621-bc6e-2521a44c257d] +tempest.api.image.v2.admin.test_images.BasicOperationsImagesAdminTest.test_create_image_owner_param[id-646a6eaa-135f-4493-a0af-12583021224e] +tempest.api.object_storage.test_object_services.ObjectTest.test_list_no_object_metadata[id-170fb90e-f5c3-4b1f-ae1b-a18810821172] +tempest.api.image.v2.test_images.MultiStoresImportImagesTest.test_glance_direct_import_image_to_all_stores[id-bf04ff00-3182-47cb-833a-f1c6767b47fd] +tempest.api.network.test_networks.NetworksTest.test_show_subnet[id-bd635d81-6030-4dd1-b3b9-31ba0cfdf6cc,smoke] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_name[id-f9eb2b70-735f-416c-b260-9914ac6181e4] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_from_image[id-54a01030-c7fc-447c-86ee-c1182beae638,image,smoke] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_update_non_existent_network[id-98bfe4e3-574e-4012-8b17-b2647063de87,negative] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_head_object_using_temp_url[id-249a0111-5ad3-4534-86a7-1993d55f9185] +tempest.api.volume.admin.test_groups.GroupsV314Test.test_create_group_from_group[id-2424af8c-7851-4888-986a-794b10c3210e] +tempest.api.volume.admin.test_user_messages.UserMessagesTest.test_list_show_messages[id-50f29e6e-f363-42e1-8ad1-f67ae7fd4d5a] +tempest.api.compute.servers.test_novnc.NoVNCConsoleTestJSON.test_novnc[id-c640fdff-8ab4-45a4-a5d8-7e6146cbd0dc] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_get_nonexistent_extra_spec_name[id-c881797d-12ff-4f1a-b09d-9f6212159753,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_create_invalid_body[id-bc772c71-1ed4-4716-b945-8b5ed0f15e87,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_duplicate_name[id-0359a7f1-f002-4682-8073-0c91e4011b7c,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_volume_backed_server_confirm[id-e6c28180-7454-4b59-b188-0257af08a63b,volume] +tempest.api.compute.servers.test_device_tagging.TaggedAttachmentsTest.test_tagged_attachment[id-3e41c782-2a89-4922-a9d2-9a188c4e7c7c,image,network,volume] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_shelve_paused_server[id-8cf9f450-a871-42cf-9bef-77eba189c0b0] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_associate_floatingip_port_ext_net_unreachable[id-6b3b8797-6d43-4191-985c-c48b773eb429,negative] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_list_get_volume_attachments[id-7fa563fe-f0f7-43eb-9e22-a1ece036b513] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeTestJSON.test_attach_detach_volume[id-52e9045a-e90d-4c0d-9087-79d657faffff,slow] +tempest.api.compute.volumes.test_attach_volume_negative.AttachVolumeNegativeTest.test_delete_attached_volume[id-a313b5cd-fbd0-49cc-94de-870e99f763c7,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_wildcard[id-e9f624ee-92af-4562-8bec-437945a18dcb] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_create_with_optional_group_id[id-7f5d2899-7705-4d4b-8458-4505188ffab6] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_show_delete_security_group_rule[id-cfb99e0e-7410-4a3d-8a0c-959a63ee77e9,smoke] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_filter_by_min_disk[id-3df2743e-3034-4e57-a4cb-b6527f6eac79] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_resize_server_revert_deleted_flavor[id-33f1fec3-ba18-4470-8e4e-1d888e7c3593] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_to_port[id-ff88804d-144f-45d1-bf59-dd155838a43a,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_tag_schema[id-dde4a891-b38b-3bf0-a3b2-e03ee67b3a3e] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_without_passing_volume_id[id-aff8ba64-6d6f-4f2e-bc33-41a08ee9f115,negative] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_list_default_quotas[id-2be020a2-5fdd-423d-8d35-a7ffbc36e9f7] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenants_name_length_over_64[id-2ff18d1e-dfe3-4359-9dc3-abf582c196b9,negative] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_show_non_existent_port[id-a954861d-cbfd-44e8-b0a9-7fab111f235d,negative] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_enable_service_with_invalid_service_id[id-508671aa-c929-4479-bd10-8680d40dd0a6,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_non_existent_role[id-ab32d759-cd16-41f1-a86e-44405fa9f6d2,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_list_subnet_with_no_gw64_one_network[id-a9653883-b2a4-469b-8c3c-4518430a7e55] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_with_params[id-59f41f43-aebf-48a9-ab5d-d76340fab32b] +tempest.api.network.test_agent_management_negative.AgentManagementNegativeTest.test_list_agents_non_admin[id-e335be47-b9a1-46fd-be30-0874c0b751e6,negative] +tempest.api.volume.admin.test_volume_retype.VolumeRetypeWithMigrationTest.test_available_volume_retype_with_migration[id-a1a41f3f-9dad-493e-9f09-3ff197d477cd] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsStaticTestJSON.test_list_projects_with_domains[id-fab13f3c-f6a6-4b9f-829b-d32fd44fdf10] +tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_list_usage_all_tenants[id-062c8ae9-9912-4249-8b51-e38d664e926e] +tempest.api.compute.servers.test_server_rescue.ServerBootFromVolumeStableRescueTest.test_stable_device_rescue_bfv_blank_volume[id-48f123cb-922a-4065-8db6-b9a9074a556b,slow] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_id[id-00579617-fe04-4e1c-9d08-ca7467d2e34b,negative] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_aggregate_name_length_less_than_1[id-3b8a1929-3793-4e92-bcb4-dfa572ee6c1d,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_with_existing_server_name[id-8fea6be7-065e-47cf-89b8-496e6f96c699] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_two_subnets[id-4544adf7-bb5f-4bdc-b769-b3e77026cef2] +tempest.api.identity.admin.v3.test_policies.PoliciesTestJSON.test_create_update_delete_policy[id-e544703a-2f03-4cf2-9b0f-350782fdb0d3,smoke] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_all_attributes[id-a4d9ec4c-0306-4111-a75c-db01a709030b] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_remote_ip_prefix[id-5f8daf69-3c5f-4aaa-88c9-db1d66f68679,negative] +tempest.api.image.v1.test_images.UpdateImageMetaTest.test_update_image_metadata[id-d6d7649c-08ce-440d-9ea7-e3dda552f33c] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_deactivate_reactivate_image[id-951ebe01-969f-4ea9-9898-8a3f1f442ab0] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_disable_log_reason_with_invalid_service_id[id-f46a9d91-1e85-4b96-8e7a-db7706fa2e9a,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_duplicate_security_group_rule_fails[id-8fde898f-ce88-493b-adc9-4e4692879fc5,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_specify_keypair[id-f9e15296-d7f9-4e62-b53f-a04e89160833] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_resize_with_qos_min_bw_allocation[compute,id-c29e7fd3-035d-4993-880f-70819847683f,network] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_add_router_interfaces_on_overlapping_subnets_returns_400[id-957751a3-3c68-4fa2-93b6-eb52ea10db6e,negative] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_readonly_update[id-fff74e1e-5bd3-4b33-9ea9-24c103bc3f59] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_x_object_manifest[id-08854588-6449-4bb7-8cca-f2e1040f5e6f] +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_update_tags_for_non_existing_image[id-8cd30f82-6f9a-4c6e-8034-c1b51fba43d9,negative] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_list_qos[id-75e04226-bcf7-4595-a34b-fdf0736f38fc] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_roles_by_unauthorized_user[id-d5d5f1df-f8ca-4de0-b2ef-259c1cc67025,negative] +tempest.api.volume.admin.test_volume_hosts.VolumeHostsAdminTestsJSON.test_show_host[id-21168d57-b373-4b71-a3ac-f2c88f0c5d31] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_show_nonexistent_hypervisor[id-c136086a-0f67-4b2b-bc61-8482bd68989f,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_stop_start_server[id-af8eafd4-38a7-4a4b-bdbc-75145a580560] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_policy[compute,id-79fdaa1c-df62-4738-a0f0-1cff9dc415f6,network] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_create_external_network[id-462be770-b310-4df9-9c42-773217e4c8b1] +tempest.scenario.test_shelve_instance.TestShelveInstance.test_shelve_volume_backed_instance[compute,id-c1b6318c-b9da-490b-9c67-9339b627271f,image,network,slow,volume] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts[id-9bfaf98d-e2cb-44b0-a07e-2558b2821e4f] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output[id-4b8867e6-fffa-4d54-b1d1-6fdda57be2f3] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list[id-0b6ddd39-b948-471f-8038-4787978747c4,smoke] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_check_revoke_roles_on_domains_group[id-c7a8dda2-be50-4fb4-9a9c-e830771078b1] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_reserve_unreserve_volume[id-92c4ef64-51b2-40c0-9f7e-4749fbaaba33] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_get_service_by_invalid_params[id-d0884a69-f693-4e79-a9af-232d15643bf7,negative] +tempest.api.image.v2.admin.test_images_metadefs_namespaces.MetadataNamespacesTest.test_basic_metadata_definition_namespaces[id-319b765e-7f3d-4b3d-8b37-3ca3876ee768] +tempest.api.image.v2.test_images.ImportImagesTest.test_remote_import[id-e04761a1-22af-42c2-b8bc-a34a3f12b585] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_display_name_and_status[id-856ab8ca-6009-4c37-b691-be1065528ad4] +tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_subnet[id-8936533b-c0aa-4f29-8e53-6cc873aec489,smoke] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_property_schema[id-dae4a891-b38b-3bf0-a3b2-e03ee67b3a3d] +tempest.api.network.test_networks.NetworksTest.test_delete_network_with_subnet[id-f04f61a9-b7f3-4194-90b2-9bcf660d1bfe] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_servers_on_same_host[id-26a9d5df-6890-45f2-abc4-a659290cb130] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_list_groups[id-cc9a57a5-a9ed-4f2d-a29f-4f979a06ec71] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output_server_id_in_shutoff_status[id-5b65d4e7-4ecd-437c-83c0-d6b79d927568] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_get_user[id-c10dcd90-461d-4b16-8e23-4eb836c00644] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_token_auth_creation_existence_deletion[id-0f9f5a5f-d5cd-4a86-8a5b-c5ded151f212] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_delete_external_networks_with_floating_ip[id-82068503-2cf2-4ed4-b3be-ecb89432e4bb] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_set_gateway_with_fixed_ip[id-cbe42f84-04c2-11e7-8adb-fa163e4fa634] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_name_desc[id-96ba6f4d-1f18-47e1-b4bc-76edc6c21250] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_nonexistent_floating_ip[id-595fa616-1a71-4670-9614-46564ac49a4c,negative] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsV319Test.test_reset_group_snapshot_status[id-3b42c9b9-c984-4444-816e-ca2e1ed30b40] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_format_xml[id-655a53ca-4d15-408c-a377-f4c6dbd0a1fa] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_host_routes_and_dns_nameservers[id-d830de0a-be47-468f-8f02-1fd996118289] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_creating[id-21737d5a-92f2-46d7-b009-a0cc0ee7a570] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_show_server_group[id-b3545034-dd78-48f0-bdc2-a4adfa6d0ead] +tempest.api.object_storage.test_object_formpost_negative.ObjectFormPostNegativeTest.test_post_object_using_form_invalid_signature[id-b277257f-113c-4499-b8d1-5fead79f7360,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_detach_volumes_with_invalid_volume_id[id-9f9c24e4-011d-46b5-b992-952140ce237a,negative] +tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_list_dhcp_agent_hosting_network[id-5032b1fe-eb42-4a64-8f3b-6e189d8b5c7d] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_subnet_with_v6_attributes_slaac[id-176b030f-a923-4040-a755-9dc94329e60c] +tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_get_usage_tenant_with_non_admin_user[id-9d00a412-b40e-4fd9-8eba-97b496316116] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_list_ports[id-cf95b358-3e92-4a29-a148-52445e1ac50e,smoke] +tempest.api.compute.admin.test_volume_swap.TestMultiAttachVolumeSwap.test_volume_swap_with_multiattach[id-e8f8f9d1-d7b7-4cd2-8213-ab85ef697b6e,slow,volume] +tempest.api.identity.admin.v3.test_endpoint_groups.EndPointGroupsTest.test_update_endpoint_group[id-51c8fc38-fa84-4e76-b5b6-6fc37770fb26] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_image_defined_boot_from_volume[compute,id-36c34c67-7b54-4b59-b188-02a2f458a63b,image,volume] +tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_slaac[compute,id-dec222b1-180c-4098-b8c5-cc1b8342d611,network,slow] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_list_non_public_flavor[id-be6cc18c-7c5d-48c0-ac16-17eaf03c54eb] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsStaticTestJSON.test_list_projects_with_name[id-fa178524-4e6d-4925-907c-7ab9f42c7e26] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_modified_since[id-be133639-e5d2-4313-9b1f-2d59fc054a16] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_shelve_unshelve_server[id-77eba8e0-036e-4635-944b-f7a8f3b78dc9,image] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_sort_key_dir[id-9faaa0c2-c3a5-43e1-8f61-61c54b409a49] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_with_nonexistent_volume_id[id-e3e466af-70ab-4f4b-a967-ab04e3532ea7,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_pause_paused_server[id-d1417e7f-a509-41b5-a102-d5eed8613369,negative] +tempest.api.compute.admin.test_server_diagnostics.ServerDiagnosticsTest.test_get_server_diagnostics[id-31ff3486-b8a0-4f56-a6c0-aab460531db3] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_create_image_reserved_property[id-e3fb7df8-2742-4143-8976-f1b26870f0a0,negative] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_volume_with_private_volume_type[id-8c09f849-f225-4d78-ba87-bffd9a5e0c6f,negative] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_get_volume_attachment[compute,id-9516a2c8-9135-488c-8dd6-5677a7e5f371] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_get_users[id-a149c02e-e5e0-4b89-809e-7e8faf33ccda] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_set_reserve_with_non_admin_user[id-ce60042c-fa60-4836-8d43-1c8e3359dc47,negative] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_status[id-a3f5b513-aeb3-42a9-b18e-f091ef73254d] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_list_update_show_delete_security_group[id-bfd128e5-3c92-44b6-9d66-7fe29d22c802,smoke] +tempest.api.compute.admin.test_volumes_negative.VolumesAdminNegativeTest.test_update_attached_volume_with_nonexistent_volume_in_uri[id-309b5ecd-0585-4a7e-a36f-d2b2bf55259d,negative] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_unset_gateway[id-ad81b7ee-4f81-407b-a19c-17e623f763e8] +tempest.api.volume.test_extensions.ExtensionsTestJSON.test_list_extensions[id-94607eb0-43a5-47ca-82aa-736b41bd2e2c] +tempest.api.image.v1.test_images.ListImagesTest.test_index_disk_format[id-f1755589-63d6-4468-b098-589820eb4031] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_delimiter[id-fe323a32-57b9-4704-a996-2e68f83b09bc] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminUnderV252Test.test_get_hypervisor_show_servers[id-e81bba3f-6215-4e39-a286-d52d2f906862] +tempest.api.volume.test_volumes_clone.VolumesCloneTest.test_create_from_volume[id-9adae371-a257-43a5-9555-dc7c88e66e0e] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_icmp_type_code[id-c9463db8-b44d-4f52-b6c0-8dbda99f26ce] +tempest.api.compute.test_versions.TestVersions.test_get_version_details[id-b953a29e-929c-4a8e-81be-ec3a7e03cb76,smoke] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_delete_non_existent_port[id-49ec2bbd-ac2e-46fd-8054-798e679ff894,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_list_account_metadata[id-4894c312-6056-4587-8d6f-86ffbf861f80,smoke] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_with_id_over_character_limit[id-b340030d-82cd-4066-a314-c72fb7c59277,negative] +tempest.api.network.test_routers.RoutersTest.test_update_router_admin_state[id-a8902683-c788-4246-95c7-ad9c6d63a4d9] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_block_migration_paused[id-1e107f21-61b2-4988-8f22-b196e938ab88] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_across_containers[id-aa467252-44f3-472a-b5ae-5b57c3c9c147] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_reserve_volume_with_negative_volume_status[id-449c4ed2-ecdd-47bb-98dc-072aeccf158c,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_delete_invalid_volume_id[id-62972737-124b-4513-b6cf-2f019f178494,negative] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_access_rules[id-529936eb-aa5d-463d-9f79-01c113d3b88f] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_server_create_metadata_key_too_long[id-fe114a8f-3a57-4eff-9ee2-4e14628df049,negative] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_identity_v3_existence[id-79aec9ae-710f-4c54-a4fc-3aa25b4feac3] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_remove_nonexistent_host[id-95d6a6fa-8da9-4426-84d0-eec0329f2e4d,negative] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_update_nonexistent_image_metadata[id-a403ef9e-9f95-427c-b70a-3ce3388796f1,negative] +tempest.api.compute.test_versions.TestVersions.test_list_api_versions[id-6c0a0990-43b6-4529-9b61-5fd8daf7c55c,smoke] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_remove_host_as_user[id-7a53af20-137a-4e44-a4ae-e19260e626d9,negative] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_verify_resize_state[id-ab0c38b4-cdd8-49d3-9b92-0cb898723c01] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw[id-9393b468-186d-496d-aa36-732348cd76e7] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_delete_non_existent_domain[id-43781c07-764f-4cf2-a405-953c1916f605,negative] +tempest.api.compute.servers.test_server_rescue.ServerBootFromVolumeStableRescueTest.test_stable_device_rescue_bfv_image_volume[id-e4636333-c928-40fc-98b7-70a23eef4224,slow] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_image_param_owner[id-e9a44b91-31c8-4b40-a332-e0a39ffb4dbb] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_name[id-33163b73-79f5-4d07-a7ea-9213bcc468ff] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_list_image_metadata[id-37ec6edd-cf30-4c53-bd45-ae74db6b0531] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_expire_invalid[id-3e48f95d-e660-4fa9-85e0-5a3d85594384] +tempest.api.identity.admin.v3.test_policies.PoliciesTestJSON.test_list_policies[id-1a0ad286-2d06-4123-ab0d-728893a76201] +tempest.api.network.test_routers.RoutersIpV6Test.test_update_router_admin_state[id-a8902683-c788-4246-95c7-ad9c6d63a4d9] +tempest.api.compute.servers.test_server_addresses_negative.ServerAddressesNegativeTestJSON.test_list_server_addresses_invalid_server_id[id-02c3f645-2d2e-4417-8525-68c0407d001b,negative,network] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_in_tenant_traffic[compute,id-63163892-bbf6-4249-aa12-d5ea1f8f421b,network] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_is_public_string_variations[id-fb9cbde6-3a0e-41f2-a983-bdb0a823c44e] +tempest.api.network.test_ports.PortsTestJSON.test_show_port_fields[id-45fcdaf2-dab0-4c13-ac6c-fcddfb579dbd] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_non_existent_id[id-1d507e98-7951-469b-82c3-23f1e6b8c254,negative] +tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_create_service_without_description[id-d1dcb1a1-2b6b-4da8-bbb8-5532ef6e8269] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_existent_host[id-19dd44e1-c435-4ee1-a402-88c4f90b5950,negative] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_user_on_project[id-c6b80012-fe4a-498b-9ce8-eb391c05169f] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_domain_roles_create_delete[id-d92a41d2-5501-497a-84bb-6e294330e8f8] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_show_non_existent_network[id-9293e937-824d-42d2-8d5b-e985ea67002a,negative] +tempest.scenario.test_volume_migrate_attached.TestVolumeMigrateRetypeAttached.test_volume_retype_attached[compute,id-deadd2c2-beef-4dce-98be-f86765ff311b,slow,volume] +tempest.api.compute.servers.test_instance_actions.InstanceActionsTestJSON.test_get_instance_action[id-aacc71ca-1d70-4aa5-bbf6-0ff71470e43c] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_not_enabled[id-78f96a9c-e0e0-4ee6-a3ba-fbf6dfd03207] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_non_existing_image[id-6fe40f1c-57bd-4918-89cc-8500f850f3de,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_format_json[id-1c7efa35-e8a2-4b0b-b5ff-862c7fd83704] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_status[id-7fc9e369-0f58-4d05-9aa5-0969e2d59d15] +tempest.api.volume.admin.test_groups.GroupsTest.test_group_create_show_list_delete[id-4b111d28-b73d-4908-9bd2-03dc2992e4d4] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_non_existent_server[id-904b13dc-0ef2-4e4c-91cd-3b4a0f2f49d8,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_create_and_delete_metadata[id-8e5fc073-59b9-42ee-984a-29ed11b2c749] +tempest.api.volume.admin.test_volumes_backup.VolumesBackupsAdminTest.test_volume_backup_export_import[id-a99c54a1-dd80-4724-8a13-13bf58d4068d] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_create_delete_metering_label_with_filters[id-ec8e15ff-95d0-433b-b8a6-b466bddb1e50] +tempest.api.object_storage.test_account_services.AccountTest.test_list_extensions[id-6eb04a6a-4860-4e31-ba91-ea3347d76b58] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_marker_and_end_marker[id-ac8502c2-d4e4-4f68-85a6-40befea2ef5e] +tempest.api.identity.v2.test_extension.ExtensionTestJSON.test_list_extensions[id-85f3f661-f54c-4d48-b563-72ae952b9383] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_non_exist_host[id-0ef07828-12b4-45ba-87cc-41425faf5711,negative] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_from_port[id-12bbc875-1045-4f7a-be46-751277baedb9,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_bad_protocol[id-981bdc22-ce48-41ed-900a-73148b583958,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_create_nonexistent_type_id[id-49d5472c-a53d-4eab-a4d3-450c4db1c545,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_non_number_size[id-5d0b480d-e833-439f-8a5a-96ad2ed6f22f,negative] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_list_roles[id-75d9593f-50b7-4fcf-bd64-e3fb4a278e23] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_image[id-ff01387d-c7ad-47b4-ae9e-64fa214638fe,negative] +tempest.api.image.v1.test_image_members.ImageMembersTest.test_add_image_member[id-1d6ef640-3a20-4c84-8710-d95828fdb6ad] +tempest.api.compute.admin.test_fixed_ips.FixedIPsTestJson.test_set_reserve[id-5485077b-7e46-4cec-b402-91dc3173433b] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_confirm[id-1499262a-9328-4eda-9068-db1ac57498d2] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_x_newest[id-94587078-475f-48f9-a40f-389c246e31cd] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_id_asc[id-c5513ada-64c1-4d28-83b9-af3307ec1388] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_role_create_delete[id-c62d909d-6c21-48c0-ae40-0a0760e6db5e] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_volume_type[id-10254ed8-3849-454e-862e-3ab8e6aa01d2,negative] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_update_port_with_cidr_address_pair[id-4d6d178f-34f6-4bff-a01c-0a2f8fe909e4] +tempest.api.compute.servers.test_device_tagging.TaggedBootDevicesTest.test_tagged_boot_devices[id-a2e65a6c-66f1-4442-aaa8-498c31778d96,image,network,slow,volume] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_create_update_and_delete_domain_config_groups_and_opts[id-c7510fa2-6661-4170-9c6b-4783a80651e9] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server_with_volume_attached[id-b68bd8d6-855d-4212-b59b-2e704044dace,slow,volume] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_without_passing_size[id-9387686f-334f-4d31-a439-33494b9e2683,negative] +tempest.api.identity.admin.v3.test_endpoints.EndPointsTestJSON.test_update_endpoint[id-37e8f15e-ee7c-4657-a1e7-f6b61e375eff,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_server_id[id-9f238683-c763-45aa-b848-232ec3ce3105] +tempest.api.compute.admin.test_services.ServicesAdminTestJSON.test_get_service_by_host_name[id-affb42d5-5b4b-43c8-8b0b-6dca054abcca] +tempest.api.compute.servers.test_instance_actions.InstanceActionsV221TestJSON.test_get_list_deleted_instance_actions[id-0a0f85d4-10fa-41f6-bf80-a54fb4aa2ae1] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_list_details_with_non_admin_user[id-dc02db05-e801-4c5f-bc8e-d915290ab345,negative] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_add_gateway_invalid_network_returns_404[id-37a94fc0-a834-45b9-bd23-9a81d2fd1e22,negative] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_create_ec2_credential[id-b580fab9-7ae9-46e8-8138-417260cb6f9f] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_change_server_password[id-6158df09-4b82-4ab3-af6d-29cf36af858d] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_name_exceeds_max_length[id-30686921-4bed-4764-a038-40d741ed4e78,negative] +tempest.api.compute.servers.test_server_password.ServerPasswordTestJSON.test_get_server_password[id-f83b582f-62a8-4f22-85b0-0dee50ff783a] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_delete_object_with_non_authorized_user[id-7343ac3d-cfed-4198-9bb0-00149741a492,negative] +tempest.api.network.admin.test_routers_dvr.RoutersTestDVR.test_centralized_router_creation[id-8a0a72b4-7290-4677-afeb-b4ffe37bc352] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_reset_state_server[id-ee8ae470-db70-474d-b752-690b7892cab1] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_properties_schema[id-dce4a891-b38b-3bf0-a3b2-e03ee67b3a3e] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_implied_domain_roles[id-eb1e1c24-1bc4-4d47-9748-e127a1852c82] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_server_status[id-de2612ab-b7dd-4044-b0b1-d2539601911f] +tempest.api.volume.admin.test_volume_pools.VolumePoolsAdminTestsJSON.test_get_pools_with_details[id-d4bb61f7-762d-4437-b8a4-5785759a0ced] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_virtio[id-16865750-1417-4854-bcf7-496e6753c01e] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_create_metadata[id-e2a08b5f-3115-4768-a3ee-d4287acd6c08] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_impersonate[id-5a0a91a4-baef-4a14-baba-59bf4d7fcace] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unpause_server_invalid_state[id-c8e639a7-ece8-42dd-a2e0-49615917ba4f,negative] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_image_with_min_ram[id-05b19d55-140c-40d0-b36b-fafd774d421b] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_name_of_non_existent_server[id-aa8eed43-e2cb-4ebf-930b-da14f6a21d81,negative] +tempest.api.compute.servers.test_server_rescue.ServerRescueTestJSON.test_rescue_unrescue_instance[id-fd032140-714c-42e4-a8fd-adcd8df06be6] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_min_max_size[id-4ad8c157-971a-4ba8-aa84-ed61154b1e7f] +tempest.api.volume.test_volumes_extend.VolumesExtendAttachedTest.test_extend_attached_volume[compute,id-301f5a30-1c6f-4ea0-be1a-91fd28d44354] +tempest.api.image.v2.admin.test_images_metadefs_namespace_objects.MetadataNamespaceObjectsTest.test_show_meta_namespace_objects[id-b1a3674e-3b4c-3f6a-a3b4-1ba3573ca768] +tempest.api.compute.admin.test_keypairs_v210.KeyPairsV210TestJSON.test_admin_manage_keypairs_for_other_users[id-3c8484af-cfb3-48f6-b8ba-d5d58bbf3eac] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_limit[id-297ec38b-2b61-4ff4-bcd1-7fa055e97b61] +tempest.api.compute.volumes.test_volume_snapshots.VolumesSnapshotsTestJSON.test_volume_snapshot_create_get_list_delete[id-cd4ec87d-7825-450d-8040-6e2068f2da8f] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip_regex[id-a905e287-c35e-42f2-b132-d02b09f3654a] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_attaching[id-db8d607a-aa2e-4beb-b51d-d4005c232011] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_check_tag_existence[id-81279a66-61c3-4759-b830-a2dbe64cbe08] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_policy_to_zero[compute,id-a9725a70-1d28-4e3b-ae0e-450abc235962,network] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_update_with_enabled_False[id-65e41f32-5eb7-498f-a92a-a6ccacf7439a,negative] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_set_server_metadata_item[id-58c02d4f-5c67-40be-8744-d3fa5982eb1c] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_get_volume_without_passing_volume_id[id-62bab09a-4c03-4617-8cca-8572bc94af9b,negative] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_create_server_when_memory_quota_is_full[id-6fdd7012-584d-4327-a61c-49122e0d5864,negative] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_update_server_from_auto_to_manual[id-5ef18867-358d-4de9-b3c9-94d4ba35742f] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_blank_id[id-e6e41425-af5c-4fe6-a4b5-7b7b963ffda5,negative] +tempest.api.identity.admin.v2.test_tokens.TokensTestJSON.test_create_check_get_delete_token[id-453ad4d5-e486-4b2f-be72-cffc8149e586] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_object_schema[id-dff4a891-b38b-3bf0-a3b2-e03ee67b3a3b] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_details[id-adcbb5a7-5ad8-4b61-bd10-5380e111a877] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_marker[id-05489dde-44bc-4961-a1f5-3ce7ee7824f7] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_delete_non_existent_project[id-7965b581-60c1-43b7-8169-95d4ab7fc6fb,negative] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_protected_image[id-ab980a34-8410-40eb-872b-f264752f46e5,negative] +tempest.api.volume.test_availability_zone.AvailabilityZoneTestJSON.test_get_availability_zone_list[id-01f1ae88-eba9-4c6b-a011-6f7ace06b725] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_update_user_for_non_existent_user[id-3d07e294-27a0-4144-b780-a2a1bf6fee19,negative] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsTest.test_delete_group_snapshots_following_updated_volumes[id-7d7fc000-0b4c-4376-a372-544116d2e127] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_update_by_unauthorized_user[id-41704dc5-c5f7-4f79-abfa-76e6fedc570b,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_filter_by_min_ram[id-935cf550-e7c8-4da6-8002-00f92d5edfaa] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_create_list_port_with_address_pair[id-86c3529b-1231-40de-803c-00e40882f043] +tempest.api.volume.admin.test_volumes_backup.VolumesBackupsAdminTest.test_volume_backup_reset_status[id-47a35425-a891-4e13-961c-c45deea21e94] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_show_hypervisor_with_non_admin_user[id-51e663d0-6b89-4817-a465-20aca0667d03,negative] +tempest.scenario.test_aggregates_basic_ops.TestAggregatesBasicOps.test_aggregate_basic_ops[compute,id-cb2b4c4f-0c7c-4164-bdde-6285b302a081,slow] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_server_addresses.ServerAddressesTestJSON.test_list_server_addresses_by_network[id-87bbc374-5538-4f64-b673-2b0e4443cc30,network,smoke] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group_rule[id-4c094c09-000b-4e41-8100-9617600c02a6,negative] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_set_reserve_with_invalid_ip[id-f51cf464-7fc5-4352-bc3e-e75cfa2cb717,negative] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_create_with_description[id-d25e9f24-1310-4d29-b61b-d91299c21d6d] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsIpV6TestJSON.test_update_show_port_with_extra_dhcp_options[id-9a6aebf4-86ee-4f47-b07a-7f7232c55607] +tempest.api.object_storage.test_container_services.ContainerTest.test_delete_container[id-95d3a249-b702-4082-a2c4-14bb860cf06a] +tempest.api.identity.admin.v2.test_endpoints.EndPointsTestJSON.test_list_endpoints[id-11f590eb-59d8-4067-8b2b-980c7f387f51] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_get_flavor[id-1f12046b-753d-40d2-abb6-d8eb8b30cb2f,smoke] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_delete_nonexistent_image_metadata_item[id-848e157f-6bcf-4b2e-a5dd-5124025a8518,negative] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_image_specify_invalid_metadata[id-55d1d38c-dd66-4933-9c8e-7d92aeb60ddc,negative] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_set_nonexistent_image_metadata[id-dc64f2ce-77e8-45b0-88c8-e15041d08eaf,negative] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration_with_attached_volume[id-e19c0cc6-6720-4ed8-be83-b6603ed5c812,volume] +tempest.api.compute.admin.test_availability_zone.AZAdminV2TestJSON.test_get_availability_zone_list[id-d3431479-8a09-4f76-aa2d-26dc580cb27c] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_nonexistent_host[id-23c92146-2100-4d68-b2d6-c7ade970c9c1,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_match[id-c05b4013-e4de-47af-be84-e598062b16fc] +tempest.api.volume.admin.test_volume_types_extra_specs.VolumeTypesExtraSpecsTest.test_volume_type_extra_specs_update[id-0806db36-b4a0-47a1-b6f3-c2e7f194d017] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_protocol_integer_value[id-0a307599-6655-4220-bebc-fd70c64f2290] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_from_deactivated_image[id-d15e7f35-2cfc-48c8-9418-c8223a89bcbb,image,negative] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_rebuild_server_in_error_state[id-682cb127-e5bb-4f53-87ce-cb9003604442] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_delete_non_existent_network[id-03795047-4a94-4120-a0a1-bd376e36fd4e,negative] +tempest.api.compute.volumes.test_volumes_get.VolumesGetTestJSON.test_volume_create_get_delete[id-f10f25eb-9775-4d9d-9cbe-1cf54dae9d5f] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_create_delete_qos_with_back_end_consumer[id-b115cded-8f58-4ee4-aab5-9192cfada08f] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_offset[id-ca96d551-17c6-4e11-b0e8-52d3bb8a63c7] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_user_update_own_password[id-ad71bd23-12ad-426b-bb8b-195d2b635f27] +tempest.api.compute.certificates.test_certificates.CertificatesV2TestJSON.test_get_root_certificate[id-3ac273d0-92d2-4632-bdfc-afbc21d4606c] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_delete_image_metadata_item[id-a013796c-ba37-4bb5-8602-d944511def14] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_metadata[id-1ca92d3c-4a8e-4b43-93f5-e4c7fb3b291d] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_get_equals_list[id-d1db68b6-aebe-4fa0-b79d-d724d2e21162] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_create_server_from_volume_snapshot[compute,id-05795fb2-b2a7-4c9f-8fac-ff25aedb1489,image,slow,volume] +tempest.api.volume.test_volumes_backup.VolumesBackupsTest.test_volume_backup_create_get_detailed_list_restore_delete[id-a66eb488-8ee1-47d4-8e9f-575a095728c6] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_rebuild_server_with_auto_disk_config[id-9c9fae77-4feb-402f-8450-bf1c8b609713] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_bootable[id-63e21b4c-0a0c-41f6-bfc3-7c2816815599] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_console_output_of_non_existent_server[id-7dd919e7-413f-4198-bebb-35e2a01b13e9,negative] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_noimpersonate[id-ed2a8779-a7ac-49dc-afd7-30f32f936ed2] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_multiple_security_groups[compute,id-d2f77418-fcc4-439d-b935-72eca704e293,network,slow] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_freeze_host_with_invalid_host[id-712bfab8-1f44-4eb5-a632-fa70bf78f05e,negative] +tempest.api.network.test_ports.PortsTestJSON.test_list_ports[id-cf95b358-3e92-4a29-a148-52445e1ac50e,smoke] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_get_nonexistent_volume_type_id[id-9f402cbd-1838-4eb4-9554-126a6b1908c9,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminV228Test.test_get_list_hypervisor_details[id-d46bab64-0fbe-4eb8-9133-e6ee56188cc5] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_delete_member_with_non_existing_image[id-e1559f05-b667-4f1b-a7af-518b52dc0c0f,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_image[id-b3304c3b-97df-46d2-8cd3-e2b6659724e7] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_lock_unlock_server[id-80a8094c-211e-440a-ab88-9e59d556c7ee] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_list_ports_binding_ext_attr[compute,id-1c82a44a-6c6e-48ff-89e1-abe7eaf8f9f8] +tempest.api.network.test_versions.NetworksApiDiscovery.test_api_version_resources[id-cac8a836-c2e0-4304-b556-cd299c7281d1,smoke] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_image_specify_name_over_character_limit[id-084f0cbc-500a-4963-8a4e-312905862581,negative] +tempest.api.image.v2.test_images_member_negative.ImagesMemberNegativeTest.test_image_share_invalid_status[id-b79efb37-820d-4cf0-b54c-308b00cf842c,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_zero[id-8cce995e-0a83-479a-b94d-e1e40b8a09d1,negative] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_block_migration[id-1dce86b8-eb04-4c03-a9d8-9c1dc3ee0c7b] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_update_server_metadata[id-344d981e-0c33-4997-8a5d-6c1d803e4134] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_update_more_key[id-f5889590-bf66-41cc-b4b1-6e6370cfd93f,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_active_status[id-ca78e20e-fddb-4ce6-b7f7-bcbf8605e66e] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_allocate_floating_ip[id-f7bfb946-297e-41b8-9e8c-aba8e9bb5194] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_password_history_not_enforced_in_admin_reset[id-568cd46c-ee6c-4ab4-a33a-d3791931979e] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_set_gateway_without_snat[id-96536bc7-8262-4fb2-9967-5c46940fa279] +tempest.api.identity.v3.test_access_rules.AccessRulesV3Test.test_show_access_rule[id-795dd507-ca1e-40e9-ba90-ff0a08689ba4] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_server_ref[id-05a377b8-28cf-4734-a1e6-2ab5c38bf606] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_list_user_roles[id-262e1e3e-ed71-4edd-a0e5-d64e83d66d05] +tempest.api.volume.admin.test_volume_quotas_negative.VolumeQuotasNegativeTestJSON.test_volume_extend_gigabytes_quota_deviation[id-d321dc21-d8c6-401f-95fe-49f4845f1a6d,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_update_port_with_second_ip[id-63aeadd4-3b49-427f-a3b1-19ca81f06270] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_prefix[id-77e742c7-caf2-4ec9-8aa4-f7d509a3344c] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_delete_volume_without_passing_volume_id[id-0d1417c5-4ae8-4c2c-adc5-5f0b864253e5,negative] +tempest.api.compute.servers.test_device_tagging.TaggedBootDevicesTest_v242.test_tagged_boot_devices[id-a2e65a6c-66f1-4442-aaa8-498c31778d96,image,network,slow,volume] +tempest.api.compute.admin.test_quotas.QuotaClassesAdminTestJSON.test_update_default_quotas[id-7932ab0f-5136-4075-b201-c0e2338df51a] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_protocol_integer_value[id-0a307599-6655-4220-bebc-fd70c64f2290] +tempest.api.network.test_service_providers.ServiceProvidersTest.test_service_providers_list[id-2cbbeea9-f010-40f6-8df5-4eaa0c918ea6] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_set_unreserve_with_non_admin_user[id-f1f7a35b-0390-48c5-9803-5f27461439db,negative] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_create_server_metadata_blank_key[id-92431555-4d8b-467c-b95b-b17daa5e57ff,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypairs_create_list_delete[id-1d1dbedb-d7a0-432a-9d09-83f543c3c19b] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesV270Test.test_create_get_list_interfaces[id-2853f095-8277-4067-92bd-9f10bd4f8e0c,network] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_server_with_scheduler_hint_group_affinity[id-9d2e924a-baf4-11e7-b856-fa163e65f5ce] +tempest.scenario.test_shelve_instance.TestShelveInstance.test_shelve_instance[compute,id-1164e700-0af0-4a4c-8792-35909a88743c,image,network,slow] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_delete_agent[id-470e0b89-386f-407b-91fd-819737d0b335] +tempest.api.object_storage.test_object_expiry.ObjectExpiryTest.test_get_object_at_expiry_time[id-e592f18d-679c-48fe-9e36-4be5f47102c5] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_max_count_non_integer[id-9c5698d1-d7af-4c80-b971-9d403135eea2,negative] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_detail_param_marker[id-46eff077-100b-427f-914e-3db2abcdb7e2] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_string[id-679bc053-5e70-4514-9800-3dfab1a380a6,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_metadata[id-96e68f0e-19ec-4aa2-86f3-adc6a45e14dd,smoke] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_specify_uuid_35_characters_or_less[id-ec176029-73dc-4037-8d72-2e4ff60cf538,negative] +tempest.api.network.admin.test_external_networks_negative.ExternalNetworksAdminNegativeTestJSON.test_create_port_with_precreated_floatingip_as_fixed_ip[id-d402ae6c-0be0-4d8e-833b-a738895d98d0,negative] +tempest.api.compute.flavors.test_flavors_negative.FlavorsV2NegativeTest.test_boot_with_low_ram[id-90f0d93a-91c1-450c-91e6-07d18172cefe,image,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_non_hex_string_id[id-924540c3-f1f1-444c-8f58-718958b6724e,negative] +tempest.api.compute.admin.test_volume_swap.TestVolumeSwap.test_volume_swap[id-1769f00d-a693-4d67-a631-6a3496773813,slow,volume] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_attached_volume[id-d0f3f0d6-d9b6-4a32-8da4-23015dcab23c,volume] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_ip[id-43a1242e-7b31-48d1-88f2-3f72aa9f2077] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_unreserve_volume_with_nonexistent_volume_id[id-eb467654-3dc1-4a72-9b46-47c29d22654c,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_list_object_metadata[id-9a447cf6-de06-48de-8226-a8c6ed31caf2,smoke] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_show_host_detail_with_non_admin_user[id-19ebe09c-bfd4-4b7c-81a2-e2e0710f59cc,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_no_container_metadata[id-a2faf936-6b13-4f8d-92a2-c2278355821e] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_list_with_non_admin_user[id-51b3d536-9b14-409c-9bce-c6f7c794994e,negative] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_int_id[id-8b4330e1-12c4-4554-9390-e6639971f086] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_list_all_container_objects_on_deleted_container[id-86b2ab08-92d5-493d-acd2-85f0c848819e,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_ip_v6_address[id-5226dd80-1e9c-4d8a-b5f9-b26ca4763fd0,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_resources_types_schema[id-dfe4a891-b38b-3bf0-a3b2-e03ee67b3a3a] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_list_tenant_request_without_token[id-df33926c-1c96-4d8d-a762-79cc6b0c3cf4,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_setting_project_id[id-e54dd3a3-4352-4921-b09d-44369ae17397] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_delete_member_with_non_existing_tenant[id-f5720333-dd69-4194-bb76-d2f048addd56,negative] +tempest.api.compute.admin.test_auto_allocate_network.AutoAllocateNetworkTest.test_server_create_no_allocate[id-5eb7b8fa-9c23-47a2-9d7d-02ed5809dd34] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_delete[id-0d148aa3-d54c-4317-aa8d-42040a475e20] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_request_without_token[id-cac81cf4-c1d2-47dc-90d3-f2b7eb572286,negative] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_subnet_with_v6_attributes_stateful[id-da40cd1b-a833-4354-9a85-cd9b8a3b74ca] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_update_metadata_get_details[id-36ec92ca-7a73-43bc-b920-7531809e8540] +tempest.api.network.test_subnetpools_extensions.SubnetPoolsTestJSON.test_create_list_show_update_delete_subnetpools[id-62595970-ab1c-4b7f-8fcc-fddfe55e9811,smoke] +tempest.api.identity.admin.v2.test_services.ServicesTestJSON.test_create_service_without_description[id-5d3252c8-e555-494b-a6c8-e11d7335da42] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_list_ports_fields[id-ff7f117f-f034-4e0e-abff-ccef05c454b4] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_suspend_server_invalid_state[id-7f323206-05a9-4bf8-996b-dd5b2036501b,negative] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_domain_create_duplicate[id-e6f9e4a2-4f36-4be8-bdbc-4e199ae29427,negative] +tempest.api.volume.test_volume_transfers.VolumesTransfersV355Test.test_create_list_delete_volume_transfer[id-af4a5b97-0859-4f31-aa3c-85b05bb63322] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_negative_id[id-75f79124-277c-45e6-a373-a1d6803f4cc4,negative] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_group_on_project[id-cbf11737-1904-4690-9613-97bcbb3df1c4] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_list_agents_with_filter[id-eabadde4-3cd7-4ec4-a4b5-5a936d2d4408] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_limit_results[id-ba2fa9a9-b672-47cc-b354-3b4c0600e2cb] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_non_admin_set_keys[id-a00a3b81-5641-45a8-ab2b-4a8ec41e1d7d,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_without_rights[id-a3a585a7-d8cf-4b65-a1a0-edc2b1204f85,negative] +tempest.api.object_storage.test_object_services.PublicObjectTest.test_access_public_container_object_without_using_creds[id-07c9cf95-c0d4-4b49-b9c8-0ef2c9b27193] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server[id-aaa6cdf3-55a7-461a-add9-1c8596b9a07c] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resume_non_existent_server[id-221cd282-bddb-4837-a683-89c2487389b6,negative] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_port_security_disable_security_group[compute,id-7c811dcc-263b-49a3-92d2-1b4d8405f50c,network,slow] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_put_object_using_temp_url[id-9b08dade-3571-4152-8a4f-a4f2a873a735] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenant_request_without_token[id-a3ee9d7e-6920-4dd5-9321-d4b2b7f0a638,negative] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_update_all_tags[id-a2c1af8c-127d-417d-974b-8115f7e3d831] +tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_add_remove_network_from_dhcp_agent[id-a0856713-6549-470c-a656-e97c8df9a14d] +tempest.api.compute.admin.test_live_migration.LiveMigrationRemoteConsolesV26Test.test_live_migration_serial_console[id-6190af80-513e-4f0f-90f2-9714e84955d7] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_list_agents[id-6a326c69-654b-438a-80a3-34bcc454e138] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin_with_specified_tenant[id-7e5d6b8f-454a-4ba1-8ae2-da857af8338b] +tempest.api.compute.images.test_list_images.ListImagesTestJSON.test_list_images[id-fd51b7f4-d4a3-4331-9885-866658112a6f] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_create_port_binding_ext_attr[compute,id-8e8569c1-9ac7-44db-8bc1-f5fb2814f29b] +tempest.api.compute.servers.test_availability_zone.AZV2TestJSON.test_get_availability_zone_list_with_non_admin_user[id-a8333aa2-205c-449f-a828-d38c2489bf25] +tempest.api.volume.test_volume_delete_cascade.VolumesDeleteCascade.test_volume_from_snapshot_cascade_delete[id-59a77ede-609b-4ee8-9f68-fc3c6ffe97b5] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_filter_by_exist_host[id-86c7a8f7-50cf-43a9-9bac-5b985317134f] +tempest.api.network.test_ports.PortsTestJSON.test_port_list_filter_by_ip[id-e7fe260b-1e79-4dd3-86d9-bec6a7959fc5] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_update_consumer[id-080a9b1a-c009-47c0-9979-5305bf72e3dc] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_max_count_less_than_one[id-a6f9c2ab-e060-4b82-b23c-4532cb9390ff,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_additional_default_security_group_fails[id-2323061e-9fbf-4eb0-b547-7e8fafc90849,negative] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_get_metadata_headers_with_invalid_container_name[id-1a95ab2e-b712-4a98-8a4d-8ce21b7557d6,negative] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_delete_security_group_rule_with_non_existent_id[id-56fddcca-dbb8-4494-a0db-96e9f869527c,negative] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_reporting_with_prefix[id-f38e647f-ab42-4a31-a2e7-ca86a6485215] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_volume_upload[id-d8f1ca95-3d5b-44a3-b8ca-909691c9532d,image] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_count_uploading_quota[id-b103788b-5329-4aa9-8b0d-97f8733460db,identity,image] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_using_marker[id-6db2f0c0-ddee-4162-9c84-0703d3dd1107] +tempest.api.object_storage.test_account_services.AccountTest.test_list_no_account_metadata[id-b904c2e3-24c2-4dba-ad7d-04e90a761be5] +tempest.api.compute.admin.test_volumes_negative.VolumesAdminNegativeTest.test_update_attached_volume_with_nonexistent_volume_in_body[id-7dcac15a-b107-46d3-a5f6-cb863f4e454a,negative] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_list_large_object_metadata[id-e69ad766-e1aa-44a2-bdd2-bf62c09c1456] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_show_default_group_config_and_options[id-11a02bf0-6f94-4380-b3b0-c8dc18fc0d22] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_reset_gateway_without_snat[id-f2faf994-97f4-410b-a831-9bc977b64374] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_zero[id-41331caa-eaf4-4001-869d-bc18c1869360,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_server_name[id-5e6ccff8-349d-4852-a8b3-055df7988dd2] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_add_router_interfaces_on_overlapping_subnets_returns_400[id-957751a3-3c68-4fa2-93b6-eb52ea10db6e,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_server_pass_id_exceeding_length_limit[id-f4d7279b-5fd2-4bf2-9ba4-ae35df0d18c5,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_port_list_filter_by_router_id[id-5ad01ed0-0e6e-4c5d-8194-232801b15c72] +tempest.api.compute.admin.test_flavors_extra_specs.FlavorsExtraSpecsTestJSON.test_flavor_non_admin_get_specific_key[id-12805a7f-39a3-4042-b989-701d5cad9c90] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_list_hosts_with_a_blank_zone[id-9af3c171-fbf4-4150-a624-22109733c2a6] +tempest.api.object_storage.test_account_bulk.BulkTest.test_extract_archive[id-a407de51-1983-47cc-9f14-47c2b059413c] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_reset_state_server_nonexistent_server[id-e741298b-8df2-46f0-81cb-8f814ff2504c,negative] +tempest.api.object_storage.test_container_sync.ContainerSyncTest.test_container_synchronization[id-be008325-1bba-4925-b7dd-93b58f22ce9b,slow] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_specify_uuid_37_characters_or_more[id-36741560-510e-4cc2-8641-55fe4dfb2437,negative] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_image_share_accept[id-5934c6ea-27dc-4d6e-9421-eeb5e045494a] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_resize_server_using_overlimit_vcpus[id-7368a427-2f26-4ad9-9ba9-911a0ec2b0db,negative] +tempest.api.identity.admin.v3.test_default_project_id.TestDefaultProjectId.test_default_project_id[id-d6110661-6a71-49a7-a453-b5e26640ff6d] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_slaac_subnet_with_ports[id-88554555-ebf8-41ef-9300-4926d45e06e9] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_host_as_user[id-7324c334-bd13-4c93-8521-5877322c3d51,negative] +tempest.scenario.test_snapshot_pattern.TestSnapshotPattern.test_snapshot_pattern[compute,id-608e604b-1d63-4a82-8e3e-91bc665c90b4,image,network,slow] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_filter_by_error_status[id-06f960bb-15bb-48dc-873d-f96e89be7870] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_show_non_existent_subnet[id-d746b40c-5e09-4043-99f7-cba1be8b70df,negative] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_invalid_key[id-0025fbd6-a4ba-4cde-b8c2-96805dcfdabc,negative] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips_outrange[id-98244d88-d990-4570-91d4-6b25d70d08af] +tempest.api.compute.servers.test_novnc.NoVNCConsoleTestJSON.test_novnc_bad_token[id-f9c79937-addc-4aaa-9e0e-841eef02aeb7] +tempest.api.object_storage.test_object_formpost_negative.ObjectFormPostNegativeTest.test_post_object_using_form_expired[id-d3fb3c4d-e627-48ce-9379-a1631f21336d,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_overwrite[id-49f866ed-d6af-4395-93e7-4187eb56d322] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_invalid_group_name[id-1759c3cb-b0fc-44b7-86ce-c99236be911d,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_restore_server_invalid_state[id-7fcadfab-bd6a-4753-8db7-4a51e51aade9,negative] +tempest.api.network.admin.test_routers_dvr.RoutersTestDVR.test_distributed_router_creation[id-08a2a0a8-f1e4-4b34-8e30-e522e836c44e] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_fresh_metadata[id-0f3d62a6-47e3-4554-b0e5-1a5dc372d501] +tempest.api.volume.admin.test_volume_quotas_negative.VolumeQuotasNegativeTestJSON.test_quota_volume_gigabytes[id-2dc27eee-8659-4298-b900-169d71a91374,negative] +tempest.api.volume.test_image_metadata.VolumesImageMetadata.test_update_show_delete_image_metadata[id-03efff0b-5c75-4822-8f10-8789ac15b13e,image] +tempest.api.volume.test_volume_absolute_limits.AbsoluteLimitsTests.test_get_volume_absolute_limits[id-8e943f53-e9d6-4272-b2e9-adcf2f7c29ad] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_show_metering_label_rule[id-b7354489-96ea-41f3-9452-bace120fb4a7] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_remove_server_all_security_groups[id-1d1c9104-1b0a-11e7-a3d4-fa163e65f5ce] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_backup[id-bbcfa285-af7f-479e-8c1a-8c34fc16543c] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_create_with_enabled_False[id-ac6c137e-4d3d-448f-8c83-4f13d0942651,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_external_network_visibility[id-af774677-42a9-4e4b-bb58-16fe6a5bc1ec,smoke] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips[id-51a5e97f-f02e-4e4e-9a17-a69811d300e3] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_update_port_with_multiple_ip_mac_address_pair[id-b3f20091-6cd5-472b-8487-3516137df933] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_unset_gateway[id-ad81b7ee-4f81-407b-a19c-17e623f763e8] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_service_and_host_name[id-ffa6167c-4497-4944-a464-226bbdb53908] +tempest.api.network.test_ports.PortsTestJSON.test_create_show_delete_port_user_defined_mac[id-13e95171-6cbd-489c-9d7c-3f9c58215c18] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_metadata_value[id-e1e8df32-7b22-44e1-aa08-ccfd8d446b58] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_nonexistent_volume_id[id-0186422c-999a-480e-a026-6a665744c30c,negative] +tempest.api.volume.test_versions.VersionsTest.test_list_versions[id-77838fc4-b49b-4c64-9533-166762517369,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_name[id-644ea267-9bd9-4f3b-af9f-dffa02396a17] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_filter_by_flavor[id-80c574cc-0925-44ba-8602-299028357dd9] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_get_server_metadata_item[id-3043c57d-7e0e-49a6-9a96-ad569c265e6a] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_get_default_quotas[id-3b0a7c8f-cf58-46b8-a60c-715a32a8ba7d] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_nonexistent_type_id[id-474090d2-0824-eb3b-9335-f506b4aa49d8,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_invalid_volume_id[id-e66e40d6-65e6-4e75-bdc7-636792fa152d,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_without_gateway[id-d2d596e2-8e76-47a9-ac51-d4648009f4d3] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_subnet_with_v6_attributes_stateless[id-7d410310-8c86-4902-adf9-865d08e31adb] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_gw_and_allocation_pools[id-8217a149-0c6c-4cfb-93db-0486f707d13f] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_rebuild_server_in_stop_state[id-30449a88-5aff-4f9b-9866-6ee9b17f906d] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_show_quota_usage[id-18c51ae9-cb03-48fc-b234-14a19374dbed] +tempest.api.identity.admin.v3.test_domain_configuration.DomainConfigurationTestJSON.test_create_update_and_delete_domain_config[id-7161023e-5dd0-4612-9da0-1bac6ac30b63] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_users_with_name[id-c285bb37-7325-4c02-bff3-3da5d946d683] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_show_ec2_credential[id-8b8d1010-5958-48df-a6cd-5e3df72e6bcf] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_create_volume_without_passing_size[id-131cb3a1-75cc-4d40-b4c3-1317f64719b0,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_server_with_null_flavor[id-45436a7d-a388-4a35-a9d8-3adc5d0d940b,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_list_details[id-1e7fdac2-b672-4ad1-97a4-bad0e3030118] +tempest.api.compute.admin.test_simple_tenant_usage_negative.TenantUsagesNegativeTestJSON.test_list_usage_all_tenants_with_non_admin_user[id-bbe6fe2c-15d8-404c-a0a2-44fad0ad5cc7,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_x_object_manifest[id-11b4515b-7ba7-4ca8-8838-357ded86fc10] +tempest.api.image.v2.test_images.MultiStoresImportImagesTest.test_glance_direct_import_image_to_specific_stores[id-82fb131a-dd2b-11ea-aec7-340286b6c574] +tempest.api.identity.v3.test_users.IdentityV3UsersTest.test_password_history_check_self_service_api[id-941784ee-5342-4571-959b-b80dd2cea516] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_remove_flavor_access_not_found[id-1f710927-3bc7-4381-9f82-0ca6e42644b7,negative] +tempest.api.compute.volumes.test_attach_volume_negative.AttachVolumeNegativeTest.test_attach_attached_volume_to_different_server[id-ee37a796-2afb-11e7-bc0f-fa163e65f5ce,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminUnderV252Test.test_search_hypervisor[id-d7e1805b-3b14-4a3b-b6fd-50ec6d9f361f] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_empty_name[id-d80d0c2f-4514-4d1e-806d-0930dfc5a187,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_remote_ip_and_group[id-b5c4b247-6b02-435b-b088-d10d45650881,negative] +tempest.api.object_storage.test_object_formpost.ObjectFormPostTest.test_post_object_using_form[id-80fac02b-6e54-4f7b-be0d-a965b5cbef76] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_startup_nonexistent_host[id-0d981ac3-4320-4898-b674-82b61fbb60e4,negative] +tempest.api.network.admin.test_routers_negative.RoutersAdminNegativeTest.test_router_set_gateway_used_ip_returns_409[id-7101cc02-058a-11e7-93e1-fa163e4fa634,negative] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_maintenance[id-b957cabd-1486-4e21-90cf-a9ed3c39dfb2] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_ipv6_addr_only[id-38fb1d02-c3c5-41de-91d3-9bc2025a75eb] +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ip_pools[id-df389fc8-56f5-43cc-b290-20eda39854d3] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_volume_from_snapshot[id-677863d1-3142-456d-b6ac-9924f667a7f4] +tempest.api.object_storage.test_account_quotas.AccountQuotasTest.test_admin_modify_quota[id-63f51f9f-5f1d-4fc6-b5be-d454d70949d6,smoke] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_non_existent_security_group[id-27edee9c-873d-4da6-a68a-3c256efebe8f,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_nonexistent_name[id-0f4aa809-8c7b-418f-8fb3-84c7a5dfc52f,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_remove_object_metakey[id-ad21e342-7916-4f9e-ab62-a1f885f2aaf9] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_as_user[id-86a1cb14-da37-4a70-b056-903fd56dfe29,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_with_invalid_status[id-143b279b-7522-466b-81be-34a87d564a7c,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_remove_object_meta[id-ce798afc-b278-45de-a5ce-2ea124b98b99] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image[id-f848bb94-1c6e-45a4-8726-39e3a5b23535,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_changes_since[id-18bac3ae-da27-436c-92a9-b22474d13aab] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_list_services_with_non_admin_user[id-1126d1f8-266e-485f-a687-adc547492646,negative] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_update_mismatch_key[id-25b822b8-9f49-44f6-80de-d99f0482e5cb,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_delete_object_without_using_creds[id-af85af0b-a025-4e72-a90e-121babf55720,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_filter_by_min_ram[id-09fe7509-b4ee-4b34-bf8b-39532dc47292] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_update_port_binding_ext_attr[compute,id-6f6c412c-711f-444d-8502-0ac30fbf5dd5] +tempest.api.compute.images.test_list_image_filters_negative.ListImageFiltersNegativeTestJSON.test_get_nonexistent_image[id-391b0440-432c-4d4b-b5da-c5096aa247eb,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_show_delete_port_user_defined_mac[id-13e95171-6cbd-489c-9d7c-3f9c58215c18] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin[id-51717b38-bdc1-458b-b636-1cf82d99f62f] +tempest.api.compute.servers.test_server_rescue.ServerRescueTestJSONUnderV235.test_rescued_vm_add_remove_security_group[id-affca41f-7195-492d-8065-e09eee245404] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_none_extra_spec_id[id-9bf7a657-b011-4aec-866d-81c496fbe5c8,negative] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_get_list_update_delete[id-2a8abbe4-d871-46db-b049-c41f5af8216e] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsTestJSON.test_update_show_port_with_extra_dhcp_options[id-9a6aebf4-86ee-4f47-b07a-7f7232c55607] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_with_floating_ip[id-58a80596-ffb2-11e6-9393-fa163e4fa634,negative] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_migrate_server_invalid_state[id-b0b17f83-d14e-4fc4-8f31-bcc9f3cfa629,negative] +tempest.api.identity.admin.v2.test_endpoints.EndPointsTestJSON.test_create_list_delete_endpoint[id-9974530a-aa28-4362-8403-f06db02b26c1] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_non_existent_image[id-ec652588-7e3c-4b67-a2f2-0fa96f57c8fc,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_from_image_with_decreasing_size[id-5b810c91-0ad1-47ce-aee8-615f789be78f,image,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_content_disposition[id-5daebb1d-f0d5-4dc9-b541-69672eff00b0] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_tags_schema[id-cde4a891-b38b-3bf0-a3b2-e03ee67b3a3a] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_update_name[id-781f2266-d128-47f3-8bdb-f70970add238] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_update_non_existent_tenant[id-bd20dc2a-9557-4db7-b755-f48d952ad706,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_volume_without_passing_volume_id[id-441a1550-5d44-4b30-af0f-a6d402f52026,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_with_default_snat_value[id-847257cc-6afd-4154-b8fb-af49f5670ce8] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_additional_args[id-87dfbcf9-1849-43ea-b1e4-efa3eeae9f71] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_with_invalid_tenant[id-921f1ad6-7907-40b8-853f-637e7ee52178,negative] +tempest.api.network.test_networks.NetworksTest.test_list_networks[id-f7ffdeda-e200-4a7a-bcbe-05716e86bf43,smoke] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] +tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_suspended_server[id-8ca07fec-0262-11e7-907e-fa163e4fa634] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_ip_prefix[id-16459776-5da2-4634-bce4-4b55ee3ec188] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_get_image_metadata_item[id-4f5db52f-6685-4c75-b848-f4bb363f9aa6] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_set_image_metadata_item[id-f2de776a-4778-4d90-a5da-aae63aee64ae] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_delete_user[id-29ed26f4-a74e-4425-9a85-fdb49fa269d2] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_error_deleting[id-bf89080f-8129-465e-9327-b2f922666ba5] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_count_quota[id-9b74fe24-183b-41e6-bf42-84c2958a7be8,identity,image] +tempest.api.image.v2.test_images_tags_negative.ImagesTagsNegativeTest.test_delete_non_existing_tag[id-39c023a2-325a-433a-9eea-649bf1414b19,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_non_existent_keypair[id-7a2efc39-530c-47de-b875-2dd01c8d39bd,negative] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_thaw_host_with_invalid_host[id-7c6287c9-d655-47e1-9a11-76f6657a6dce,negative] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_list_api_versions[id-721f480f-35b6-46c7-846e-047e6acea0dc,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_show_metering_label[id-30abb445-0eea-472e-bd02-8649f54a5968] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_x_remove_object_metakey[id-9a88dca4-b684-425b-806f-306cd0e57e42] +tempest.api.network.admin.test_negative_quotas.QuotasNegativeTest.test_network_quota_exceeding[id-644f4e1b-1bf9-4af0-9fd8-eb56ac0f51cf,negative] +tempest.scenario.test_object_storage_basic_ops.TestObjectStorageBasicOps.test_swift_basic_ops[id-b920faf1-7b8a-4657-b9fe-9c4512bfb381,object_storage] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container[id-92139d73-7819-4db1-85f8-3f2f22a8d91f,smoke] +tempest.api.identity.admin.v3.test_application_credentials.ApplicationCredentialsV3AdminTest.test_create_application_credential_with_roles[id-3b3dd48f-3388-406a-a9e6-4d078a552d0e] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_delete_non_existent_security_group[id-1f1bb89d-5664-4956-9fcd-83ee0fa603df,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_keypair_create_with_invalid_pub_key[id-29cca892-46ae-4d48-bc32-8fe7e731eb81,negative] +tempest.api.compute.admin.test_quotas.QuotasAdminTestV257.test_get_updated_quotas[id-e641e6c6-e86c-41a4-9e5c-9493c0ae47ad] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_pause_state[id-943bd6e8-4d7a-4904-be83-7a6cc2d4213b] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_specify_multibyte_character_name[id-defbaca5-d611-49f5-ae21-56ee25d2db49] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_size_negative[id-8b472729-9eba-446e-a83b-916bdb34bef7,negative] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_add_member_with_non_existing_image[id-147a9536-18e3-45da-91ea-b037a028f364,negative] +tempest.api.volume.test_volume_transfers.VolumesTransfersTest.test_create_list_delete_volume_transfer[id-ab526943-b725-4c07-b875-8e8ef87a2c30] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_group_on_system[id-c888fe4f-8018-48db-b959-542225c1b4b6] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_source_volid[id-47c73e08-4be8-45bb-bfdf-0c4e79b88344,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_create_and_remove_metadata[id-f726174b-2ded-4708-bff7-729d12ce1f84] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_router_admin_state[compute,id-04b9fe4e-85e8-4aea-b937-ea93885ac59f,network,slow] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_update_non_existent_router_returns_404[id-b23d1569-8b0c-4169-8d4b-6abd34fad5c7,negative] +tempest.api.network.admin.test_dhcp_agent_scheduler.DHCPAgentSchedulersTestJSON.test_list_networks_hosted_by_one_dhcp[id-30c48f98-e45d-4ffb-841c-b8aad57c7587] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_enabled_non_bool[id-23a2f3da-4a1a-41da-abdd-632328a861ad,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_negative_image_id[id-68e2c175-bd26-4407-ac0f-4ea9ce2139ea,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_with_non_authorized_user[id-63d84e37-55a6-42e2-9e5f-276e60e26a00,negative] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_from_deleted_server[id-6cd5a89d-5b47-46a7-93bc-3916f0d84973,negative] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_revert_cold_migration[id-caa1aa8b-f4ef-4374-be0d-95f001c2ac2d] +tempest.api.compute.admin.test_fixed_ips.FixedIPsTestJson.test_list_fixed_ip_details[id-16b7d848-2f7c-4709-85a3-2dfb4576cc52] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_visibility[id-7a95bb92-d99e-4b12-9718-7bc6ab73e6d2] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_delete_non_existent_subnet[id-a176c859-99fb-42ec-a208-8a85b552a239,negative] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_reserve_volume_with_nonexistent_volume_id[id-ac6084c0-0546-45f9-b284-38a367e0e0e2,negative] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_update_policy_from_zero[compute,id-9cfc3bb8-f433-4c91-87b6-747cadc8958a,network] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_active_server[id-925fdfb4-5b13-47ea-ac8a-c36ae6fddb05] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id[id-32248db1-ab88-4821-9604-c7c369f1f88c,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_invalid_ports[id-0d9c7791-f2ad-4e2f-ac73-abf2373b0d2d,negative] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTestIDE.test_stable_device_rescue_cdrom_ide[id-947004c3-e8ef-47d9-9f00-97b74f9eaf96] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_networks[id-f7ffdeda-e200-4a7a-bcbe-05716e86bf43,smoke] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_create_ec2_credential[id-b0f55a29-54e5-4166-999d-712347e0c920] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_live_migration[compute,id-03fd1562-faad-11e7-9ea0-fa163e65f5ce,network,slow] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_list_domains[id-8cf516ef-2114-48f1-907b-d32726c734d4] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_with_invalid_username[id-bde9aecd-3b1c-4079-858f-beb5deaa5b5e,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_delete_nonexistent_volume_id[id-555efa6e-efcd-44ef-8a3b-4a7ca4837a29,negative] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_list_migrations[id-75c0b83d-72a0-4cf8-a153-631e83e7d53f] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_floating_ip_delete_port[id-e1f6bffd-442f-4668-b30e-df13f2705e77] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_from_non_bootable_volume[id-74085be3-a370-4ca2-bc51-2d0e10e0f573,image,negative,volume] +tempest.api.network.admin.test_floating_ips_admin_actions.FloatingIPAdminTestJSON.test_create_list_show_floating_ip_with_tenant_id_by_admin[id-32727cc3-abe2-4485-a16e-48f2d54c14f2] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_availability_zone[id-c0cfa863-3020-40d7-b587-e35f597d5d87] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_group_create_update_get[id-2e80343b-6c81-4ac3-88c7-452f3e9d5129] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_add_gateway_net_not_external_returns_400[id-11836a18-0b15-4327-a50b-f0d9dc66bddd,negative] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_revoke_user_roles_on_domain[id-3acf666e-5354-42ac-8e17-8b68893bcd36] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_with_multiple_params[id-2a7064eb-b9c3-429b-b888-33928fc5edd3] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_in_same_container[id-1a9ab572-1b66-4981-8c21-416e2a5e6011] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_shelved_state[id-bb0cb402-09dd-4947-b6e5-5e7e1cfa61ad] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_delete_with_nonexistent_container_name[id-65387dbf-a0e2-4aac-9ddc-16eb3f1f69ba,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_namespaces_schema[id-ffe44891-678b-3ba0-a3e2-e0a3967b3aeb] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_create_server_with_personality[id-3cfe87fd-115b-4a02-b942-7dc36a337fdf,slow] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_by_unauthorized_user[id-d0537987-0977-448f-a435-904c15de7298,negative] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_update_desc[id-f138b715-255e-4a7d-871d-351e1ef2e153] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_set_nonexistent_image_metadata_item[id-2154fd03-ab54-457c-8874-e6e3eb56e9cf,negative] +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_quotas[id-f1ef0a97-dbbb-4cca-adc5-c9fbc4f76107] +tempest.api.compute.servers.test_virtual_interfaces_negative.VirtualInterfacesNegativeTestJSON.test_list_virtual_interfaces_invalid_server_id[id-64ebd03c-1089-4306-93fa-60f5eb5c803c,negative,network] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_encryption_create_get_update_delete[id-7830abd0-ff99-4793-a265-405684a54d46] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_update_agent[id-dc9ffd51-1c50-4f0e-a820-ae6d2a568a9e] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_create_projects_name_length_over_64[id-502b6ceb-b0c8-4422-bf53-f08fdb21e2f0,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_create_none_body[id-c821bdc8-43a4-4bf4-86c8-82f3858d5f7d,negative] +tempest.api.network.test_networks.NetworksTest.test_create_update_network_description[id-c72c1c0c-2193-4aca-ccc4-b1442640bbbb] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit[id-5cfa4ab2-4373-48dd-a41f-a532b12b08b2] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_without_gateway[id-d2d596e2-8e76-47a9-ac51-d4648009f4d3] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_create_with_enabled_True[id-9c43181e-0627-484a-8c79-923e8a59598b,negative] +tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_subnet_id[id-b42e6e39-2e37-49cc-a6f4-8467e940900a,smoke] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_role_create_blank_name[id-c0b89e56-accc-4c73-85f8-9c0f866104c1,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_update_volume_with_empty_volume_id[id-72aeca85-57a5-4c1f-9057-f320f9ea575b,negative] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_search_hypervisor_with_non_admin_user[id-5b6a6c79-5dc1-4fa5-9c58-9c8085948e74,negative] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_marker[id-f7064ae8-dbcc-48da-b594-82feef6ea5af] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_rebuild[id-70cdb8a1-89f8-437d-9448-8844fd82bf46,negative] +tempest.api.network.test_versions.NetworksApiDiscovery.test_show_api_v2_details[id-e64b7216-3178-4263-967c-d389290988bf,smoke] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_delete_server_metadata_item[id-127642d6-4c7b-4486-b7cd-07265a378658] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_suspend_resume_server[id-0d8ee21e-b749-462d-83da-b85b41c86c7f] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypairsv22_create_list_show[id-8726fa85-7f98-4b20-af9e-f710a4f3391c] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_pagination[id-e9138a2c-f67b-4796-8efa-635c196d01de] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_update_floatingip_with_port_multiple_ip_address[id-45c4c683-ea97-41ef-9c51-5e9802f2f3d7] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_listing[id-941814cf-db9e-4b21-8112-2b6d0af10ee5] +tempest.api.image.v1.test_image_members.ImageMembersTest.test_get_shared_images[id-6a5328a5-80e8-4b82-bd32-6c061f128da9] +tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_show_host_detail[id-38adbb12-aee2-4498-8aec-329c72423aa4] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_register_with_invalid_container_format[id-036ede36-6160-4463-8c01-c781eee6369d,negative] +tempest.api.compute.admin.test_flavors_microversions.FlavorsV255TestJSON.test_crud_flavor[id-61976b25-488d-41dc-9dcb-cb9693a7b075] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_list_consumers[id-09ca50de-78f2-4ffb-ac71-f2254036b2b8] +tempest.api.network.test_networks.NetworksTest.test_show_subnet_fields[id-270fff0b-8bfc-411f-a184-1e8fd35286f0] +tempest.api.volume.admin.test_volume_snapshot_quotas_negative.VolumeSnapshotQuotasNegativeTestJSON.test_quota_volume_gigabytes_snapshots[id-c99a1ca9-6cdf-498d-9fdf-25832babef27,negative] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsStaticTestJSON.test_list_projects[id-1d830662-22ad-427c-8c3e-4ec854b0af44] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_resources[id-b9232f5e-d9e5-4d97-b96c-28d3db4de1bd,smoke] +tempest.api.compute.admin.test_delete_server.DeleteServersAdminTestJSON.test_delete_server_while_in_error_state[id-99774678-e072-49d1-9d2a-49a59bc56063] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.test_create_list_show_delete_interfaces_by_fixed_ip[id-d290c06c-f5b3-11e7-8ec8-002293781009,network] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_image_null_id[id-ef45000d-0a72-4781-866d-4cb7bf2562ad,negative] +tempest.api.object_storage.test_object_services.PublicObjectTest.test_access_public_object_with_another_user_creds[id-54e2a2fe-42dc-491b-8270-8e4217dd4cdc] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_revoke_user_roles_on_project_tree[id-9f02ccd9-9b57-46b4-8f77-dd5a736f3a06] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_non_existent_security_group[id-be308db6-a7cf-4d5c-9baf-71bafd73f35e,negative] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_snapshot_create_delete_with_volume_in_use[compute,id-8567b54c-4455-446d-a1cf-651ddeaa3ff2] +tempest.scenario.test_volume_backup_restore.TestVolumeBackupRestore.test_volume_backup_restore[compute,id-2ce5e55c-4085-43c1-98c6-582525334ad7,image,slow,volume] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_volume_type_encryption_nonexistent_type_id[id-a5924b5f-b6c1-49ba-994c-b4af55d26e52,negative] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_detailed_filter_by_invalid_status[id-d56e9540-73ed-45e0-9b88-98fc419087eb] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unshelve_non_existent_server[id-23d23b37-afaf-40d7-aa5d-5726f82d8821,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resume_server_invalid_state[id-ccb6294d-c4c9-498f-8a43-554c098bfadb,negative] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_stage_quota[id-fc76b8d9-aae5-46fb-9285-099e37f311f7,identity,image] +tempest.api.compute.admin.test_flavors_access.FlavorsAccessTestJSON.test_flavor_access_add_remove[id-59e622f6-bdf6-45e3-8ba8-fedad905a6b4] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_trust_expire[id-0ed14b66-cefd-4b5c-a964-65759453e292] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_get_nonexistent_volume_id[id-f131c586-9448-44a4-a8b0-54ca838aa43e,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway[id-6cc285d8-46bf-4f36-9b1a-783e3008ba79] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_network[id-2bf13842-c93f-4a69-83ed-717d2ec3b44e,smoke] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_list_user_groups[id-64573281-d26a-4a52-b899-503cb0f4e4ec] +tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_get_available_project_scopes[id-08ed85ce-2ba8-4864-b442-bcc61f16ae89] +tempest.api.image.v2.admin.test_images.ImportCopyImagesTest.test_image_copy_image_import[id-9b3b644e-03d1-11eb-a036-fa163e2eaf49] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unshelve_server_invalid_state[id-8f198ded-1cca-4228-9e65-c6b449c54880,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_attach_volumes_with_nonexistent_volume_id[compute,id-f5e56b0a-5d02-43c1-a2a7-c9b792c2e3f6,negative] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_get_service_by_invalid_service_and_valid_host[id-1e966d4a-226e-47c7-b601-0b18a27add54,negative] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_set_image_metadata[id-ece7befc-d3ce-42a4-b4be-c3067a418c29] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_list_server_metadata_non_existent_server[id-f408e78e-3066-4097-9299-3b0182da812e,negative] +tempest.api.volume.test_volumes_clone_negative.VolumesCloneNegativeTest.test_create_from_volume_decreasing_size[id-9adae371-a257-43a5-459a-dc7c88e66e0e,negative] +tempest.api.identity.admin.v3.test_list_projects.ListProjectsTestJSON.test_list_projects_with_enabled[id-0fe7a334-675a-4509-b00e-1c4b95d5dae8] +tempest.api.image.v1.test_images.UpdateImageMetaTest.test_list_image_metadata[id-01752c1c-0275-4de3-9e5b-876e44541928] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_status[id-2943f712-71ec-482a-bf49-d5ca06216b9f] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_bootable[id-2016a942-3020-40d7-95ce-7613bf8407ce] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_list_get_volume_attachments_multiattach[id-8d5853f7-56e7-4988-9b0c-48cea3c7049a] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_list[id-a6154130-5a55-4850-8be4-5e9e796dbf17,smoke] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_migrate_with_qos_min_bw_allocation[compute,id-8a98150c-a506-49a5-96c6-73a5e7b04ada,network] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_create_server_with_scheduling_hint[id-fdcd9b33-0903-4e00-a1f7-b5f6543068d6] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_list_external_networks[id-39be4c9b-a57e-4ff9-b7c7-b218e209dfcc] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_get_role_by_id[id-db6870bd-a6ed-43be-a9b1-2f10a5c9994f] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_create_region_with_specific_id[id-2c12c5b5-efcf-4aa5-90c5-bff1ab0cdbe2,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_metadata_exceeds_length_limit[id-7fc74810-0bd2-4cd7-8244-4f33a9db865a,negative] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_create_delete_metering_label_rule_with_filters[id-f4d547cd-3aee-408f-bf36-454f8825e045] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_update_external_network[id-4db5417a-e11c-474d-a361-af00ebef57c5] +tempest.api.network.test_ports.PortsTestJSON.test_update_port_with_security_group_and_extra_attributes[id-58091b66-4ff4-4cc1-a549-05d60c7acd1a] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_get_nonexistent_image_metadata_item[id-41ae052c-6ee6-405c-985e-5712393a620d,negative] +tempest.api.compute.admin.test_simple_tenant_usage.TenantUsagesTestJSON.test_get_usage_tenant[id-94135049-a4c5-4934-ad39-08fa7da4f22e] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_to_itself[id-2248abba-415d-410b-9c30-22dff9cd6e67] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_preserve_preexisting_port[compute,id-759462e1-8535-46b0-ab3a-33aa45c55aaa,network,slow] +tempest.api.volume.admin.test_volumes_list.VolumesListAdminTestJSON.test_volume_list_param_tenant[id-5866286f-3290-4cfd-a414-088aa6cdc469] +tempest.api.identity.v3.test_catalog.IdentityCatalogTest.test_catalog_standardization[id-56b57ced-22b8-4127-9b8a-565dfb0207e2] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_valid_object[id-9a0fb034-86af-4df0-86fa-f8bd7db21ae0,smoke] +tempest.api.volume.admin.test_groups.GroupsTest.test_group_update[id-4a8a6fd2-8b3b-4641-8f54-6a6f99320006] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_when_public_key_bits_exceeds_maximum[id-fc100c19-2926-4b9c-8fdc-d0589ee2f9ff,negative] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_associate_user_to_project[id-59398d4a-5dc5-4f86-9a4c-c26cc804d6c6] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_create_router_with_snat_explicit[id-ea74068d-09e9-4fd7-8995-9b6a1ace920f] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_metadata[id-005f9bf6-e06d-41ec-968e-96c78e0b1d82] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_non_admin_unset_keys[id-28f12249-27c7-44c1-8810-1f382f316b11,negative] +tempest.api.compute.servers.test_create_server_multi_nic.ServersTestMultiNic.test_verify_multiple_nics_order[id-0578d144-ed74-43f8-8e57-ab10dbf9b3c2] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_check_revoke_roles_on_projects_group[id-26021436-d5a4-4256-943c-ded01e0d4b45] +tempest.api.volume.admin.test_volume_quotas_negative.VolumeQuotasNegativeTestJSON.test_quota_volumes[id-bf544854-d62a-47f2-a681-90f7a47d86b6,negative] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_list_projects_by_unauthorized_user[id-24c49279-45dd-4155-887a-cb738c2385aa,negative] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_list_fixed_ip_details_with_non_admin_user[id-9f17f47d-daad-4adc-986e-12370c93e407,negative] +tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_rescope_token[id-565fa210-1da1-4563-999b-f7b5b67cf112] +tempest.api.identity.admin.v3.test_endpoints_negative.EndpointsNegativeTestJSON.test_update_with_enabled_True[id-faba3587-f066-4757-a48e-b4a3f01803bb,negative] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_port_range[id-00296fa9-0576-496a-ae15-fbab843189e0,negative] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_shutdown_nonexistent_host[id-9e637444-29cf-4244-88c8-831ae82c31b6,negative] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_create_floatingip_in_private_network[id-50b9aeb4-9f0b-48ee-aa31-fa955a48ff54,negative] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os[compute,id-76f26acd-9688-42b4-bc3e-cd134c4cb09e,network,slow] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_create_domain_without_description[id-2abf8764-309a-4fa9-bc58-201b799817ad] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_eui64[id-e5517e62-6f16-430d-a672-f80875493d4c] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_create_port_binding_ext_attr[compute,id-8e8569c1-9ac7-44db-8bc1-f5fb2814f29b] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object[id-02610ba7-86b7-4272-9ed8-aa8d417cb3cd,smoke] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete[id-27fb0e9f-fb64-41dd-8bdb-1ffa762f0d51,smoke] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_invalid_status[id-ba94b27b-be3f-496c-a00e-0283b373fa75,negative] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_delete_large_object[id-87b6dfa1-abe9-404d-8bf0-6c3751e6aa77] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_using_marker[id-e800f879-9828-4bd0-8eae-4f17189951fb] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_delete_ec2_credential[id-6aba0d4c-b76b-4e46-aa42-add79bc1551d] +tempest.api.network.test_networks.NetworksTest.test_update_subnet_gw_dns_host_routes_dhcp[id-3d3852eb-3009-49ec-97ac-5ce83b73010a] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_disk_format[id-70c6040c-5a97-4111-9e13-e73665264ce1,negative] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_server_with_scheduler_hint_group[id-ed20d3fb-9d1f-4329-b160-543fbd5d9811] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_create_duplicate[id-af16f44b-a849-46cb-9f13-a751c388f739,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_ethertype[id-5666968c-fff3-40d6-9efc-df1c8bd01abb,negative] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_resize_server_with_multiattached_volume[id-f01c7169-a124-4fc7-ae60-5e380e247c9c] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_router_remove_interface_in_use_returns_409[id-04df80f9-224d-47f5-837a-bf23e33d1c20,negative] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_cross_tenant_traffic[compute,id-e79f879e-debb-440c-a7e4-efeda05b6848,network] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_nonexistent_extra_spec_id[id-25e5a0ee-89b3-4c53-8310-236f76c75365,negative] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_role_create_update_show_list[id-18afc6c0-46cf-4911-824e-9989cc056c3a,smoke] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_create_list_port_with_address_pair[id-86c3529b-1231-40de-803c-00e40882f043] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_host_list[id-7f6a1cc5-2446-4cdb-9baa-b6ae0a919b72] +tempest.api.compute.admin.test_create_server.ServersWithSpecificFlavorTestJSON.test_verify_created_server_ephemeral_disk[id-b3c7bcfc-bb5b-4e22-b517-c7f686b802ca] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_fixedips_duplicate[id-57b8302b-cba9-4fbb-8835-9168df029051] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_delete_image_that_is_not_yet_active[id-0894954d-2db2-4195-a45b-ffec0bc0187e,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_flavor[id-18f5227f-d155-4429-807c-ccb103887537,negative] +tempest.api.compute.admin.test_agents.AgentsAdminTestJSON.test_create_agent[id-1fc6bdc8-0b6d-4cc7-9f30-9b04fabe5b90] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_show_servers_with_nonexistent_hypervisor[id-02463d69-0ace-4d33-a4a8-93d7883a2bba,negative] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_flavor_access_list_with_public_flavor[id-0621c53e-d45d-40e7-951d-43e5e257b272,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_suspend_resume[compute,id-5cdf9499-541d-4923-804e-b9a60620a7f0,network,slow] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_bulk_port[id-67f1b811-f8db-43e2-86bd-72c074d4a42c] +tempest.api.volume.test_volumes_actions.VolumesActionsTest.test_attach_detach_volume_to_instance[compute,id-fff42874-7db5-4487-a8e1-ddda5fb5288d,smoke] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_disassociate_floating_ip[id-307efa27-dc6f-48a0-8cd2-162ce3ef0b52] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_bootable[id-2016a939-72ec-482a-bf49-d5ca06216b9f] +tempest.api.identity.v2.test_api_discovery.TestApiDiscovery.test_api_media_types[id-007a0be0-78fe-4fdb-bbee-e9216cc17bb2,smoke] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_duplicate_security_group_rule_fails[id-8fde898f-ce88-493b-adc9-4e4692879fc5,negative] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_personality_files_exceed_limit[id-176cd8c9-b9e8-48ee-a480-180beab292bf] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_remote_ip_and_group[id-b5c4b247-6b02-435b-b088-d10d45650881,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_nonexistent_security_group[id-c5fa6041-80cd-483b-aa6d-4e45f19d093c,negative] +tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_port_id[id-2b7d2f37-6748-4d78-92e5-1d590234f0d5,smoke] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_deleting[id-92ce8597-b992-43a1-8868-6316b22a969e] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_resource_type_schema[id-fde34891-678b-3b40-ae32-e0a3e67b6beb] +tempest.api.identity.admin.v2.test_tokens.TokensTestJSON.test_rescope_token[id-25ba82ee-8a32-4ceb-8f50-8b8c71e8765e] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_create_delete_qos_with_front_end_consumer[id-7e15f883-4bef-49a9-95eb-f94209a1ced1] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_port_update_new_security_group[compute,id-f4d556d7-1526-42ad-bafb-6bebf48568f6,network,slow] +tempest.api.image.v1.test_image_members.ImageMembersTest.test_remove_member[id-a76a3191-8948-4b44-a9d6-4053e5f2b138] +tempest.api.network.test_ports.PortsTestJSON.test_create_update_port_with_second_ip[id-63aeadd4-3b49-427f-a3b1-19ca81f06270] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_show_details[id-94ff9eae-a183-428e-9cdb-79fde71211cc] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_disable_service_with_invalid_service_id[id-a9eeeade-42b3-419f-87aa-c9342aa068cf,negative] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_get_service_by_invalid_service_and_valid_host[id-1e966d4a-226e-47c7-b601-0b18a27add54,negative] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_create_delete_tag[id-8d95abe2-c658-4c42-9a44-c0258500306b] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_server_security_groups[id-fe4abc0d-83f5-4c50-ad11-57a1127297a2] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_delete_server_group_with_affinity_policy[id-5dc57eda-35b7-4af7-9e5f-3c2be3d2d68b] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_create_user_with_enabled[id-89d9fdb8-15c2-4304-a429-48715d0af33d] +tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_get_available_domain_scopes[id-ec5ecb05-af64-4c04-ac86-4d9f6f12f185] +tempest.api.image.v2.test_images.ListUserImagesTest.test_get_images_schema[id-25c8d7b2-df21-460f-87ac-93130bcdc684] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_remote_group_id[id-c2ed2deb-7a0c-44d8-8b4c-a5825b5c310b] +tempest.scenario.test_network_v6.TestGettingAddress.test_dhcp6_stateless_from_os[compute,id-d7e1f858-187c-45a6-89c9-bdafde619a9f,network,slow] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_keypair_create_delete[id-6c1d3123-4519-4742-9194-622cb1714b7d] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_show_port_fields[id-45fcdaf2-dab0-4c13-ac6c-fcddfb579dbd] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_index[id-c1f055ab-621d-4a6a-831f-846fcb578b8b] +tempest.api.compute.limits.test_absolute_limits_negative.AbsoluteLimitsNegativeTestJSON.test_max_metadata_exceed_limit[id-215cd465-d8ae-49c9-bf33-9c911913a5c8,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_set_empty_name[id-38204696-17c6-44da-9590-40f87fb5a899,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_vnc_console[id-c6bc11bf-592e-4015-9319-1c98dc64daf5] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_list_migrations_in_flavor_resize_situation[id-1b512062-8093-438e-b47a-37d2f597cd64] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_with_duplicate_name[id-57ae8558-120c-4723-9308-3751474e7ecf,negative] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_error[id-645a4a67-a1eb-4e8e-a547-600abac1525d] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_user_roles_by_unauthorized_user[id-7391ab4c-06f3-477a-a64a-c8e55ce89837,negative] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_migrate_non_existent_server[id-46a4e1ca-87ae-4d28-987a-1b6b136a0221,negative] +tempest.api.network.test_ports.PortsTestJSON.test_create_port_in_allowed_allocation_pools[id-0435f278-40ae-48cb-a404-b8a087bc09b1,smoke] +tempest.api.compute.admin.test_server_diagnostics.ServerDiagnosticsV248Test.test_get_server_diagnostics[id-64d0d48c-dff1-11e6-bf01-fe55135034f3] +tempest.api.compute.admin.test_volume.AttachSCSIVolumeTestJSON.test_attach_scsi_disk_with_config_drive[id-777e468f-17ca-4da4-b93d-b7dbf56c0494] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_list_as_user[id-b7d475a6-5dcd-4ff4-b70a-cd9de66a6672,negative] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_additional_args[id-87dfbcf9-1849-43ea-b1e4-efa3eeae9f71] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_floating_ip_update_different_router[id-1bb2f731-fe5a-4b8c-8409-799ade1bed4d] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_remove_image_member[id-72989bc7-2268-48ed-af22-8821e835c914] +tempest.api.compute.servers.test_server_addresses_negative.ServerAddressesNegativeTestJSON.test_list_server_addresses_by_network_neg[id-a2ab5144-78c0-4942-a0ed-cc8edccfd9ba,negative,network] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_show_non_existent_security_group_rule[id-4c094c09-000b-4e41-8100-9617600c02a6,negative] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_dissociate_nonexistent_floating_ip[id-0a081a66-e568-4e6b-aa62-9587a876dca8,negative] +tempest.scenario.test_volume_migrate_attached.TestVolumeMigrateRetypeAttached.test_volume_migrate_attached[compute,id-fe47b1ed-640e-4e3b-a090-200e25607362,slow,volume] +tempest.api.identity.v2.test_tokens.TokensTest.test_create_token[id-65ae3b78-91ff-467b-a705-f6678863b8ec] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_create_project_with_empty_name[id-7828db17-95e5-475b-9432-9a51b4aa79a9,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypair_create_with_pub_key[id-39c90c6a-304a-49dd-95ec-2366129def05] +tempest.api.identity.v3.test_domains.DefaultDomainTestJSON.test_default_domain_exists[id-17a5de24-e6a0-4e4a-a9ee-d85b6e5612b5,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_deleted_server[id-581a397d-5eab-486f-9cf9-1014bbd4c984,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_created_at_desc[id-dcbbe24a-f3c0-4ec8-9274-55d48db8d1cf] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_with_detail[id-6e85fde4-b3cd-4137-ab72-ed5f418e8c24] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_list_hosts_with_non_admin_user[id-dd032027-0210-4d9c-860e-69b1b8deed5f,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_without_write_rights[id-8ba512ad-aa6e-444e-b882-2906a0ea2052,negative] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_register_upload_get_image_file[id-139b765e-7f3d-4b3d-8b37-3ca3876ee318,smoke] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_update_name_default[id-966e2b96-023a-11e7-a9e4-fa163e4fa634,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_delete_invalid_volume_id[id-1f035827-7c32-4019-9240-b4ec2dbd9dfd,negative] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_metadata_exceeds_overall_metadata_count[id-ac666539-d566-4f02-8ceb-58e968dfb732,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_reboot_non_existent_server[id-d4c023a0-9c55-4747-9dd5-413b820143c7,negative] +tempest.api.volume.admin.test_backends_capabilities.BackendsCapabilitiesAdminTestsJSON.test_compare_volume_stats_values[id-a9035743-d46a-47c5-9cb7-3c80ea16dea0] +tempest.api.object_storage.test_object_expiry.ObjectExpiryTest.test_get_object_after_expiry_time[id-fb024a42-37f3-4ba5-9684-4f40a7910b41] +tempest.api.volume.test_volume_metadata.VolumesMetadataTest.test_crud_volume_metadata[id-6f5b125b-f664-44bf-910f-751591fe5769] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_create_metadata_key[id-64fd53f3-adbd-4639-af54-436e4982dbfb] +tempest.api.object_storage.test_account_bulk.BulkTest.test_bulk_delete[id-c075e682-0d2a-43b2-808d-4116200d736d] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_distinction[id-46435ab1-a0af-4401-8373-f14e66b0dd58] +tempest.api.compute.admin.test_quotas_negative.QuotasSecurityGroupAdminNegativeTest.test_security_groups_exceed_limit[id-7c6c8f3b-2bf6-4918-b240-57b136a66aa0,negative,network] +tempest.api.volume.test_volumes_get.VolumesGetTest.test_volume_create_get_update_delete_as_clone[id-3f591b4a-7dc6-444c-bd51-77469506b3a1] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_get_object_using_temp_url_key_2[id-671f9583-86bd-4128-a034-be282a68c5d8] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_limit_results[id-8d7691b3-6ed4-411a-abc9-2839a765adab] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_additional_default_security_group_fails[id-2323061e-9fbf-4eb0-b547-7e8fafc90849,negative] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_delete_when_peer_group_deleted[id-fc5c5acf-2091-43a6-a6ae-e42760e9ffaf] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_mtu_sized_frames[compute,id-b158ea55-472e-4086-8fa9-c64ac0c6c1d0,network,slow] +tempest.api.volume.test_volume_transfers.VolumesTransfersV355Test.test_create_get_list_accept_volume_transfer[id-9f36bb2b-619f-4507-b246-76aeb9a28851] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_format_xml[id-4477b609-1ca6-4d4b-b25d-ad3f01086089] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_no_params[id-1e341d7a-90a9-494c-b143-2cdf2aeb6aee] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_filter_by_type[id-e3356918-4d3e-4756-81d5-abc4524ba29f] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_flavor_non_admin_add[id-41eaaade-6d37-4f28-9c74-f21b46ca67bd,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_remove_user_role_non_existent_tenant[id-67a679ec-03dd-4551-bbfc-d1c93284f023,negative] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_list_domains_filter_by_enabled[id-3fd19840-65c1-43f8-b48c-51bdd066dff9] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_max_count_less_than_min_count[id-476da616-f1ef-4271-a9b1-b9fc87727cdf,negative] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_with_details[id-bad0567a-5a4f-420b-851e-780b55bb867c] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_invalid_name[id-45fbe5e0-acb5-49aa-837a-ff8d0719db91,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_delete_metadata_key[id-a2e36378-6f1f-43f4-840a-ffd9cfd61914] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_volume_backed_live_migration[id-5071cf17-3004-4257-ae61-73a84e28badd,volume] +tempest.api.compute.servers.test_create_server.ServersTestFqdnHostnames.test_create_server_with_fqdn_name[id-622066d2-39fc-4c09-9eeb-35903c114a0a] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_update_enable[id-b6b25683-c97f-474d-a595-55d410b68100] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_with_description[id-0ecf465c-0dc4-4532-ab53-91ffeb74d12d] +tempest.api.compute.admin.test_fixed_ips_negative.FixedIPsNegativeTestJson.test_fixed_ip_with_invalid_action[id-fd26ef50-f135-4232-9d32-281aab3f9176,negative] +tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_stopped_server[id-aaacd1d0-55a2-4ce8-818a-b5439df8adc9] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_create_server_with_admin_password[id-b92d5ec7-b1dd-44a2-87e4-45e888c46ef0] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_error[id-f18b4bef-212e-45e7-b3ca-59af3a465f82] +tempest.api.network.test_ports.PortsTestJSON.test_port_list_filter_by_router_id[id-5ad01ed0-0e6e-4c5d-8194-232801b15c72] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filtered_by_name_regex[id-24a89b0c-0d55-4a28-847f-45075f19b27b] +tempest.api.compute.admin.test_security_groups.SecurityGroupsTestAdminJSON.test_list_security_groups_list_all_tenants_filter[id-49667619-5af9-4c63-ab5d-2cfdd1c8f7f1,network] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_list_regions[id-d180bf99-544a-445c-ad0d-0c0d27663796] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_disk_format[id-4a4735a7-f22f-49b6-b0d9-66e1ef7453eb] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_servers_on_different_hosts_with_list_of_servers[id-7869cc84-d661-4e14-9f00-c18cdc89cf57] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_assignments_for_domain_roles[id-3859df7e-5b78-4e4d-b10e-214c8953842a] +tempest.api.compute.certificates.test_certificates.CertificatesV2TestJSON.test_create_root_certificate[id-c070a441-b08e-447e-a733-905909535b1b] +tempest.api.network.test_networks.NetworksIpV6Test.test_delete_network_with_subnet[id-f04f61a9-b7f3-4194-90b2-9bcf660d1bfe] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_stats[id-797e4f28-b6e0-454d-a548-80cc77c00816] +tempest.api.volume.admin.test_group_type_specs.GroupTypeSpecsTest.test_group_type_specs_create_show_update_list_delete[id-bb4e30d0-de6e-4f4d-866c-dcc48d023b4e] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_remove_metadata_key[id-24d16451-1c0c-4e4f-b59c-9840a3aba40e] +tempest.api.network.test_networks.NetworksTest.test_show_network_fields[id-867819bb-c4b6-45f7-acf9-90edcf70aa5e] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_gw_and_allocation_pools[id-8217a149-0c6c-4cfb-93db-0486f707d13f] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_hotplug_nic[compute,id-c5adff73-e961-41f1-b4a9-343614f18cfa,network] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration[compute,id-a4858f6c-401e-4155-9a49-d5cd053d1a2f,network,slow] +tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_paused_server[id-71bcb732-0261-11e7-9086-fa163e4fa634] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_when_tenant_is_disabled[id-440a7a8d-9328-4b7b-83e0-d717010495e4,negative] +tempest.api.compute.servers.test_server_metadata.ServerMetadataTestJSON.test_list_server_metadata[id-479da087-92b3-4dcf-aeb3-fd293b2d14ce] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_no_object[id-4646ac2d-9bfb-4c7d-a3c5-0f527402b3df] +tempest.api.network.test_networks.NetworksTest.test_create_update_delete_network_subnet[id-0e269138-0da6-4efc-a46d-578161e7b221,smoke] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_pause_unpause_server[id-bd61a9fd-062f-4670-972b-2d6c3e3b9e73] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_uptime_with_non_admin_user[id-6c3461f9-c04c-4e2a-bebb-71dc9cb47df2,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_object_upload_in_segments[id-e3e6a64a-9f50-4955-b987-6ce6767c97fb] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_zero_limit[id-b1495414-2d93-414c-8019-849afe8d319e] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details[compute,id-d8bb918e-e2df-48b2-97cd-b73c95450980,network,slow] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_cold_migration_revert[compute,id-25b188d7-0183-4b1e-a11d-15840c8e2fd6,network,slow] +tempest.api.compute.servers.test_server_addresses.ServerAddressesTestJSON.test_list_server_addresses[id-6eb718c0-02d9-4d5e-acd1-4e0c269cef39,network,smoke] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_service_binary_name[id-63a3e1ca-37ee-4983-826d-83276a370d25] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_details_by_name[id-2de3a6d4-12aa-403b-a8f2-fdeb42a89623] +tempest.scenario.test_server_multinode.TestServerMultinode.test_schedule_to_all_nodes[compute,id-9cecbe35-b9d4-48da-a37e-7ce70aa43d30,network,smoke] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_details_by_availability_zone[id-e1b80d13-94f0-4ba2-a40e-386af29f8db1] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_pause_non_existent_server[id-6a8dc0c6-6cd4-4c0a-9f32-413881828091,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_personality_file_contents_not_encoded[id-b8a7235e-5246-4a8f-a08e-b34877c6586f,negative] +tempest.api.compute.test_tenant_networks.ComputeTenantNetworksTest.test_list_show_tenant_networks[id-edfea98e-bbe3-4c7a-9739-87b986baff26,network] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_list_ec2_credentials[id-9e2ea42f-0a4f-468c-a768-51859ce492e0] +tempest.api.compute.test_extensions.ExtensionsTest.test_get_extension[id-05762f39-bdfa-4cdb-9b46-b78f8e78e2fd] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_check_revoke_roles_on_projects_user[id-18b70e45-7687-4b72-8277-b8f1a47d7591] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_update_name_default[id-966e2b96-023a-11e7-a9e4-fa163e4fa634,negative] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_router_rescheduling[compute,id-2e788c46-fb3f-4ac9-8f82-0561555bea73,network,slow] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_write_object_without_using_creds[id-af587587-0c24-4e15-9822-8352ce711013,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_gw[id-e41a4888-65a6-418c-a095-f7c2ef4ad59a] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_create_user[id-2d55a71e-da1d-4b43-9c03-d269fd93d905,smoke] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_reset_gateway_without_snat[id-f2faf994-97f4-410b-a831-9bc977b64374] +tempest.api.volume.admin.test_volume_types_extra_specs.VolumeTypesExtraSpecsTest.test_volume_type_extra_specs_list[id-b42923e9-0452-4945-be5b-d362ae533e60] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_user_authentication[id-aca696c3-d645-4f45-b728-63646045beb1] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object[id-5b4ce26f-3545-46c9-a2ba-5754358a4c62,smoke] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_get_image_members_schema[id-6ae916ef-1052-4e11-8d36-b3ae14853cbb] +tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_dhcpv6_stateless[compute,id-7ab23f41-833b-4a16-a7c9-5b42fe6d4123,network,slow] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeShelveTestJSON.test_detach_volume_shelved_or_offload_server[id-b54e86dd-a070-49c4-9c07-59ae6dae15aa,slow] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_without_param[id-0cd85f75-6992-4a4a-b1bd-d11e37fd0eee,negative] +tempest.api.identity.admin.v2.test_services.ServicesTestJSON.test_list_services[id-34ea6489-012d-4a86-9038-1287cadd5eca,smoke] +tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_network[id-d4f9024d-1e28-4fc1-a6b1-25dbc6fa11e2,smoke] +tempest.api.network.test_networks.NetworksIpV6Test.test_update_subnet_gw_dns_host_routes_dhcp[id-3d3852eb-3009-49ec-97ac-5ce83b73010a] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_query_application_credentials[id-9bb5e5cc-5250-493a-8869-8b665f6aa5f6] +tempest.api.object_storage.test_container_acl.ObjectTestACLs.test_read_object_with_rights[id-a3270f3f-7640-4944-8448-c7ea783ea5b6] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_update_user_request_without_a_token[id-3cc2a64b-83aa-4b02-88f0-d6ab737c4466,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_image_with_invalid_image_id[id-381acb65-785a-4942-94ce-d8f8c84f1f0f,negative] +tempest.api.volume.admin.test_volume_services.VolumesServicesTestJSON.test_get_service_by_host_name[id-178710e4-7596-4e08-9333-745cb8bc4f8d] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_revert_with_volume_attached[id-fbbf075f-a812-4022-bc5c-ccb8047eef12,volume] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_create_snapshot_without_passing_volume_id[id-bb9da53e-d335-4309-9c15-7e76fd5e4d6d,negative] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_type[id-888c0cc0-7223-43c5-9db0-b125fd0a393b] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_limit[id-e914a891-3cc8-4b40-ad32-e0a39ffbddbb] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_end_marker[id-888a3f0e-7214-4806-8e50-5e0c9a69bb5e] +tempest.api.identity.v2.test_ec2_credentials.EC2CredentialsTest.test_show_ec2_credential[id-cb284075-b613-440d-83ca-fe0b33b3c2b8] +tempest.scenario.test_dashboard_basic_ops.TestDashboardBasicOps.test_basic_scenario[dashboard,id-4f8851b1-0e69-482b-b63b-84c6e76f6c80,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_name_length_exceeds_256[id-c3e0fb12-07fc-4d76-a22e-37409887afe8,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_unpause_non_existent_server[id-705b8e3a-e8a7-477c-a19b-6868fc24ac75,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_nonexistent_volume_id[id-8f05a943-013c-4063-ac71-7baf561e82eb,negative] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_search_nonexistent_hypervisor[id-19a45cc1-1000-4055-b6d2-28e8b2ec4faa,negative] +tempest.api.volume.admin.test_volume_retype.VolumeRetypeWithoutMigrationTest.test_available_volume_retype[id-b90412ee-465d-46e9-b249-ec84a47d5f25] +tempest.scenario.test_server_advanced_ops.TestServerAdvancedOps.test_server_sequence_suspend_resume[compute,id-949da7d5-72c8-4808-8802-e3d70df98e2c,slow] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential[id-8080c75c-eddc-4786-941a-c2da7039ae61] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_verify_entry_in_list_details[id-8261d7b0-be58-43ec-a2e5-300573c3f6c5] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeShelveTestJSON.test_attach_volume_shelved_or_offload_server[id-13a940b6-3474-4c3c-b03f-29b89112bfee,slow] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_get_keypair_detail[id-a4233d5d-52d8-47cc-9a25-e1864527e3df] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_force_delete_nonexistent_server_id[id-6f47992b-5144-4250-9f8b-f00aa33950f3,negative] +tempest.api.network.test_networks.BulkNetworkOpsTest.test_bulk_create_delete_port[id-48037ff2-e889-4c3b-b86a-8e3f34d2d060,smoke] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_update_snapshot_status[id-41288afd-d463-485e-8f6e-4eea159413eb] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_create_delete_qos_with_both_consumer[id-f88d65eb-ea0d-487d-af8d-71f4011575a4] +tempest.api.compute.admin.test_instance_usage_audit_log_negative.InstanceUsageAuditLogNegativeTestJSON.test_instance_usage_audit_logs_with_nonadmin_user[id-a9d33178-d2c9-4131-ad3b-f4ca8d0308a2,negative] +tempest.api.image.v2.test_images.ImportImagesTest.test_image_glance_direct_import[id-32ca0c20-e16f-44ac-8590-07869c9b4cc2] +tempest.api.volume.test_volumes_extend.VolumesExtendTest.test_volume_extend[id-9a36df71-a257-43a5-9555-dc7c88e66e0e] +tempest.api.compute.servers.test_virtual_interfaces.VirtualInterfacesTestJSON.test_list_virtual_interfaces[id-96c4e2ef-5e4d-4d7f-87f5-fed6dca18016,network] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_delete_as_user[id-cd6de795-c15d-45f1-8d9e-813c6bb72a3d,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_role_create_duplicate[id-c0cde2c8-81c1-4bb0-8fe2-cf615a3547a8,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_bad_remote_ip_prefix[id-5f8daf69-3c5f-4aaa-88c9-db1d66f68679,negative] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_list_users_with_invalid_tenant[id-f5d39046-fc5f-425c-b29e-bac2632da28e,negative] +tempest.api.compute.servers.test_server_personality.ServerPersonalityTestJSON.test_can_create_server_with_max_number_personality_files[id-52f12ee8-5180-40cc-b417-31572ea3d555,slow] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_update_port_with_multiple_ip_mac_address_pair[id-b3f20091-6cd5-472b-8487-3516137df933] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_duplicate[id-8bd56d02-3ffa-4d67-9933-b6b9a01d6089,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_default_gw[id-ebb4fd95-524f-46af-83c1-0305b239338f] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_with_param_metadata[id-b5ebea1b-0603-40a0-bb41-15fcd0a53214] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesUnderV243Test.test_add_remove_fixed_ip[id-c7e0e60b-ee45-43d0-abeb-8596fd42a2f9,network,smoke] +tempest.api.network.test_floating_ips.FloatingIPTestJSON.test_create_floating_ip_specifying_a_fixed_ip_address[id-36de4bd0-f09c-43e3-a8e1-1decc1ffd3a5,smoke] +tempest.api.compute.images.test_image_metadata_negative.ImagesMetadataNegativeTestJSON.test_list_nonexistent_image_metadata[id-94069db2-792f-4fa8-8bd3-2271a6e0c095,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_invalid_date[id-87d12517-e20a-4c9c-97b6-dd1628d6d6c9,negative] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_with_invalid_maintenance_mode[id-ab1e230e-5e22-41a9-8699-82b9947915d4,negative] +tempest.api.compute.admin.test_flavors_extra_specs.FlavorsExtraSpecsTestJSON.test_flavor_non_admin_get_all_keys[id-a99dad88-ae1c-4fba-aeb4-32f898218bd0] +tempest.api.network.test_ports.PortsTestJSON.test_create_port_with_no_securitygroups[id-4179dcb9-1382-4ced-84fe-1b91c54f5735,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_list_metering_labels[id-e2fb2f8c-45bf-429a-9f17-171c70444612] +tempest.api.compute.images.test_images_negative.ImagesNegativeTestJSON.test_create_image_from_invalid_server[id-82c5b0c4-9dbd-463c-872b-20c4755aae7f,negative] +tempest.api.volume.test_snapshot_metadata.SnapshotMetadataTestJSON.test_update_show_snapshot_metadata_item[id-e8ff85c5-8f97-477f-806a-3ac364a949ed] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_host_routes_and_dns_nameservers[id-d830de0a-be47-468f-8f02-1fd996118289] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_delete_active_domain[gate,id-1f3fbff5-4e44-400d-9ca1-d953f05f609b,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_show_port[id-c9a685bd-e83f-499c-939f-9f7863ca259f,smoke] +tempest.api.image.v2.admin.test_images_metadefs_namespace_objects.MetadataNamespaceObjectsTest.test_create_update_delete_meta_namespace_objects[id-b1a3775e-3b5c-4f6a-a3b4-1ba3574ae718] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_invalid_access_ip_v4_address[id-7f70a4d1-608f-4794-9e56-cb182765972c,negative] +tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_list_security_groups_by_server[id-79517d60-535a-438f-af3d-e6feab1cbea7] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_create_server_when_cpu_quota_is_full[id-91058876-9947-4807-9f22-f6eb17140d9b,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_path[id-58ca6cc9-6af0-408d-aaec-2a6a7b2f0df9] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_create_with_empty_name[id-878b4e57-faa2-4659-b0d1-ce740a06ae81,negative] +tempest.api.compute.admin.test_availability_zone_negative.AZAdminNegativeTestJSON.test_get_availability_zone_list_detail_with_non_admin_user[id-bf34dca2-fdc3-4073-9c02-7648d9eae0d7,negative] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_non_admin_update_specific_key[id-1ebf4ef8-759e-48fe-a801-d451d80476fb,negative] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_volume_from_snapshot_decreasing_size[id-677863d1-34f9-456d-b6ac-9924f667a7f4,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_create_role_by_unauthorized_user[id-585c8998-a8a4-4641-a5dd-abef7a8ced00,negative] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_get_user[id-b4baa3ae-ac00-4b4e-9e27-80deaad7771f] +tempest.api.compute.images.test_list_images.ListImagesTestJSON.test_list_images_with_detail[id-9f94cb6b-7f10-48c5-b911-a0b84d7d4cd6] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_startup_host_with_non_admin_user[id-9f4ebb7e-b2ae-4e5b-a38f-0fd1bb0ddfca,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_delete_role_by_unauthorized_user[id-15347635-b5b1-4a87-a280-deb2bd6d865e,negative] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_uptime[id-91a50d7d-1c2b-4f24-b55a-a1fe20efca70] +tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.test_reassign_port_between_servers[id-2f3a0127-95c7-4977-92d2-bc5aec602fb4] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_delete_non_existent_security_group[id-1f1bb89d-5664-4956-9fcd-83ee0fa603df,negative] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_create_not_enabled[id-3be22093-b30f-499d-b772-38340e5e16fb] +tempest.api.network.test_floating_ips_negative.FloatingIPNegativeTestJSON.test_create_floatingip_with_port_ext_net_unreachable[id-22996ea8-4a81-4b27-b6e1-fa5df92fa5e8,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_reboot[compute,id-7b6860c2-afa3-4846-9522-adeb38dfbe08,network,slow] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_delete_metadata_key[id-d4d884d3-4696-4b85-bc98-4f57c4dd2bf1] +tempest.scenario.test_encrypted_cinder_volumes.TestEncryptedCinderVolumes.test_encrypted_cinder_volumes_luks[compute,id-79165fb4-5534-4b9d-8429-97ccffb8f86e,image,slow,volume] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_update_user_by_unauthorized_user[id-424868d5-18a7-43e1-8903-a64f95ee3aac,negative] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_param_display_name_and_status[id-777c87c1-2fc4-4883-8b8e-5c0b951d1ec8] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_allocation_pools[id-bec949c4-3147-4ba6-af5f-cd2306118404] +tempest.api.volume.test_volume_delete_cascade.VolumesDeleteCascade.test_volume_delete_cascade[id-994e2d40-de37-46e8-b328-a58fba7e4a95] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsTestJSON.test_create_list_port_with_extra_dhcp_options[id-d2c17063-3767-4a24-be4f-a23dbfa133c9] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_shutoff_status[id-451dbbb2-f330-4a9f-b0e1-5f5d2cb0f34c] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_no_ra[id-ae2f4a5d-03ff-4c42-a3b0-ce2fcb7ea832] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_with_non_admin_user[id-e40c72b1-0239-4ed6-ba21-81a184df1f7c,negative] +tempest.api.compute.limits.test_absolute_limits.AbsoluteLimitsTestJSON.test_absLimits_get[id-b54c66af-6ab6-4cf0-a9e5-a0cb58d75e0b] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_reset_state_server_invalid_state[id-b0b4d8af-1256-41ef-9ee7-25f1c19dde80,negative] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_force_delete_when_volume_is_error[id-3e33a8a8-afd4-4d64-a86b-c27a185c5a4a] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_delete_non_existent_server[id-1041b4e6-514b-4855-96a5-e974b60870a3,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_created_at_asc[id-4052c3a0-2415-440a-a8cc-305a875331b0] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_unmodified_since[id-0aa1201c-10aa-467a-bee7-63cbdd463152] +tempest.api.compute.floating_ips.test_list_floating_ips_negative.FloatingIPDetailsNegativeTestJSON.test_get_nonexistent_floating_ip_details[id-7ab18834-4a4b-4f28-a2c5-440579866695,negative] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_enable_service_with_invalid_host[id-3246ce65-ba70-4159-aa3b-082c28e4b484,negative] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_create_application_credential_expires[id-852daf0c-42b5-4239-8466-d193d0543ed3] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_list_security_groups[id-e30abd17-fef9-4739-8617-dc26da88e686,smoke] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenant_with_empty_name[id-5a2e4ca9-b0c0-486c-9c48-64a94fba2395,negative] +tempest.api.object_storage.test_object_temp_url.ObjectTempUrlTest.test_get_object_using_temp_url[id-f91c96d4-1230-4bba-8eb9-84476d18d991] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_metadata_key[id-31f40a5f-6a52-4314-8794-cd89baed3040] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_qos_min_bw_allocation_basic[compute,id-78625d92-212c-400e-8695-dd51706858b8,network] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_user_domains[id-08f9aabb-dcfe-41d0-8172-82b5fa0bd73d] +tempest.api.network.test_ports.PortsTestJSON.test_create_bulk_port[id-67f1b811-f8db-43e2-86bd-72c074d4a42c] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_list_domains_filter_by_name[id-c6aee07b-4981-440c-bb0b-eb598f58ffe9] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_set_gateway[id-6cc285d8-46bf-4f36-9b1a-783e3008ba79] +tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_slaac_from_os[compute,id-b6399d76-4438-4658-bcf5-0d6c8584fde2,network,slow] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_non_existent_server[id-d86141a7-906e-4731-b187-d64a2ea61422,negative] +tempest.api.image.v2.test_images_metadefs_schema.MetadataSchemaTest.test_get_metadata_namespace_schema[id-e9e44891-3cb8-3b40-a532-e0a39fea3dab] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_metadata_non_existent_server[id-0df38c2a-3d4e-4db5-98d8-d4d9fa843a12,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_detailed_limit_results[id-b26f6327-2886-467a-82be-cef7a27709cb] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_too_many_objects[id-3a387039-697a-44fc-a9c0-935de31f426b,smoke] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_delete_metadata[id-3a5ce7d4-6e4b-47d0-9d87-7cd42c325094] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_limit_and_marker_and_end_marker[id-8cf98d9c-e3a0-4e44-971b-c87656fdddbd] +tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_delete_stateless_subnet_with_ports[id-2de6ab5a-fcf0-4144-9813-f91a940291f1] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,slow,volume] +tempest.api.network.test_security_groups.SecGroupTest.test_create_list_update_show_delete_security_group[id-bfd128e5-3c92-44b6-9d66-7fe29d22c802,smoke] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_create_with_duplicate_name[id-9fdb4abc-6b66-4b27-b89c-eb215a956168,negative] +tempest.api.compute.volumes.test_attach_volume.AttachVolumeMultiAttachTest.test_boot_from_multiattach_volume[id-65e33aa2-185b-44c8-b22e-e524973ed625] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_update[id-cf9f07c6-db9e-4462-a243-5933ad65e9c8] +tempest.api.volume.admin.test_backends_capabilities.BackendsCapabilitiesAdminTestsJSON.test_get_capabilities_backend[id-3750af44-5ea2-4cd4-bc3e-56e7e6caf854] +tempest.api.compute.test_extensions.ExtensionsTest.test_list_extensions[id-3bb27738-b759-4e0d-a5fa-37d7a6df07d1] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_get_service_with_valid_service_and_invalid_host[id-64e7e7fb-69e8-4cb6-a71d-8d5eb0c98655,negative] +tempest.api.compute.servers.test_servers.ServersTestJSON.test_update_access_server_address[id-89b90870-bc13-4b73-96af-f9d4f2b70077] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_create_backup[id-b963d4f1-94b3-4c40-9e97-7b583f46e470,image] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_revert[id-c03aab19-adb1-44f5-917d-c419577e9e68] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_usb[id-647d04cf-ad35-4956-89ab-b05c5c16f30c] +tempest.api.object_storage.test_container_services.ContainerTest.test_update_container_metadata_with_create_and_delete_metadata[id-cf19bc0b-7e16-4a5a-aaed-cb0c2fe8deef] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_shelve_shelved_server[id-443e4f9b-e6bf-4389-b601-3a710f15fddd,negative] +tempest.api.network.test_routers.RoutersTest.test_add_multiple_router_interfaces[id-802c73c9-c937-4cef-824b-2191e24a6aab,smoke] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_marker[id-638f876d-6a43-482a-bbb3-0840bca101c6] +tempest.api.object_storage.test_object_services.ObjectTest.test_list_object_metadata_with_x_object_manifest[id-23a3674c-d6de-46c3-86af-ff92bfc8a3da] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_user_on_system[id-e5a81737-d294-424d-8189-8664858aae4c] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_user_roles_request_without_token[id-682adfb2-fd5f-4b0a-a9ca-322e9bebb907,negative] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_update_port_with_security_group_and_extra_attributes[id-58091b66-4ff4-4cc1-a549-05d60c7acd1a] +tempest.api.image.v2.admin.test_images_metadefs_resource_types.MetadataResourceTypesTest.test_basic_meta_def_resource_type_association[id-6f358a4e-5ef0-11e6-a795-080027d0d606] +tempest.api.network.test_security_groups.SecGroupTest.test_list_security_groups[id-e30abd17-fef9-4739-8617-dc26da88e686,smoke] +tempest.api.identity.v2.test_users.IdentityUsersTest.test_user_update_own_password[id-165859c9-277f-4124-9479-a7d1627b0ca7] +tempest.api.compute.images.test_images.ImagesTestJSON.test_delete_saving_image[id-aa06b52b-2db5-4807-b218-9441f75d74e3] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_blank_key[id-96100343-7fa9-40d8-80fa-d29ef588ce1c,negative] +tempest.api.image.v1.test_image_members_negative.ImageMembersNegativeTest.test_get_image_without_membership[id-f25f89e4-0b6c-453b-a853-1f80b9d7ef26,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_restore_nonexistent_server_id[id-9c6d38cc-fcfb-437a-85b9-7b788af8bf01,negative] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_list_snapshots_invalid_param_marker[id-b68deeda-ca79-4a32-81af-5c51179e553a,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_subnet[id-bd635d81-6030-4dd1-b3b9-31ba0cfdf6cc,smoke] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit_equals_zero[id-e3b44b7f-ae87-45b5-8a8c-66110eb24d0a] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_list_quotas[id-59eada70-403c-4cef-a2a3-a8ce2f1b07a0] +tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os[compute,id-2c92df61-29f0-4eaa-bee3-7c65bef62a43,network,slow] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_update_user[id-39d05857-e8a5-4ed4-ba83-0b52d3ab97ee] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_nonexistent_snapshot_id[id-0c36f6ae-4604-4017-b0a9-34fdc63096f9,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors[id-e36c0eaa-dff5-4082-ad1f-3f9a80aa3f59,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_image[id-fcba1052-0a50-4cf3-b1ac-fae241edf02f,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_status_active_detail[id-e5dc26d9-9aa2-48dd-bda5-748e1445da98] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_by_name[id-a28e8da4-0b56-472f-87a8-0f4d3f819c02] +tempest.api.identity.admin.v3.test_trusts.TrustsV3TestJSON.test_get_trusts_query[id-6268b345-87ca-47c0-9ce3-37792b43403a] +tempest.api.network.admin.test_routers_negative.RoutersAdminNegativeIpV6Test.test_router_set_gateway_used_ip_returns_409[id-7101cc02-058a-11e7-93e1-fa163e4fa634,negative] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairTestJSON.test_update_port_with_address_pair[id-9599b337-272c-47fd-b3cf-509414414ac4] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_delete_role_non_existent[id-38373691-8551-453a-b074-4260ad8298ef,negative] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_with_invalid_password[id-d5308b33-3574-43c3-8d87-1c090c5e1eca,negative] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_servers_on_different_hosts[id-cc7ca884-6e3e-42a3-a92f-c522fcf25e8e] +tempest.api.volume.admin.test_user_messages.UserMessagesTest.test_delete_message[id-c6eb6901-cdcc-490f-b735-4fe251842aed] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_2d_way[id-06f90388-2d0e-40aa-934c-e9a8833e958a] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata[id-7a94c25d-66e6-434c-9c38-97d4e2c29945,smoke] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_delete_request_without_token[id-e450db62-2e9d-418f-893a-54772d6386b1,negative] +tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON.test_credentials_create_get_update_delete[id-7cd59bf9-bda4-4c72-9467-d21cab278355,smoke] +tempest.api.identity.admin.v2.test_tokens_negative.TokensAdminTestNegative.test_check_token_existence_negative[id-a0a0a600-4292-4364-99c5-922c834fdf05,negative] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_uuid_id[id-94c9bb4e-2c2a-4f3c-bb1f-5f0daf918e6d] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_reverse_order[id-b1811cff-d1ed-4c15-a52e-efd8de41cf34] +tempest.api.compute.security_groups.test_security_group_rules.SecurityGroupRulesTestJSON.test_security_group_rules_create_with_optional_cidr[id-7a01873e-3c38-4f30-80be-31a043cfe2fd] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_get_non_existent_server[id-3436b02f-1b1e-4f03-881e-c6a602327439,negative] +tempest.api.identity.admin.v3.test_services.ServicesTestJSON.test_create_update_get_service[id-5193aad5-bcb7-411d-85b0-b3b61b96ef06,smoke] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_with_domain[id-5f50fe07-8166-430b-a882-3b2ee0abe26f] +tempest.scenario.test_encrypted_cinder_volumes.TestEncryptedCinderVolumes.test_encrypted_cinder_volumes_cryptsetup[compute,id-cbc752ed-b716-4717-910f-956cce965722,image,slow,volume] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeV253TestJSON.test_get_service_by_invalid_params[id-d0884a69-f693-4e79-a9af-232d15643bf7,negative] +tempest.api.compute.admin.test_quotas.QuotasAdminTestV236.test_get_updated_quotas[id-4268b5c9-92e5-4adc-acf1-3a2798f3d803] +tempest.api.object_storage.test_container_staticweb.StaticWebTest.test_web_listing_css[id-bc37ec94-43c8-4990-842e-0e5e02fc8926] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_allocate_floating_ip_from_nonexistent_pool[id-6e0f059b-e4dd-48fb-8207-06e3bba5b074,negative] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshots_list_details_with_params[id-220a1022-1fcd-4a74-a7bd-6b859156cda2] +tempest.api.identity.admin.v3.test_domains_negative.DomainsNegativeTestJSON.test_create_domain_with_empty_name[id-9018461d-7d24-408d-b3fe-ae37e8cd5c9e,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_name[id-097af10a-bae8-4342-bff4-edf89969ed2a] +tempest.api.compute.admin.test_volumes_negative.UpdateMultiattachVolumeNegativeTest.test_multiattach_rw_volume_update_failure[id-7576d497-b7c6-44bd-9cc5-c5b4e50fec71,negative,volume] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_host_name_is_same_as_server_name[id-ac1ad47f-984b-4441-9274-c9079b7a0666] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_numeric_server_name[id-fd57f159-68d6-4c2a-902b-03070828a87e,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_long_keynames[id-3faa916f-779f-4103-aca7-dc3538eee1b7,negative] +tempest.api.network.test_routers.RoutersTest.test_add_remove_router_interface_with_port_id[id-2b7d2f37-6748-4d78-92e5-1d590234f0d5,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringTestJSON.test_list_metering_label_rules[id-cc832399-6681-493b-9d79-0202831a1281] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_get_users_request_without_token[id-a73591ec-1903-4ffe-be42-282b39fefc9d,negative] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_min_count_non_integer[id-999aa722-d624-4423-b813-0d1ac9884d7a,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_with_a_deleted_server[id-24a26f1a-1ddc-4eea-b0d7-a90cc874ad8f,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_server_name_blank[id-dbbfd247-c40c-449e-8f6c-d2aa7c7da7cf,negative] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_volume_without_passing_volume_id[id-c6c3db06-29ad-4e91-beb0-2ab195fe49e3,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_volume_get_nonexistent_volume_id[id-c03ea686-905b-41a2-8748-9635154b7c57,negative] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_version_statuses[id-8879a470-abfb-47bb-bb8d-5a7fd279ad1e,smoke] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_get_image_member[id-a6ee18b9-4378-465e-9ad9-9a6de58a3287] +tempest.api.compute.admin.test_flavors_access.FlavorsAccessTestJSON.test_flavor_access_list_with_private_flavor[id-ea2c2211-29fa-4db9-97c3-906d36fad3e0] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_show_non_existent_security_group[id-424fd5c3-9ddc-486a-b45f-39bf0c820fc6,negative] +tempest.api.object_storage.test_account_services_negative.AccountNegativeTest.test_list_containers_with_non_authorized_user[id-070e6aca-6152-4867-868d-1118d68fb38c,negative] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_with_snat_explicit[id-ea74068d-09e9-4fd7-8995-9b6a1ace920f] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_get_service_with_valid_service_and_invalid_host[id-64e7e7fb-69e8-4cb6-a71d-8d5eb0c98655,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_with_x_object_metakey[id-a28a8b99-e701-4d7e-9d84-3b66f121460b] +tempest.api.volume.admin.test_volume_services_negative.VolumeServicesNegativeTest.test_disable_log_reason_with_no_reason[id-77767b36-5e8f-4c68-a0b5-2308cc21ec64,negative] +tempest.api.compute.admin.test_live_migration.LiveAutoBlockMigrationV225Test.test_live_migration_with_trunk[id-0022c12e-a482-42b0-be2d-396b5f0cffe3,network] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_list_snapshot_invalid_param_limit[id-8fd92339-e22f-4591-86b4-1e2215372a40,negative] +tempest.api.identity.v3.test_api_discovery.TestApiDiscovery.test_api_media_types[id-657c1970-4722-4189-8831-7325f3bc4265,smoke] +tempest.api.object_storage.test_container_services.ContainerTest.test_create_container_with_metadata_key[id-c2ac4d59-d0f5-40d5-ba19-0635056d48cd] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_get_invalid_volume_id[id-30799cfd-7ee4-446c-b66c-45b383ed211b,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_flavor[id-5913660b-223b-44d4-a651-a0fbfd44ca75,negative] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_update_metadata_with_nonexistent_container_name[id-125a24fa-90a7-4cfc-b604-44e49d788390,negative] +tempest.api.network.test_ports.PortsTestJSON.test_create_update_delete_port[id-c72c1c0c-2193-4aca-aaa4-b1442640f51c,smoke] +tempest.api.compute.servers.test_multiple_create.MultipleCreateTestJSON.test_multiple_create_with_reservation_return[id-864777fb-2f1e-44e3-b5b9-3eb6fa84f2f7] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_by_unauthorized_user[id-391df5cf-3ec3-46c9-bbe5-5cb58dd4dc41,negative] +tempest.api.identity.admin.v3.test_inherits.InheritsV3TestJSON.test_inherit_assign_list_check_revoke_roles_on_domains_user[id-4e6f0366-97c8-423c-b2be-41eae6ac91c8] +tempest.api.compute.images.test_images_oneserver.ImagesOneServerTestJSON.test_create_image_specify_multibyte_character_image_name[id-3b7c6fe4-dfe7-477c-9243-b06359db51e6] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcp_stateful_router[id-e98f65db-68f4-4330-9fea-abd8c5192d4d] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_show_non_existent_security_group[id-424fd5c3-9ddc-486a-b45f-39bf0c820fc6,negative] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_attach_volume[id-d0ccac79-0091-4cf4-a1ce-26162d0cc55f,negative,volume] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_show_metering_label[id-30abb445-0eea-472e-bd02-8649f54a5968] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_in_allowed_allocation_pools[id-0435f278-40ae-48cb-a404-b8a087bc09b1,smoke] +tempest.api.network.admin.test_floating_ips_admin_actions.FloatingIPAdminTestJSON.test_list_floating_ips_from_admin_and_nonadmin[id-64f2100b-5471-4ded-b46c-ddeeeb4f231b] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_list_regions_filter_by_parent_region_id[id-2d1057cb-bbde-413a-acdf-e2d265284542] +tempest.api.compute.images.test_image_metadata.ImagesMetadataTestJSON.test_update_image_metadata[id-7b491c11-a9d5-40fe-a696-7f7e03d3fea2] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_update_desc[id-859fcfe1-3a03-41ef-86f9-b19a47d1cd87] +tempest.api.network.test_routers.RoutersTest.test_update_delete_extra_route[id-c86ac3a8-50bd-4b00-a6b8-62af84a0765c] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_show_host_detail_with_nonexistent_hostname[id-e75b0a1a-041f-47a1-8b4a-b72a6ff36d3f,negative] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_volume_backed_live_migration[id-5071cf17-3004-4257-ae61-73a84e28badd,volume] +tempest.scenario.test_unified_limits.ImageQuotaTest.test_image_size_quota[id-05e8d064-c39a-4801-8c6a-465df375ec5b,identity,image] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_no_body[id-08961d20-5cbb-4910-ac0f-89ad6dbb2da1,negative] +tempest.api.compute.admin.test_networks.NetworksTest.test_list_all_networks[id-df3d1046-6fa5-4b2c-ad0c-cfa46a351cb9] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_update_name[id-f608f368-048c-496b-ad63-d286c26dab6b] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_delete_user_request_without_a_token[id-57fe1df8-0aa7-46c0-ae9f-c2e785c7504a,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_with_x_fresh_metadata[id-5a9e2cc6-85b6-46fc-916d-0cbb7a88e5fd] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_update_enable[id-8fc8981f-f12d-4c66-9972-2bdcf2bc2e1a] +tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image[id-10407036-6059-4f95-a2cd-cbbbee7ed329] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_show_metering_label_rule[id-b7354489-96ea-41f3-9452-bace120fb4a7] +tempest.api.identity.admin.v3.test_endpoints.EndPointsTestJSON.test_list_endpoints[id-c19ecf90-240e-4e23-9966-21cee3f6a618] +tempest.api.identity.admin.v2.test_services.ServicesTestJSON.test_create_get_delete_service[id-84521085-c6e6-491c-9a08-ec9f70f90110] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_networks_fields[id-6ae6d24f-9194-4869-9c85-c313cb20e080] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_delete_multiple_server_groups_with_same_name_policy[id-154dc5a4-a2fe-44b5-b99e-f15806a4a113] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list[id-bc2dd1a0-15af-48e5-9990-f2e75a48325d] +tempest.api.compute.admin.test_services_negative.ServicesAdminNegativeTestJSON.test_list_services_with_non_admin_user[id-1126d1f8-266e-485f-a687-adc547492646,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_keypair_delete_nonexistent_key[id-7cc32e47-4c42-489d-9623-c5e2cb5a2fa5,negative] +tempest.api.network.test_tags.TagsExtTest.test_update_and_delete_all_tags[id-663a90f5-f334-4b44-afe0-c5fc1d408791] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_users_with_not_enabled[id-bff8bf2f-9408-4ef5-b63a-753c8c2124eb] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsAssociationNegativeTestJSON.test_associate_ip_to_server_without_passing_floating_ip[id-804b4fcb-bbf5-412f-925d-896672b61eb3,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_detailed_limit_results[id-67aec2d0-35fe-4503-9f92-f13272b867ed] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_update_metadata_with_blank_key[id-a452f38c-05c2-4b47-bd44-a4f0bf5a5e48,negative] +tempest.api.network.test_ports.PortsTestJSON.test_port_list_filter_by_ip_substr[id-79895408-85d5-460d-94e7-9531c5fd9123] +tempest.api.volume.admin.test_group_snapshots.GroupSnapshotsTest.test_create_group_from_group_snapshot[id-eff52c70-efc7-45ed-b47a-4ad675d09b81] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_scsi[id-12340157-6306-4745-bdda-cfa019908b48] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_x_object_metakey[id-0dbbe89c-6811-4d84-a2df-eca2bdd40c0e] +tempest.api.compute.volumes.test_attach_volume_negative.AttachVolumeNegativeTest.test_attach_attached_volume_to_same_server[id-aab919e2-d992-4cbb-a4ed-745c2475398c,negative] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_create_get_delete[id-4e955c3b-49db-4515-9590-0c99f8e471ad] +tempest.api.image.v2.test_images.ImportImagesTest.test_image_web_download_import[id-f6feb7a4-b04f-4706-a011-206129f83e62] +tempest.api.identity.admin.v3.test_project_tags.IdentityV3ProjectTagsTest.test_list_update_delete_project_tags[id-7c123aac-999d-416a-a0fb-84b915ab10de] +tempest.api.object_storage.test_account_services.AccountTest.test_update_account_metadata_with_delete_metadata[id-9f60348d-c46f-4465-ae06-d51dbd470953] +tempest.api.network.test_security_groups.SecGroupTest.test_create_security_group_rule_with_icmp_type_code[id-c9463db8-b44d-4f52-b6c0-8dbda99f26ce] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_content_encoding[id-605f8317-f945-4bee-ae91-013f1da8f0a0] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_wrong_ip_prefix_version[id-7607439c-af73-499e-bf64-f687fd12a842,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_with_invalid_network_uuid[id-4e72dc2d-44c5-4336-9667-f7972e95c402,negative] +tempest.api.identity.v3.test_application_credentials.ApplicationCredentialsV3Test.test_list_application_credentials[id-ff0cd457-6224-46e7-b79e-0ada4964a8a6] +tempest.api.compute.admin.test_migrations.MigrationsAdminTest.test_cold_migration[id-4bf0be52-3b6f-4746-9a27-3143636fe30d] +tempest.api.volume.test_volumes_backup.VolumesBackupsTest.test_bootable_volume_backup_and_restore[id-2a8ba340-dff2-4511-9db7-646f07156b15,image] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_blank_id[id-04f72aa3-fcec-45a3-81a3-308ef7cc82bc,negative] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_created_server_vcpus[id-cbc0f52f-05aa-492b-bdc1-84b575ca294b] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_create_role_request_without_token[id-a7edd17a-e34a-4aab-8bb7-fa6f498645b8,negative] +tempest.api.identity.admin.v3.test_endpoints.EndPointsTestJSON.test_create_list_show_delete_endpoint[id-0e2446d2-c1fd-461b-a729-b9e73e3e3b37] +tempest.api.compute.servers.test_servers_microversions.ServerShowV257Test.test_rebuild_server[id-803df848-080a-4261-8f11-b020cd9b6f60] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_nonexistent_hypervisor_uptime[id-f60aa680-9a3a-4c7d-90e1-fae3a4891303,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_security_group_get_nonexistent_group[id-673eaec1-9b3e-48ed-bdf1-2786c1b9661c,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_update_object_metadata_with_remove_metadata[id-48650ed0-c189-4e1e-ad6b-1d4770c6e134] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_list_flavor_without_extra_data[id-63dc64e6-2e79-4fdf-868f-85500d308d66] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_list_volumes_detail_with_nonexistent_name[id-9ca17820-a0e7-4cbd-a7fa-f4468735e359,negative] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_rebuild[compute,id-88a529c2-1daa-4c85-9aec-d541ba3eb699,network,slow] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_shutoff_state[id-546d368c-bb6c-4645-979a-83ed16f3a6be] +tempest.api.identity.admin.v3.test_list_users.UsersV3TestJSON.test_list_users[id-b30d4651-a2ea-4666-8551-0c0e49692635] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeTestJSON.test_get_hypervisor_stats_with_non_admin_user[id-e2b061bb-13f9-40d8-9d6e-d5bf17595849,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_list_subnets[id-db68ba48-f4ea-49e9-81d1-e367f6d0b20a,smoke] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_show_port_binding_ext_attr[id-b54ac0ff-35fc-4c79-9ca3-c7dbd4ea4f13] +tempest.api.image.v1.test_images.CreateRegisterImagesTest.test_register_then_upload[id-3027f8e6-3492-4a11-8575-c3293017af4d] +tempest.api.object_storage.test_container_acl.ObjectTestACLs.test_write_object_with_rights[id-aa58bfa5-40d9-4bc3-82b4-d07f4a9e392a] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_type_list[id-9d9b28e3-1b2e-4483-a2cc-24aa0ea1de54] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_migration_with_trunk[id-0022c12e-a482-42b0-be2d-396b5f0cffe3,network] +tempest.api.volume.admin.test_volume_manage.VolumeManageAdminTest.test_unmanage_manage_volume[id-70076c71-0ce1-4208-a8ff-36a66e65cc1e] +tempest.api.volume.admin.test_volume_pools.VolumePoolsAdminTestsJSON.test_get_pools_without_details[id-0248a46c-e226-4933-be10-ad6fca8227e7] +tempest.scenario.test_minbw_allocation_placement.MinBwAllocationPlacementTest.test_empty_update[compute,id-0805779e-e03c-44fb-900f-ce97a790653b,network] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_authentication_request_without_token[id-5d1fa498-4c2d-4732-a8fe-2b054598cfdd] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_non_existent_remote_groupid[id-4bf786fd-2f02-443c-9716-5b98e159a49a,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_router_add_gateway_invalid_network_returns_404[id-37a94fc0-a834-45b9-bd23-9a81d2fd1e22,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_status_non_existing[id-fcdf192d-0f74-4d89-911f-1ec002b822c4,negative] +tempest.api.network.test_routers.RoutersTest.test_create_show_list_update_delete_router[id-f64403e2-8483-4b34-8ccd-b09a87bcc68c,smoke] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_enabled[id-1f66dc76-50cc-4741-a200-af984509e480] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_get_keypair_detail[id-a4233d5d-52d8-47cc-9a25-e1864527e3df] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_ip_protocol[id-84c81249-9f6e-439c-9bbf-cbb0d2cddbdf,negative] +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_list_floating_ips[id-16db31c3-fb85-40c9-bbe2-8cf7b67ff99f] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_set_gateway_with_fixed_ip[id-cbe42f84-04c2-11e7-8adb-fa163e4fa634] +tempest.api.volume.test_volumes_negative.VolumesNegativeTest.test_volume_extend_with_None_size[id-355218f1-8991-400a-a6bb-971239287d92,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_max_size[id-feb32ac6-22bb-4a16-afd8-9454bb714b14] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_connectivity_between_vms_on_different_networks[compute,id-1546850e-fbaa-42f5-8b5f-03d8a6a95f15,network,slow] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_delete_with_az[id-5873a6f8-671a-43ff-8838-7ce430bb6d0b] +tempest.api.compute.admin.test_instance_usage_audit_log.InstanceUsageAuditLogTestJSON.test_get_instance_usage_audit_log[id-6e40459d-7c5f-400b-9e83-449fbc8e7feb] +tempest.api.image.v2.admin.test_images_metadefs_namespace_objects.MetadataNamespaceObjectsTest.test_list_meta_namespace_objects[id-a2a3615e-3b5c-3f6a-a2b1-1ba3574ae738] +tempest.api.network.test_ports.PortsTestJSON.test_show_port[id-c9a685bd-e83f-499c-939f-9f7863ca259f,smoke] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_quota_usage_after_volume_transfer[id-8911036f-9d54-4720-80cc-a1c9796a8805] +tempest.api.compute.test_quotas.QuotasTestJSON.test_compare_tenant_quotas_with_default_quotas[id-cd65d997-f7e4-4966-a7e9-d5001b674fdc] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_add_flavor_access_duplicate[id-f3592cc0-0306-483c-b210-9a7b5346eddc,negative] +tempest.api.compute.admin.test_floating_ips_bulk.FloatingIPsBulkAdminTestJSON.test_create_list_delete_floating_ips_bulk[id-2c8f145f-8012-4cb8-ac7e-95a587f0e4ab,network] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_delete_role_request_without_token[id-44b60b20-70de-4dac-beaf-a3fc2650a16b,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_for_non_existent_role[id-99b297f6-2b5d-47c7-97a9-8b6bb4f91042,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_router_remove_interface_in_use_returns_409[id-04df80f9-224d-47f5-837a-bf23e33d1c20,negative] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_status[id-9b0ea018-6185-4f71-948a-a123a107988e] +tempest.api.network.admin.test_quotas.QuotasTest.test_show_quota_details[id-7b05ec5f-bf44-43cb-b28f-ddd72a824288] +tempest.api.object_storage.test_account_bulk.BulkTest.test_bulk_delete_by_POST[id-dbea2bcb-efbb-4674-ac8a-a5a0e33d1d79] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_register_with_invalid_disk_format[id-993face5-921d-4e84-aabf-c1bba4234a67,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypairsv22_create_list_show_with_type[id-89d59d43-f735-441a-abcf-0601727f47b6] +tempest.api.compute.admin.test_quotas_negative.QuotasAdminNegativeTest.test_create_server_when_instances_quota_is_full[id-7c6be468-0274-449a-81c3-ac1c32ee0161,negative] +tempest.api.volume.admin.test_group_types.GroupTypesTest.test_group_type_create_list_update_show[id-dd71e5f9-393e-4d4f-90e9-fa1b8d278864] +tempest.api.object_storage.test_account_quotas.AccountQuotasTest.test_upload_valid_object[id-a22ef352-a342-4587-8f47-3bbdb5b039c4,smoke] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_delete_quota[id-874b35a9-51f1-4258-bec5-cd561b6690d3] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_list_user_projects[id-a831e70c-e35b-430b-92ed-81ebbc5437b8] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_reboot_nonexistent_host[id-f86bfd7b-0b13-4849-ae29-0322e83ee58b,negative] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_negative_image_id[id-4ed757cd-450c-44b1-9fd1-c819748c650d,negative] +tempest.api.volume.admin.test_volume_quota_classes.VolumeQuotaClassesTest.test_show_default_quota[id-abb9198e-67d0-4b09-859f-4f4a1418f176] +tempest.api.volume.admin.test_snapshot_manage.SnapshotManageAdminTest.test_unmanage_manage_snapshot[id-0132f42d-0147-4b45-8501-cc504bbf7810] +tempest.api.compute.servers.test_create_server.ServersTestBootFromVolume.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_for_non_existent_tenant[id-0132cc22-7c4f-42e1-9e50-ac6aad31d59a,negative] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_detail_server_is_deleted[id-93055106-2d34-46fe-af68-d9ddbf7ee570,negative] +tempest.api.object_storage.test_container_acl_negative.ObjectACLsNegativeTest.test_read_object_with_non_authorized_user[id-abf63359-be52-4feb-87dd-447689fc77fd,negative] +tempest.api.compute.flavors.test_flavors.FlavorsV2TestJSON.test_list_flavors_filter_by_min_disk[id-10645a4d-96f5-443f-831b-730711e11dd4] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_list_nonexistent_volume_type_id[id-dee5cf0c-cdd6-4353-b70c-e847050d71fb,negative] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_create_and_show_consumer[id-c8307ea6-a86c-47fd-ae7b-5b3b2caca76d] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_image[id-05e8a8e7-9659-459a-989d-92c2f501f4ba] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_flavor[id-573637f5-7325-47bb-9144-3476d0416908] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_marker[id-c31ddc63-2a58-4f6b-b25c-94d2937e6867] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_sort[id-eeadce49-04e0-43b7-aec7-52535d903e7a] +tempest.api.network.test_extra_dhcp_options.ExtraDHCPOptionsIpV6TestJSON.test_create_list_port_with_extra_dhcp_options[id-d2c17063-3767-4a24-be4f-a23dbfa133c9] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsTestJSON.test_list_ports_binding_ext_attr[compute,id-1c82a44a-6c6e-48ff-89e1-abe7eaf8f9f8] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_resize_server_using_overlimit_ram[id-28dcec23-f807-49da-822c-56a92ea3c687,negative] +tempest.api.network.test_networks.NetworksTest.test_list_subnets[id-db68ba48-f4ea-49e9-81d1-e367f6d0b20a,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_shelve_non_existent_server[id-abca56e2-a892-48ea-b5e5-e07e69774816,negative] +tempest.api.image.v2.admin.test_images.BasicOperationsImagesAdminTest.test_update_image_owner_param[id-525ba546-10ef-4aad-bba1-1858095ce553] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volume_list_pagination[id-af55e775-8e4b-4feb-8719-215c43b0238c] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_create_is_domain_project[id-a7eb9416-6f9b-4dbb-b71b-7f73aaef59d5] +tempest.api.network.test_routers_negative.DvrRoutersNegativeTest.test_router_create_tenant_distributed_returns_forbidden[id-4990b055-8fc7-48ab-bba7-aa28beaad0b9,negative] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_suspended_state[id-1f82ebd3-8253-4f4e-b93f-de9b7df56d8b] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_resize_server_from_manual_to_auto[id-414e7e93-45b5-44bc-8e03-55159c6bfc97] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_with_none_id[id-f83fe669-6758-448a-a85e-32d351f36fe0] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_with_no_securitygroups[id-4179dcb9-1382-4ced-84fe-1b91c54f5735,smoke] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_changes_since[id-7d439e18-ac2e-4827-b049-7e18004712c4] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_stop_start[compute,id-61f1aa9a-1573-410e-9054-afa557cab021,network,slow] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_block_migration[id-1dce86b8-eb04-4c03-a9d8-9c1dc3ee0c7b] +tempest.api.compute.admin.test_live_migration_negative.LiveMigrationNegativeTest.test_invalid_host_for_migration[id-7fb7856e-ae92-44c9-861a-af62d7830bcb,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_the_default_security_group[id-36a1629f-c6da-4a26-b8b8-55e7e5d5cd58,negative] +tempest.api.object_storage.test_container_services.ContainerTest.test_list_container_contents_with_end_marker[id-55b4fa5c-e12e-4ca9-8fcf-a79afe118522] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_server_with_non_public_flavor[id-bcc418ef-799b-47cc-baa1-ce01368b8987] +tempest.api.network.test_networks.NetworksTest.test_list_subnets_fields[id-842589e3-9663-46b0-85e4-7f01273b0412] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_by_unauthorized_user[id-60a1f5fa-5744-4cdf-82bf-60b7de2d29a4,negative] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_id_over_character_limit[id-a4a448ab-3db2-4d2d-b9b2-6a1271241dfe,negative] +tempest.api.compute.admin.test_flavors_access_negative.FlavorsAccessNegativeTestJSON.test_flavor_non_admin_remove[id-073e79a6-c311-4525-82dc-6083d919cb3a,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_update_multiple_extra_spec[id-a77dfda2-9100-448e-9076-ed1711f4bdfc,negative] +tempest.api.compute.servers.test_servers.ServerShowV263Test.test_show_update_rebuild_list_server[id-71b8e3d5-11d2-494f-b917-b094a4afed3c] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_server_metadata_non_existent_server[id-4d9cd7a3-2010-4b41-b8fe-3bbf0b169466,negative] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_with_dhcp_enabled[id-94ce038d-ff0a-4a4c-a56b-09da3ca0b55d] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_invalid_ports[id-0d9c7791-f2ad-4e2f-ac73-abf2373b0d2d,negative] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_list_users_with_roles_for_tenant[id-a8b54974-40e1-41c0-b812-50fc90827971] +tempest.api.network.admin.test_metering_extensions.MeteringIpV6TestJSON.test_list_metering_label_rules[id-cc832399-6681-493b-9d79-0202831a1281] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_distinction_with_prefix[id-4236305b-b65a-4bfc-a9d2-69cb5b2bf2ed] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_limit[id-614cdfc1-d557-4bac-915b-3e67b48eee76] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_image_specify_metadata_over_limits[id-3d24d11f-5366-4536-bd28-cff32b748eca,negative] +tempest.api.volume.test_volume_transfers.VolumesTransfersTest.test_create_get_list_accept_volume_transfer[id-4d75b645-a478-48b1-97c8-503f64242f1a] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_delete_non_existent_tenant[id-9c9a2aed-6e3c-467a-8f5c-89da9d1b516b,negative] +tempest.api.image.v1.test_images.ListImagesTest.test_index_no_params[id-246178ab-3b33-4212-9a4b-a7fe8261794d] +tempest.api.network.test_routers.RoutersIpV6Test.test_router_interface_port_update_with_fixed_ip[id-96522edf-b4b5-45d9-8443-fa11c26e6eff] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_list_public_flavor_with_other_user[id-b345b196-bfbd-4231-8ac1-6d7fe15ff3a3] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_tag[id-aa8ac4df-cff9-418b-8d0f-dd9c67b072c9] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_delete_nonexistent_security_group[id-6727c00b-214c-4f9e-9a52-017ac3e98411,negative] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_set_server_metadata_missing_metadata[id-64a91aee-9723-4863-be44-4c9d9f1e7d0e,negative] +tempest.api.identity.admin.v3.test_oauth_consumers.OAUTHConsumersV3Test.test_delete_consumer[id-fdfa1b7f-2a31-4354-b2c7-f6ae20554f93] +tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_boot_server_from_encrypted_volume_luks[compute,id-cb78919a-e553-4bab-b73b-10cf4d2eb125,volume] +tempest.api.compute.servers.test_delete_server.DeleteServersTestJSON.test_delete_server_while_in_building_state[id-9e6e0c87-3352-42f7-9faf-5d6210dbd159] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_update_instance_port_admin_state[compute,id-f5dfcc22-45fd-409f-954c-5bd500d7890b,network,slow] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_update_security_groups[id-7d4e1d3c-3209-4d6d-b020-986304ebad1f] +tempest.api.network.test_security_groups.SecGroupTest.test_create_show_delete_security_group_rule[id-cfb99e0e-7410-4a3d-8a0c-959a63ee77e9,smoke] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_server_with_non_existent_flavor[id-ced1a1d7-2ab6-45c9-b90f-b27d87b30efd,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_list_roles_request_without_token[id-11a3c7da-df6c-40c2-abc2-badd682edf9f,negative] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_shutdown_host_with_non_admin_user[id-a803529c-7e3f-4d3c-a7d6-8e1c203d27f6,negative] +tempest.api.object_storage.test_object_slo.ObjectSloTest.test_retrieve_large_object[id-49bc49bc-dd1b-4c0f-904e-d9f10b830ee8] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestV241.test_create_update_show_aggregate_add_remove_host[id-fdf24d9e-8afa-4700-b6aa-9c498351504f] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_create_enabled[id-670bdddc-1cd7-41c7-b8e2-751cfb67df50] +tempest.api.volume.admin.test_volume_retype.VolumeRetypeWithMigrationTest.test_volume_from_snapshot_retype_with_migration[id-d0d9554f-e7a5-4104-8973-f35b27ccb60d] +tempest.api.volume.admin.test_volume_types.VolumeTypesTest.test_volume_crud_with_volume_type_and_extra_specs[id-c03cc62c-f4e9-4623-91ec-64ce2f9c1260] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_delete_non_existent_router_returns_404[id-c7edc5ad-d09d-41e6-a344-5c0c31e2e3e4,negative] +tempest.api.identity.v3.test_projects.IdentityV3ProjectsTest.test_list_projects_returns_only_authorized_projects[id-86128d46-e170-4644-866a-cc487f699e1d] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_object_metakey[id-e4183917-33db-4153-85cc-4dacbb938865] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_exceed_limit[id-37791bbd-90c0-4de0-831e-5f38cba9c6b3] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_security_groups_create_list_delete[id-eb2b087d-633d-4d0d-a7bd-9e6ba35b32de,smoke] +tempest.api.object_storage.test_account_services.AccountTest.test_list_no_containers[id-884ec421-fbad-4fcc-916b-0580f2699565] +tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers_with_detail[id-585e934c-448e-43c4-acbf-d06a9b899997] +tempest.api.image.v2.test_images.ListUserImagesTest.test_get_image_schema[id-622b925c-479f-4736-860d-adeaf13bc371] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_create_router_setting_project_id[id-e54dd3a3-4352-4921-b09d-44369ae17397] +tempest.api.identity.admin.v3.test_regions.RegionsTestJSON.test_create_update_get_delete_region[id-56186092-82e4-43f2-b954-91013218ba42] +tempest.api.volume.admin.test_multi_backend.VolumeMultiBackendTest.test_backend_name_reporting[id-c1a41f3f-9dad-493e-9f09-3ff197d477cc] +tempest.api.volume.admin.test_volume_quotas.VolumeQuotasAdminTestJSON.test_update_all_quota_resources_for_tenant[id-3d45c99e-cc42-4424-a56e-5cbd212b63a6] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port[compute,id-7c0bb1a2-d053-49a4-98f9-ca1a1d849f63,network,slow] +tempest.api.identity.v3.test_access_rules.AccessRulesV3Test.test_list_access_rules[id-2354c498-5119-4ba5-9f0d-44f16f78fb0e] +tempest.api.identity.v3.test_ec2_credentials.EC2CredentialsTest.test_delete_ec2_credential[id-9408d61b-8be0-4a8d-9b85-14f61edb456b] +tempest.api.object_storage.test_container_services_negative.ContainerNegativeTest.test_create_container_metadata_name_exceeds_max_length[id-41e645bf-2e68-4f84-bf7b-c71aa5cd76ce,negative] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_unset_nonexistent_key[id-440b9f3f-3c7f-4293-a106-0ceda350f8de,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_des[id-97d12b1c-a610-4194-93f1-ba859e718b45,negative] +tempest.api.compute.servers.test_instance_actions_negative.InstanceActionsNegativeTestJSON.test_list_instance_actions_non_existent_server[id-67e1fce6-7ec2-45c6-92d4-0a8f1a632910,negative] +tempest.api.volume.admin.test_volume_snapshot_quotas_negative.VolumeSnapshotQuotasNegativeTestJSON.test_quota_volume_snapshots[id-02bbf63f-6c05-4357-9d98-2926a94064ff,negative] +tempest.api.compute.admin.test_instance_usage_audit_log_negative.InstanceUsageAuditLogNegativeTestJSON.test_get_instance_usage_audit_logs_with_invalid_time[id-9b952047-3641-41c7-ba91-a809fc5974c8,negative] +tempest.api.compute.admin.test_quotas_negative.QuotasSecurityGroupAdminNegativeTest.test_security_groups_rules_exceed_limit[id-6e9f436d-f1ed-4f8e-a493-7275dfaa4b4d,negative,network] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_reset_network_inject_network_info[id-7a1323b4-a6a2-497a-96cb-76c07b945c71] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_transfer_encoding[id-4f84422a-e2f2-4403-b601-726a4220b54e] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_roles_hierarchy[id-dc6f5959-b74d-4e30-a9e5-a8255494ff00] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_security_group[id-be308db6-a7cf-4d5c-9baf-71bafd73f35e,negative] +tempest.api.volume.admin.test_volume_hosts.VolumeHostsAdminTestsJSON.test_list_hosts[id-d5f3efa2-6684-4190-9ced-1c2f526352ad] +tempest.api.compute.admin.test_services.ServicesAdminTestJSON.test_list_services[id-5be41ef4-53d1-41cc-8839-5c2a48a1b283] +tempest.api.network.test_tags.TagsExtTest.test_create_check_list_and_delete_tags[id-c6231efa-9a89-4adf-b050-2a3156b8a1d9] +tempest.api.compute.servers.test_multiple_create.MultipleCreateTestJSON.test_multiple_create[id-61e03386-89c3-449c-9bb1-a06f423fd9d1] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeTestJSON.test_project_delete_by_unauthorized_user[id-8d68c012-89e0-4394-8d6b-ccd7196def97,negative] +tempest.api.network.test_security_groups_negative.NegativeSecGroupIPv6Test.test_create_security_group_rule_with_bad_ethertype[id-5666968c-fff3-40d6-9efc-df1c8bd01abb,negative] +tempest.api.compute.admin.test_flavors.FlavorsAdminTestJSON.test_create_flavor_using_string_ram[id-3b541a2e-2ac2-4b42-8b8d-ba6e22fcd4da] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_update_server_name_length_exceeds_256[id-5c8e244c-dada-4590-9944-749c455b431f,negative] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_create_tenant_by_unauthorized_user[id-d26b278a-6389-4702-8d6e-5980d80137e0,negative] +tempest.api.compute.servers.test_server_rescue.ServerRescueTestJSONUnderV235.test_rescued_vm_associate_dissociate_floating_ip[id-4842e0cf-e87d-4d9d-b61f-f4791da3cacc] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_param_offset_and_limit[id-51c22651-a074-4ea7-af0b-094f9331303e] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_stateless_no_ra_no_dhcp[id-21635b6f-165a-4d42-bf49-7d195e47342f] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_update_image_reserved_property[id-a0ae75d4-ce9c-4576-b823-aba04c8acabd,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_router_add_gateway_net_not_external_returns_400[id-11836a18-0b15-4327-a50b-f0d9dc66bddd,negative] +tempest.api.compute.keypairs.test_keypairs_v22.KeyPairsV22TestJSON.test_keypair_create_delete[id-6c1d3123-4519-4742-9194-622cb1714b7d] +tempest.api.object_storage.test_healthcheck.HealthcheckTest.test_get_healthcheck[id-db5723b1-f25c-49a9-bfeb-7b5640caf337] +tempest.api.network.test_routers.RoutersIpV6Test.test_add_multiple_router_interfaces[id-802c73c9-c937-4cef-824b-2191e24a6aab,smoke] +tempest.api.compute.images.test_images_oneserver_negative.ImagesOneServerNegativeTestJSON.test_create_second_image_when_first_image_is_being_saved[id-0460efcf-ee88-4f94-acef-1bf658695456,negative] +tempest.api.network.test_allowed_address_pair.AllowedAddressPairIpV6TestJSON.test_update_port_with_address_pair[id-9599b337-272c-47fd-b3cf-509414414ac4] +tempest.api.compute.admin.test_flavors_extra_specs_negative.FlavorsExtraSpecsNegativeTestJSON.test_flavor_get_nonexistent_key[id-329a7be3-54b2-48be-8052-bf2ce4afd898,negative] +tempest.api.compute.admin.test_instance_usage_audit_log.InstanceUsageAuditLogTestJSON.test_list_instance_usage_audit_logs[id-25319919-33d9-424f-9f99-2c203ee48b9d] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_host_get_details[id-eeef473c-7c52-494d-9f09-2ed7fc8fc036] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_stop_non_existent_server[id-a31460a9-49e1-42aa-82ee-06e0bb7c2d03,negative] +tempest.api.compute.floating_ips.test_floating_ips_actions_negative.FloatingIPsNegativeTestJSON.test_delete_nonexistent_floating_ip[id-ae1c55a8-552b-44d4-bfb6-2a115a15d0ba,negative] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_get_qos[id-7aa214cc-ac1a-4397-931f-3bb2e83bb0fd] +tempest.api.compute.servers.test_server_password.ServerPasswordTestJSON.test_delete_server_password[id-f8229e8b-b625-4493-800a-bde86ac611ea] +tempest.api.network.admin.test_external_network_extension.ExternalNetworksTestJSON.test_show_external_networks_attribute[id-2ac50ab2-7ebd-4e27-b3ce-a9e399faaea2] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_list_server_groups[id-d4874179-27b4-4d7d-80e4-6c560cdfe321] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_force_detach_volume[compute,id-d38285d9-929d-478f-96a5-00e66a115b81] +tempest.scenario.test_object_storage_basic_ops.TestObjectStorageBasicOps.test_swift_acl_anonymous_download[id-916c7111-cb1f-44b2-816d-8f760e4ea910,object_storage,slow] +tempest.api.identity.v2.test_tenants.IdentityTenantsTest.test_list_tenants_returns_only_authorized_tenants[id-ecae2459-243d-4ba1-ad02-65f15dc82b78] +tempest.api.image.v1.test_images_negative.CreateDeleteImagesNegativeTest.test_delete_image_non_hex_string_id[id-950e5054-a3c7-4dee-ada5-e576f1087abd,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_allocation_pools[id-bec949c4-3147-4ba6-af5f-cd2306118404] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_x_object_meta[id-1c7ed3e4-2099-406b-b843-5301d4811baf] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_bad_protocol[id-981bdc22-ce48-41ed-900a-73148b583958,negative] +tempest.api.compute.admin.test_services.ServicesAdminTestJSON.test_get_service_by_service_binary_name[id-f345b1ec-bc6e-4c38-a527-3ca2bc00bef5] +tempest.api.network.test_networks.NetworksTest.test_create_delete_subnet_all_attributes[id-a4d9ec4c-0306-4111-a75c-db01a709030b] +tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_create_aggregate_name_length_exceeds_255[id-4c194563-543b-4e70-a719-557bbe947fac,negative] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_resize_server_confirm_from_stopped[id-138b131d-66df-48c9-a171-64f45eb92962] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit_equals_infinite[id-a1427f61-420e-48a5-b6e3-0b394fa95400] +tempest.api.compute.admin.test_servers_on_multinodes.ServersOnMultiNodesTest.test_create_server_with_scheduler_hint_group_anti_affinity[id-f8bd0867-e459-45f5-ba53-59134552fe04] +tempest.api.compute.servers.test_servers_microversions.ServerShowV254Test.test_rebuild_server[id-09170a98-4940-4637-add7-1a35121f1a5a] +tempest.scenario.test_minimum_basic.TestMinimumBasicScenario.test_minimum_basic_scenario[compute,id-bdbb5441-9204-419d-a225-b4fdbfb1a1a8,image,network,volume] +tempest.api.object_storage.test_account_services.AccountTest.test_list_containers_with_prefix[id-365e6fc7-1cfe-463b-a37c-8bd08d47b6aa] +tempest.api.network.test_routers.RoutersTest.test_router_interface_port_update_with_fixed_ip[id-96522edf-b4b5-45d9-8443-fa11c26e6eff] +tempest.api.object_storage.test_container_sync_middleware.ContainerSyncMiddlewareTest.test_container_synchronization[id-ea4645a1-d147-4976-82f7-e5a7a3065f80,slow] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_create_server_invalid_bdm_in_2nd_dict[id-12146ac1-d7df-4928-ad25-b1f99e5286cd,negative,volume] +tempest.api.object_storage.test_container_quotas.ContainerQuotasTest.test_upload_large_object[id-22eeeb2b-3668-4160-baef-44790f65a5a0,smoke] +tempest.api.compute.floating_ips.test_list_floating_ips.FloatingIPDetailsTestJSON.test_get_floating_ip_details[id-eef497e0-8ff7-43c8-85ef-558440574f84] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_with_if_none_match[id-641500d5-1612-4042-a04d-01fc4528bc30] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_reboot_host_with_non_admin_user[id-02d79bb9-eb57-4612-abf6-2cb38897d2f8,negative] +tempest.api.network.test_tags.TagsTest.test_create_list_show_update_delete_tags[id-ee76bfaf-ac94-4d74-9ecc-4bbd4c583cb1] +tempest.api.identity.admin.v2.test_users.UsersTestJSON.test_list_users_for_tenant[id-6e317209-383a-4bed-9f10-075b7c82c79a] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_limits_pass_negative_value[id-62610dd9-4713-4ee0-8beb-fd2c1aa7f950,negative] +tempest.api.object_storage.test_object_services.ObjectTest.test_create_object_with_etag[id-73820093-0503-40b1-a478-edf0e69c7d1f] +tempest.api.compute.admin.test_flavors_microversions.FlavorsV261TestJSON.test_crud_flavor[id-61976b25-488d-41dc-9dcb-cb9693a7b075] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_group_on_domain[id-4bf8a70b-e785-413a-ad53-9f91ce02faa7] +tempest.api.identity.admin.v3.test_groups.GroupsV3TestJSON.test_group_users_add_list_delete[id-1598521a-2f36-4606-8df9-30772bd51339,smoke] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_limit[id-db4d8e0a-7a2e-41cc-a712-961f6844e896] +tempest.api.compute.servers.test_multiple_create_negative.MultipleCreateNegativeTestJSON.test_min_count_less_than_one[id-daf29d8d-e928-4a01-9a8c-b129603f3fc0,negative] +tempest.api.identity.admin.v3.test_projects.ProjectsTestJSON.test_project_create_with_parent[id-1854f9c0-70bc-4d11-a08a-1c789d339e3d] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_non_existing_server_name[id-e2c77c4a-000a-4af3-a0bd-629a328bde7c,negative] +tempest.api.volume.admin.test_qos.QosSpecsTestJSON.test_set_unset_qos_key[id-ed00fd85-4494-45f2-8ceb-9e2048919aed] +tempest.api.object_storage.test_object_services.ObjectTest.test_get_object_if_different[id-50d01f12-526f-4360-9ac2-75dd508d7b68] +tempest.api.network.test_security_groups_negative.NegativeSecGroupTest.test_create_security_group_rule_with_non_existent_remote_groupid[id-4bf786fd-2f02-443c-9716-5b98e159a49a,negative] +tempest.api.compute.images.test_images_negative.ImagesDeleteNegativeTestJSON.test_delete_non_existent_image[id-137aef61-39f7-44a1-8ddf-0adf82511701,negative] +tempest.api.identity.admin.v3.test_domains.DomainsTestJSON.test_create_update_delete_domain[id-f2f5b44a-82e8-4dad-8084-0661ea3b18cf,smoke] +tempest.api.compute.servers.test_server_rescue.ServerStableDeviceRescueTest.test_stable_device_rescue_disk_virtio_with_volume_attached[id-a3772b42-00bf-4310-a90b-1cc6fd3e7eab,volume] +tempest.api.compute.test_quotas.QuotasTestJSON.test_get_default_quotas[id-9bfecac7-b966-4f47-913f-1a9e2c12134a] +tempest.api.identity.v2.test_api_discovery.TestApiDiscovery.test_api_version_resources[id-ea889a68-a15f-4166-bfb1-c12456eae853,smoke] +tempest.api.compute.admin.test_quotas.QuotasAdminTestJSON.test_delete_quota[id-389d04f0-3a41-405f-9317-e5f86e3c44f0] +tempest.api.network.test_extensions.ExtensionsTestJSON.test_list_show_extensions[id-ef28c7e6-e646-4979-9d67-deb207bc5564,smoke] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_create_user_request_without_a_token[id-55bbb103-d1ae-437b-989b-bcdf8175c1f4,negative] +tempest.api.compute.admin.test_servers_negative.ServersAdminNegativeTestJSON.test_reset_state_server_invalid_type[id-4cdcc984-fab0-4577-9a9d-6d558527ee9d,negative] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_with_detail_param_limit[id-33985568-4965-49d5-9bcc-0aa007ca5b7a] +tempest.api.compute.servers.test_server_metadata_negative.ServerMetadataNegativeTestJSON.test_delete_metadata_non_existent_server[id-6bbd88e1-f8b3-424d-ba10-ae21c45ada8d,negative] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_keypair_create_with_pub_key[id-39c90c6a-304a-49dd-95ec-2366129def05] +tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON.test_credentials_list_delete[id-13202c00-0021-42a1-88d4-81b44d448aab] +tempest.api.object_storage.test_object_services.ObjectTest.test_delete_object[id-17738d45-03bd-4d45-9e0b-7b2f58f98687] +tempest.api.network.test_networks_negative.NetworksNegativeTestJSON.test_update_non_existent_subnet[id-1cc47884-ac52-4415-a31c-e7ce5474a868,negative] +tempest.api.compute.volumes.test_volumes_list.VolumesTestJSON.test_volume_list_param_limit[id-1048ed81-2baf-487a-b284-c0622b86e7b8] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescue_paused_instance[id-cc3a883f-43c0-4fb6-a9bb-5579d64984ed,negative] +tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway_with_snat_explicit[id-b386c111-3b21-466d-880c-5e72b01e1a33] +tempest.api.network.test_networks.NetworksTest.test_show_network[id-2bf13842-c93f-4a69-83ed-717d2ec3b44e,smoke] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_name[id-55c8f5f5-bfed-409d-a6d5-4caeda985d7b] +tempest.api.image.v2.test_images_member_negative.ImagesMemberNegativeTest.test_image_share_owner_cannot_accept[id-27002f74-109e-4a37-acd0-f91cd4597967,negative] +tempest.api.volume.admin.test_volume_types_extra_specs_negative.ExtraSpecsNegativeTest.test_delete_nonexistent_volume_type_id[id-031cda8b-7d23-4246-8bf6-bbe73fd67074,negative] +tempest.api.compute.servers.test_list_server_filters.ListServerFiltersTestJSON.test_list_servers_filter_by_server_name[id-9b067a7b-7fee-4f6a-b29c-be43fe18fc5a] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32,smoke] +tempest.api.identity.admin.v3.test_users_negative.UsersNegativeTest.test_create_user_for_non_existent_domain[id-e75f006c-89cc-477b-874d-588e4eab4b17,negative] +tempest.api.compute.keypairs.test_keypairs.KeyPairsV2TestJSON.test_keypairs_create_list_delete[id-1d1dbedb-d7a0-432a-9d09-83f543c3c19b] +tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON.test_grant_list_revoke_role_to_user_on_domain[id-6c9a2940-3625-43a3-ac02-5dcec62ef3bd] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_update_port_with_two_security_groups_and_extra_attributes[id-edf6766d-3d40-4621-bc6e-2521a44c257d] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_resize_nonexistent_server[id-7ea45b3e-e770-46fa-bfcc-9daaf6d987c0,negative] +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke] +tempest.api.compute.security_groups.test_security_group_rules_negative.SecurityGroupRulesNegativeTestJSON.test_create_security_group_rule_with_invalid_id[id-2244d7e4-adb7-4ecb-9930-2d77e123ce4f,negative] +tempest.api.volume.admin.test_volume_type_access.VolumeTypesAccessTest.test_volume_type_access_add[id-d4dd0027-835f-4554-a6e5-50903fb79184] +tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_update_user_password[id-2d223a0e-e457-4a70-9fb1-febe027a0ff9] +tempest.api.volume.test_volumes_snapshots_negative.VolumesSnapshotNegativeTestJSON.test_list_snapshots_invalid_param_sort[id-27b5f37f-bf69-4e8c-986e-c44f3d6819b8,negative] +tempest.api.network.test_dhcp_ipv6.NetworksTestDHCPv6.test_dhcpv6_64_subnets[id-4256c61d-c538-41ea-9147-3c450c36669e] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_update_network_description[id-c72c1c0c-2193-4aca-ccc4-b1442640bbbb] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescued_vm_detach_volume[id-f56e465b-fe10-48bf-b75d-646cda3a8bc9,negative,volume] +tempest.api.volume.test_volumes_list.VolumesListTestJSON.test_volumes_list_by_status[id-39654e13-734c-4dab-95ce-7613bf8407ce] +tempest.api.network.test_routers_negative.RoutersNegativeTest.test_show_non_existent_router_returns_404[id-c2a70d72-8826-43a7-8208-0209e6360c47,negative] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_create_update_with_az[id-4d2b2004-40fa-40a1-aab2-66f4dab81beb] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_get_non_existent_image[id-668743d5-08ad-4480-b2b8-15da34f81d9f,negative] +tempest.api.compute.volumes.test_volumes_negative.VolumesNegativeTest.test_create_volume_with_invalid_size[id-5125ae14-152b-40a7-b3c5-eae15e9022ef,negative] +tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin_with_all_tenants[id-9f5579ae-19b4-4985-a091-2a5d56106580] +tempest.api.identity.admin.v2.test_roles.RolesTestJSON.test_assign_user_role[id-0146f675-ffbd-4208-b3a4-60eb628dbc5e] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_limit_results[id-3a484ca9-67ba-451e-b494-7fcf28d32d62] +tempest.scenario.test_stamp_pattern.TestStampPattern.test_stamp_pattern[compute,id-10fd234a-515c-41e5-b092-8323060598c5,image,network,slow,volume] +tempest.api.compute.admin.test_hypervisor.HypervisorAdminTestJSON.test_get_hypervisor_list[id-7f0ceacd-c64d-4e96-b8ee-d02943142cc5] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsAssociationTestJSON.test_associate_already_associated_floating_ip[id-6edef4b2-aaf1-4abc-bbe3-993e2561e0fe] +tempest.api.image.v2.test_images_negative.ImportImagesNegativeTest.test_image_web_download_import_with_bad_url[id-c52f6a77-f522-4411-8dbe-9d14f2b1ba6b,negative] +tempest.api.compute.admin.test_hypervisor_negative.HypervisorAdminNegativeUnderV252Test.test_show_servers_with_non_admin_user[id-2a0a3938-832e-4859-95bf-1c57c236b924,negative] +tempest.api.identity.admin.v3.test_projects_negative.ProjectsNegativeStaticTestJSON.test_project_create_duplicate[id-874c3e84-d174-4348-a16b-8c01f599561b,negative] +tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON.test_aggregate_add_host_create_server_with_az[id-96be03c7-570d-409c-90f8-e4db3c646996] +tempest.scenario.test_security_groups_basic_ops.TestSecurityGroupsBasicOps.test_boot_into_disabled_port_security_network_without_secgroup[compute,id-13ccf253-e5ad-424b-9c4a-97b88a026699,network,slow] +tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON.test_security_group_create_get_delete[id-ecc0da4a-2117-48af-91af-993cca39a615] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_rebuild_server_with_manual_disk_config[id-bef56b09-2e8c-4883-a370-4950812f430e] +tempest.api.object_storage.test_object_services.ObjectTest.test_copy_object_with_x_object_meta[id-edabedca-24c3-4322-9b70-d6d9f942a074] +tempest.api.network.test_networks.NetworksIpV6Test.test_show_subnet_fields[id-270fff0b-8bfc-411f-a184-1e8fd35286f0] +tempest.api.compute.servers.test_list_servers_negative.ListServersNegativeTestJSON.test_list_servers_by_changes_since_future_date[id-74745ad8-b346-45b5-b9b8-509d7447fc1f,negative] +tempest.api.volume.admin.test_snapshots_actions.SnapshotsActionsTest.test_snapshot_force_delete_when_snapshot_is_creating[id-05f711b6-e629-4895-8103-7ca069f2073a] +tempest.api.compute.servers.test_server_tags.ServerTagsTestJSON.test_delete_all_tags[id-a63b2a74-e918-4b7c-bcab-10c855f3a57e] +tempest.api.image.v2.test_images.ListUserImagesTest.test_list_images_param_size[id-cf1b9a48-8340-480e-af7b-fe7e17690876] +tempest.api.identity.admin.v3.test_endpoint_groups.EndPointGroupsTest.test_create_list_show_check_delete_endpoint_group[id-7c69e7a1-f865-402d-a2ea-44493017315a] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_show_non_existent_router_returns_404[id-c2a70d72-8826-43a7-8208-0209e6360c47,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestMultiTenantJSON.test_update_server_of_another_tenant[id-543d84c1-dd2e-4c6d-8cb2-b9da0efaa384,negative] +tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image[id-f66891a7-a35c-41a8-b590-a065c2a1caa6,smoke] +tempest.api.identity.admin.v2.test_tenants.TenantsTestJSON.test_tenant_list_delete[id-16c6e05c-6112-4b0e-b83f-5e43f221b6b0] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_duplicate_user_role[id-5c3132cd-c4c8-4402-b5ea-71eb44e97793,negative] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_list_tenants_by_unauthorized_user[id-ca9bb202-63dd-4240-8a07-8ef9c19c04bb,negative] +tempest.api.volume.admin.test_volume_types_extra_specs.VolumeTypesExtraSpecsTest.test_volume_type_extra_spec_create_get_delete[id-d4772798-601f-408a-b2a5-29e8a59d1220] +tempest.api.volume.admin.test_volume_types_negative.VolumeTypesNegativeTest.test_get_nonexistent_type_id[id-994610d6-0476-4018-a644-a2602ef5d4aa,negative] +tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token[id-6f8e4436-fc96-4282-8122-e41df57197a9] +tempest.api.image.v1.test_images.ListImagesTest.test_index_min_size[id-6ffc16d0-4cbf-4401-95c8-4ac63eac34d8] +tempest.api.compute.admin.test_live_migration.LiveMigrationTest.test_live_block_migration_with_attached_volume[id-e19c0cc6-6720-4ed8-be83-b6603ed5c812,volume] +tempest.api.volume.test_volumes_snapshots.VolumesSnapshotTestJSON.test_volume_from_snapshot_no_size[id-053d8870-8282-4fff-9dbb-99cb58bb5e0a] +tempest.api.compute.admin.test_live_migration_negative.LiveMigrationNegativeTest.test_live_block_migration_suspended[id-6e2f94f5-2ee8-4830-bef5-5bc95bb0795b,negative] +tempest.api.compute.admin.test_availability_zone.AZAdminV2TestJSON.test_get_availability_zone_list_detail[id-ef726c58-530f-44c2-968c-c7bed22d5b8c] +tempest.api.compute.servers.test_server_group.ServerGroupTestJSON.test_create_delete_server_group_with_anti_affinity_policy[id-3645a102-372f-4140-afad-13698d850d23] +tempest.api.identity.admin.v2.test_users_negative.UsersNegativeTestJSON.test_authentication_for_disabled_user[id-593a4981-f6d4-460a-99a1-57a78bf20829,negative] +tempest.api.network.admin.test_routers.RoutersIpV6AdminTest.test_update_router_set_gateway_with_snat_explicit[id-b386c111-3b21-466d-880c-5e72b01e1a33] +tempest.api.compute.admin.test_hosts_negative.HostsAdminNegativeTestJSON.test_update_host_with_invalid_status[id-fbe2bf3e-3246-4a95-a59f-94e4e298ec77,negative] +tempest.api.volume.test_volumes_get.VolumesSummaryTest.test_show_volume_summary[id-c4f2431e-4920-4736-9e00-4040386b6feb] +tempest.api.identity.admin.v3.test_users_negative.UsersNegativeTest.test_authentication_for_disabled_user[id-b3c9fccc-4134-46f5-b600-1da6fb0a3b1f,negative] +tempest.api.identity.admin.v2.test_roles_negative.RolesNegativeTestJSON.test_assign_user_role_request_without_token[id-f0d2683c-5603-4aee-95d7-21420e87cfd8,negative] +tempest.api.network.test_routers_negative.RoutersNegativeIpV6Test.test_update_non_existent_router_returns_404[id-b23d1569-8b0c-4169-8d4b-6abd34fad5c7,negative] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestMultiTenantJSON.test_delete_a_server_of_another_tenant[id-5c75009d-3eea-423e-bea3-61b09fd25f9c,negative] +tempest.api.compute.security_groups.test_security_groups_negative.SecurityGroupsNegativeTestJSON.test_update_security_group_with_invalid_sg_name[id-cda8d8b4-59f8-4087-821d-20cf5a03b3b1,negative] +tempest.api.compute.servers.test_server_rescue_negative.ServerRescueNegativeTestJSON.test_rescue_non_existent_server[id-6dfc0a55-3a77-4564-a144-1587b7971dde,negative] +tempest.api.compute.keypairs.test_keypairs_negative.KeyPairsNegativeTestJSON.test_create_keypair_with_empty_name_string[id-1398abe1-4a84-45fb-9294-89f514daff00,negative] +tempest.api.compute.admin.test_delete_server.DeleteServersAdminTestJSON.test_admin_delete_servers_of_others[id-73177903-6737-4f27-a60c-379e8ae8cf48] +tempest.api.image.v2.test_versions.VersionsTest.test_list_versions[id-659ea30a-a17c-4317-832c-0f68ed23c31d,smoke] +tempest.api.image.v2.test_images_member.ImagesMemberTest.test_get_image_member_schema[id-634dcc3f-f6e2-4409-b8fd-354a0bb25d83] +tempest.api.compute.servers.test_disk_config.ServerDiskConfigTestJSON.test_resize_server_from_auto_to_manual[id-693d16f3-556c-489a-8bac-3d0ca2490bad] +tempest.api.volume.admin.test_volumes_actions.VolumesActionsTest.test_volume_reset_status[id-d063f96e-a2e0-4f34-8b8a-395c42de1845] +tempest.api.compute.test_networks.ComputeNetworksTest.test_list_networks[id-3fe07175-312e-49a5-a623-5f52eeada4c2] +tempest.api.network.test_ports.PortsIpV6TestJSON.test_port_list_filter_by_ip[id-e7fe260b-1e79-4dd3-86d9-bec6a7959fc5] +tempest.api.network.test_networks.NetworksTest.test_external_network_visibility[id-af774677-42a9-4e4b-bb58-16fe6a5bc1ec,smoke] +tempest.api.volume.test_volumes_snapshots_list.VolumesSnapshotListTestJSON.test_snapshot_list_param_sort_id_desc[id-8a7fe058-0b41-402a-8afd-2dbc5a4a718b] +tempest.api.compute.admin.test_server_diagnostics_negative.ServerDiagnosticsNegativeTest.test_get_server_diagnostics_by_non_admin[id-e84e2234-60d2-42fa-8b30-e2d3049724ac,negative] +tempest.api.network.admin.test_ports.PortsAdminExtendedAttrsIpV6TestJSON.test_update_port_binding_ext_attr[compute,id-6f6c412c-711f-444d-8502-0ac30fbf5dd5] +tempest.api.compute.servers.test_servers_negative.ServersNegativeTestJSON.test_rebuild_deleted_server[id-98fa0458-1485-440f-873b-fe7f0d714930,negative] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_delete_subnet_with_dhcp_enabled[id-94ce038d-ff0a-4a4c-a56b-09da3ca0b55d] +tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_container_format[id-292bd310-369b-41c7-a7a3-10276ef76753,negative] +tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_security_group_rule_with_remote_ip_prefix[id-16459776-5da2-4634-bce4-4b55ee3ec188] +tempest.api.image.v2.admin.test_images_metadefs_namespace_properties.MetadataNamespacePropertiesTest.test_basic_meta_def_namespace_property[id-b1a3765e-3a5d-4f6d-a3a7-3ca3476ae768] +tempest.api.network.test_networks.NetworksIpV6Test.test_create_update_delete_network_subnet[id-0e269138-0da6-4efc-a46d-578161e7b221,smoke] +tempest.api.compute.floating_ips.test_floating_ips_actions.FloatingIPsTestJSON.test_delete_floating_ip[id-de45e989-b5ca-4a9b-916b-04a52e7bbb8b] +tempest.api.identity.admin.v2.test_tenant_negative.TenantsNegativeTestJSON.test_tenant_delete_by_unauthorized_user[id-162ba316-f18b-4987-8c0c-fd9140cd63ed,negative] +tempest.api.identity.admin.v2.test_tokens.TokensTestJSON.test_list_endpoints_for_token[id-ca3ea6f7-ed08-4a61-adbd-96906456ad31] +tempest.scenario.test_network_advanced_server_ops.TestNetworkAdvancedServerOps.test_server_connectivity_pause_unpause[compute,id-2b2642db-6568-4b35-b812-eceed3fa20ce,network,slow] +tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON.test_list_images_with_detail_filter_by_server_ref[id-8c78f822-203b-4bf6-8bba-56ebd551cf84] +tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output_with_unlimited_size[id-89104062-69d8-4b19-a71b-f47b7af093d7] +tempest.api.compute.servers.test_create_server_multi_nic.ServersTestMultiNic.test_verify_duplicate_network_nics[id-1678d144-ed74-43f8-8e57-ab10dbf9b3c2] +tempest.api.compute.servers.test_servers.ServerShowV247Test.test_update_rebuild_list_server[id-8de397c2-57d0-4b90-aa30-e5d668f21a8b] diff --git a/.automation.conf/tempest/skip-lists/ci-multinode-tempest-full b/.automation.conf/tempest/skip-lists/ci-multinode-tempest-full new file mode 100644 index 000000000..94c494687 --- /dev/null +++ b/.automation.conf/tempest/skip-lists/ci-multinode-tempest-full @@ -0,0 +1 @@ +tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details.*: "Cirros image doesn't have '/var/run/udhcpc.eth0.pid" diff --git a/.automation.conf/tempest/tempest-ci-aio.overrides.conf b/.automation.conf/tempest/tempest-ci-aio.overrides.conf new file mode 100644 index 000000000..16a1b891d --- /dev/null +++ b/.automation.conf/tempest/tempest-ci-aio.overrides.conf @@ -0,0 +1,2 @@ +[volume] +catalog_type = volumev3 diff --git a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf new file mode 100644 index 000000000..12cb9f692 --- /dev/null +++ b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf @@ -0,0 +1,36 @@ +[DEFAULT] +openstack_client_http_timeout = 300 + +[auth] +compute_quotas = cores:-1,ram:-1 + +[identity] +v3_endpoint_type = publicURL + +[compute] +min_compute_nodes = 2 +min_microversion = 2.1 +max_microversion = 2.96 + +[service-clients] +http_timeout = 600 + +[compute-feature-enabled] +resize = true +live_migration = true +block_migration_for_live_migration = false +volume_backed_live_migration = true +console_output = true + +[volume] +catalog_type = volumev3 +storage_protocol = ceph +build_timeout = 600 +min_microversion = 3.0 +max_microversion = 3.71 + +[image] +build_timeout = 600 + +[dashboard] +dashboard_url = https://192.168.39.2 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..e9a948a0d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @stackhpc/kayobe diff --git a/.github/auto-label.yaml b/.github/auto-label.yaml new file mode 100644 index 000000000..0adaf7284 --- /dev/null +++ b/.github/auto-label.yaml @@ -0,0 +1,55 @@ +--- +path: + pullrequest: true + multipleLabelPaths: + - paths: + .automation: 'kayobe-automation' + .automation.conf: 'kayobe-automation' + - paths: + doc: 'documentation' + - paths: + .github: 'workflows' + etc: + kayobe: + enviromnents: + ci-aio: 'workflows' + ci-builder: 'workflows' + ci-multinode: 'workflows' + trivy: 'workflows' + terraform: 'workflows' + tools: + scan-images.sh: "workflows" + tox.ini: 'workflows' + - paths: + etc: + kayobe: + ansible: 'ansible' + ansible.cfg: 'ansible' + - paths: + etc: + kayobe: + kolla: + config: + fluentd: "monitoring" + grafana: "monitoring" + prometheus: "monitoring" + inventory: + group_vars: + prometheus-blackbox-exporter: "monitoring" + - paths: + etc: + kayobe: + kolla: + kolla-build.conf: "kolla" + kolla-image-tags.yml: "kolla" + kolla.yml: "kolla" + tools: + kolla-images.py: "kolla" + +staleness: + pullrequest: true + old: 30 + extraold: 90 + +requestsize: + enabled: true diff --git a/.github/cherry-pick-bot.yml b/.github/cherry-pick-bot.yml new file mode 100644 index 000000000..4e1570437 --- /dev/null +++ b/.github/cherry-pick-bot.yml @@ -0,0 +1,3 @@ +--- +enabled: true +preservePullRequestTitle: true diff --git a/.github/path-filters.yml b/.github/path-filters.yml new file mode 100644 index 000000000..574969c0c --- /dev/null +++ b/.github/path-filters.yml @@ -0,0 +1,31 @@ +# This file is a list of path filters for the PR workflow in .github/workflows/stackhpc-pull-request.yml. +aio: &aio + - '.automation' + - '.automation.conf/config.sh' + - '.automation.conf/tempest/load-lists/default' + - '.automation.conf/tempest/load-lists/platform.2022.11-test-list.txt' + - '.automation.conf/tempest/load-lists/refstack-2019.11-test-list.txt' + - '.automation.conf/tempest/tempest-ci-aio.overrides.conf' + - '.github/workflows/stackhpc-all-in-one.yml' + - '.github/workflows/stackhpc-build-kayobe-image.yml' + - '.github/workflows/stackhpc-pull-request.yml' + - 'etc/kayobe/*.yml' + - 'etc/kayobe/ansible/configure-aio-resources.yml' + - 'etc/kayobe/ansible/stackhpc-openstack-tests.yml' + - 'etc/kayobe/ansible/growroot.yml' + - 'etc/kayobe/ansible/requirements.yml' + - 'etc/kayobe/ansible/scripts/aio-init.sh' + - 'etc/kayobe/environments/ci-aio/**' + - 'etc/kayobe/inventory/**' + - 'etc/kayobe/kolla/**' + - 'kayobe-env' + - 'requirements.txt' + - 'terraform/aio/**' +check-tags: &check-tags + - '.github/workflows/stackhpc-check-tags.yml' + - 'etc/kayobe/kolla-image-tags.yml' + - 'etc/kayobe/pulp.yml' + - 'tools/kolla-images.py' +build-kayobe-image: + - *aio + - *check-tags diff --git a/.github/workflows/amphora-image-build.yml b/.github/workflows/amphora-image-build.yml new file mode 100644 index 000000000..2347c688a --- /dev/null +++ b/.github/workflows/amphora-image-build.yml @@ -0,0 +1,271 @@ +--- +name: Build Amphora image +on: + workflow_dispatch: + inputs: + runner_env: + description: Which cloud to run on? + type: choice + default: SMS Lab + options: + - SMS Lab + - Leafcloud + secrets: + KAYOBE_VAULT_PASSWORD_CI_BUILDER: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} +jobs: + runner-selection: + uses: ./.github/workflows/runner-selector.yml + with: + runner_env: ${{ inputs.runner_env }} + amphora-image-build: + name: Build Amphora image + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - runner-selection + permissions: {} + steps: + + - name: Install Package dependencies + run: | + sudo apt update && + sudo apt install -y git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "BRANCH=$BRANCH" >> $GITHUB_OUTPUT + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + + # Generate a tag to apply to all built Amphora image. + - name: Generate Amphora image tag + id: image_tag + run: | + echo "image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + + - name: Display Amphora image tag + run: | + echo "${{ steps.image_tag.outputs.image_tag }}" + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "ubuntu" + aio_vm_name = "skc-amphora-image-builder" + # Must be an Ubuntu Jammy host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" + aio_vm_interface = "ens3" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=ubuntu --skip-tags network + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Create Amphora image output directory + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "mkdir -p /opt/kayobe/images/amphora" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Build Octavia Amphora image + id: build_amphora + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/octavia-amphora-image-build.yml -e amphora_image_dest=/opt/kayobe/images/amphora/amphora-x64-haproxy.qcow2 + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /var/log/octavia-amphora-image-build.log" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_amphora.outcome == 'failure' + + - name: Upload Octavia Amphora image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/amphora \ + -e artifact_tag=${{ steps.image_tag.outputs.image_tag }} \ + -e file_regex="*.qcow2" \ + -e repository_name="amphora-images-${{ steps.openstack_release.outputs.openstack_release }}" \ + -e pulp_base_path="amphora-images/${{ steps.openstack_release.outputs.openstack_release }}" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_amphora.outcome == 'success' + + - name: Copy logs back to runner + continue-on-error: true + run: | + mkdir artifact + scp stack@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/var/log/octavia-amphora-image-build.log ./artifact + if: always() + + - name: Fail if Amphora image builds failed + run: | + echo "Builds failed. See workflow artifacts for details." && + exit 1 + if: steps.build_amphora.outcome == 'failure' + + - name: Upload logs & image artifact + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: amphora-image-build-log + path: ./artifact + if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() diff --git a/.github/workflows/amphora-image-promote.yml b/.github/workflows/amphora-image-promote.yml new file mode 100644 index 000000000..ebdce743a --- /dev/null +++ b/.github/workflows/amphora-image-promote.yml @@ -0,0 +1,69 @@ +--- +name: Promote Amphora image +on: + workflow_dispatch: + inputs: + image_tag: + description: Tag to promote + type: string + required: true +env: + ANSIBLE_FORCE_COLOR: True +jobs: + amphora-image-promote: + name: Promote Amphora image + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + working-directory: src/kayobe-config + + - name: Setup networking + run: | + if ! ip l show breth1 >/dev/null 2>&1; then + sudo ip l add breth1 type bridge + fi + sudo ip l set breth1 up + if ! ip a show breth1 | grep 192.168.33.3/24; then + sudo ip a add 192.168.33.3/24 dev breth1 + fi + if ! ip l show dummy1 >/dev/null 2>&1; then + sudo ip l add dummy1 type dummy + fi + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Promote Amphora image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e repository_name="amphora-images-${{ steps.openstack_release.outputs.openstack_release }}" \ + -e pulp_base_path="amphora-images/${{ steps.openstack_release.outputs.openstack_release }}" + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} diff --git a/.github/workflows/ipa-image-build.yml b/.github/workflows/ipa-image-build.yml new file mode 100644 index 000000000..4b298bf12 --- /dev/null +++ b/.github/workflows/ipa-image-build.yml @@ -0,0 +1,424 @@ +--- +name: Build IPA images +on: + workflow_dispatch: + inputs: + rocky9: + description: Build Rocky Linux 9 + type: boolean + default: true + ubuntu-jammy: + description: Build Ubuntu 22.04 Jammy + type: boolean + default: true + ubuntu-noble: + description: Build Ubuntu 24.04 Noble + type: boolean + default: true + runner_env: + description: Which cloud to run on? + type: choice + default: SMS Lab + options: + - SMS Lab + - Leafcloud + secrets: + KAYOBE_VAULT_PASSWORD_CI_BUILDER: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} +jobs: + runner-selection: + uses: ./.github/workflows/runner-selector.yml + with: + runner_env: ${{ inputs.runner_env }} + + ipa-image-build: + name: Build IPA images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - runner-selection + permissions: {} + steps: + - name: Install Package dependencies + run: | + sudo apt update && + sudo apt install -y git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Output image tag of the builder + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + + - name: Generate IPA image tag + id: ipa_image_tag + run: | + echo "ipa_image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + + - name: Display IPA image tag + run: | + echo "${{ steps.ipa_image_tag.outputs.ipa_image_tag }}" + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "ubuntu" + aio_vm_name = "skc-ipa-image-builder" + aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" + aio_vm_interface = "ens3" + aio_vm_volume_size = "${{ vars.HOST_IMAGE_BUILD_VOLUME }}" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure \ + -e seed_bootstrap_user=ubuntu \ + --skip-tags network,apt,docker,docker-registry + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Build a Ubuntu 22.04 Jammy IPA image + id: build_ubuntu_jammy_ipa + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud deployment image build --force-rebuild \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" \ + -e ipa_ci_builder_distribution="ubuntu" \ + -e ipa_ci_builder_release="jammy" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/ipa/ipa.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_ubuntu_jammy_ipa.outcome == 'failure' + + - name: Upload Ubuntu 22.04 Jammy IPA kernel image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/ipa \ + -e artifact_type=ipa-images \ + -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" \ + -e file_regex='*.kernel' \ + -e upload_checksum=true + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy_ipa.outcome == 'success' + + - name: Upload Ubuntu 22.04 Jammy IPA ramdisk image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/ipa \ + -e artifact_type=ipa-images \ + -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" \ + -e file_regex='*.initramfs' \ + -e upload_checksum=true + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy_ipa.outcome == 'success' + + - name: Build a Ubuntu 24.04 Noble IPA image + id: build_ubuntu_noble_ipa + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud deployment image build --force-rebuild \ + -e os_distribution="ubuntu" \ + -e os_release="noble" \ + -e ipa_ci_builder_distribution="ubuntu" \ + -e ipa_ci_builder_release="noble" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/ipa/ipa.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_ubuntu_noble_ipa.outcome == 'failure' + + - name: Upload Ubuntu 24.04 Noble IPA kernel image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/ipa \ + -e artifact_type=ipa-images \ + -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \ + -e os_distribution="ubuntu" \ + -e os_release="noble" \ + -e file_regex='*.kernel' \ + -e upload_checksum=true + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble_ipa.outcome == 'success' + + - name: Upload Ubuntu 24.04 Noble IPA ramdisk image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/ipa \ + -e artifact_type=ipa-images \ + -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \ + -e os_distribution="ubuntu" \ + -e os_release="noble" \ + -e file_regex='*.initramfs' \ + -e upload_checksum=true + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble_ipa.outcome == 'success' + + - name: Build a Rocky 9 IPA image + id: build_rocky_9_ipa + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud deployment image build --force-rebuild \ + -e os_distribution="rocky" \ + -e os_release="9" \ + -e ipa_ci_builder_distribution="rocky" \ + -e ipa_ci_builder_release="9" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/ipa/ipa.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_rocky_9_ipa.outcome == 'failure' + + - name: Upload Rocky 9 IPA kernel image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/ipa \ + -e artifact_type=ipa-images \ + -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \ + -e os_distribution="rocky" \ + -e os_release="9" \ + -e file_regex='*.kernel' \ + -e upload_checksum=true + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 && steps.build_rocky_9_ipa.outcome == 'success' + + - name: Upload Rocky 9 IPA ramdisk image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/ipa \ + -e artifact_type=ipa-images \ + -e artifact_tag=${{ steps.ipa_image_tag.outputs.ipa_image_tag }} \ + -e os_distribution="rocky" \ + -e os_release="9" \ + -e file_regex='*.initramfs' \ + -e upload_checksum=true + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 && steps.build_rocky_9_ipa.outcome == 'success' + + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + if: always() + + - name: Upload logs artifact + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: Build logs + path: ./logs + + - name: Fail if the IPA image build failed + run: | + echo "Builds failed. See workflow artifacts for details." && + exit 1 + if: steps.build_rocky_9_ipa.outcome == 'failure' || + steps.build_ubuntu_jammy_ipa.outcome == 'failure' || + steps.build_ubuntu_noble_ipa.outcome == 'failure' + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() diff --git a/.github/workflows/ipa-image-promote.yml b/.github/workflows/ipa-image-promote.yml new file mode 100644 index 000000000..cf533f777 --- /dev/null +++ b/.github/workflows/ipa-image-promote.yml @@ -0,0 +1,118 @@ +--- +name: Promote IPA image +on: + workflow_dispatch: + inputs: + rocky9: + description: Promote Rocky Linux 9 + type: boolean + default: true + ubuntu-jammy: + description: Promote Ubuntu 22.04 Jammy + type: boolean + default: true + ubuntu-noble: + description: Promote Ubuntu 24.04 Noble + type: boolean + default: true + image_tag: + description: Tag to promote + type: string + required: true +env: + ANSIBLE_FORCE_COLOR: True +jobs: + ipa-image-promote: + name: Promote IPA image + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-22.04 + steps: + - name: Validate inputs + run: | + if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' && ${{ inputs.ubuntu-noble }} == 'false' ]]; then + echo "At least one distribution must be selected" + exit 1 + fi + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + working-directory: src/kayobe-config + + - name: Setup networking + run: | + if ! ip l show breth1 >/dev/null 2>&1; then + sudo ip l add breth1 type bridge + fi + sudo ip l set breth1 up + if ! ip a show breth1 | grep 192.168.33.3/24; then + sudo ip a add 192.168.33.3/24 dev breth1 + fi + if ! ip l show dummy1 >/dev/null 2>&1; then + sudo ip l add dummy1 type dummy + fi + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Promote Rocky Linux 9 IPA image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e artifact_type="ipa-images" \ + -e os_distribution='rocky' \ + -e os_release='9' + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 + + - name: Promote Ubuntu Jammy 22.04 IPA image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e artifact_type="ipa-images" \ + -e os_distribution='ubuntu' \ + -e os_release='jammy' + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy + + - name: Promote Ubuntu Noble 24.04 IPA image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e artifact_type="ipa-images" \ + -e os_distribution='ubuntu' \ + -e os_release='noble' + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble diff --git a/.github/workflows/multinode-inputs.py b/.github/workflows/multinode-inputs.py new file mode 100644 index 000000000..930b4c776 --- /dev/null +++ b/.github/workflows/multinode-inputs.py @@ -0,0 +1,83 @@ +# Generate inputs for the reusable multinode.yml workflow. +# The test scenario is randomly selected. +# The inputs are printed to stdout in GitHub step output key=value format. + +from dataclasses import dataclass +import random +import typing as t + + +@dataclass +class OSRelease: + distribution: str + release: str + ssh_username: str + + +@dataclass +class OpenStackRelease: + version: str + previous_version: str + os_releases: t.List[OSRelease] + + +@dataclass +class Scenario: + openstack_release: OpenStackRelease + os_release: OSRelease + neutron_plugin: str + upgrade: str + + +ROCKY_9 = OSRelease("rocky", "9", "cloud-user") +UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu") +UBUNTU_NOBLE = OSRelease("ubuntu", "noble", "ubuntu") +# NOTE(upgrade): Add supported releases here. +OPENSTACK_RELEASES = [ + OpenStackRelease("2023.1", "zed", [ROCKY_9, UBUNTU_JAMMY]), + OpenStackRelease("2024.1", "2023.1", [ROCKY_9, UBUNTU_JAMMY]), + OpenStackRelease("2025.1", "2024.1", [ROCKY_9, UBUNTU_NOBLE]), +] +NEUTRON_PLUGINS = ["ovs", "ovn"] + + +def main() -> None: + scenario = random_scenario() + inputs = generate_inputs(scenario) + for name, value in inputs.items(): + write_output(name, value) + + +def random_scenario() -> Scenario: + openstack_release = random.choice(OPENSTACK_RELEASES) + os_release = random.choice(openstack_release.os_releases) + neutron_plugin = random.choice(NEUTRON_PLUGINS) + upgrade = 'major' if random.random() > 0.6 else 'none' + return Scenario(openstack_release, os_release, neutron_plugin, upgrade) + + +def generate_inputs(scenario: Scenario) -> t.Dict[str, str]: + branch = get_branch(scenario.openstack_release.version) + previous_branch = get_branch(scenario.openstack_release.previous_version) + inputs = { + "os_distribution": scenario.os_release.distribution, + "os_release": scenario.os_release.release, + "ssh_username": scenario.os_release.ssh_username, + "neutron_plugin": scenario.neutron_plugin, + "upgrade": scenario.upgrade, + "stackhpc_kayobe_config_version": branch, + "stackhpc_kayobe_config_previous_version": previous_branch, + } + return inputs + + +def get_branch(version: str) -> str: + return f"stackhpc/{version}" + + +def write_output(name: str, value: str) -> None: + print(f"{name}={value}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml new file mode 100644 index 000000000..7a9891cd7 --- /dev/null +++ b/.github/workflows/overcloud-host-image-build.yml @@ -0,0 +1,466 @@ +--- +name: Build overcloud host images +on: + workflow_dispatch: + inputs: + rocky9: + description: Build Rocky Linux 9 + type: boolean + default: true + ubuntu-jammy: + description: Build Ubuntu 22.04 Jammy + type: boolean + default: true + ubuntu-noble: + description: Build Ubuntu 24.04 Noble + type: boolean + default: true + runner_env: + description: Which cloud to run on? + type: choice + default: SMS Lab + options: + - SMS Lab + - Leafcloud + secrets: + KAYOBE_VAULT_PASSWORD_CI_BUILDER: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} +jobs: + runner-selection: + uses: ./.github/workflows/runner-selector.yml + with: + runner_env: ${{ inputs.runner_env }} + overcloud-host-image-build: + name: Build overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - runner-selection + permissions: {} + steps: + - name: Validate inputs + run: | + if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' && ${{ inputs.ubuntu-noble }} == 'false' ]]; then + echo "At least one distribution must be selected" + exit 1 + fi + + - name: Install Package dependencies + run: | + sudo apt update && + sudo apt install -y git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq gh + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Output image tag of the builder + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + + # Generate a tag to apply to all built overcloud host images. + - name: Generate overcloud host image tag + id: host_image_tag + run: | + echo "host_image_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + + - name: Display overcloud host image tag + run: | + echo "${{ steps.host_image_tag.outputs.host_image_tag }}" + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "ubuntu" + aio_vm_name = "skc-host-image-builder" + # Must be an Ubuntu Jammy host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" + aio_vm_interface = "ens3" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=ubuntu --skip-tags network + + - name: Install dependencies + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run \ + --command "sudo apt update && sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Create bifrost_httpboot Docker volume + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Build a Rocky Linux 9 overcloud host image + id: build_rocky_9 + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution="rocky" \ + -e os_release="9" \ + -e stackhpc_overcloud_dib_name=overcloud-rocky-9 + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_rocky_9.outcome == 'failure' + + - name: Upload Rocky Linux 9 overcloud host image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/overcloud-rocky-9 \ + -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \ + -e artifact_type="kayobe-images" \ + -e file_regex="*.qcow2" \ + -e os_distribution="rocky" \ + -e os_release="9" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' + + - name: Upload Rocky Linux 9 overcloud host image to current Dev Cloud (SMS/Leafcloud) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \ + -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' + + - name: Upload Rocky Linux 9 overcloud host image to other Dev Cloud (Leafcloud/SMS) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \ + -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }} + if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' + + - name: Build an Ubuntu Jammy 22.04 overcloud host image + id: build_ubuntu_jammy + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" \ + -e stackhpc_overcloud_dib_name=overcloud-ubuntu-jammy + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_ubuntu_jammy.outcome == 'failure' + + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/overcloud-ubuntu-jammy \ + -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \ + -e artifact_type="kayobe-images" \ + -e file_regex="*.qcow2" \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' + + - name: Upload Ubuntu Jammy overcloud host image to current Dev Cloud (SMS/Leafcloud) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \ + -e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' + + - name: Upload Ubuntu Jammy overcloud host image to other Dev Cloud (Leafcloud/SMS) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \ + -e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }} + if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' + + - name: Build an Ubuntu Noble 24.04 overcloud host image + id: build_ubuntu_noble + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud host image build --force-rebuild \ + -e os_distribution="ubuntu" \ + -e os_release="noble" \ + -e stackhpc_overcloud_dib_name=overcloud-ubuntu-noble + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble + + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-noble/overcloud-ubuntu-noble.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: steps.build_ubuntu_noble.outcome == 'failure' + + - name: Upload Ubuntu Noble 24.04 overcloud host image to Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-upload.yml \ + -e artifact_path=/opt/kayobe/images/overcloud-ubuntu-noble \ + -e artifact_tag=${{ steps.host_image_tag.outputs.host_image_tag }} \ + -e artifact_type="kayobe-images" \ + -e file_regex="*.qcow2" \ + -e os_distribution="ubuntu" \ + -e os_release="noble" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble.outcome == 'success' + + - name: Upload Ubuntu Noble overcloud host image to current Dev Cloud (SMS/Leafcloud) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-noble/overcloud-ubuntu-noble.qcow2" \ + -e image_name=overcloud-ubuntu-noble-${{ steps.host_image_tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble.outcome == 'success' + + - name: Upload Ubuntu Noble overcloud host image to other Dev Cloud (Leafcloud/SMS) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-noble/overcloud-ubuntu-noble.qcow2" \ + -e image_name=overcloud-ubuntu-noble-${{ steps.host_image_tag.outputs.host_image_tag }} + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID_OTHER_CLOUD }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET_OTHER_CLOUD }} + if: inputs.ubuntu-noble && steps.build_ubuntu_noble.outcome == 'success' + + + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r ubuntu@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true + if: always() + + - name: Fail if any overcloud host image builds failed + run: | + echo "Builds failed. See workflow artifacts for details." && + exit 1 + if: steps.build_rocky_9.outcome == 'failure' || + steps.build_ubuntu_jammy.outcome == 'failure' || + steps.build_ubuntu_noble.outcome == 'failure' + + - name: Upload logs artifact + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: Build logs + path: ./logs + if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml new file mode 100644 index 000000000..2acf1fb48 --- /dev/null +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -0,0 +1,118 @@ +--- +name: Promote overcloud host image +on: + workflow_dispatch: + inputs: + rocky9: + description: Promote Rocky Linux 9 + type: boolean + default: true + ubuntu-jammy: + description: Promote Ubuntu 22.04 Jammy + type: boolean + default: true + ubuntu-noble: + description: Promote Ubuntu 24.04 Noble + type: boolean + default: true + image_tag: + description: Tag to promote + type: string + required: true +env: + ANSIBLE_FORCE_COLOR: True +jobs: + overcloud-host-image-promote: + name: Promote overcloud host image + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-22.04 + steps: + - name: Validate inputs + run: | + if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' && ${{ inputs.ubuntu-noble }} == 'false' ]]; then + echo "At least one distribution must be selected" + exit 1 + fi + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + working-directory: src/kayobe-config + + - name: Setup networking + run: | + if ! ip l show breth1 >/dev/null 2>&1; then + sudo ip l add breth1 type bridge + fi + sudo ip l set breth1 up + if ! ip a show breth1 | grep 192.168.33.3/24; then + sudo ip a add 192.168.33.3/24 dev breth1 + fi + if ! ip l show dummy1 >/dev/null 2>&1; then + sudo ip l add dummy1 type dummy + fi + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe control host bootstrap + + - name: Promote Rocky Linux 9 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e artifact_type="kayobe-images" \ + -e os_distribution='rocky' \ + -e os_release='9' + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 + + - name: Promote Ubuntu Jammy 22.04 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e artifact_type="kayobe-images" \ + -e os_distribution='ubuntu' \ + -e os_release='jammy' + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy + + - name: Promote Ubuntu Noble 24.04 overcloud host image artifact + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-artifact-promote.yml \ + -e artifact_type="kayobe-images" \ + -e os_distribution='ubuntu' \ + -e os_release='noble' + env: + ARTIFACT_TAG: ${{ inputs.image_tag }} + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml new file mode 100644 index 000000000..65a12f086 --- /dev/null +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -0,0 +1,236 @@ +--- +name: Upload overcloud host images +on: + workflow_dispatch: + inputs: + rocky9: + description: Upload Rocky Linux 9 + type: boolean + default: true + ubuntu-jammy: + description: Upload Ubuntu 22.04 Jammy + type: boolean + default: true + ubuntu-noble: + description: Upload Ubuntu 24.04 Noble + type: boolean + default: true + kayobe-environment: + description: Kayobe environment to use + type: string + default: "ci-builder" + runner_env: + description: Which cloud to run on? + type: choice + default: SMS Lab + options: + - SMS Lab + - Leafcloud + secrets: + KAYOBE_VAULT_PASSWORD_CI_BUILDER: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + runner-selection: + uses: ./.github/workflows/runner-selector.yml + with: + runner_env: ${{ inputs.runner_env }} + overcloud-host-image-upload: + name: Upload overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: ${{ inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_image_build }} + needs: + - runner-selection + permissions: {} + steps: + - name: Validate inputs + run: | + if [[ ${{ inputs.rocky9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' && ${{ inputs.ubuntu-noble }} == 'false' ]]; then + echo "At least one distribution must be selected" + exit 1 + fi + + - name: Install package dependencies + run: | + sudo apt update + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe control host bootstrap --skip-tags bootstrap + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + + - name: Install OpenStack client + run: | + source venvs/kayobe/bin/activate && + pip install python-openstackclient -c https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt + + - name: Output Rocky Linux 9 image tag + id: rocky_9_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: rocky_9_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true + + - name: Download Rocky Linux 9 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="rocky" \ + -e os_release="9" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' + + - name: Upload Rocky Linux 9 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/rocky-9.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' + + - name: Output Ubuntu Jammy image tag + id: ubuntu_jammy_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_jammy_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: ubuntu_jammy_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true + + - name: Download Ubuntu Jammy 22.04 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure' + + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/ubuntu-jammy.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure' + + - name: Output Ubuntu Noble image tag + id: ubuntu_noble_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_noble_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: ubuntu_noble_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-ubuntu-noble-${{ steps.ubuntu_noble_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true + + - name: Download Ubuntu Noble 24.04 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="ubuntu" \ + -e os_release="noble" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.ubuntu-noble && steps.ubuntu_noble_image_exists.outcome == 'failure' + + - name: Upload Ubuntu Noble 24.04 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-ubuntu-noble-${{ steps.ubuntu_noble_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/ubuntu-noble.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-noble && steps.ubuntu_noble_image_exists.outcome == 'failure' diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml new file mode 100644 index 000000000..dfb00d9df --- /dev/null +++ b/.github/workflows/package-build-ofed.yml @@ -0,0 +1,248 @@ +--- +name: Build OFED kernel modules +on: + workflow_dispatch: + inputs: + rocky9: + description: Build Rocky Linux 9 + type: boolean + default: true + secrets: + KAYOBE_VAULT_PASSWORD_CI_BUILDER: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-doca-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} +jobs: + overcloud-ofed-packages: + name: Build OFED kernel modules + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: arc-skc-host-image-builder-runner + permissions: {} + steps: + - name: Generate OFED tag + id: ofed_tag + run: | + echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + + - name: Install Package dependencies + run: | + sudo apt update && + sudo apt install -y git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + - name: Install terraform + uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Output image tag + id: image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + working-directory: ${{ github.workspace }}/src/kayobe-config + + # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag + - name: Output image name + id: image_name + run: | + echo image_name=overcloud-rocky-9-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "cloud-user" + aio_vm_name = "skc-ofed-builder" + aio_vm_image = "${{ env.VM_IMAGE }}" + aio_vm_flavor = "en1.medium" + aio_vm_network = "stackhpc-ci" + aio_vm_subnet = "stackhpc-ci" + aio_vm_interface = "ens3" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + VM_IMAGE: ${{ steps.image_name.outputs.image_name }} + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-doca-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-doca-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe control host bootstrap + + - name: Run growroot playbook + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe seed host configure --skip-tags network,docker,docker-registry + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Run a distro-sync + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Reset BLS entries on the seed host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reset-bls-entries.yml \ + -e "reset_bls_host=ofed-builder" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Disable noexec in /var/tmp + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe seed host command run --become --command "sed -i 's/noexec,//g' /etc/fstab" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Reboot to apply the kernel update + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/reboot.yml + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Run OFED builder playbook + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Run OFED upload playbook + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-doca-builder && + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \ + -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() diff --git a/.github/workflows/runner-selector.yml b/.github/workflows/runner-selector.yml new file mode 100644 index 000000000..f825ec604 --- /dev/null +++ b/.github/workflows/runner-selector.yml @@ -0,0 +1,48 @@ +--- +name: Create output for `runs-on` variable + +on: + workflow_call: + inputs: + runner_env: + description: 'The environment input from caller workflow' + default: 'SMS Lab' + required: True + type: string + outputs: + runner_name_image_build: + description: "Image builder runner name" + value: ${{ jobs.define_runner.outputs.image-build-runner }} + runner_name_container_image_build: + description: "Container image build runner name" + value: ${{ jobs.define_runner.outputs.container-image-build-runner }} + runner_name_aio: + description: "AiO runner name" + value: ${{ jobs.define_runner.outputs.aio-runner }} + +jobs: + define_runner: + environment: ${{ inputs.runner_env }} + runs-on: ubuntu-latest + outputs: + image-build-runner: ${{ steps.builder-runner.outputs.runner_name_image_build }} + container-image-build-runner: ${{ steps.container-image-build-runner.outputs.runner_name_container_image_build }} + aio-runner: ${{ steps.aio-runner.outputs.runner_name_aio }} + steps: + - name: Set output for image builder runner + run: echo "Setting runner for ${{ inputs.runner_env }} -> ${{ vars.RUNS_ON_TARGET_IMAGE_BUILDER }}" + + - id: builder-runner + run: echo "runner_name_image_build=${{ vars.RUNS_ON_TARGET_IMAGE_BUILDER }}" >> $GITHUB_OUTPUT + + - name: Set output for container image build runner + run: echo "Setting runner for ${{ inputs.runner_env }} -> ${{ vars.RUNS_ON_TARGET_CONTAINER_IMAGE_BUILDER }}" + + - id: container-image-build-runner + run: echo "runner_name_container_image_build=${{ vars.RUNS_ON_TARGET_CONTAINER_IMAGE_BUILDER }}" >> $GITHUB_OUTPUT + + - name: Set output for aio runner + run: echo "Setting runner for ${{ inputs.runner_env }} -> ${{ vars.RUNS_ON_TARGET_AIO }}" + + - id: aio-runner + run: echo "runner_name_aio=${{ vars.RUNS_ON_TARGET_AIO }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml new file mode 100644 index 000000000..2316afd61 --- /dev/null +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -0,0 +1,534 @@ +--- +# This reusable workflow deploys a VM on a cloud using Terraform, then deploys +# OpenStack in the VM via Kayobe. Tempest is then used to test the cloud. + +name: All in one + +on: + workflow_call: + inputs: + runner_env: + description: Which cloud to run on? + type: string + default: SMS Lab + kayobe_image: + description: Kayobe container image + type: string + required: true + os_distribution: + description: Host OS distribution + type: string + default: rocky + os_release: + description: Host OS release + type: string + default: '9' + ssh_username: + description: User for terraform to access the all-in-one VM + type: string + default: cloud-user + neutron_plugin: + description: Neutron ML2 plugin + type: string + required: true + vm_image_override: + description: Full name of an image to use instead of the default + type: string + default: "" + vm_interface: + description: Default network interface name + type: string + default: ens3 + OS_CLOUD: + description: Name of cloud in clouds.yaml + type: string + required: true + if: + description: Whether to run the workflow (workaround for required status checks issue) + type: boolean + default: true + upgrade: + description: Whether to perform an upgrade + type: boolean + default: false + stackhpc_cloud_tests_version: + description: Git version of https://github.com/stackhpc/stackhpc-cloud-tests to use for testing + type: string + default: main + repository: + description: SKC repository to checkout (convenience for external CI) + type: string + default: ${{ github.repository }} + github_ref: + description: Git ref to checkout (convenience for external CI) + type: string + default: ${{ github.ref }} + secrets: + KAYOBE_VAULT_PASSWORD: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +jobs: + runner-selection: + uses: ./.github/workflows/runner-selector.yml + with: + runner_env: ${{ inputs.upgrade == true && 'Leafcloud' || inputs.runner_env }} + # NOTE: Runner needs unzip and nodejs packages. + all-in-one: + name: All in one + if: ${{ inputs.if && !cancelled() }} + environment: ${{ inputs.upgrade == true && 'Leafcloud' || inputs.runner_env }} + runs-on: ${{ needs.runner-selection.outputs.runner_name_aio }} + needs: + - runner-selection + permissions: {} + env: + KAYOBE_ENVIRONMENT: ci-aio + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + KAYOBE_IMAGE: ${{ inputs.kayobe_image }} + # NOTE(upgrade): Reference the PREVIOUS release here. + PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-2023.1 + # NOTE(upgrade): Reference the PREVIOUS release branch here. + PREVIOUS_BRANCH: stackhpc/2023.1 + steps: + - name: Install Package dependencies + run: | + sudo apt update && + sudo apt install -y git unzip nodejs openssh-client + + # If testing upgrade, checkout previous release, otherwise checkout current branch + - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.upgrade && env.PREVIOUS_BRANCH || inputs.github_ref }} + submodules: true + + - name: Output Kayobe image + id: kayobe_image + run: | + if ${{ inputs.upgrade }}; then + kayobe_image=$PREVIOUS_KAYOBE_IMAGE + else + kayobe_image=$KAYOBE_IMAGE + fi + echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT + + - name: Make sure dockerd is running and test Docker + run: | + docker ps + + - name: Output image tag + id: image_tag + run: | + echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_$(sed s/-/_/ <(echo "${{ inputs.os_release }}"))_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag + - name: Output image name + id: image_name + run: | + if [ -z "${{ inputs.vm_image_override }}" ]; then + echo image_name=overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT + else + echo image_name=${{ inputs.vm_image_override }} >> $GITHUB_OUTPUT + fi + + - name: Install terraform + uses: hashicorp/setup-terraform@5e8dbf3c6d9deaf4193ca7a8fb23f2ac83bb6c85 # v4.0.0 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "${{ env.SSH_USERNAME }}" + aio_vm_interface = "${{ env.VM_INTERFACE }}" + aio_vm_name = "${{ env.VM_NAME }}" + aio_vm_image = "${{ env.VM_IMAGE }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" + aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}" + aio_vm_tags = ${{ env.VM_TAGS }} + EOF + working-directory: ${{ github.workspace }}/terraform/aio + env: + SSH_USERNAME: "${{ inputs.ssh_username }}" + VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}" + VM_IMAGE: ${{ steps.image_name.outputs.image_name }} + VM_INTERFACE: ${{ inputs.vm_interface }} + VM_VOLUME_SIZE: ${{ inputs.upgrade && '100' || '50' }} + VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]' + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + id: tf_apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml + admin_oc_net_name: ethernet + ethernet_cidr: "{{ access_cidr.value }}" + ethernet_allocation_pool_start: 0.0.0.0 + ethernet_allocation_pool_end: 0.0.0.0 + ethernet_ips: + controller0: "{{ access_ip_v4.value }}" + EOF + + - name: Write all-in-one scenario config + run: | + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/zz-aio-scenario.yml + --- + os_distribution: ${{ env.OS_DISTRIBUTION }} + os_release: "${{ env.OS_RELEASE }}" + kolla_enable_ovn: ${{ env.ENABLE_OVN }} + EOF + env: + ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }} + OS_DISTRIBUTION: ${{ inputs.os_distribution }} + OS_RELEASE: ${{ inputs.os_release }} + + # Use a heredoc to define a multiline string output + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + - name: Set SSH key output + id: ssh_key + run: | + echo "ssh_key<> $GITHUB_OUTPUT + cat terraform/aio/id_rsa >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image. + - name: Pull previous Kayobe image + run: | + docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }} + if: inputs.upgrade + + # The same tag may be reused (e.g. pr-123), so ensure we have the latest image. + - name: Pull current Kayobe image + run: | + docker image pull $KAYOBE_IMAGE + + # Rocky 9 OVN deployments will fail when the hostname contains a '.' + - name: Fix hostname + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/fix-hostname.yml + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + # Reboot to Apply hostname change + - name: Reboot + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/reboot.yml -e reboot_with_bootstrap_user=true + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: Run growroot + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml' + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: Host configure + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: Pull container images + run: | + # Try up to 3 times to pull container images. + for attempt in $(seq 3); do + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-container-image-pull.sh + if [ "$?" = "0" ]; then + echo "Pulled container images on attempt $attempt" + exit 0 + fi + echo "Failed to pull container images on attempt $attempt" + done + echo "Failed to pull container imagers after $attempt attempts" + exit 1 + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: Service deploy + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: Configure aio resources + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: Upgrade to RabbitMQ 3.12 + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/kolla-ansible-run.sh "rabbitmq-upgrade 3.12" + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade + + - name: Upgrade to RabbitMQ 3.13 + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/kolla-ansible-run.sh "rabbitmq-upgrade 3.13" + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade + + - name: Change rabbit queues from HA to Quorum + run: | + sed -i -e 's/om_enable_rabbitmq_high_availability: true/om_enable_rabbitmq_high_availability: false/' \ + -e 's/om_enable_rabbitmq_quorum_queues: false/om_enable_rabbitmq_quorum_queues: true/' \ + etc/kayobe/environments/ci-aio/kolla/globals.yml + if: inputs.upgrade + + - name: Migrate RabbitMQ queues + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/script-run.sh tools/rabbitmq-quorum-migration.sh + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade + + # If testing upgrade, checkout the current release branch + # Stash changes to tracked files, and set clean=false to avoid removing untracked files. + # Revert changes to RabbitMQ Queue types to avoid a merge conflict + - name: Stash config changes + run: git restore etc/kayobe/environments/ci-aio/kolla/globals.yml && git stash + if: inputs.upgrade + + - name: Checkout current release config + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.github_ref }} + submodules: true + clean: false + if: inputs.upgrade + + - name: Pop stashed config changes + run: git stash pop + if: inputs.upgrade + + # Now begin upgrade + - name: Host upgrade + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-upgrade.sh + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade + + # NOTE(Alex-Welsh): configure host with selinux_state=disabled, because + # Antelope images are not built as permissive + - name: Host configure + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh -e selinux_state=disabled + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade + + - name: Service upgrade + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-upgrade.sh + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade + + - name: Ensure we have IP on breth1 to reach the instances + # NOTE(wszumski): Whilst we don't need to create resources again, in some circumstances + # we can lose the IP address that allows us to connect to the instances. This playbook + # also fixes that issue. + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade + + - name: Tempest tests + id: tempest + run: | + mkdir -p tempest-artifacts + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: StackHPC Cloud tests + id: stackhpc-cloud-tests + continue-on-error: true + run: | + mkdir -p sct-results + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -v $(pwd)/sct-results:/stack/sct-results \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-cloud-tests.yml' \ + ${{ (inputs.upgrade && '-e selinux_state=disabled') || '' }} \ + -e sct_version=${{ inputs.stackhpc_cloud_tests_version }} + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + + - name: Collect diagnostic information + id: diagnostics + run: | + mkdir -p diagnostics + sudo -E docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -v $(pwd)/diagnostics:/stack/diagnostics \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/diagnostics.yml' + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: ${{ !cancelled() && steps.tf_apply.outcome == 'success' }} + + - name: Upload test result artifacts + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: test-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }} + path: | + diagnostics/ + tempest-artifacts/ + sct-results/ + if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-cloud-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }} + + - name: Fail if any tests failed + run: | + rc=0 + if [[ $(wc -l < tempest-artifacts/failed-tests) -ne 0 ]]; then + echo "Some Tempest tests failed." + echo "See HTML results artifact (tempest-artifacts) for details." + rc=1 + fi + if [[ $(wc -l < sct-results/failed-tests) -ne 0 ]]; then + echo "Some StackHPC Cloud tests failed." + echo "See HTML results artifact (sct-results) for details." + rc=1 + fi + exit $rc + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/terraform/aio + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: always() diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml new file mode 100644 index 000000000..4704a3feb --- /dev/null +++ b/.github/workflows/stackhpc-build-kayobe-image.yml @@ -0,0 +1,123 @@ +--- +# This reusable workflow builds a Kayobe container image using the Dockerfile +# in kayobe-automation, then pushes it to a registry. + +name: Build kayobe image + +on: + push: + branches: + # NOTE(upgrade): Reference only the current release branch here. + - stackhpc/2024.1 + + workflow_call: + inputs: + http_proxy: + type: string + required: false + https_proxy: + type: string + required: false + no_proxy: + type: string + required: false + base_image: + type: string + required: false + default: "rockylinux:9" + if: + description: Whether to run the workflow (workaround for required status checks issue) + type: boolean + default: true + outputs: + kayobe_image: + description: Reference of Kayobe image that was built + value: ${{ jobs.build-kayobe-image.outputs.kayobe_image }} + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-kayobe-image: + name: Build kayobe image + if: inputs.if || github.repository == 'stackhpc/stackhpc-kayobe-config' && github.event_name == 'push' + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + outputs: + kayobe_image: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - name: Checkout kayobe config + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + + - name: Log in to the Container registry + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + with: + driver-opts: | + image=moby/buildkit:master + env.http_proxy=${{ env.http_proxy }} + env.https_proxy=${{ env.https_proxy }} + # Doesn't like commas: invalid value "127.0.0.1", expecting k=v + # env.no_proxy='${{ env.no_proxy }}' + env: + http_proxy: ${{ inputs.http_proxy }} + https_proxy: ${{ inputs.https_proxy }} + no_proxy: ${{ inputs.no_proxy }} + + # Setting KAYOBE_USER_UID and KAYOBE_USER_GID to 1001 to match docker's defaults + # so that docker can run as a privileged user within the Kayobe image. + - name: Build and push Docker image + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + with: + file: ./.automation/docker/kayobe/Dockerfile + context: . + build-args: | + http_proxy=${{ inputs.http_proxy }} + https_proxy=${{ inputs.https_proxy }} + BASE_IMAGE=${{ inputs.base_image || 'rockylinux:9' }} + KAYOBE_USER_UID=1001 + KAYOBE_USER_GID=1001 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Send message to Slack via Workflow Builder + uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1 + with: + webhook-type: "incoming-webhook" + payload: | + { + "channel-id": "${{ env.SLACK_CHANNEL_ID }}", + "inputs": "${{ env.INPUTS }}", + "message": "${{ env.MESSAGE }}", + "results-url": "${{ env.RESULTS_URL }}", + "workflow-url": "${{ env.WORKFLOW_URL }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # #release-train-alerts + SLACK_CHANNEL_ID: C03B28HRP53 + INPUTS: >- + branch: ${{ github.ref_name }} + MESSAGE: "SKC Build Kayobe Image workflow failed :sob:" + RESULTS_URL: "N/A" + WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + if: failure() && github.event_name == 'push' diff --git a/.github/workflows/stackhpc-check-tags.yml b/.github/workflows/stackhpc-check-tags.yml new file mode 100644 index 000000000..7a5aecfd9 --- /dev/null +++ b/.github/workflows/stackhpc-check-tags.yml @@ -0,0 +1,65 @@ +--- +# This workflow queries the Test Pulp server to check that all image tags +# specified in kolla_image_tags are present. + +name: Check container image tags +on: + workflow_call: + inputs: + kayobe_image: + description: Kayobe container image + type: string + required: true + if: + description: Whether to run the workflow (workaround for required status checks issue) + type: boolean + default: true + secrets: + KAYOBE_VAULT_PASSWORD: + required: true + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + check-tags: + name: Check container image tags + if: ${{ inputs.if && ! cancelled() }} + runs-on: arc-skc-aio-runner + permissions: {} + env: + KAYOBE_ENVIRONMENT: ci-aio + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + KAYOBE_IMAGE: ${{ inputs.kayobe_image }} + steps: + - name: Install package dependencies + run: | + sudo apt update + sudo apt install -y git unzip nodejs + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + + # The same tag may be reused (e.g. pr-123), so ensure we have the latest image. + - name: Pull latest Kayobe image + run: | + docker image pull $KAYOBE_IMAGE + + - name: Check kolla-images.py image map and tag hierarchy + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \ + '$KAYOBE_CONFIG_PATH/ansible/check-kolla-images-py.yml' + + - name: Check container image tags + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + $KAYOBE_IMAGE \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \ + '$KAYOBE_CONFIG_PATH/ansible/check-tags.yml' diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml new file mode 100644 index 000000000..4a0973127 --- /dev/null +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -0,0 +1,123 @@ +--- +name: Clean up stale CI resources +on: + schedule: + # Every 2 hours at quarter past + - cron: '15 0/2 * * *' + +jobs: + ci-cleanup: + name: Clean up stale CI resources + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-latest + permissions: {} + strategy: + matrix: + environment: [SMS Lab, Leafcloud] + environment: ${{ matrix.environment }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + + - name: Install OpenStack client + run: | + pip install python-openstackclient -c https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}/upper-constraints.txt + + - name: Clean up aio instances over 3 hours old + run: | + result=0 + changes_before=$(date -Imin -d -3hours) + for status in ACTIVE BUILD ERROR SHUTOFF; do + for instance in $(openstack server list --tags skc-ci-aio --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do + echo "Cleaning up $status instance $instance" + openstack server show $instance + if ! openstack server delete $instance; then + echo "Failed to delete $status instance $instance" + result=1 + fi + done + done + exit $result + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Clean up host image builder instances over 5 hours old + run: | + result=0 + changes_before=$(date -Imin -d -5hours) + for status in ACTIVE BUILD ERROR SHUTOFF; do + for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do + echo "Cleaning up $status instance $instance" + openstack server show $instance + if ! openstack server delete $instance; then + echo "Failed to delete $status instance $instance" + result=1 + fi + done + done + exit $result + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Clean up GH actions multinode instances over 12 hours old + run: | + result=0 + changes_before=$(date -Imin -d -12hours) + for status in ACTIVE BUILD ERROR SHUTOFF; do + for instance in $(openstack server list --tags gh-actions-multinode --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do + echo "Cleaning up $status instance $instance" + openstack server show $instance + if ! openstack server delete $instance; then + echo "Failed to delete $status instance $instance" + result=1 + fi + done + done + exit $result + env: + OS_CLOUD: ${{ vars.OS_CLOUD }} + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Send message to Slack via Workflow Builder + uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1 + with: + webhook-type: "incoming-webhook" + payload: | + { + "channel-id": "${{ env.SLACK_CHANNEL_ID }}", + "inputs": "${{ env.INPUTS }}", + "message": "${{ env.MESSAGE }}", + "results-url": "${{ env.RESULTS_URL }}", + "workflow-url": "${{ env.WORKFLOW_URL }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # #release-train-alerts + SLACK_CHANNEL_ID: C03B28HRP53 + INPUTS: "N/A" + MESSAGE: "SKC CI Cleanup workflow failed :sob:" + RESULTS_URL: "N/A" + WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + if: failure() diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml new file mode 100644 index 000000000..ce2b0063e --- /dev/null +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -0,0 +1,424 @@ +--- +name: Build Kolla container images +on: + workflow_dispatch: + inputs: + regexes: + description: Space-separated list of regular expressions matching overcloud images to build + type: string + required: false + default: "" + overcloud: + description: Build container images for overcloud services? + type: boolean + required: false + default: true + seed: + description: Build container images for seed services? + type: boolean + required: false + default: false + rocky-linux-9: + description: Build Rocky Linux 9 images? + type: boolean + required: false + default: true + ubuntu-jammy: + description: Build Ubuntu Jammy 22.04 images? + type: boolean + required: false + default: true + ubuntu-noble: + description: Build Ubuntu Noble 24.04 images? + type: boolean + required: false + default: true + push: + description: Whether to push images + type: boolean + required: false + default: true + sbom: + description: Generate SBOM? + type: boolean + required: false + default: true + push-critical: + description: Push scanned images that have critical vulnerabilities? + type: boolean + required: false + default: false + runner_env: + description: Which cloud to run on? + type: choice + default: SMS Lab + options: + - SMS Lab + - Leafcloud + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + generate-tag: + name: Generate container image tag + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-latest + permissions: {} + outputs: + datetime_tag: ${{ steps.datetime_tag.outputs.datetime_tag }} + matrix: ${{ steps.set-matrix.outputs.matrix }} + openstack_release: ${{ steps.openstack_release.outputs.openstack_release }} + steps: + - name: Validate inputs + run: | + if [[ ${{ inputs.rocky-linux-9 }} == 'false' && ${{ inputs.ubuntu-jammy }} == 'false' && ${{ inputs.ubuntu-noble }} == 'false' ]]; then + echo "At least one distribution must be selected" + exit 1 + fi + if [[ ${{ inputs.overcloud }} == 'false' && ${{ inputs.seed }} == 'false' ]]; then + echo "At least one of overcloud or seed must be selected" + exit 1 + fi + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) + echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," | tee -a "$GITHUB_OUTPUT" + + # Generate a tag to apply to all built container images. + # Without this, each kayobe * container image build command would use a different tag. + - name: Generate container datetime tag + id: datetime_tag + run: | + echo "datetime_tag=$(date +%Y%m%dT%H%M%S)" | tee -a "$GITHUB_OUTPUT" + + # Dynamically define job matrix. + # We need a separate matrix entry for each distribution, when the relevant input is true. + # https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional + # NOTE(bbezak): Both amd64 and aarch64 need to be built in a single workflow to create a multi-architecture manifest. + # For now include only RL9 in aarch64 + - name: Generate build matrix + id: set-matrix + run: | + output="{'distro': [" + if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then + output+="{'name': 'rocky', 'release': 9, 'arch': 'amd64'}," + output+="{'name': 'rocky', 'release': 9, 'arch': 'aarch64'}," + fi + if [[ ${{ inputs.ubuntu-jammy }} == 'true' ]]; then + output+="{'name': 'ubuntu', 'release': 'jammy', 'arch': 'amd64'}," + fi + if [[ ${{ inputs.ubuntu-noble }} == 'true' ]]; then + output+="{'name': 'ubuntu', 'release': 'noble', 'arch': 'amd64'}," + fi + # remove trailing comma + output="${output%,}" + output+="]}" + echo "matrix=$output" | tee -a "$GITHUB_OUTPUT" + + - name: Display container datetime tag + run: | + echo "${{ steps.datetime_tag.outputs.datetime_tag }}" + + runner-selection: + uses: ./.github/workflows/runner-selector.yml + with: + runner_env: ${{ inputs.runner_env }} + + container-image-build: + name: Build Kolla container images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ${{ matrix.distro.arch == 'aarch64' + && fromJson('["self-hosted","sms","arm64"]') + || needs.runner-selection.outputs.runner_name_container_image_build }} + timeout-minutes: 720 + permissions: {} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }} + needs: + - generate-tag + - runner-selection + steps: + - name: Purge workspace + run: sudo rm -rf "$GITHUB_WORKSPACE"/* + + - name: Install package dependencies + run: | + sudo apt update + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Make sure dockerd is running and test Docker + run: | + docker ps + + - name: Install Trivy + run: | + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.69.2 + + - name: Install yq + run: | + ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') + curl -sL "https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_${ARCH}.tar.gz" | tar xz && sudo mv yq_linux_${ARCH} /usr/bin/yq + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install -r ../src/kayobe-config/requirements.txt + + # Required for Pulp auth proxy deployment and Docker registry login. + # Normally installed during host configure. + - name: Install Docker Python SDK + run: | + . /etc/os-release + if [[ $VERSION_ID == 22.04 ]]; then + sudo pip install docker 'requests<2.32.0' + else + sudo pip install docker 'requests<2.32.0' --break-system-packages + fi + + - name: Get Kolla tag + id: write-kolla-tag + run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ needs.generate-tag.outputs.datetime_tag }}" | tee -a "$GITHUB_OUTPUT" + + - name: Configure localhost as a seed + run: | + cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF + # A 'seed' host used for building images. + # Use localhost for container image builds. + [seed] + localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3 + EOF + + - name: Create build logs output directory + run: mkdir image-build-logs + + - name: Build kolla overcloud images + id: build_overcloud_images + continue-on-error: true + run: | + args="${{ inputs.regexes }}" + if [[ "${{ matrix.distro.arch }}" == 'aarch64' ]]; then + args="$args -e kolla_base_arch=${{ matrix.distro.arch }}" + fi + args="$args -e kolla_base_distro=${{ matrix.distro.name }}" + args="$args -e kolla_base_distro_version=${{ matrix.distro.release }}" + if [[ "${{ matrix.distro.name }}" == 'rocky' ]]; then + args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}-${{ matrix.distro.arch }}" + else + args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}" + fi + args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" + args="$args -e kolla_build_log_path=$GITHUB_WORKSPACE/image-build-logs/kolla-build-overcloud.log" + args="$args -e base_path=$GITHUB_WORKSPACE/opt/kayobe" + # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the + # runner and dind containers. + args="$args -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy" + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe overcloud container image build $args + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.overcloud + + - name: Copy build configs to output directory + run: sudo cp -rnL "$GITHUB_WORKSPACE/opt/kayobe/etc/kolla/"* image-build-logs/ + if: inputs.overcloud + + - name: Build kolla seed images + id: build_seed_images + continue-on-error: true + run: | + args="-e kolla_base_distro=${{ matrix.distro.name }}" + args="$args -e kolla_base_distro_version=${{ matrix.distro.release }}" + args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}" + args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true" + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed container image build $args + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.seed && matrix.distro.arch == 'amd64' + + - name: Copy seed container image build logs to output directory + run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log + if: inputs.seed && matrix.distro.arch == 'amd64' + + - name: Get built container images + run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" | tee "${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images" + + - name: Fail if no images have been built + run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi + + - name: Scan built container images + run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} ${{ inputs.sbom && '--sbom' }} + + - name: Move image scan logs to output artifact + run: mv image-scan-output image-build-logs/image-scan-output + if: ${{ !cancelled() }} + + - name: Fail if any images have critical vulnerabilities + run: if [ -e image-build-logs/image-scan-output/critical-images.txt ] && [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi + if: ${{ !inputs.push-critical }} + + - name: Copy clean images to push-attempt-images list + run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt + if: inputs.push + + # NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical. + # This should be reverted when it's decided to filter high level CVEs as well. + - name: Append dirty images to push list + run: | + cat image-build-logs/image-scan-output/high-images.txt >> image-build-logs/push-attempt-images.txt + if: ${{ inputs.push }} + + - name: Append images with critical vulnerabilities to push list + run: | + cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt + if: ${{ inputs.push && inputs.push-critical }} + + - name: Push images + run: | + touch image-build-logs/push-failed-images.txt + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml && + + while read -r image; do + # Retries! + for i in {1..5}; do + if docker push $image; then + echo "Pushed $image" + break + elif [ $i -eq 5 ] ; then + echo "Failed to push $image" + echo $image >> image-build-logs/push-failed-images.txt + else + echo "Failed on retry $i" + sleep 5 + fi; + done + done < image-build-logs/push-attempt-images.txt + shell: bash + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_BUILDER }} + if: inputs.push + + - name: Upload output artifact + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ matrix.distro.name }}-${{ matrix.distro.release }}-${{ matrix.distro.arch }}-logs + path: image-build-logs + retention-days: 7 + if: ${{ !cancelled() }} + + - name: Fail when images failed to build + run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1 + if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }} + + - name: Fail when images failed to push + run: if [ -e image-build-logs/push-failed-images.txt ] && [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi + if: ${{ inputs.push && !cancelled() }} + + # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected. + # This can be used again instead of "Fail when critical vulnerabilities are found" when it's + # decided to fail the job on detecting high CVEs as well. + # - name: Fail when images failed scanning + # run: if [ $(wc -l < image-build-logs/image-scan-output/high-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/high-images.txt && exit 1; fi + # if: ${{ !inputs.push-critical && !cancelled() }} + + - name: Fail when critical vulnerabilities are found + run: if [ -e image-build-logs/image-scan-output/critical-images.txt ] && [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi + if: ${{ !inputs.push-critical && !cancelled() }} + + - name: Remove locally built images for this run + if: always() && runner.arch == 'ARM64' + run: | + docker images --format '{{.Repository}}:{{.Tag}}' \ + --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" \ + | xargs -r -n1 docker rmi -f + + create-manifests: + # Only for Rocky Linux for now + name: Create Multiarch Docker Manifests + if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && inputs.rocky-linux-9 + runs-on: ${{ needs.runner-selection.outputs.runner_name_container_image_build }} + permissions: {} + needs: + - container-image-build + - runner-selection + steps: + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + + - name: Combine pushed images lists + run: | + find . -name 'push-attempt-images.txt' -exec cat {} + > all-pushed-images.txt + + - name: Log in to container registry + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + registry: ark.stackhpc.com + username: ${{ secrets.RLS_TRAIN_CI_ARK_REGISTRY_USER }} + password: ${{ secrets.RLS_TRAIN_CI_ARK_REGISTRY_PASS }} + + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + path: src/kayobe-config + + - name: Create and push Docker manifests + run: src/kayobe-config/tools/multiarch-manifests.sh + + - name: Upload manifest logs + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: manifest-logs + path: | + all-pushed-images.txt + logs/manifest-creation.log + retention-days: 7 + if: ${{ !cancelled() }} + + trigger-image-sync: + name: Trigger container image repository sync + needs: + - container-image-build + - create-manifests + if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() + runs-on: ubuntu-latest + permissions: {} + steps: + # NOTE(mgoddard): Trigger another CI workflow in the + # stackhpc-release-train repository. + - name: Trigger container image repository sync + run: | + filter='${{ inputs.regexes }}' + if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then + filter="$filter bifrost" + fi + gh workflow run \ + container-sync.yml \ + --repo stackhpc/stackhpc-release-train \ + --ref main \ + -f filter="$filter" \ + -f sync-old-images=false + env: + GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} + + - name: Display link to container image repository sync workflows + run: | + echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml" diff --git a/.github/workflows/stackhpc-multinode-periodic.yml b/.github/workflows/stackhpc-multinode-periodic.yml new file mode 100644 index 000000000..d307c7255 --- /dev/null +++ b/.github/workflows/stackhpc-multinode-periodic.yml @@ -0,0 +1,50 @@ +--- +# This workflow provides a periodic deploy of a multi-node test cluster. +# The test scenario is randomly selected. + +name: Multinode periodic +'on': + schedule: + # Runs nightly at 2:42 AM. + - cron: "42 2 * * *" +jobs: + generate-inputs: + name: Generate inputs + runs-on: ubuntu-latest + outputs: + os_distribution: ${{ steps.generate-inputs.outputs.os_distribution }} + os_release: ${{ steps.generate-inputs.outputs.os_release }} + ssh_username: ${{ steps.generate-inputs.outputs.ssh_username }} + neutron_plugin: ${{ steps.generate-inputs.outputs.neutron_plugin }} + upgrade: ${{ steps.generate-inputs.outputs.upgrade }} + stackhpc_kayobe_config_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_version }} + stackhpc_kayobe_config_previous_version: ${{ steps.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Generate inputs for multinode workflow + id: generate-inputs + run: | + python3 .github/workflows/multinode-inputs.py >> $GITHUB_OUTPUT + + - name: Display generated inputs + run: | + echo '${{ toJSON(steps.generate-inputs.outputs) }}' + multinode: + name: Multinode periodic + needs: + - generate-inputs + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.4.0 + with: + multinode_name: mn-periodic-${{ github.run_id }} + os_distribution: ${{ needs.generate-inputs.outputs.os_distribution }} + os_release: ${{ needs.generate-inputs.outputs.os_release }} + ssh_username: ${{ needs.generate-inputs.outputs.ssh_username }} + neutron_plugin: ${{ needs.generate-inputs.outputs.neutron_plugin }} + upgrade: ${{ needs.generate-inputs.outputs.upgrade }} + stackhpc_kayobe_config_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_version }} + stackhpc_kayobe_config_previous_version: ${{ needs.generate-inputs.outputs.stackhpc_kayobe_config_previous_version }} + enable_slack_alert: true + secrets: inherit + if: github.repository == 'stackhpc/stackhpc-kayobe-config' diff --git a/.github/workflows/stackhpc-multinode.yml b/.github/workflows/stackhpc-multinode.yml new file mode 100644 index 000000000..ce9b0d8b5 --- /dev/null +++ b/.github/workflows/stackhpc-multinode.yml @@ -0,0 +1,76 @@ +--- +# This workflow provides a workflow_dispatch (manual) trigger to deploy a +# multi-node test cluster. + +name: Multinode +'on': + workflow_dispatch: + # NOTE: workflow_dispatch is limited to 10 inputs. + inputs: + multinode_name: + description: Multinode cluster name + type: string + required: true + os_distribution: + description: Host OS distribution + type: choice + default: rocky + options: + - rocky + - ubuntu + neutron_plugin: + description: Neutron ML2 plugin + type: choice + default: ovn + options: + - ovn + - ovs + upgrade: + description: Whether to perform an upgrade + default: none + type: choice + options: + - none + - minor + - major + break_on: + description: When to break execution for manual interaction + type: choice + default: never + options: + - always + - failure + - never + - success + break_duration: + description: How long to break execution for (minutes) (note that instances are cleaned up after 12h) + type: number + default: 60 + ssh_key: + description: SSH public key to authorise on Ansible control host + type: string + terraform_kayobe_multinode_version: + description: terraform-kayobe-multinode version + type: string + default: stackhpc/2024.1 +jobs: + multinode: + name: Multinode + uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.5.0 + with: + multinode_name: ${{ inputs.multinode_name }} + os_distribution: ${{ inputs.os_distribution }} + os_release: ${{ inputs.os_distribution == 'rocky' && '9' || 'jammy' }} + ssh_username: ${{ inputs.os_distribution == 'rocky' && 'cloud-user' || 'ubuntu' }} + neutron_plugin: ${{ inputs.neutron_plugin }} + upgrade: ${{ inputs.upgrade }} + break_on: ${{ inputs.break_on }} + # Workaround loss of number type using fromJSON: https://github.com/orgs/community/discussions/67182 + break_duration: ${{ fromJSON(inputs.break_duration) }} + ssh_key: ${{ inputs.ssh_key }} + stackhpc_kayobe_config_version: ${{ github.ref_name }} + # NOTE(upgrade): Reference the PREVIOUS release here. + stackhpc_kayobe_config_previous_version: ${{ inputs.upgrade == 'major' && 'stackhpc/2023.1' || 'stackhpc/2024.1' }} + terraform_kayobe_multinode_version: ${{ inputs.terraform_kayobe_multinode_version }} + terraform_kayobe_multinode_previous_version: ${{ inputs.upgrade == 'major' && 'stackhpc/2023.1' || 'stackhpc/2024.1' }} + secrets: inherit diff --git a/.github/workflows/stackhpc-promote.yml b/.github/workflows/stackhpc-promote.yml new file mode 100644 index 000000000..2c66454a5 --- /dev/null +++ b/.github/workflows/stackhpc-promote.yml @@ -0,0 +1,68 @@ +--- +name: Promote Pulp repositories +on: + push: + branches: + # NOTE(upgrade): Reference only the current release branch here. + - stackhpc/2024.1 +jobs: + promote: + name: Trigger Pulp promotion workflows + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-latest + permissions: {} + steps: + # NOTE(mgoddard): Trigger another CI workflow in the + # stackhpc-release-train repository. + - name: Trigger package repository promotion + run: | + gh workflow run \ + package-promote.yml \ + --repo stackhpc/stackhpc-release-train \ + --ref main \ + -f kayobe_config_branch=${{ github.ref_name }} + env: + GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} + + - name: Display link to package repository promotion workflows + run: | + echo "::notice Package repository promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-promote.yml" + + # NOTE(mgoddard): Trigger another CI workflow in the + # stackhpc-release-train repository. + - name: Trigger container image promotion + run: | + gh workflow run \ + container-promote.yml \ + --repo stackhpc/stackhpc-release-train \ + --ref main \ + -f kayobe_config_branch=${{ github.ref_name }} + env: + GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }} + + - name: Display link to container image promotion workflows + run: | + echo "::notice Container image promote workflow: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-promote.yml" + + - name: Send message to Slack via Workflow Builder + uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1 + with: + webhook-type: "incoming-webhook" + payload: | + { + "channel-id": "${{ env.SLACK_CHANNEL_ID }}", + "inputs": "${{ env.INPUTS }}", + "message": "${{ env.MESSAGE }}", + "results-url": "${{ env.RESULTS_URL }}", + "workflow-url": "${{ env.WORKFLOW_URL }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # #release-train-alerts + SLACK_CHANNEL_ID: C03B28HRP53 + INPUTS: >- + branch: ${{ github.ref_name }} + MESSAGE: "SKC promote workflow failed :sob:" + RESULTS_URL: "N/A" + WORKFLOW_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + if: failure() diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml new file mode 100644 index 000000000..c330e6d1e --- /dev/null +++ b/.github/workflows/stackhpc-pull-request.yml @@ -0,0 +1,266 @@ +--- + +concurrency: + group: stackhpc-pull-request-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +name: Pull request +'on': + pull_request: +jobs: + # Detect which files have changed and use this to run jobs conditionally. + # Note that we can't use the workflow-level paths attribute since this + # would skip the workflow entirely, and would prevent us from making the + # aio jobs required to pass (a skip counts as a pass). + check-changes: + runs-on: ubuntu-22.04 + permissions: + pull-requests: read + packages: none + name: Check changed files + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + needs: + - lint + - tox + outputs: + aio: ${{ steps.changes.outputs.aio }} + build-kayobe-image: ${{ steps.changes.outputs.build-kayobe-image }} + check-tags: ${{ steps.changes.outputs.check-tags }} + steps: + - name: GitHub Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Check changed files + uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 + id: changes + with: + # Filters are defined in this file. + filters: .github/path-filters.yml + + tox: + runs-on: ubuntu-22.04 + permissions: {} + strategy: + matrix: + include: + - environment: pep8 + python-version: "3.10" + - environment: releasenotes + python-version: "3.10" + - environment: docs + python-version: "3.10" + name: Tox ${{ matrix.environment }} with Python ${{ matrix.python-version }} + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + steps: + - name: GitHub Checkout 🛎 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + - name: Setup Python ${{ matrix.python-version }} 🐍 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install Tox 📦 + run: pip install tox + - name: Run Tox ${{ matrix.environment }} 🧪 + run: tox -e ${{ matrix.environment }} + + lint: + runs-on: ubuntu-22.04 + permissions: {} + strategy: + fail-fast: false + matrix: + include: + # NOTE(upgrade): Keep these in sync with Kayobe's supported Ansible and Python versions (see release notes). + - ansible: "2.16" + python: "3.12" + - ansible: "2.15" + python: "3.10" + name: Ansible ${{ matrix.ansible }} lint with Python ${{ matrix.python }} + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + steps: + - name: GitHub Checkout 🛎 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup Python ${{ matrix.python-version }} 🐍 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ matrix.python }} + + - name: Install dependencies 📦 + run: | + python -m pip install --upgrade pip + pip install ansible-core==${{ matrix.ansible }}.* ansible-lint -r requirements.txt + + - name: Install Ansible Galaxy collections and roles + run: | + ansible-galaxy collection install -r etc/kayobe/ansible/requirements.yml + ansible-galaxy role install -r etc/kayobe/ansible/requirements.yml + + - name: Linting code 🧪 + run: | + ansible-lint -v --force-color etc/kayobe/ansible/. + + # A skipped job is treated as success when used as a required status check. + # The registered required status checks refer to the name of the job in the + # called reusable workflow rather than the jobs in this file. The following + # jobs need to run unconditionally to allow GitHub required status checks to + # pass even when there are changed files. The `! failure()` condition runs + # when the parent jobs completed successfully or were skipped. We pass an + # 'if' argument to the called workflow to allow running it conditionally. + + build-kayobe-image: + name: Build Kayobe Image + permissions: + contents: read + packages: write # required by docker/build-push-action + needs: + - check-changes + uses: ./.github/workflows/stackhpc-build-kayobe-image.yml + with: + if: ${{ needs.check-changes.outputs.build-kayobe-image == 'true' }} + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + + check-tags: + name: Check container image tags + permissions: {} + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-check-tags.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + if: ${{ needs.check-changes.outputs.check-tags == 'true' }} + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + + all-in-one-ubuntu-jammy-ovn: + name: aio (Ubuntu Jammy OVN) + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: ubuntu + os_release: jammy + ssh_username: ubuntu + neutron_plugin: ovn + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + + all-in-one-ubuntu-noble-ovn: + name: aio (Ubuntu Noble OVN) + permissions: {} + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: ubuntu + os_release: noble + ssh_username: ubuntu + neutron_plugin: ovn + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + + all-in-one-rocky-9-ovs: + name: aio (Rocky 9 OVS) + permissions: {} + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: rocky + os_release: "9" + ssh_username: cloud-user + neutron_plugin: ovs + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + + all-in-one-rocky-9-ovn: + name: aio (Rocky 9 OVN) + permissions: {} + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: rocky + os_release: "9" + ssh_username: cloud-user + neutron_plugin: ovn + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + + # Test upgrade scenarios + + all-in-one-upgrade-ubuntu-jammy-ovn: + name: aio upgrade (Ubuntu Jammy OVN) + permissions: {} + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: ubuntu + os_release: jammy + ssh_username: ubuntu + neutron_plugin: ovn + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + upgrade: true + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + + all-in-one-upgrade-rocky-9-ovn: + name: aio upgrade (Rocky 9 OVN) + permissions: {} + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: rocky + os_release: "9" + ssh_username: cloud-user + neutron_plugin: ovn + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + upgrade: true + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} + + all-in-one-upgrade-rocky-9-ovs: + name: aio upgrade (Rocky 9 OVS) + permissions: {} + needs: + - check-changes + - build-kayobe-image + uses: ./.github/workflows/stackhpc-all-in-one.yml + with: + kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} + os_distribution: rocky + os_release: "9" + ssh_username: cloud-user + neutron_plugin: ovs + OS_CLOUD: openstack + if: ${{ needs.check-changes.outputs.aio == 'true' }} + upgrade: true + secrets: inherit + if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} diff --git a/.github/workflows/stackhpc-update-kolla.yml b/.github/workflows/stackhpc-update-kolla.yml new file mode 100644 index 000000000..45e926c41 --- /dev/null +++ b/.github/workflows/stackhpc-update-kolla.yml @@ -0,0 +1,29 @@ +name: Update Kolla versions + +on: + # Allow manual executions + workflow_dispatch: + # Run weekly on Tuesday + schedule: + - cron: '0 0 * * 2' + +jobs: + update-from-branch: + name: Update dependencies + strategy: + matrix: + include: + - version: stackhpc/2023.1 + codename: Antelope + - version: stackhpc/2024.1 + codename: Caracal + - version: stackhpc/2025.1 + codename: Epoxy + uses: ./.github/workflows/update-dependencies.yml + with: + branch: ${{ matrix.version }} + openstack_codename: ${{ matrix.codename }} + permissions: + contents: write + pull-requests: write + if: github.repository == 'stackhpc/stackhpc-kayobe-config' diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml new file mode 100644 index 000000000..98f3f0c84 --- /dev/null +++ b/.github/workflows/tag-and-release.yml @@ -0,0 +1,14 @@ +--- +name: Tag & Release +'on': + push: + branches: + - stackhpc/2024.1 +permissions: + actions: read + contents: write +jobs: + tag-and-release: + uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main + with: + upstream: https://github.com/openstack/kayobe-config diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml new file mode 100644 index 000000000..097d8926c --- /dev/null +++ b/.github/workflows/update-dependencies.yml @@ -0,0 +1,189 @@ +name: Update dependencies + +on: + workflow_call: + inputs: + branch: + description: Branch to update. Must exist in all repositories. e.g. stackhpc/2025.1 + type: string + required: true + openstack_codename: + description: OpenStack codename e.g. Epoxy + type: string + required: true + workflow_dispatch: + inputs: + branch: + description: Branch to update. Must exist in all repositories. e.g. stackhpc/2025.1 + type: string + required: true + openstack_codename: + description: OpenStack codename e.g. Epoxy + type: string + required: true + +jobs: + propose-dependency-updates: + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: ubuntu-24.04 + permissions: + contents: write + pull-requests: write + name: Propose dependency updates + outputs: + kolla-tag: ${{ steps.latest_kolla_tag.outputs.latest_tag || steps.current_kolla_version.outputs.version }} + kolla-ansible-tag: ${{ steps.latest_kolla_ansible_tag.outputs.latest_tag || steps.current_kolla_ansible_version.outputs.version }} + kayobe-tag: ${{ steps.latest_kayobe_tag.outputs.latest_tag || steps.current_kayobe_version.outputs.version }} + steps: + - name: Checkout Kayobe-config + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.branch }} + path: src/kayobe-config + + - name: Set sanitised branch name + id: branch_name + run: | + sanitised_name=$(echo "update-dependencies-${{ inputs.branch }}" | tr '/' '-') + echo "name=${sanitised_name}" >> $GITHUB_OUTPUT + + - name: Set up branch and Git config + run: | + git checkout -b ${{ steps.branch_name.outputs.name }} + git config user.name "stackhpc-ci" + git config user.email "22933334+stackhpc-ci@users.noreply.github.com" + working-directory: src/kayobe-config + + - name: Initialise PR Body + run: | + echo "This PR was created automatically to update dependencies for the ${{ inputs.branch }} release." > pr_body.md + echo "" >> pr_body.md + echo "### Changes" >> pr_body.md + + - name: Checkout Kolla repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: stackhpc/kolla + ref: ${{ inputs.branch }} + fetch-tags: true + path: src/kolla + + - name: Get latest Kolla tag + id: latest_kolla_tag + run: echo "latest_tag=$(git describe --tags --abbrev=0 --match stackhpc/\*)" >> $GITHUB_OUTPUT + working-directory: ${{ github.workspace }}/src/kolla + + - name: Get current Kolla version + id: current_kolla_version + run: | + VERSION=$(awk -F': ' '/stackhpc_kolla_source_version:/ {print $2}' src/kayobe-config/etc/kayobe/stackhpc.yml | xargs) + echo "version=${VERSION}" >> $GITHUB_OUTPUT + + - name: Update and commit Kolla version if needed + if: steps.latest_kolla_tag.outputs.latest_tag != steps.current_kolla_version.outputs.version + run: | + sed -i "s/stackhpc_kolla_source_version\:.*$/stackhpc_kolla_source_version\: $(echo $TAG | sed 's|/|\\/|g')/g" etc/kayobe/stackhpc.yml + echo "- **Kolla** bumped from \`${{ steps.current_kolla_version.outputs.version }}\` to \`${{ steps.latest_kolla_tag.outputs.latest_tag }}\`" >> ../../pr_body.md + echo " - Changelog: https://github.com/stackhpc/kolla/releases/tag/${{ steps.latest_kolla_tag.outputs.latest_tag }}" >> ../../pr_body.md + git add etc/kayobe/stackhpc.yml + git commit -m "(automated) Bump kolla to ${{ steps.latest_kolla_tag.outputs.latest_tag }}" + env: + TAG: ${{ steps.latest_kolla_tag.outputs.latest_tag }} + working-directory: src/kayobe-config + + - name: Checkout Kolla Ansible repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: stackhpc/kolla-ansible + ref: ${{ inputs.branch }} + fetch-tags: true + path: src/kolla-ansible + + - name: Get latest Kolla Ansible tag + id: latest_kolla_ansible_tag + run: echo "latest_tag=$(git describe --tags --abbrev=0 --match stackhpc/\*)" >> $GITHUB_OUTPUT + working-directory: ${{ github.workspace }}/src/kolla-ansible + + - name: Get current Kolla Ansible version + id: current_kolla_ansible_version + run: | + VERSION=$(awk -F': ' '/stackhpc_kolla_ansible_source_version:/ {print $2}' src/kayobe-config/etc/kayobe/stackhpc.yml | xargs) + echo "version=${VERSION}" >> $GITHUB_OUTPUT + + - name: Update and commit Kolla Ansible version if needed + if: steps.latest_kolla_ansible_tag.outputs.latest_tag != steps.current_kolla_ansible_version.outputs.version + run: | + sed -i "s/stackhpc_kolla_ansible_source_version\:.*$/stackhpc_kolla_ansible_source_version\: $(echo $TAG | sed 's|/|\\/|g')/g" etc/kayobe/stackhpc.yml + echo "- **Kolla-Ansible** bumped from \`${{ steps.current_kolla_ansible_version.outputs.version }}\` to \`${{ steps.latest_kolla_ansible_tag.outputs.latest_tag }}\`" >> ../../pr_body.md + echo " - Changelog: https://github.com/stackhpc/kolla-ansible/releases/tag/${{ steps.latest_kolla_ansible_tag.outputs.latest_tag }}" >> ../../pr_body.md + git add etc/kayobe/stackhpc.yml + git commit -m "(automated) Bump kolla-ansible to ${{ steps.latest_kolla_ansible_tag.outputs.latest_tag }}" + env: + TAG: ${{ steps.latest_kolla_ansible_tag.outputs.latest_tag }} + working-directory: src/kayobe-config + + - name: Checkout Kayobe repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: stackhpc/kayobe + ref: ${{ inputs.branch }} + fetch-tags: true + path: src/kayobe + + - name: Get latest Kayobe tag + id: latest_kayobe_tag + run: echo "latest_tag=$(git describe --tags --abbrev=0 --match stackhpc/\*)" >> $GITHUB_OUTPUT + working-directory: ${{ github.workspace }}/src/kayobe + + - name: Get current Kayobe version + id: current_kayobe_version + run: | + VERSION=$(grep 'kayobe@stackhpc/' src/kayobe-config/requirements.txt | sed 's/.*@//' | xargs) + echo "version=${VERSION}" >> $GITHUB_OUTPUT + + - name: Update and commit Kayobe version if needed + if: steps.latest_kayobe_tag.outputs.latest_tag != steps.current_kayobe_version.outputs.version + run: | + sed -i "s|kayobe@stackhpc/.*$|kayobe@$(echo $TAG | sed 's|/|\\/|g')|g" requirements.txt + echo "- **Kayobe** bumped from \`${{ steps.current_kayobe_version.outputs.version }}\` to \`${{ steps.latest_kayobe_tag.outputs.latest_tag }}\`" >> ../../pr_body.md + echo " - Changelog: https://github.com/stackhpc/kayobe/releases/tag/${{ steps.latest_kayobe_tag.outputs.latest_tag }}" >> ../../pr_body.md + git add requirements.txt + git commit -m "(automated) Bump kayobe to ${{ steps.latest_kayobe_tag.outputs.latest_tag }}" + env: + TAG: ${{ steps.latest_kayobe_tag.outputs.latest_tag }} + working-directory: src/kayobe-config + + - name: Check for new commits + id: check_commits + run: | + count=$(git rev-list --count ${{ inputs.branch }}..HEAD) + if [ "$count" -gt 0 ]; then + echo "has_commits=true" >> $GITHUB_OUTPUT + else + echo "has_commits=false" >> $GITHUB_OUTPUT + fi + working-directory: src/kayobe-config + + - name: Push commits + if: steps.check_commits.outputs.has_commits == 'true' + run: git push --force origin ${{ steps.branch_name.outputs.name }} + working-directory: src/kayobe-config + + - name: Create or Update Pull Request + if: steps.check_commits.outputs.has_commits == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: src/kayobe-config + run: | + EXISTING_PR=$(gh pr list --head "${{ steps.branch_name.outputs.name }}" --json number -q '.[0].number') + if [ -n "$EXISTING_PR" ]; then + gh pr close $EXISTING_PR + fi + gh pr create \ + --base "${{ inputs.branch }}" \ + --head "${{ steps.branch_name.outputs.name }}" \ + --title "(automated) Bump dependencies for OpenStack ${{ inputs.branch }}" \ + --body-file ../../pr_body.md \ + --label "automated" \ + --label "${{ inputs.openstack_codename }}" + diff --git a/.github/workflows/upstream-sync.yml b/.github/workflows/upstream-sync.yml new file mode 100644 index 000000000..92e5c1f54 --- /dev/null +++ b/.github/workflows/upstream-sync.yml @@ -0,0 +1,38 @@ +--- +name: Upstream Sync +'on': + schedule: + - cron: "15 8 * * 1" + workflow_dispatch: +permissions: + contents: write + pull-requests: write +jobs: + synchronise-2023-1: + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + name: Synchronise 2023.1 + uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main + with: + release_series: 2023.1 + upstream: openstack/kayobe-config + synchronise-2024-1: + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + name: Synchronise 2024.1 + uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main + with: + release_series: 2024.1 + upstream: openstack/kayobe-config + synchronise-2025-1: + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + name: Synchronise 2025.1 + uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main + with: + release_series: 2025.1 + upstream: openstack/kayobe-config + synchronise-master: + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + name: Synchronise master + uses: stackhpc/.github/.github/workflows/upstream-sync.yml@main + with: + release_series: master + upstream: openstack/kayobe-config diff --git a/.gitignore b/.gitignore index 2fd3e0fea..8dd567812 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ pip-log.txt *~ .*.swp .*sw? +.vscode/* # Files generated by Ansible ansible/*.retry @@ -42,6 +43,20 @@ etc/kayobe/inventory/group_vars/seed/ansible-host # Ignore kolla configuration. etc/kolla +# Ignore wazuh +etc/kayobe/ansible/roles/wazuh-ansible/ + +# Ignore ceph generated config in AUFN env +etc/kayobe/environments/aufn-ceph/kolla/config/glance/ceph.conf +etc/kayobe/environments/aufn-ceph/kolla/config/glance/ceph.client.glance.keyring +etc/kayobe/environments/aufn-ceph/kolla/config/cinder/ceph.conf +etc/kayobe/environments/aufn-ceph/kolla/config/cinder/ceph.client.glance.keyring +etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.conf +etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring + +# Tempest logs +tempest-artifacts + # Ansible Galaxy roles & collections etc/kayobe/ansible/roles/*\.*/ etc/kayobe/ansible/collections/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..ae07caa84 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule ".automation"] + path = .automation + url = https://github.com/stackhpc/kayobe-automation.git diff --git a/.gitreview b/.gitreview index 088e01d84..bbccd1ae9 100644 --- a/.gitreview +++ b/.gitreview @@ -2,4 +2,4 @@ host=review.opendev.org port=29418 project=openstack/kayobe-config.git -defaultbranch=stable/2024.1 +defaultbranch=unmaintained/2024.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..498ddee33 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/sirwart/ripsecrets + rev: v0.1.7 + hooks: + - id: ripsecrets diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..d830e9eb1 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,27 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + apt_packages: + - tox + os: ubuntu-24.04 + tools: + python: "3.12" + jobs: + post_checkout: + - git remote set-branches origin master stackhpc/2024.1 stackhpc/2023.1 stackhpc/zed stackhpc/yoga stackhpc/xena stackhpc/wallaby + - git fetch --unshallow + +# Build documentation in the doc/source/ directory with Sphinx +sphinx: + configuration: doc/source/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: doc/requirements.txt diff --git a/.yamllint b/.yamllint index 479488dee..e02351b2a 100644 --- a/.yamllint +++ b/.yamllint @@ -9,4 +9,14 @@ rules: level: error comments: require-starting-space: false + line-length: disable truthy: disable + +ignore: | + *etc/kayobe/environments/ci-multinode/secrets.yml + .automation/ + releasenotes/ + .github/ + .gitlab/ + .gitlab-ci.yml + etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml diff --git a/README.rst b/README.rst index 10002e894..0f5cf6d74 100644 --- a/README.rst +++ b/README.rst @@ -1,36 +1,18 @@ -==================== -Kayobe Configuration -==================== +============================= +StackHPC Kayobe Configuration +============================= -This repository provides configuration for the `kayobe -`_ project. It is intended to encourage -version control of site configuration. +This repository provides a base Kayobe configuration for the Caracal release +of StackHPC OpenStack. -Kayobe enables deployment of containerised OpenStack to bare metal. +Documentation is hosted on `readthedocs.io +`__, +and includes release notes. -Containers offer a compelling solution for isolating OpenStack services, but -running the control plane on an orchestrator such as Kubernetes or Docker -Swarm adds significant complexity and operational overheads. +Resources +========= -The hosts in an OpenStack control plane must somehow be provisioned, but -deploying a secondary OpenStack cloud to do this seems like overkill. - -Kayobe stands on the shoulders of giants: - -* OpenStack bifrost discovers and provisions the cloud -* OpenStack kolla builds container images for OpenStack services -* OpenStack kolla-ansible delivers painless deployment and upgrade of - containerised OpenStack services - -To this solid base, kayobe adds: - -* Configuration of cloud host OS & flexible networking -* Management of physical network devices -* A friendly openstack-like CLI - -All this and more, automated from top to bottom using Ansible. - -* Documentation: https://docs.openstack.org/kayobe/latest/ -* Source: https://opendev.org/openstack/kayobe -* Bugs: https://storyboard.openstack.org/#!/project/openstack/kayobe-config +* Kayobe documentation: https://docs.openstack.org/kayobe/2024.1/ +* Kayobe source: https://opendev.org/openstack/kayobe +* Kayobe bugs: https://storyboard.openstack.org/#!/project/openstack/kayobe-config * IRC: #openstack-kolla diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 000000000..96ae3225f --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,8 @@ +# TODO(priteau): Remove myst-parser dependency and bump Sphinx once +# compatibility is resolved +myst-parser>=4.0.0,<5 # MIT +reno>=3.4.0 # Apache-2.0 +sphinx>=4.2.0,<9 # BSD +sphinx-immaterial # MIT +sphinx-substitution-extensions # MIT +sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD diff --git a/doc/source/_static/images/capi-architecture-diagram.png b/doc/source/_static/images/capi-architecture-diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..259cb89390a22560ce0fcc3e9167d24a5a15d76f GIT binary patch literal 267742 zcmeFZguBaxDPpmglm ziPU4qj`N-*0zb)h{8R=0!n0A5xOXhO?dlBp#{&a3DMLBAW4FNTlgIG9OpXyk9|8YU zg8z>lBS^qIMgacCgMOBP|JS=GQWFUNdVQQ1`ryZ*SiWP&#EwZlxck@vZ*J(si;LoA zH9rFwSt%|vOwlPm5F&l_;L)&06Z5U>FRxBtxL_4yB=#)?O+fhBmgd&dnS>J`)99G4 zUZ;EMbJ9op{42((n3d&8#PkPrOWQ(Qi~J_m*=7h1{#(Oo*X*T!T8>#`ymsxyG5nKs zfBu8?nn&o#9+{$l`W*BvSMUi_9mz<=j^Pph`H$GPE-#|u6PEO9ds_SjTskxzY)vAMi=K&IwS>H3k$s;de09|6&)|l$> z5?aJpMcuT9-FD2krjr5~goTBTX^Ahh-ZURTh2aqpN%~Rpz6_tx*Yz+#r#8k3JMSL< zq>^>Zyua`#ZGq=ekuI-*uZ|asO zufUsNVlPZcb)AEDn1|K{hyAUoiP6uVsGX_S_o_7RyU$Ax_IDT+;)GB(PLDFxI2{*C zr^XZCu?K@CP^591zD99d_eI84ilb8s!^>kHGY=emR`H<8r^6E3&G$2+Q{`-ibg=Ih zI+MIuMxuVEiS9}D7h7^5aTsJAmw&NY-*t%qnl&NST;03rN+~`EgH^6O`agestFs;L zXBSpYlczT6&HX@ja)Z1@f2t*lJOAldBK1=1luV7Hrkk!mzmZd!XQ~&ZjyFdXxr=~T z55w6sZ%}4p1GIG##PFCk3O_#1)>3bZ;E0!cH|+H6k<_*%oh0S%6wBPKvB^Q8Vf ze{)`!X}`ze_7?L!>9X|sg!T5Si6-LHlZ+cfWypewfz{c21!qbYQ)p8Z{E6{@_2Q`5v?r{ zlNRESLmeJay1)Va)0bJTc)xSFZ^?9we!{(^qcCcZBXO`>nQRRX3cAK^*m64+Z-ZDG z_YD@T;n^7Pwrx^V7S5`Eof>8v7$CBJQ}>Yu2ZnApl2b2PuihUV7oQhv>I&18gP*(B z=LaUB^WJ%aT6n)lRp&S>O$+^~_+5E@6&(inWb1n-{vliCnuJ0==e_NxJ#kgKOR-to zLyaUG*D#k!D2nu1%!Zdc`FGf%#0lCz+=z862Y0%>hSl+MlB#VI@0u8}tPazzbSlN% z*qCkkaNldZa$|S}CQft2aVO=+mz|*I5Ab<&9SU}}(JzRsk)|eyF5Y*_N?$fAIbz+M zLEx@qvvo43HbBU68?{~JUef&d=vwLSa`Qy640})!0^0<(e5SY`Zv^kC^IDJU&I<1% zVTUDMtnIjl7&ZhGKs1UR&(5AVlPhD-Cn`VK##U<-z;_IqLW77%C>t6>nD%H%P%!Lk zK8|j_nU5>&q6b>Gl-3AoIXT{kc7oW*IBp|<^P!pR;yw*ws@f)%-8R(^-ky)!3bJzq zmouO5c{uHC(`OJt(f%k)B}bd>G&ws@m29qY=k<2G;I~I@qIV!tI5;X5UO0>tlZ3WD zW=5aD>E~vVsK5W_%uk)1&srofP4wyv+V6g4j{H~nhkM5lOe$50;1)q|rR$yjYpt+|r4iJY_VhN|1bkN#6E& zN>N3&myl`q%@rlR4D?$Np!>qfXYV!!Gl~;5z|%Yz-pC8M z4wx59O*DnUiy~N5<$cIGESignQa9%M^r#Fnk2`NHuz^Tk1lJQb9x5v`uPurz@PCX7 zZ*6ibGVK}vSx;LG2dfnR0_{3L{AP=;iyii?Kiancr{F@#*Jy(Q%dX_Ng$V2)k|L=k zNL*v{Z#%UQ6IUcz!+ZXm3^whgkMP%DVxBj%6%%GEroCC{_~3bvbXT6+@p!19{eoHC z*^*Wg9zToF-W|eR5@}+!CJvI*(;CgQG(vYq1#*sM5)EK;bZD0- zg+C+}W;JHe;2dpq(x|aBrN!n0hM7^B17r08@8oCTNe@Y+WMsJCZ5M0QQOa(_dyo37 zx__C-N8(f>baKgN4f!h{*1n(WJzSOFr1hT=P@suBZ~g#+Yu-{GAb^$j7dfsNs%kTE zTXP{xu#!`J=KcDmrC>TjeSObJN&SrTHdvqQ<8oRxRxh!7T2tn}`Taz@O&eW_(TCec z54*$(&~1-VX6i=aIJGnbrlacNpRC~0aq%?f640dqM(IkCDPN$SCqt9NZG$pMJ{Zmo z6mxVZ-ff;S{vmdE=KEK#W?W7+&FZ$AAca%}yxgv|?0t0fY?F=QcwGF?2FW^R!el~kJ}ut@`BZ;#xDr(%Ni)uH8=fK^mdlg%=)=9ZHQ8t$ zX5&XNohp|^>?~in5(7MD@!h&0|HIey!TjcTe~WHC#!0=iMhKa_1S${bQ<4;q`ddH9 zB~kpAan%wa3Uj~AgJ7sAN3fu{&&5W&HL$&{;RBV<_KmZvqpwes+w~-UR*9_|0BL%b z0o-HQ_8N}ruB#2hl0UWJ_*A@mIDis@9F3TtQQSuP$XM-*Qsg;BiA9SH|AzdBH~H^B z=sJ&|lx5n!km`Q8G083&cP74OyPd@kg2aaJF+^t0X{~ZQ^9Vnpl$|x)R8zZYozEo} zfWwFE`aI8x+*Awar0tIQMU@S0aR;v zL@Ys05vu2wTC||OMK1bo`;!cnRy*w-tIegSn%W;v1le|`H_L?GGC%;lqCDQ6u8b0z z`kX5FK1TiOFFf(THTH8xu-md}$o<3*H#)BemqAklNVgOd5yd6TG^#H%;gqf7HJ9kT;6fsdPeWMuFo%0YX0;N4KYB znK2b5ucWVT4~`fUA8Epyiat* zISVB45HfOQK7qAi{?{iq3HMjx4eTXhhyoF-a^izi z>5tQc$y`L8RzH&6#OUS_+4CldfaHxWf;}s}>q(TYTY*}uqq4jh+oj=|O;R={hbj>N8mMc0f205eC1;XM1K zpDoYck8kfNdGXp|L^`g zSz9Pp6=^~3cmME+(8&|-FL{Yx@o1Bvb?vr!oNITV;2rE%FG4>z?qKr0rR|skZsgWK};Iyk7bf5KPv?o|QGa{RWCA-Pn*7HR5?eS$8rdf7yx@C89q>1>VR#LO zknGT6d;TAX&>#%LZ>dpx?9uehzsBp4OiFh<_F_Pk{82FK?|oUy1oF(rTq*ZI-d5m6 zfJj4raH6G-$lQPDQo0Sold@BB!uAjU{@D?)g73Yku~iZ>gdJ5g{XJLXabT_n3#Qus z9{mxo#FS2wM#}rt%QF4*Tn&OkAl7+(km4T~K#~|NfYi^(8HRtJD`$w$YmXB=Dj@wm zaH(rx0hnpHV-FG7-#hL%udEfoTrKq37#)@bSI*yg{^J#^PL&oWOOuI$f`YS~D>~R0 zzapS-m-~kRz{LV)0j1)S#t1T;bSN?e5-JZl9xAqL}oM%Wh6B9pK4Ha%Fsv z+o#7@U3W(g_H+asmJ+I;KoqAJjZ=er%uuX?$ku}!@-bJsCUUB_KRpJ-S#tixd%s;| zfHJ`pUu}=lKHQFkq<5^xy;pHq1t_U=XJ`6EbpfWR^R_iA0$?^g-hE%4_~Ok`p)j8MiU6sV zk25!ayVg=nHI!(njfR=q(AzJb(DCy2I8Be73ti->kw+b`+Fh1uM+DFa-)h`{{X%RJ zp5UU;aMB<(d8xhbWTz6_@4xd}jpdbppDzzFe29Gm?02dBh}Bn;Y#}ad_FNP64fO&; zxf&n*<5>jSfx1l3wp{1XX#t#J#qv|;dqTTQUr()HoId;83!Hoaoi&EX>A%%Bc+4ln z4 zlHSH52vk2kmLeu+zq+KovpydJisqg6srGnL8TzLRu`tH!{p}1$IVfb`o8HgUt4~(T zd#04HFEua)5mOb8E8B}xgPm)B0konEE$c$rTBZ3HZ#`~UiQ%>Za5(XWHK;%P52g@6GT za9mD4Sk7FncUO$Gui9A<9P5kfneFaqjphx>$f?o*RFt{lJO57$JsA-PcRJ(?hsGvc z&y1&R0gMz32qenIr2RQQ)%pnjwCoJn6_m20`s=BQ^~1Osc|KHQZy-g!Y_rZ+E1l%v>wweKt?k7TNRtm)1V zuOc{QXw_7l^LSo8O$h#RVl{=lfZuJiWB8mm0Q}@!7c+(qCTRvfAcLYRmNN}vref9W!t zS0+)4Ioir@Ue*UzH*ll zc>8A7Mi0kXC_bP3n4^|={k+@<`QJl;lOZ5O-@dCv0>vM(HbUFzaT;N#1`uZoo&?Fq zQ?p2=EdXLwUb2-!(9W!eti$m5h}%vUS>`!|)lI-0>w{R#Nv6d!kfTXGpXFhL;37c7 znks>khEcAw9Y*<*p8YZtC%Kp;&s6R3^9L!RcwGQ3$KKAyxpcE+;u-I)wphV;@;x-B zUCdR;@E70>FQ-!zzo!7{njjk5z{iC3|!3 z75ImC2Nh9zcOlcL_{^pqvv^mc5!sxWbkyWbmH*`)lbWw#A_u!R?oTLfd)BPbU0uGs zW&D=IbE0X2uxRAc=;uS8vubS=Q2CeOM%ZX2C8hEFX7&&uv<#lShofwT$m3a{T|^fd z%gzZ($l$s=*KpX4V!1RCEbqtGaaPO869aG1!a>6lxbiHW(GCv0>FBYt$ua<)lS_H! z;zfow^Iv86(ah`b3r)8dbI&++Z3Fe~cGY-bT$}|+6|b{S{s}Gf(y4X z4L!PLH%LCtXf#oJqi&Ei&;W&$=h&hwjJTIFw`axlXO|2|V=MoN8A~>O2Cd^b*+736 z3cu80^b8MzP6KPZG}yyY}d9K-+*0IhAF}3gL<=2Zc7vp;xl~(X^*os zcqv;aX#O)%{SDXyj|1AL4ga$EVYIu>q)XjnGAvG@$EH!3n=cU{jLGK|G`9DC>;|jn$)b!_G8ba8) zkv#!zEf^xNKph}O&~Z+5*QzG9*m7hT*;cxruP!Dbs~O^7g$e5QPzt#rmkffYhy0Xu zl{~rY^0?GMW$Ri>YYT`ilpe?xwPJM=SueD3=dc8|6z$C!$-RZ4@^bkDaI4uBl7`cv z+ie`i)8qoPm@KL#6bQYSr^?4O=|{iC{Lc^%p_<1>s0zNt&wofUoMcGA-4;aeZH#cZ zZ=>TMNLWUrc?6qle>?}`T?&{zy!R?8(wDHMAm*1Y+pbQ5i<@sz3))?T7sJ%L6esgS zWAG*+y!3F|)HxQfZn646M5bL>YoR~@}FBCcgw7s?ivfs$tGVkt|**nRYv zd1Th&Y=DwW-r!_oV=GvooLZ~Q^AAQXcs;&*_Ozp)tpyOn!x2NJHfj3$`rh*M@<-GO zy=0^Oc6lqT#-~6;4qV00sopQ!owsxkd4!))X805ErW263o4Vtll3w-o66pY-`Eaq4 za&pKC0q*7T#<7;BBE2b|;eOKC#61vl)^^oAyEN<;220CMF~4pes!&g?2q=4Fdz0CH ze;4)4>nwFXwZwJ}0=6?Zs4SuGwF7UE7c3$AT^)f(I0q`ad77(->cA_-(WUQS)j!D* zNCQ%?m=#neb%6VXm)VlR`R^1Yqg~Hky>q+Z*>n3UAR1=#38t~xUGmO51C33Hde3dN z2CqG!U~2$n9s>3UtRTrcMs_i&O0_L$KJ3;LAD;rp3pqvDRvY!cy4i~d>N5lezr|sa z4+)rK=+$*)m62y$P*|1$n2vi}yQg%o%7vO!6_gc@wo-Xp z&hJHG)^ZAh0YHuk`w@CbPXEv8_P7c75C8K##gOW7UQtY0mL`8^ zP%J10!mr(bBMC^M`Sog0F5Nqx%A%5KjEkqq)Ttg0BhR1!0$vd(43;*Ju35c*PB;{8 z`o+%mdR2h7ayB+Ym4kshhpcJ4z`c<%;+1Ws`bS{8n9O$1xI5#vx4jk6cZA@vc?pEZ z>URHds4!Q&Bq<|fGsK}5X#36Q5@fKz@(Q$etc8G$#k8}r7$HI9SgSGwK!JJvPqzfT zYNZtEMj&VG!J)!MXlp#=ht|1XotokTP?zDJX3qbmn46gkEpy3@CV;+cSX&}KPp*$2x5F~~}AiXCd#NJC0NI_(4x^h~yIx5RQ z5u+Q)J+nFp;O^^8?Me+GCrt{?i)sUAd>7OhfdF{7#TzX+$Q=g>8aof9pJ~(We<=Xj zE^z0aD5JO_fE-KP#0Bq5vJj8S^q7Knsq){Fs=uA45YYCUaTUc?oSfMS^V&P!#S-3<~Kzz;V?nP)!T`|*yW zY?(f+6Byz;#pZ!puvgqcR!cy1CNPLzf)ZhgvmK!%)CN_S8pd{gcYUdKhR`i%`#wI1peFBAxOLewu z7#mnvdE1wS`16d?SCUGRrW*iKQM$8FcYoi09RV-^6@5(>HvfpiD-R;q_~Hn%pcwVz zo|weqY=3oIP;S2ibrwR7QfRinD1&pkD>`R~GR8XFbMETQ$Is7lUgQ<9uc)Q%WC4Gb zqw&CObP+~z2o-{N*4D>5$D|1WjJ=#4)xa9g?_#wQL9n0!uKZS@!Yqw zJ~xWB*V}#HBA|wwlgJ`OBY-ulExolen?ynz%+&VEHrP_HYZK{1fWSenq>i{2%lMfh z60-v0G|$UXP!cZgZWYH>ue1qHfN2jFQ|`i*Q%~=mMJ-gW(Hx<2h^-!1N_3Z5@P-Z_ zX%2qbn-!3x;L7u`ous4)9k(rmQ@q~57OAZ_o>UO`$RA0VMP!cbAAKMPGu232z$nf{ zF;YL@29i~okGRdAISJ)Hh+8PWS1`X~#~|#qsZs6#E7gKIS5?6FHxIyEyz{rlq*oL~ z+twD^m@nxf2~W%7+6z(QqW#&K4Deeoj-cK1n4vX^Kb-}kfM+UV5|7(r1kjOHnrXdu zH>fXxw4_!FW;R*BF}MvKS5Mh+hH=`oauaOX@Om0&*{c0*1XT0_%4@9xsgw3*-87>} z_TvtWF^^kdC8oY@IXykSjaBcHo@y#%5@5@?T(@oYh{u7n(HwQU9JW`KO&${z^!(b& zZk&7uO;$2aMi5K3LCkXkp#Yp}6TteHf2nirZC2yVoDeF!XYbL_g?U$7SY$oy(z-ze z{068hBLr+FuPlZ;vI`EVYCDQ+18)p7Nd6toyuF#qTe$AI&%yE=S@T_FgD@rSwCnAx zQOd84RQy+d8yV?HT|I=Vbcl%#vUFDmYN-!X{f`G?X$?@Ug9PSC!!m$t6Y<2UIkTj& zo)|J;O49BCqO)0=M!9OuxX=N}>HthLSJ1~02~q?~PFdM0nRhw%UY$?&xlm`utBmc2 zSGkmO4mp-uPxuz4vOM{u-<^V^$7xhx3iTV$e{GMhceC9;XQ6uCm63(sdxx3AsiEc*FX&v}BG`bcIuddqa!A)XKt93uW zOf0|y+|uW1RURXvlk7F>zo!bU0QR@@vHGQuh+REBz0qn>@|~ztvDvVTu4=8f#pozu z62!|w!aF<{lim%rp%YssV_pgN$izEcT zjLW&Hp`jN$k;%L#;gF?uup|%op5y^ozpz9p|2v@Y>u03q;7st|y04kc)z3EO_OFAZ zdlC@b=kkr(X@HHp$d&lgErv_C9tVPydHwbOLXLlZyO`{0fJSHP?~nbS`2ZvT>kW@h za>=&`>>ULE{>k6({hwDJYL@|$Y*z3>9pxAL`>>yaOiOq)*Wv$zl>g6{f_SJ7D0)^= z>i#!J-p^leFuVYWiqOH^_Rx0xA9wLTUrOrzUl#VizTmgB9|-kxTo}F9b^Kp*AnD=( ztf7r>NT~F&*>LO0mZ;jr+rqyWt2|zvZ-{tQ`GNb_y}yzYBX|M4t{<~d`+GYw87f(3 zj#ueOnLPkE(6sq;%mA>~`F8UfkcB-)#I?T!E-meooRT61;I)2BB&SlAMv)}1r_*nX zA5ih+0+sLVY5K6k8)1m90nmJ6XR*2%i2#f_YD|(AxaZ+^i^J5Y0pr^Rz-I9@Bf#f6 zz18}hderTwFV9eLJ@CIO@WU|#_+t7Bi~@c_u4y9*uA{D}b{;IbjDjcX*UagXIsD5r zA0894$gWlL2m+5QJ|C}L#|MQ|AYjiLK`B6`vj{uY>Ea6}#>Ma#S-gNGhlXAU|2aM|MFm z90pPQK-;1NWp6N`Kz!t9_Q4dKrf0 zP4$emfAa9*HC8MR--QOhXU`vF0fQQYq}TL1)tF-J#rz)U~-^f>*8Xc1(xDBJ)hDdJ!r@PJti!#V~pG#DQh!|emNjXo$@>e`h-&p|S%l)Dn{ z5|mCSzPpz#5WKKG2h@Oh%Xd|X?I~ZBBj8Qy_Wk&Wjdum~E^h;eieLP9u)q|HlTTFP zjN3*hPW^ABfhP8u0$r*53XOBS=Vd51a&p`@Xu9Y)UxdX+DHEXOqIrUVL9+o2#ey&d z|1v5DBo|Le?TP0G&Kr}SPhq=T?e5_KxxMwjiA)79%VuED=9nKSwvQur$o!lGDhkTa z$eeqC&f`z5$JChQGp^AB1z9#0*0autm*Gj2vRFDr<2aXB@5{lU)-JFC@DbE?fHb74 z=QZuV{nF3rHn1L#2`v28bP(CLTZYRnh<&F&H0gJJE&Kf%5RSf`F(R&~1orwxzPKJx zRGHn?!BwuxP`K|5jEujbvdA@bUXW;lS6d~z8>*q*4|a5BfE1gYmbOy>&e*MULQ5|o zj^JT6WHdSu80zFG@VzRa#Pi~abLY`}9zcv~(USa9Qn?0HRO75p^fbu;mIQZgB?lMtWi+PPHt<%#jZtz5&tq ztAlFA%PtKS2_*#GE?Pm9mxC7-lsQnLSU8NVj8{D-n&vSBNCGNRPn*e>Q&&PU&N$=w z6SOXmK#S6q)jmX@aS6_qXC^{9^X`mI>2+Y83pXCw-&=HlufCr}R1)tvNg)7O5YYhY z-)CEI5+E|>{k&>?9I9vd@hKh7^3HUB1{f(65QDI8ncWH7SqqYnuY&>iM#a;ciF5(t zZx^73AZ;=A2@UY=4FhgLwwY1dz-u|yzrEhDeZ2}=JgOK>9k)aJ}dm3|t-4hH= zPFnRmE}n&am9b!4iR-rmL|wFzyg|;6I@Ep;3W!l}|H!kvm7w-tt{2_s?EaB${oR_M z1w)3@mRd~GQR(2|b6}ib`SSX7rF_t#qV&IUuk0pveRd7=9Wzz4kuoS2j>U~mBH zM30(snPkv)yz<1)wbi>59CZ!Y7q5lnU4Y2BuSL?qYcbdxIjf4zv%67Q54x9P`%q)5 zI<8M_=k*cNQc^)cNzz)V+|Z_{!va$`5C;Mj^#OXrB%Ug_>{rLG^9p7lc2NL$Wmm3i z=ZBDo>h@91T@fZi%SzavKv*lrVIXm4EM%?S{z+OSdG)V-3_GhOcqAZTO zE2seIZ?`c_n3)7#W0M=Px75-GFhx4|`M*TPp0rB9oA#o0yHlec1US#~wk(K~PNO(%;s{OguRR+Y&13RIAl5cv z`Db)al5{NaC3cLk+%#WTOdIEuV0-V_25Z@EL2Vnn9Iz;tt4KBCP|c2+E<|HO!+-?` zhe9!5T`ViAidjH(IO8x`C{s`?8Ty(Q8^nn|nqr5SmB>?<4JZ8dEPy@GP?=rvi0jKt zHvI+xBTgVEXvia*p7!_o7udwe{xS-Byt-U6m*1?lr&;M#WZc0Y`nbx)F&!AaY7$|4 z1>7L1wns6-673It;{Y%CWNT8h&cG^w*MeAuVWIt3TS<&bwXNI;2=+V8GZz>7L6W27e4}9%fPd?ty-2gU(h3w85?X{9yX9LODHjWiD zQT=|z&mMz)>ML?38mi)I%U;O$WuR>=0rPR?L%Jq>Og&8i^TJv~BRb=>#*OaLMC?rs z#B11u?M*3(fZ3b!2x7kLy{r*^d#%<#E7pi*p~g;pKOz2|+11rg^{iB8nt%1HTyI0x z5?o%rL)KRIbt@YP@0c}gYpOeNZ3B1x#g&fU*=b}>`b_H~w6NdP%rOe;+sR%_PECbS z-b>7wD1|;#%)(uZugpi}tot4vZW***)JctpB~TDqBwz%5X@oT{vuj;hA`;zQ@|+Sd z5*{{8(ui_PV&dd4c#)l&q zgW7n2y*Hf=3n8n21Fy$lttw$VM!XFa9~Cj_39Vc(7*9dy863t*&NVLT9oCWBAU1Xi z*(px$_=G&XA_ha ztH3Mx>Qw!qS6oxc4}aW88p&g#1S~J|S#V%XI)(I^H{-dc84-F@@e2~J^3bB3T|VJV z4zjEOE$q6`Clz$&*<|v9gDZB8!ohEKgIGl{KB7Pt)Qt=ocTKXmnHJjko*zMVyk)HM zo~d<;TRlWs-ca}9wuRAbcY28JeE*YKYz_zIF+Kuyz(WQ?c0gOd!9jU#gg9bH@;cP~ z!fCrWEZmIdglyGM6mC+0wwO~XYN#gt;spmc?4>4LGtoe?_ zWYc9szz76)Y>ZR{oiM7!NWW14o;VgOayzj5TXtUQCVK4s2sFx#H}Yy zb_c+%-T;!BnPZzLkShcV&M&BWt^=4=xQ5u>{F!e)z^AFrMP0j6p55Nak{|>~SWsds z)YNYROjg?Fc)SGuy5XoYEFU6FPODvnG_T+%qrjn-olPt1HW+4PXsl}kT3RNbW?Mp1 z7Q&NgMr40ePcv>s7s%LcnXd#f=~y7&8Q~)2X9j`R*jMRX%qFUS_QxKaLp?v42Uy?M zOXTB(n1LjAF{h{pxXo^X=G^qG3L3};32c^skwO0_RC&YzwxR7U4HlXlsq2r5E#RsO zV1qceE7pSu9sUx>a&(t{pfbcTq|;#`MR#I6)CdbirE37_3E_ZSo3k%M3F!H!f<8z8ERO!T^7x>O`L#D zs^|z});5+8Jg^cUw`QBQZ|mDlLBvbO2HiyBbBy(HWU~& zZUJgxU5HxBzMbl*cH&cvz*Tr^?amAog6TXQ1_ z7vD;&oan6gt~c7W3p0K7I(P2zt0N+q7@Y*JJ#=7kS<&}aPQ+aDK3u&pTumWFZA9z%_FJNXUK12O2k|5Jhw7iC!ACx>F zO9s!CPy)=$X{KjZ?IzwpACyP{L&UAvNegiEu58swYOu{QV-Jm=qm54h(WhCY}|eU+8? z=g*DjSKdD0qaSbE zT=;#K{f}Hj=Lof!E*7Ls{lnG4W_}EbBCn&~|9P0uyNys7af}xh{=J9nKS!N-9engw z%OK?0_>Xb@hQpWC25Z&0GdCmfPcubFdI@|~@#*A^f4-?@po7T|8!5E^^QwLXSb% z-AkvY3x^Ii>qX5j3p=gm{==`{4zvGTEPA1k0G`YNJyfMy9FV+zW8K}4-Dw1mo?<}3 z1La_Arh^ZfT#saOfWd>=Zf4N|BM9lsE~x|4V@D0VHVpjg*zEI8wZmW4KYs@>+l40R zhwBSNxJ5g_f9iwm^5+>l0nOM~35lS)NcG0jU8ij4dKJUj;j@eONe}(T3))3i z{_vot7h*!EK+i~A+-37aLzX152PmfdUUv1HP_rGu>q3WwP8)=rdR$1-(BaLVVg|l! zc8GMzvIM4s5FpnM9IZ-#Uyp;_*=TfP)~{dZIGih5&{NI`)++aCl?B{k1!Tr6W_sv+ zc-YUY=z=D6F5m&xJg_BR1W(86|Ko*+3Bf}GQydUH^S=yebe)7yvkiEjRS?u%QJz{4 zWadB5SOwH&qaA;B{;(TR@+MyFmjTL9OxujH$@!xLz<)my${XPUoOO*rr8mIcuXfu@ z29N7O2^~T3KylPt6?h~6Zm~W8Fab0KC4M&H?#YI0LL4Pc{_h=ihpV-4{0b-aMnK@% z!=O}K@WMY7MU+y+0VdmfioMhB=wPIU?;B|s6!@=4A=whz%FRG(2P6{lSFB+)Cnh3v zfTDHelfYhxT|Wt?>)+49@|Jhus4il#6tp|19K5E51ZbR`vHt_MU6&1Vf0fsFN?0^Ob`+NJ(_y?^# zD?oSCpQr9Ibp5~Qfbmx(gn?(?ECUNx{bMSVBZwSmdj>(OG_(5bukAcqQ3EP<=rIU< z=VHH@G)AtyfzHTO&>R;3MP%Ldu8+e7Pg+cuh4(j{uqpY-t=cE&ylh|p6$o)eT z#Q3Kg&?;|uS{k7j`9)Bb_7wk-P^jTJgh5}lu^k4b;QP9O%jL2jm((6f3*0Jpr_YFtAwK^(m(FL+XaAHCVt#FvOJ`#gYM<%Ep zZ9o`Fya=?6KSyDT!(gaCebe=q|A9^eA2@(QpaTwQ;=SSu&}1_aYd`#_NBzAJ`;x&%UZdd1jUW2eR8MC2YK9hG)~m~0Bqu7d8kTM$)Q z-3AoqGEDS(y#8NKtN-&o;GZ50WV*Gs!9vDmhb`Qrm!bAv7H)d_uVLh-b-j6d=0KzL zXRf&4lXUf{a4&WpTF~({nUTZqe@P6U^u>W%LC$su9cHL|!o5JjpYHF%beGx~nP9VH zdEdGI-+TIdXu(PVUn+^%!45-Z0)AV~YD$E5wD}bh>3A5j$=wxbV+J)Z@cw)%4Lw{M z2OYN$;cVa_{&!+oAdkbL=D)Aq%oRW@Ks1lhap4=at2$$fZB!2GBYY2f^bSf-Z)q^N z4+{nJGq!IP2n<{mdHzs7rYUz&X3=@SZQU=&b!x!?k;ai>{JBD@Lf2tm9fdqNC1AHa z7`i%TX*AgFsL;wYpf^XA(q?dZttiD><5N^f(!8ahe-meXzzy7B^) z{Ia@l)@w7kv0`8n;CLa4gj_?n5mm$*u=pUT`(pZ|>q4SB<>%t_izIMZeN%^`uR4=1 zAHE!4mgXF3SG&_CHBxS~yW76=+Bx15y|p7SU??Q0lNzN%r$ZxEYW4h1v^e!)M7Q)~MAW9xluycCNab!fwh zKDAWX4A?kQhCZLfadf@MvXWi?!7REEe7|0S>RS5gvr{jX`t6aNB2}*L<+dZ_ES;#C z#xGSh_49JgQjK5w4l-IOBIBydLs<`|`J4CFRQeN7(@a@f7RqRbd`FB9suCekqD6rD za$2985p8Q~E7Y-TJ~6VmbmCc?fR?e>d~eO|J1%Sji<9M1;ZiKY3JaBG!Mh4ymDEcs zGv;0`qM8SEEW-%uUD}&f7fW~I%OfuHMJH^9?@(=Fght$c-rE{qP`Nbj!50SKXppYU z&@l_;)G-g*t6GoTXlzy z#slpNZV6;0xxJ;L_Cwp!UZEDu`mnuf6JX`2{kuy|)VCyx%LG2%ikYA3-MmhLE^zbj zN8_?CaoKFeUF~_QJ@`S<#BpbVX!0ov1s~D6wW6}vjwxz;XHi;_>08EP)Sl74Q{ALL zGD7}clA8VJ)?f&CycLS+s;ExD*8c96Y9#n$B!i`C#4}+b9!d0G}H&Ivv*6c?%DSaW!;j-!TL0bznN{TW~@`pE7BggvTg7$w;cqm z?N6<{nhWWu98`V~)e+ds_jfaqJlO5zn7xAB8A#kH(@mZ#d=X`Y`sz}>x;KNVh*=yl z*ecRTg}8kmOZp}_zVv)#XRbsicPF03xlN*ZhTIFr%ZzZY>oX4(;U1f6?xZo=Syi)i zaI55VJ8844)#1othQJHS;Vu?zOawCpS*jgHVxn^^zs=a`wQj1dG zX7UY(%3N670r_yX!jy35W77z>Py^M<(RvaumjjVqW=*UUJh?#9Wvh2F**bl$RW{-7 z%vlZ&&i#SU8T*-&J*>)-h9X_Hn_VU?DrrFyY9ND2fAfO6mI5 z@&?1v!82O8=gcGIa%(cN{i!j@c`K-fozp0)gFOq9U@APPk%O3KG~cGkwtL%L)umKnGvdO3!M^d>3hn{cl&i=Gstbmmo#iFl3%3--lrHeS6$%x zxieo!E?Vd})t_8oUqaGiL*DPYAVcJoS*=hqvCO5Es)kpP(bqq__-J*J1TA4iR%#`> z=!&3lO>oN8c~A=T(xK!M%r~>6avzuToV{z4R@+q$k?L z&zrcf$A4Ao;HyfSDG$LXCl30P&LWW}gUX)H2kpa-$0QZ?4-TZ>FisT`-nG8?i0sT2 zPC6n|Va{g9RPMwy2Pb#;t8rw)m%iCfUcK|pmPqzR5FJHO%+p3ziv6|u4@Tt~2qpXW zxt3*Dn5;b#`D884*~xH6bFp&z&h_hQ!k4?hsl9Z1^h0fwjgaGID=F-WJ1KhZ`Ts}Q zTR=tGt!==HNFyaBodSZCbeD8@i?nnzq$s75(kv%w3j^>w<7$-o4}3zO0py8E)-c6L{+B z6-w8t;mKx@oexi~ET}R4de(u%yGLuqj@Kh_yWf(BY)>GNmV7*rF;y(I*tj(H2szR; z_QS)t<&U(G9dBZPE1{BkesK^Z+!y<%3OBeDH`j^Mn9&3qYkN$fB)xp&3pQRnL-l9T!)~}#4 zim=R@@!-wF%lFo*jZM6oyi`c*M$}*&?Vaa~QRLVHk=FOgc5;dC@wDwNR?xc5H?MVF z2AZIQ9rSIw`pE<%>K1D>~uG)-IqY zb->H-I=F~8tFdhaf6QDz48N7>+9q2L_Oo+;br{kh0>{tum+yl0#L}hsK9Iwm{H|8= zuSWKO&61l|><_4Q)q$_Q(^V&$=HO)2VNf~0Jnl`bWh2Gc{F<9H)^}YV0`uL7IwF^O zH>D)NVBAMfAW-KTSK@v7sFU@sm(%{z3uJPO)|jLHtR&AiH0N=^M*om^3%27u$GK*h zZeN<-2$a#G2LCz2^NGCQXg|wU-xze3ME8#_?$|nHKQm-O20Ef0vVHu{GTVAqzxms~ zRG>u}VpFwQEM3Zp9((!jITPLcc7Nqcoe=iC{5B&dily-$kHH8w-MRT|0eQ=MH2I0E6 z9Z#3}@D1`}jr<}D4|%G$fAdE_kqY@-M5v{CDQawD310UxeD#7t6OlX43#nA101U6` zEx7+?tL>&C9yJVCY2UxAxs-!P6!k+@fm<|!^^fB}`GDC*=@sl&H4c#tAWDGIs`df6qnC7<{J zSD`M|@pU&Af~Ozl)HFFAx0qxty!tsf%bCuxsB&(jlY-IRFp!>PvAV4WGqE1a$mr^% zrp?hqa;LTzZ*A8RTI!@?lgJF|sE!@k~}ybLz6IDO6IenR%erf9G?39)Ob?3yI@H-qC38Ed6?n0%Wc+U~4fn2wy9ZhIg? zfr}CJo~I=zu4<#Cmq^xEU|w_GMZE5pxYFqh5Lv!^<#!um=a)g^6}bBfIbX15Vsgb9 zTFqKsmg=(WR#v5MqACfTPJQhC<`NJBYQ=^o}#?*$;*?qzv3i&Vd0sfE?t7kA#9K^LnNd}VpHxBk*R zFOC;F_>i`pTZ`_65N)1^rEijkw1y$i`eA~0yXq#MrW0)Fiq2Y~A_|i%{Ys|_w8MUT ztH)3Of<+|;wR>Aq4CTRugnr=)KW~zq9Vsiqm=(gR(H+c&IW{Y-xh|llcE#>VkDMfv z5BpZPd;_XAD5uW2u1uKDiDrA9^#K=^ULNC=uku!k>yOazHwsUJIsLov@i9w8+JlS| z2F!cU^lz4_Qd%`(-pE-3ZmMH@Y}Z|pz8s~ZhZ+&ICd9%NEBB=-$vcoE2MdKg^hu{@ zV-LdZxy47@>xo1@^E7LUw;d}?6h5p>%DPlL~Yx0c#Xm-j~p$AStpSZ-?_} zN}|8H2z&J;$LludyMkFL3M~3UhA&=qFLmd$xxBI}9^%(~dgE$Pf?QgjT5-Wq`VLWv z@Diz1Jr3$31jtEJ2{jox?7<)b8K$C@ zHow;`dmGX~8@q7yMxBmS(#y(<-sW~ej!P8QTl-%AQZu~c2XJ!lbCci|O-_9Fp*{jt zI}@%`avbeabGqUA1r@@fI#B~z)Zg2M*nuI>J=wlwQiXtBR#;-f_af;%o90a?g43p$ z1RW{v)QfZWl9|MAq50x5CV8$zNTVC}LQ^mdg<%y;&T|F}eF@hK5`J6+5RhZ9-fEE3 z_LuL%uwuq6nYj4O;IbfDAP6kWF&?lMAWsEH*?zFvR=1J*a_eQQU5Oi+W>l9tq1gEb zYlL}R89qa?+{F%7vf5_F;Eceb@}yAZ`{HDb(Z$AjpK5cf z0A3?>?aWu4BH}>wE>OkyYArl!*@TGx#q(>4Csc;fJRD)EnY`qlTUVI-Kkl5MRNFf{ z4)J>JJSs@l{uS#UZ~PAQ&HwmoBxyx3q1+)W7g2IfI58j3dj!9)Y0`vlL{becIz&{y z=bkl{=JoQt-}oJB{B`H@7aAl?2K7qf2>}^$JS>toR9o1N?y*pVB$92pNoVs5UL1994$K9um`=~v= z)DNmuGn&>j8lBmq*E2EG$`zXUV~mhKxQ*AZkuvU^oqIfevoe_Xl5oD{z^;M9D35%H zVt0}R%Xb65?vT%|Ua0W>P{bY1E5PP$NRh%tu4$EEDyU7qAQN#)%uV9Eb}wi;r)6`h zGQLse2=r=>}SQHz^Kz&Qkt>+P&XDNbzM6#{a2fO7dNjQ z5X4=nlAkx`OQ`An3g{gSTnyUEySiz^*2odr3XNp1?GXo@I7ZSvx3;T&UUk9RoH6{*)CYSBellUml>W6HRymKMWdLxF-8v zyYiq8oMIIBbBbZcNJjH-j{1!};|j7w9rG{0H_{rzJHG8pmo#ZDj#sr%uBwQi#(U?h z_H2-S4d0(4HJ2U_kzJC?SKnGo>l4N_fG)XkytBS8wF_x!_Pok=x%Cw0){&yL&YRFj zVwYU}TK$U)N6PJU+kunh%@09}n!Z3Ad|0!ydFM6s<#)F5knny~{FxHrz;AcXJDF{* zU)nV|hwu5^#iGvE=V5kScw31x(-7N?b!2hz5IQ;24_9oyMfZ*&I;(wtN)lExE85o& zCXH?5?XcX84dohTev1R1V(7zrCoA=bI5035~HM!AR3X#i{Ow)S0K#9wBG)tn=+m zwS6JwJyzF#+#dqi#Y2C0XX<^Ap-AB#AtPS9afyt(QPh)cKW4A%=Up@&-LVT?-^4ufK!IhIkVVnW5KgbRk#k$K86oeM#x21-7S{bMN=sC|0bJ? zigB}A$k2|1S!Vm7r&4?ZLps$!^)v%io?}X#9_b56Q<$!ZZGE{I#mf! zk6U33j&D!;0#=HRi%6IIHQVg8&DN!dPYQ0L?QrZbc@NmNRqDBIp8r`TWn zu<5n<;9Z_+`!~Fm_S16@Sfo!!H8XX;O7{F-sXH>Blr>WIH(s5S+)QlkbBKJ7f3QFI zwiQ;d6)O-yXqw-f^+t^RZTvcdkVPeW9?#Hjr!0|m(syKJ~IzDkHkkxnKS++2sb zA%uKm!-z>lDzcd`UpxCIt2vnv6vI(ahr5hi(5`y-ie8Iq9ov~9#N@aaokNv4=4Mrl zIFCv*$yh)`ab2;;lt0OD{@w9*T3B5Q_=W$v^xY>&3~^oMi|+3grC)9Ymdoeb@ox_u z<{jYoN9L1tlIHt$!4J25{hQ&SrxQEf{4>ORMqAp6!Rnf`+-s4TZ9&#P+JV-NQP-aH zdh*BTlW(b$gX!e5{UvFFJox=QuGmph0c5;Gfg@|I5|hrl3sH9Hyhf9V?bV@-jZll> zSnGkpK)Cd4)E5#umtSv|CPbGq?%r1fsBf)WE{-aS^#<>syvfa=#tfv82%MhOl>9Nx z_?|_4kmS}7*GxP}Xj!GL@=&wdybCuM8bI|ja>a2vM)SVB!fwDT|29f*zgaw0))q!e z`z~oENzQ#G4|X_mi*D(;N~EP^6o#1zzFV%T5JB`0+&~>N#o_e%BB;^x>|9_hE{M!~<$z+M!GO?x_ z@D4Pv(nuF|ahvSW z*Z|EGW0d<9!%;ekk_O&cRX3XxcmL*G1=h3m;uX?mlgQ`Lr7U{4^|%s~(R!Adg-))S zAqI!##QF*|677rk>0r70vX+RbXN1oA(}XwY%mRZYc85*z82;Kf(x2!)*JurySlox_QQnHQb%&z16ycj{NyX5G_o-omMdQ(9-B=`F_cD9-$a&0rxOQI~djvjVD!H3^x+&L1bHi?03 z#<^c55Oa1UUxpzYTt<%EGn=HIV1Unzm4_1!pQ6vo)F++Gl7wY6**# z+m|1#SjpPazve`uno{SDqT;RJzDj=+VwXs{2cF3{$?7@9yjL&2t;`!g%IscrX!`!y z&AmP`WyAi;+~n$pGIjkHZPG-#=5zg-bUXf4dO?JYQQ`NACoeFGlo~=;@u$Rb^1cRB zyd#rzddc%zkpg1iMD)v&)u8T)`Hv6HVjjDopd`ZQ7{uu+M75&zy%)RjhX zs!~eHn^d?>Rz0CkOUWx>d*%y$xKb_HcX#BL~wOJgy7tl2r)_s=q=p=CI0_ z#uBW-E;i8<=u3LM-up)XpiVuxNU%vU=2Zk;7Kx-y=nh2l9cwE5*JaKiyAPOngau3F z+_E)R!CqM>REb`NOc#NFufW{fRwv+HP$)y`RCnO7%k<={lZ%6knQo+W(>+cOrTj!= z+QV$&-arjr6n&ee44)ThCJ!R5mBlUj;yw0DnR%QAbu@k9NMvDC&SnaSC;WXcgn>Qc zo1PzLA%RhQ)mN|k7AB69XCZq;c%cow0jh`PrfJGHLTbn_ObLrMSj4DpSNXO=UG((U zTGDW248Re4oKnAg|0d{mavl-R+=(o_}hg?YF_}N6V$?wYl2jsUh;d9~;8{^FczlZtJ zisi8o z{P;d&$v>Jt{De4mgWmo@(v?X9v?Tmbh1@z6lXK#roMeYJ4;E^qgKy9eHthMXL*_BGfB6OGX^b-u5Pi5jmTxWB)rkQJ~eU+eD>mQYW%suhMuL6&Y zQqhJO!0J$l1S2qW_Oxf9E;ylh#yR)2!1|6je3Ggs)9Hwa3lyAD11Cm z%m8RKe1JY@_I@S7BZcD82F2zKkVm5e_{6SpPC9gShewOQBz@$3l^h0n zvbYl2_V1utjJ0M3zR8`=y#Ofiy?|$v`l(1}3*ZX4fs@PHcJ=Z5M5E_Z$Kn5(Y5wtd zy7(ulA5I1d1oVQA#|UgRT3diZZ=<1$xU92hA4z(HZP=gkl+DYMFxH3t5U4N5!2OUm6Ei(HMHO*ZCO(n%AH zK)b&uB}$YYC5K6YCJ+X+^izyqe>@ocb_g> zI-ZRZ{31nuob9NN0ND=!$N%SdHX)NvL=Zq^HmI;~+NFQgN+@RjGYHUk%m5LEN10*e z=|`m%Kfkv8)aU;gsDJ(#WsQ)hvV5L6danDy|L=d~Q+|?lzT7G`*$U8Gse$gBZy2C%2cU0BWY9ddx&FtchC zz%om>WB#w%@c(@=V4eB{2E>pl^-mzRf_9zAHn|o$ zF2+iLn1m~&g{*b!)y{WLZM34qg!ZJtjsFhdIZ;uu4O z!r{Vhdm4OzZBY-c93Zy}jYsB^+HF8-jQ-yf6A+ZY5>XE3n#U$x3T=Vx+2~_^z1jQr z^m42i(Nw5d->^`X@uj8hSU~l^bK5QUV=L8gngF==jrgbi%e4@SK|suIlj%Lm(R{5< z^Ps{ThxWTV2xF^52K+}eAWEpUa|rlDE!U6?{a=gY4GIwu!2>3G;_8uSvAn*rN5Q&J z!@mA|deS~0vH;!AHidvFe(^k@-6urV^5~@Z6&cHYnzQs@+ah#s3Lpx~$BvGT!E8;R z{O z1*koi-BmaP&G|z@u*D5V%m4EoXr5O0x$^SmF)CPY5N{{`JqY5jqhu1(fqwbEjqAPQ z@_0abypMDUp!qYadI2V?>HGj;Dgm9fWn6s};lFR;{~i_Gek}z2!T3{*QZlfvO~8>YfpKBQb5EVcTE50(ZC5H!x_7(HUT1+kI+*!*1dC+ zV_QT>tuSIb2lrtG;=>m4W^kG;kIHGzBMgBqRvow6SWsN*l#Y)g;5GhFC15? zqWWv}0&M9%pluiyY0z5r0|G;vfc1e3W~=|hz4IVre#2lzmE)TD8&?>hXS_KVT(N>( zZT-)52P{DB*pBvg9qI?-cELgmBeDsxd9GpkqPK5yLpox6QXmr#!>(My2Q^z9qG!A= znyYB2QKFOxLo-KFoa8A?-Z1EoI?m04_s$ni;o18J3`vOT*A_{OXKM&^)gm~~&_Q!Z z?wfh^7BA~H9Nrph{NHbEv)RbFRqOuV!J^y{6*_B#WCD*Lm7rUh0B=do;NyFM(Fh2x z+D8NWJ(Bw$GXVFQUkL0!u<2-7z33I2-wfC+#6e`^8!E0F=Oi=HC3B{nM6LFZF$ zUj!!$C2PjrOb-&*Z5sU~PdAk|BQ{FxYcal7w9$@=Spfgsa%0N#CPFrVH(wsa09V4XK{oXR z>uw9?m8XY`O{S5jg3NEQ|F9+HC@;L40r9S`2D)~n)E~v;lejnNb+vK=D4~g4I7Rgr z;>9ctOX(M0?h+%TbK!mfGA(jsVyEki z^mi|k{tihlgvvnc(2+-vU`(WG;8W>?#evsaYcAMH%;h*=3^y z1D5u7Z>7dYW&A>jxE_|v8x5xW?qCLCMa-jY0e$TT)X(LxYfGil5<>MM$^gkEnpIF z-qaLQbtyT#wIiV=y!f1v>nZ7Cy3jHm#IYHqD~m-yRg^MaMCmIq^Ycxl9W(xP_s?f@ zc`Qq~7qRZniIuiLR$-D!YFkrbqfL!%sr#^}Xp@b8&z*1oj$u^PZrnV#%0?hna+vi; zS$P9JQB}ZB6-JE5QcXB2c%~p98yvzF{$I9qvHJ?oN)t2ja`PSRE|GYp)9wUK+E zc67e`rnnj3rbri^(>tcdVJ5GYrnulf^^~O6j^`NA1+$3E*$NQ)z~KN9*y4IaBq864Hax}n@7^j zwB1}%?vMI{OLS=fY*15Ur+?lxyvZU2bjpJO@u*6hskhAs9DKl(m0kd(>lNJGHFjtX zfP3|gCSIqx{;W#B}InCUKA0b`$=4S;uhGHM-rLSIEeG|nwPWU~SqyY}-ly

Qa8@tk$lSw8%Wsnku6;kA`c5qEXdtvHCb-aV-{2wuAxez_A?vQ8UQf1^0 zN)JCD%SOS|ewxO?^bPvI0#W8!06-M)muGFI?IU#wn{JX#oRRS40gTJ{Si zPPxbu5)i|yEn|MUA5Hk#3+_y+HxI%Flw*aYQ%Vb2ovI=^EgsUTOP#vI5JW`bjLK2< zY^U(k)5Tn9j;X1~p}lAA#`0xvWN{X}rqEb+xv6O5DM>mX+WHeiPd(6J#&RRzwR6H} z(BPhOTSlK0_NrbXN<_E5dHF%YtFC}vQS*NH*+^8(T|3x~pdXgn7wv5+l3$&p9y{8K z9zpXs76A&_o&cL7dugZh91yfK1Ax{asE#Pg5#Z7AC>>z(GMBYs6#=xJ<`I}xgHEFT zUok>eXoozW)*yLzkh_i@8bJtbYJDJ##vqhQM{X(PTR>nr1D%P(9nvh;D1-dY)Pi44eT?; zx07tDi0c>rZ6367G>fT!Y5^P%q+3R_!@s|An8HJ<@RfYp~#`G91@ zr~B+kVdv5efHN}D;AFbH(%w!tSD*Z=E9JPTMU$#QkUgX%jUw5ch zzEstQeblTWbJD7oD{ftAR9NB_SLu8>qcZf`t$;CZFiy#H3LA-#LAgGOse32PTptq2 zwNTh~78Hh}OcTl$-d1bB92~~iFLfL! z2*DmK-QPwPR7)vTnJGjgK4wnO9ED8L=}*~I+p6@b<;&>?^eq>reUcZF=y#jE;!wRD zc;~>d>yK&0h+MqEO{^p6c}T@!vix2*Yj!Sptb;eMi=Z!`q>RwEzSPvu{LO*jm4EJ~ zSm?3&UbSYy-e=JEEz^YT^hp=vu*)l(#Rdxzb(031kKWh8Wbu?MCGO9;E(LdIYQ9t6 zf~Y0@?1ffX%A%E$^J|usjPd;%w1@VzcI@vs<9El@5@Ap8crV;?tU82n`D$_6NYn52 z$Su-e~Gl*@&f+cUv&kXB1xVg#4%X{<0>h85&4oZ4R zBF64scHM?ofl_$5koGj0wFl9r9ge;p6*_E}C5TLc7*tmLueq3H-t@FfpB2rtPL>fK z36ZUU8t(f>jPfUrdiUpAF=HG)F6}RJ{g~-&>3dnbXN2b5YI!5PN)Ec25y#7Wx`W9T z^J)9g*ZF;4ZqB=by`9BEwKGh*N%gFNhRIi zG!7D)U&u2kNE|opLFTFoe#vZ3!aw&PgYlC4M_%((N`CP5&I3OPfWljDzUm)1Mm*gJ zYVfYX{Z=j@EK;VuU3IZwe#|+U)=ZKB2GAa%2v1kTi0;+eg(0)kxUPks(XZ}@V&f#c zZ+&@CcjCaq=1_O?sB?Xo=q=)UljZ^iAs%RssAZKf-giWs*0 zD|(&OUx;m`Ocu$8x zaJU$AoF3p}R;Az{-`Rqd7=dg)9%V!d=^x)Jicgtjs08M3!zYovsS?2*Rf^SUoR_=dg13bWbD zUvR&7U93i;UB;GIQm*z=U7bTP69qCyPxg>Yx;y_t`XxJ zY(`2Qi5&Mf>?ZtuPoCCsaM&Fb5kc_;Edbg<)|H*N&q5o-?U~(-PfbebRMi>afIiPQW#O~R|2tZ z33hQ)=x2n!%yWti+BIo85n8P&Gi?6O$kGiMr`dd)c+`&5+WybVRrCekXTrHCLJQdd zTq$*~P>&Nq;S(R%bFjR5Ag;l2he-OG$puXYtCmp_iP=<;9dr4n=fz~J9ppWXQ>#@Z z%;R!UeE*_p(AeiI?rz!Mq)xe(?SB8Hcm7~48V*~uZ+0gO4794-%&xThSU z&-6gr+<`m!QByL*qeVdZq(4%V&J;o)uxAE|{t>nGt)%^9Ci>*DpmML4!#rJru*f9M z`7ZpUc)t(b6B-}4J4BRb4<{S0Fl11S}d0z8g+@dKV-Co~uJW10}P zXm2wq+vSS|+Lg6zRxhn;Vx24wV5hHyo(Du756sd7T@ZYu#>o}p>EP2L0a#zH|4JXa zc|uc!gDOI;{Z;(X^pPN`eA5y8oOA2`HsDY!fs?m#57N|#?F|4NML_f_>N%c^wpMzahf$c#vmaNUYLj=&A|26Mwu= zVT)qO8yrr2iK~PANX595dJkneL4{nX#@@QFeHBvW&Lc(wPFJeLhv=J)oAz1-oq`rG zFI!@h`?wrO-nf*gEQT;BJH})hl3H>gt=;FM_GD~p5-zY^s`GnEIHg1sW~1>f6}$%} z^j#duwRGXF&uaVT&;>38fz3FAy|yEg>5W%rSS)5YUDMQkeNa}yu_p#*JzRMDL)5`? z?W2_->M%C(00C$`kADIgQ|Cd);oQ+x)F}LkWK#@zGA(wHI+x_r1`S0@B9Hw{H5%Cy zE+dLi=3mO>p9)6}Iq-PbYf}pCfRfx5b#lk!A}>uAeX`(TzjK7+)c;3i;&{3?UrD)Y zGzU|WciW`%wE*i9EYY@UIc=LcB_5by9^&%Hkk5Hofw_` z`Df`@r^%iAw(zo6EnyQ+9SSv8_jFy)UqYADxHMcf;P((}qhj$;)^~xh;GHijMG_I# zW{VYqM!S$yS8Oc3yd9_R^M%dxe9(tCMsRdne$G7YFP$7wqM*nqr4;`Ck z+*7+Q7d*Jc>W!}A+s{2{7j{Pp-(b&eTGlqsZ0ITeyj<%~%j}3g^9XOU-!E!uKe7GU z^{g(P-#CP)JWsOiYl&l5zhU@T$Rs6_|LuYv-xB3%-hL`SygnhV5ybunA?D?$g4EOY zVuNO{Hmd_paYdk-hAc*KiI{(Yc_BPqutw4Ly|ZyCA)S$k0~g+doMxsIIInfZS4@_X z&rQWN0(7!mxq3(o5!g&NJ>2YgyD}G`l+T}KsS5r&%|xzM&Tl-DvMTq@9u!L%nZ4hS z^Qf_(*6_5?r+W8rQyM+C6x?YT-`DN}PaG&NJ>rR7gCDBI$&+Dhl}WZ)!?;?+(g*`V zB1aWof9N?*(n4fW@{Vx5vGv^#v{a?J=Zj_AE!WBMwl30l;i40i&$L{PD}_lWNzF^O z6nL;RMo2uqqC|i1tEt5K*Hv1q41zn;bD>y|lCO3^sZJ;7u{%Mhg&+T)<;*5qfVDgg zBT8DxxB@6WoHsw{Tn*uk_{mMJD${2503OM8qJu!Ku10@yo9z!u*lS6G{5$v%-Dy6} zo_soP<&Q3-hL*HS{lE~Z7o#4UC|e$Wy3j!zw3Am@D=V#eF~GHLknf?IgSXvL>dwb0 zt#22VH*Q=+mXN~o1dtB3|K3B|bmvCqP4f3c5IFI=%yA{7m~9A~JHl3XWO@i$E>O3m z+ctmg8rHg96Z<2=c9l7P%(5g#e>u4r(V$#svnVkDA7!A4j%1k-p0sB#We=sLVKAPp z4v2GD0Fg|JbND0r(t`TKKX?Ce_SWJ8;C92@2p#JiQ&U{KS{L zufg~RvnIBQX3PekYzfzmRlw(J?2j9BnC9{WTTZ^G@={j{m@(PYt@`v-dJrjpH~RTw zOpImkHg}T~wn;ZH-etEYpL+vagNgA`QX3X6CdUV<`Ul^RUXI4*w{l(k4X(TE*C+7< z8y3cYSM^2+^pU$}QoEB%0waUUimLNNadiKz_f^zTD5b0HvytlKP4&Sjv~peWy*Y=4kd*NZdFZ#$pj3Y1+=mRz>4aS6L26)7|K zC?b9TVk~%vY*g6gUunL1nJj-1wmuBehdjrhhNSTA`_`Mx|G5kQP!uhhWG!scB$OvF zPIN>&UiJ4u@@FiBes&>1x+Tzi*>fYDaISWDY&ACLa!p5*a@RP$2orP0i;l>47X_R7}Vmp~yF)s_7E^0V7v zOoTbgd!=HHx47}kx&#i57Um9)nlXfRg_sJRq2?EH+FQhFty$|~=7%qB0F+hP8{~{K zURmN0qL;3C)zZfqDDiOQmX<61W#8ns1=D$tCI(W@Gz^oAqu|wMo`ydW9M&d|IC>Zr zoaOw}Gw>uMAN(+(3QE`>0=PA@jEGIuWZN8qwnsh>SeDVX7_9C-T2)JJ558Y9D4NGP z6+&7 zd}kw@`JiSU(&@@nqm%Hsqsnmu7gpzhEWauFyTBbl+E@+ny$McqYY6z~ek=296WdC2 z1^(pz!I#Zg$;YZn@Zv%(KpR_fKm*f6%}s(_PhH2Xd3iPc0@}0FT$r}g%d9I(uC}yg zWnz8--PJR;?FakXM`n>0o$ zb(uLoB9$6N{_ve=--_o?@6K=+Z>P5#)ySE6U(S2RMxtCaWp2}$g1$^fHGPPpfmOTc zXF*(=K(A#?GA6#G)}esEGQH$)Maur8MIqKi=n-fcW;xn+0eTYxJJ_NR3(t(%ai%S6|$%1?1|dAQ|B- z>gI=^3eSv8MHszK+i#8~f~OBYhi`ldukyKjq1$GXUozVu5heesQ8=a8XAytU%QyA= zoBGqvTBG0SRDar)upY)_w9pF5p4lY3ycK}IGdz>$nt7}~oOlzWH+a+ISkg#&)JYN%pE_ZEAv?Vg&hG4D zeYg0gf|Xt(_3p~ybTDIK2nU()2r;DM=-HW;w@5x2p2A1A6NOxik8cC-rw7$Z#A!P( zXlz@o*Ly$2aSA8FeVFdxrQR2SpR+xbIvgE=Qn>x(VA(}aMuMf?wdLlLJI9seEWJH^JfK zZeDG~ATc}tLlebv^L4Og+JHi4$$sH@&q+xZuIJkB)_I#j;c0LuKqW^bNt6A0d{Q!~ ze{3Jx-TPHVMlD62>%3MB0s{4<1 z^L3d;0LM)lu;}2^N?e&WCtmA67QPHx{GaugZw=epYV~`Mo)RfE9X*VXAGX{e8H)!m%^~U68?@>xx~~)3 z&nA2$86)W)zT{PY&R`e&+X?FnC4d4)_4%+tStmHM*Nja6l|;3ym-!0l{C=1xLk3$t=fKlbGq2O>{I!ds(ENEHts6 z^iX38)>u}HvLADFMBy%dCzh0N@V=m75st2XwPqKYmoF^?_n3DIA1B#KJ^ zE97(;Fk12f%+~3yLP1vlxuX40MXe6{^o}hs;BDEdl!sT1uA1|MPAsmi z_VjjFY9opl@@iWegK**n((P2Jx-J$o6*UC>C}4;4cp_MZ*31j;&^T>Bo!vQCpq6yb z9BiJ~g#?tAb?Tcccniqlj*oVvjcwjbcwa-~HzS?`OoHRF0vz ztH(`}keFT0kjOzON3vm9nX-w98Ws!7!gu!$z*P2yYmOV)gYR<0cx}E6vNAtLYNA8PI#B zGtbNWoaywDyi372jR3~QVk3u$fXqX~#4?wnNILzQNZy8II&S`+42a&c-K0*}Epskh z_`~V?!%7&!kUn21xa`{xL-+Ry{LcG~ilwNpYnc+NJKeA-nnmCH)=rguY~7e$xi_!H z(1_-&`pw3OvbJ2k#BrBKNFND+2@X(e2#6xIXZF06Ail2#mY`6g+RexVa>$1_>k#Jc zvXgBiI^jK{=2%CBroQvTB`N=d_+yFfwSToY{?)Vbcb`U}%#-En9k89!R{ zi=f?#Z&gB%nUaAIafjW|d6X_P@Smvb``6F16@_Y*HI@w5at541+^sgl6lx8VBqQkS zg#)HH3~?UZ#Kp!VZ^~NoTMVf#5o^O2J}RFEufj)JPo9r@-bv6-Ko1{A!9RW)l@|#1 zMHhpuQSg0#PfnUoWQ~lj$Ht$G8^T@&H^8qF#)G*gI$2$)B&Ic8^*%rQ;hcfD$9AWf?dkexjJ&=PEPk%FtQk{X%yWK?cTR zzOCe8j}?5Xbo<*B^RLk?M0Aa`Mw_I&P&jLN1XTaykz)#Q*tU-vi?uAePtNSL6tF8k z<(S%^-S>(x*!ENQuU1IE@ha?q<1Y^%R*2V7SBDVPjf%FLX&)sYjS`L%| zSBxr~o^R3Fv8k^_Vz1x_^+SS+DW^EuW#n>1VI5}U>XTvXFd+XhQxP4k3t4~C&cLN)b>p* zyXU$f)fk^J&sU9z4dOl0J$KEF-Y>Tc1M`rFBu3US2f;Bg;r(M&it@c>Icmje#fIKm(BI2-`lpMgW}g8xA;NU zooK)Bs0uJusG6IJoyW6%Q}Ohi5rJftuvqZ3JPufAd1ZgCvmGdVV9)!*Gd5;F=Wk}- zD3;6vmlo4Nz{oXBXn_F5JLe&m`iA-T`j(0A<3qfqeERr!2i~u3a*Kg#=EiuSZI*}LQfdyX)F_sr& z4uu!X_Pi*@v&&AS3=VMLs;+56zG1Rs1!K>94r5@MFO2SLX_Ghqs%_o#3hA3we9fh5 z5??rk*tWD2^bmpv zcM0z9E(xCCZV3|H-7Po-_YmCO-8I<3f)iX1?(#ReZ{Ez@ng31|RrEPkG~Iizy?lM2 z!Nli$hav{l1Cnqaq^|w2J@uFc41)qSno3#hw)BfIqel7oDxekW=5(wQQDDI8y*g3oU2>emT%_Xam`kk6{)LhghY;9O@e*uSLl z^|M9d#*&qNjHT3Xpd2oAPdxK7vQ&H`3G56xG@mbSTeh{v^>UKoxriAoGbg_40ElV% zE^irU1SZthHEO_b$9}%qIEgg5y$ESCDTGu~!KFfK-L#VLN5OZd9p}5H)7obEUgy5ST!1KvgkFc#Icxk>9y)CV(>V=yaOOSbAfTF zNdTddMunE*G|zvivkEBg4L)~e;_+}B4?Wkl&$fQrJI>aaJu?u7K6kGl{HpGCm0L^W zvcMU98?dg?zMLwS570h+%gltBHC0q7ZjEsTlL4K6gyp0tELhRP*Dz_B+s9faVgJ*6 zY!rV%)Ij+GuV3GN_bqP*ThmlbncT=&>Z-ciSTg*u#E=XZuJ6LHSu<%E0RXVZFy`|6ASIi*x{B@CBEu~u#x zkiN8+Y3XSiOd~n~ItEnS|%QobIipOyqUJJ9>TYmy6V3u!`}!3qf(^(56_%A-Wo z&T8>V?-JM(9+9wG{gq;hl;u_!rM)uh)vei9q|lbqP@y8o957aQ_w}u`n$qtb#Kij8 zQ)zLz8Ip$iS%NZO30UJYCwEY%m4a8ra2jNnJAd>NT-n0i$wL0gfxp{ie-Lq@fB?L? zu2NkW2q>5-&(*xxRQ2VvuNA1nHgYNAW>X3d#9ycLbR&1(_#*?Lp{B00XeX=jA#@u* z$nkfYHx`>{0>xS3V}`tvA4g@3?+d5iUkOlR%(^FC`Klj;XSGxVJ?@4Y&Mv3~Ambmo zV27w(lB|||g-ph0rZ^9PMaee!bi&;k4z(Ms|>5|_MrKIexEYQQv<<)PDlY5fpzD~Q<-%Z0kr-TX|qA)p}C>choGu*h-aDC(SNW81eiiFHJVO*IZp6 zsEnLxqt0nD@9%g-&V65;&x{Rj7s*q+^IcE%-Yq!QD^FM>W~}I{q_U?ks4mdg8Y%MR zoBF2qxF}Mlx}0|G6JdhJQs0_(yqe=O&02@_V`6=1nBYcUPbs6hQry&2_~*WI+z0;! z5dY)HN}yxyi=6cJT2#;M#7*#QqWhQ`TK$bfI=CSnZV`ux$^J*4IgRz+!nF8TWy*qp zKjz~F)OA0XQ|06<{_9;s%+`*B_rzf;WNSh*UU{c{Q!vR<%A<|~^iQp{HQYe+yT>>> z#=wSih8g=;pDaLiC#lBG0ZD?vsSBbl&kk|9%%nixYcGWE#db-Ly1909LY^Gm50RJw zIwB*F4k?VXcPB}b@*lrO=4X1ji2~;fs$Ao1ycD5h+%tggE!Y?Bu8e~ zvIS7W?V#s=xEdgj3a<~VMbe#tgPXPFJKgpQJsW!(3xK{=hx(`>v~;(}mk?XO9#nyQJw zusy{r?T7g>Hr<-L-K>4u80xskC znBI*1;}8B4I#?UYu7-)(g`zQ|g{k8WQBEZNhnedi6|E>465jcvn4*pK*2)?YU=N15 zixXrMIv1h-Dx-L+JCi*RqLsQlWtFTBIj4dljk{CK!`$>&5s```zYJXjpE??Y2Y~4A zTWtJ*ZkB}w?)QjST3*2^Im}i&Sx`}v%1AsWdMads;@|ig+BxRCoX&ku_;V;-D9$vx zybLmwzyg#L9EMFZMu%r`lv3JDgSX)XC4zS$ujPXo_Eo=m7v2LjzYokeV6+R2!{75n zTP8G0S1-@~-3Pep9jiwfxU=YEe0j=9V^tIv&gJsXwidAsrmY1V3#$h-mkTJrldL#W zj_@D#T$Oz4!`jD_t0~WXV7*8;!Mpp|9Z;?5x+$w-lo03|HNp43|Dx>5*IY42@U`EPAeAzMvkA$~DE3-Xxvz^I= zK`Hyy=>srmbOX@E=4dx=Sc(5@_TX)k5Z3dsn8-VuIaq9M{qv;0x_%;e*-`HG*YuJA zG`$r-tvJ8aWbaD^mdDek^gRaYPmX^@BmaR!Mj&*g!2Wm11EjJ7!D7BnD_RQIz!A+3 ze=YcA5%VV4fFoZ2<;!MOU5(*|spT;UUkj8;DfOH4Fy?7grdWjO)GyvZf1UD8biL;9 zGvjzMTr%3o!pb2`?e;HrWBtcIF161_frq~&W*?Kx$m{3no(hHLy1HV13py_;GucM< zMOPELn3Ah$MDQ|OqIEnbzB9F?JG}w+da$sck+9$)9Iaj@BmdiBc1J)3xQ)^%g|?{8*>uJS$uKvzMA^YM+YO5_R+ng%Z}3ksCnA z@eCI}W<$z-b?{3`v(CcIbwWOwRpR`#Da4!U? zx$=*lsz!eOG5Szt(D(!W)^PJ3ioFNwMl)A2o^fnhU6){fS)Z78XAVruZyhF5CzFsS z1CIn&PhuZ#lrh^D(?*5oMI*U*q$qdhqx68Obyzl<^lS~jNpHd)0%j4v%LaeSLI9IS zQ0*E!81POLfHOTWlc_w;NP`K;08)$7jTg@|C;Z19(5C<>zq%hg+^<@5HGpOLG4|X= zgT)gjl4lDiF`vTCPd^+1F2n*D{dpd!QL6-;C63RhM}Iz+To)KnZ!_lR)70W&XI6J0^?WEWE5_D+dUB+xA2 zixXbJA#nKn$fdL80ThY*b>gS!XQrxWWOIN$yDEWB4cWx$yFG!ne7Z1D7|8MRbS8@Y z1pE7Bz2BWT0N#neSgW4e`xaoi)9$LPUIA#-`pknCfWUY1ollno-ddH}|8m%Y^ZwV* z0LjbiW5K5-yPLDHYW~(sQ(cuW1?&VhHlPkZJi~VIsWNSN1-*$t9uPDg8YD}E zp83TO&y?W+QA$bLPk<+`I=~gwc`S*{Yy5!YSJFLv10b)5a{97Z&)2*!QjH+y?W3VE zGTF$L-!Sw19YQr!T;7Ivh-k@fJ|k-*_guJ|I0ab08ulc(Y=H71$sx=+LoT!h|yn7kh)qu1~d^@rQvx=1A< zFP!kPiGFqHzP`_>?s8C!GTZp-ga@M@N80be?|PKELLv>*S z#shG99SYut7SfOU6T=_|M{-}`3;FI6yf8ef5}1<-7Y0OkbZw?nv!o4q@$G5zt&*LvGnIp zuCyq2u>7b2cUh3Q82Ntc)z}qp$Y@syx^j}$WKM`Vcd=^eH-PuBVaTscxZq%+zPtsP zVP=sqJAXOW26#rA@)~v`Y&H(bDJu#t>vOD>cJ2b!s00M* zW0QlK8g>&`jd`}F@0FW~h;K3X2OR5WR&kRB5+BG?^X!nhWuiz0WFkAbb;ud;YhPn@ zn0(CiiCZJB;QJ?WCh>g(I#}L!qK&9GUx}>U0E%;c#a)&!LE1Lrp18L1VTuxFuc=G| z%>-o4iwttLYE0L72%o9{FstUPJEdOs484Sg%b{2#>$w72^{MbV8!+lqn-tXU09za_ z!`RsK%HMKtR-?}K%$RPruDYZlp;)DuQm4tm)-m&4(oAb^Wwnxx6GvVti9>@Wx85K=ATGH=!}(*Pk5mzhCL!7jgVh@BB1?_J5Jb4|ghoKNJV~CiuzF zsU@Kr`gc_8&P679+N-pGoD#LWxVqnLLV^c@3jf6y?J+bcc#B9gP&-er|NUNG!Z`b-Z0UndREA{gr=kwHS7?IWD zGLz1sLJIqPFMyr88GoNJV+Td(zT%m#Sp|7)4#6iAp(k1^*c{PlY%qzyL+z23u{9?576;BLE?c4(038ipL10=tAL>co)7iJnh2!P z)kI)7Y6zg@^E{+i2^Z@uwbs2rN)J643Ij1xL^RGzj)_#*mA3g0fhdC4-_6;2mH;PU z2{0bS5kd6qLyk0TD#GKfel>b-r z_xJ17;0}btFGIl8m`r)XbgoV}f1z6Zj~e30Uo8rwufQT=uIo8dg-TM(UU!xu+e=M= z67_{=+k55>PJ_zX?ZJBU#^qER&><9MiXnATynEDY_5kswQ|SWTt_87SDfsmC^b=4i zA0?Z$n=R8Rt=K5*GSn6suuacCoG3w&!7gEO{SIgyx7vn%H$A6K_03DCHlVcoDF_I| zMr@3k{{F!KF|Gcu@1tM)reWp*1IVB-uM+8a+Eaj?L+ghDza>J`xM@M%~E zQ;Oi;QLyG;G3+sF)(kYrG|dN(w?8wSX9C)>Nx4XQP$jRNWW|9~BU8{*yBn~O5dgPq zze*6MPLd?Dr8Z(APRk)nqe#6A2H|*JwE5uUVY`)aL&stNr{Xi3-%? z)S|Ncd$9**{=D?=JwQ1Vc;BC#0&KH`!1-i&PCHpeGAg9=PQS8zm^UxSwHHneSO*AC zhPwh0vn`Xb00x(BTFzSkS9zz*^LhXW&W_PLN&-iK-b!bk@d(p}j(oP{Or1oOFACOR z-uv#5YgX|h%9RTdXO7;)>T{hXmN4Fl6YibAWP+NT(l2HWLeN zgOD;jFMm}5-mvKc5+3{UMq{Ma1{ClllFxZyYAdJz&w2f~gxDek(8{*68w6VXYLk-R zRhCJaEyJl_|BmlyFkww?4dz}&K!ljJL_6OE;aeRC%sc`p9}FkA4eOEi9F<-VZnI4* zm@XpwKiKD`)$Ty9rCK+qn-^qbyy~A*MLs#b0t-D}X9G#`*{?)>_=Zk*mxrE&aQu16 zO7qyfN8(UsRKf`QAf$7s2GIYp$(nFmjP=noj(6Ju zTx}{qs7gRs$tb(NXq??aM97Q{h#xvSvu6;EELBh6tADT4|9%n80+r<=>&@9J?4?^l z!@G0Xd^uz;R2JusUmuNORvx8I> z3mo(VekKCUzXvt{wo1kS?!7|;YMyn= zi6pL&mq*QOL{Q?W;vyr3HmNWAv)&BhiRVZCXJ-aon50H=IoUYI%m5Fnzs_<&JJ{^h zFQ`K+OB7bEHBr~?y=sa28tqy0x3Wt#-j-+WZ4iOWCh0Kdpaxr81)QE%YPuOmrX9d< z1Btrr8?*r68#_ElCEPEN{Q$ke4CohMw5tCI2L2rt{Otl1AxzzME)ukMuwX*{y7C=% z-rC#L5)oNX!1jd`jaAOO0O?Plbly?`jJAR?wFtBx%w&2E02Y_Xp*O;Lk%8#PJ>g%{ePT zv$#85ocL_+(F*~}SSGhroLPWSd=r?ek8Xc@fMgh?1ehr|-5oWT67-J}j(Y*xdIO+X zRFj36xloOz^KoVAXOvx*H|^1GN0)8_RD6Q~lF}ND9-z+`9cOh;0a|3&y3b-S`G>V? zzii?Sc|V>JPn;R=XH1t7K9A`~v%xT%NpXP%bby-iKkol?MwbkVt3^u~?NsiB0v_b4qQ+<1C`*m1C>zx@70@Cq!&UcFg9X$>~Ri`b9DT zh~?5Hs#^??X(^KXHwb{v`k|YKxqloaIdG{y)9h^hezPfL8hQ9HjUwi1vhBK7@V7_& z+sCTdjy%g|0q6zA9fmVy=_-;q#pNJS(GU=DqFtBd0T1jV`>a|_$_-tW0h`x7S!+Jq zVA&Dt0`P~t+B^f%h!$DZtO6{n?;Qm|p_fbV2ZThkSQD%fB^clFyDG`^}pH z0QjfW`Rg(&J5X#YXBHDAr?>=YRZCkYf!barf8^RP#F+Z`v6Qy09FA3;tMmzD zoZLL_AbWG-|MFCsq2Pke5Lmv|2duLkIwmD}n#r<8)H!xOL)LV@v$y{9KER-+@KXl$ z2l#$}(tq4$65nWq0~Ts9yho&2>>ThC(dGP{e}Hm$#5>*$rSjz2@4(m<{O?ZGe{5l| ze8+bgnVHR`Q98YO=pp5;!i{QSl=TSX9hO53O6NaZ9q7^>>>-5#M6@Vj|E(3~-%IoF z3n+{~Tla&NP?{PdX!&Je4PMQj@ZctR+PO5i(GVSfhlP;RpzcqfM59E<28-ow*+6Ob zZ_|HX`gbJuFqcOcmzc(pyW{9=?ar1|u4{LjRarLm=ZA_i^PD4Rk^DL6e=Wjl74Gvo zQj@~q3^fO&y(5DAXK|ARCGqFU1Qy5!Q_2$O*l5o;8%Tan5_JU|@sjWSLU{h;Pj6qk z$(n4Mt$qLZoBppCc3*vH`W?PD)dZjq@qzi}h`?|1*gUc3i2vLK1#pvo*)wvZ|D)LY zE>Zx6n9(`{aI$2vyR29=wF)g2$E9qx(I0?0n+{ZcY6$d}@|o~IubC)`Vq3VHb*Xxp z1BgK?<+zSW6TEBw4rnH=cequr{`~w91F%2^cr>uOhvlzc{{K{m|6N&rV)F&_%_eXn zoob8m=WFTAx<$v2+sVS?iWlAObvY!n^0?v@5^?^k0Xjz*$RyHq7^?i^d%i?hgjsIR zi)~@{YP48xlOj=ew*2^(osp+-XRnK0AWy8BXb$yx-8LwI#oyjATmSygFI*Kad?EdL zBM2ik=d<$b&2+31i?~1H`n};{Q9P<1KRk6ZAcG5)OZ600$_CH|s z{`(UD*QNgy4p^-4Ecr#Wu2CR&`_AijH@_!d!~f6QkTAXsI*kkq#{cKlct?^AW5*g- ze*~-cQPlOhVd+gSo7+t_8{iCnlNEj{qaN%ZGWtJEiLV8fB3Z!h$6UKpX8zf}HqSf$ z&;J@KK9&MD`LV&4>HpZ|cksaD%EM1b&s!|#7{6nv{OfSuz+lnqZu7o#lWn%K`Dfrz zADGa1Vfk0!CTxI{WV2Kcp%vMj@~LJ0O>5HrU%Cy?*J#Y+BOTgAI~_l!`hRG)|F7PA zjvrrvpW8lXESKK)*B8_LG*^>d5ZTeSd8?8nk1x#mXPe<)0Ou9HSSM1c^VD7b?uHh&O%B|Opd?oy+c zR;7d*dW>HMZ>AjIqjn7$_Z8zS|$h{x#rY8SnAO_?>vgu>0*`UH-kavJ6@S#pM zV=NBWFMHr#h8Oz>N5R5$ul~@>)QyXYIpPu9HfI_SrkhFHE3-EJ?ksp6Ab2(2nlVdU z1m&@)I-AD!me0Fw=FOImH|W^q1jSs)oR&{Rq}GrTq}c4l{z+fx3su>mJbyohJ4*9H z8zT1Jj;Kz0wkVUXg+tt^*$Z#Uz~GnNH7n>9hLR!&l|b;zCg@t z6^;bkp;KHDBfx)Dst$D8Em*Y}PFgFI5ojA0^#39-9lKD!u(kjg~x~pGy_y_k%Ou z&*qT@2ou*&5m-_1JtE4A39PRXjxPx9UBU%#*Oq-9XXuOedMcCt=B09eAMhPoNguSY-Y^@rWRE$a;1#7CxbMdfX^jm)25g&@-B(waAH@!32Bc4A=C0~GQqSg#bKRu>rV5jUz z?nTEKygn5mN9BMTtkRW2g25ioLtVYP+&HkRzl-1Q-yon`q?IJeK;xdZNlvifkt;MD zJ6T2ujEmXJ5Dn+^GYj11P6{{HuxxF8oM)b_=IkgyPrzR=uL6RI(kaB3iLw zAy6NRW)t}W*_Qi)MgT@vT@MXaI{U(1VP+x>pH3UPtEn1IWP)}*W_ed8b~BDK^)&K3m>Ur| zgUnFxBYix+(hQ~!|1>8Dy>U2{MOmpYh^R?O#bp2<}`6R0r^bw1web_Rwpe@PJ0OE2pvqHY&@-#ZQ>_dwU@@2Nl?;q|iEct2-iH@4RT zG>mu&YPcEx{M&HhE|Lzv-GE(M<2Yb%fqQh2LeW+)(Y#{#65MOc3cTGhQ2csGCt{$f z{XU(q<}5=X#r&7z_|IR$hU7L9S+4Zja?O%Qrm}Q6r{lfs=CaK*589}I{>@+*a3W*t ztrtXxrKN`p3JV86D1|j+exMK}GB|4ceU8hk^5H!doxDHueV-+j6oEKC-=bx6RPq6N%2^ zm@iCI6ItZ_;U7ZpoTkbZn55scWDe5{sfb)$Ao-pXp4q|b=|HW;=F=9IwF#n*j*R3G z`I)3EI!kPt2^8quo$tM8Ucw3=O)t*{tC$pgJn!6Po7S8x(>X2tE@QGsYg5@Ru98Q* z4tsISVN6EIv^o*R;_qfCK~p$myBS28r?q}EcY@(?W4AxJXB7yWLi2gm@p{LW_k&mH z=_(L8=~ioY*K@XD{+VVIlqS22SA;yO(n&@OwdVOh zn;+b+C>jdCpyOEmYSS7)QJELmf*#THbh`lcmF*^LwIpUx=l97Z%6{JWt*K$hP)Hh! z3}vyYYm!V=<8vCxElWQuG`TYDx;K_e>Xsgs|6&P`)$zjKQPQm`FGh>L52US5@yhjr zFXHnZ3!D>#&(!cGqr^CuWE1@|L*#oXi*f?c|eB#D^0w26&oX)IUMx!dV%a`_)nzKSqhviij z#kwU=l7+!mg{2awvA-rM# zu1EZluxN6)qPb~0-)U0{AF}P7k@|o)e1n$cUTY=3keQYVNc_FTYN%+(g04q#qr^-) zQoZzL1%bv~M_!8nOlp|%^MN1{L5rRZg|URIUXny}u{!zK&ydMy^hyp-Xx`S1A|QI> zqJFs^lhdQ+3*i#8{i}#ZD;Ngd+sOChZ>&5`dI2OqndM1NabqP;qc4<_^M&5FcYOm|+#NkcCTIA>ZAva;#FYANQ9g*5FrCTv-^lVF)UAD!*i%p)hx|T-QZ4!t( zjfr~LXOXUv>dm%vKa*H}^Aw1zSEtzOrA*R>+GWl2tnj1FpEjp`LYCJ8ozxM`C}K_!7irzWdiW@ z>o4HnZamUscX7K4WbiotsOwv(U61A1H+$-wxp}esctn?(A0F<|OrPL#uZtCS^9?lh%V)kB8u9sUC=a>ANJzV%J-&{aUS-R5UY$>FH@Oqpge4 zimzD0jaK|J&eb(G5tiTXTypPKigUf~)fb&E-e#jKL8IC69NTQOvTMp$64*T=)7yHk zN)&8GiM83*f|nYHsD3Q%#E`=Rl&)R|Lmk~VHvyfdH}@U(P23<(j{)HnHX~L8OB4yK zE3<-IGS8#GL^JH>LqCou{%(Di>cK8J zj~&)O7Vg<}NCHkn*L64q*S9Xb<5q5A=QF8k1=t6gb9~0@Dmy*}tLnoYi}qqlV1?ny z{;);hBsEI=w8}mwLZ^!9ONWLFzW15w(?4*_@=MuF708kC;3SK9oR@KO5FH)!n4z2m zx$47ymQiJT^5iJueHIQojT&ot=iUovQZiW15(`>?JL5@;m_@1|a;`NHbpj$q3y>Jk zL6U5~BK&Tm=Mhm+iI42cVtn86xsm%mJ4FkHm?10PoNN9S$Bf0CXvL(yl-IFU$(^AQ z`I>dQx*DhDLg5=}a)GTIg&B+SEjJ^t2SN4JXt>n<{IRrLB&|BIb*4uoGgU8J;|*U# zqLbB9# z27hZQy7?9x6XOK(ToAgne~e5QEgbUKhxg$|H~MKK z1uLGkv&8aqV~`e`>(JR6-4?ZebFE}1z@ma8eCFforO^qR=v6`k;&z6*g5Gf%Df`$E z{L3cWk&u*Q4o=Wo!IGmALIK2hE7wT5SPIw>?-jjuuDY3Q6|m64WMe#b6v>%H<% zCIw5zxN-C;2s-=%V_d$o{*>^kW>MBuEz__8^*NJ0C4?XHQp0?95d_PjId zhZ^^Nd|k@U#ac-=*8z(2g;?I)o4oJ$IUu^fiAp?U%48nyE`N;3Ru6g;yc2k((Q!7>(mTd=>c5z`1h;iRqnP~<6kf| z&AK2g_Z@y|uj1kYL*Hdf>SJHyVzwo(WjBgm%LPhS$TE{6Kh5025y}oCVa+2}MeX{3 z6Ee#oruN7V{L!LatYN`3Y6>@nY~Hg!&rlUaB|Fl^R1~wSgT7dYws)IL_%!Mlo1X!fUqf@+cT#E~%~gCLR94vS^7I64$zLf1kF4nGRv(!OT~@|(hAHb5n{@c^UlZ2j&!ul zlF3IWjQ-AeqqozlbE}b6UsI->D6XfHQFX* z$ThqD%FhecetmtCo?pFh+!n zow7h!)8}ySxENY1wX?tCojy8KlQf9OQIeE#o#|fCK+W`+E zo9S(i7qXrGH*ty8>3XNhv+|-qZo5>kn{|mP0g?Xt#e`1?!{!oIHDL=AoTv>_*VL*L zXlwYqA7f|TdnrpSf>HuV^dFJf*!3p+Zjli>AyrpG2nJZ-r0z!|QKM_|Kom2b)$pn2 zOQLa)5}qXPxk{TwOP}I!kB!XSxW_FO3xDcUUlD zNu;rfE#0%N@~GzzTMA=)B8N?o6l;BeG~6?+pt{qrr|K%v+&+cHU)mpR^;m{|#sZXx zD$OBYu~D+1%F3I^)mFra-u~X!XSp>U__c@k(}WiJ4Gu!r%Dz+z3N}t=|DG<#SjVeZ z9*b(qDc(lR$=3X!{IfJbW?KpS_L)gdQ@y)Kd{)#w$U9m6Ep~;y;Mo$^{w=awaqM-+ zuOWV1W64>B8SmeR=!wPao*unD(k+uA4@UX`%8MF5*o}=?vT6!;gmQ&sd1vbSuB6da zf&lr9bgR%E61D4b9vRME*!qd&(N2PVMy~y}@1bSJusQChC;y-lj!-SXrR}J zc*T{s+3K=b?<{+?`DAa#y|z-e%>)L=w?|w!*ZJUwc8WHdZ1K5%D>%_=Vwu^A3NbIj zl)IgYp*SmILAuxMlDDX+41+3!Nhv>)ut3HO>*8=JIWK&lob#%Sn9T7aamRecUCzR* zK81C@@rvSMH6JoT?Bx|@54TRe`D+huzGAxMmx|Nmfjw*FTm+peqp1DkS6A{Ec;}5g zxCSSU5XHATR$ppKCmQV&w*6Mg-*zRYPeGhyjasVp=aJLrdWMOEE7$zYKg0Zn_eDyr z*Gfue`P?Ew9U%^Vca^KPU7Me%eSF7naV#vy%%CP{H>WRmj6m8DLlQ6!3+{6%KRP=d z@iQ~k72!^gk2gb0+U3WAHLIe5+{d=&{=i=U@O_8GC{~?O!W3=>{tqNjS2!bZpGqcyEWv+ z;^4KG=p7bA1I@VPqW+V`)>OzmKOJI)GDN?!ueZk|>7|g1kj`gLdlW3}Wjk_AXS&z1 zT2Ha=z0Jf8@M4G;QcaAG*bg&2?HFtNk@D9%9rG9^z6Coz73XAN&`CN5;;p*6#y5n7 zN}`5sr!Wbsk*ErNKaTy}+2u)OZxNZxvIQ&3UwH3J?v$-!!4ss_`i*B}C)l*L|bd?v+rp7RqTnTE(-=mHlMdY-5N#a7kQ zxyMba5|8afP`5>FQDcz1!KeKl4c5BtWzGA#gDucX%J|korF$j~5~ol4tGp!~Hm`NR zo8HK(D$Rh&dHZPu8=CBu7MW@D&MbepuW|<-!{in=V$o*owXchDVibLugTvZ5t(v;0 zoPRcIRu)>AXkDw3gslSQt`gv65RVLYvu$YKn|Xxw9VO_mgd$zuD?q z(ExA7Ll_c{%8v^R1|%tk=d+)s8WPdm*bj|DH~27kZ3}#~-@e2xdX4jLF1jZ0!Nuqz z%!il|uhlXmOXmn<9XuWf=2G?MPUZ79XsI|qvl1~WiYsx{P?Wuu{7uuc!@WvmQ z=xYY7g2L}tJ@+4*t*9ELQ#fr2a&t8#U{A*LB{R~O-5=7ot zJ4`^n5MpB?%VDS-Zv1!c+&xRP_W@{t=y=NVL7zS-Adi;>wr6GD@2y9Rn#F-gFBgc9 zvQv-lpam{k=9zvJZG}7R%e*jWTOaa1|zpqv&G3MVlUddHdbIfSi6bsnD-79jV8)oTlf_OV7lGP zca@~cMnsg(+nlm7bdPv_w= zw4;^!M4I{0k_MWXFkkiXWN{oOMp1&;ANNF2W?c@>%c2`1g!fyt9K}(1x&hdieHb+JlX8feB8?#7Y<-D&)Y#$Z#ScBy}xC>y}%d)tv&Jn zyeieEE#Z(&=T?o3be;Rf&(_nm&0|~8`EKx1%g=^B$)-jrU3tfF9orfJPTxI(;@XXPIB302drqRC~$Slr=Vg&*57!c zn?8WU3<2`LZ_jJ4sA^vBU8PPV(^%gK5aTX$1K&7mwx!O0)r4Y(=&=Po6VdXTKC9Zy4j2~9v8@R+zUu0O zy3tMb_8qqDtZu<PV5H5k||cNF~m5Pz) zm2J0Wyvi~>8VxTQw^ZIOxJ>W#Tfs;UMmKjdpeLRW89gB`L1WH;6Xo8Q*I@uP!7emV z;`WQ8z%I+xHmtSkq`eo?LtVc57m!Gu^b; zSCyFHO5wF9867{hrqXa%kjowOmrZuTBVBJ3F3+ca>jhRN?Kr9IAV&!b47`FVBu`Hxph=S##4*_O>iz3)3O9r|0)&X>yh7UC*C#}QM|LsjP^ zGb7~TGjdLB9a5Qwij|sZFfS=w3q!zQH`ZU4KC4EA@T{MPAaAt073Ig)=AJFZznl{2 zmlmhPId{zQ>k(@Q=3epgn=i{^q2wfQDD)<2lZuh@)FCm}Z@NVYm1K6!Y$xPgGs|eh7Tb4S~`8)Nqj8w2)1J$IH@NtXnF!Fqj zV?MJ;5$73F>?CdVWX#Oq4Vl;rQIw=5Vu3F%pI}|$5b<=gsO3zV$Ovbw(?{Q9$)P_0 z%*-J!#3k_S7_x!mEGAv=Xo|{v>vqlg^7LA`^k;xVY64x z*F(?C;g!H+W1+820X_&R^KQvsv|0+M<3y^G>W$1je-LOSWhT?>UdF?(7JbB(4^1|c zT1cv2x>$=Xv^iPnHnodDA4VB8U>-<|)DIN@V0~}R^9mylEwNH*vI;!o!WvSUO0!%d zws=Nb02A3!EMe5xE9=}gUL&ftD%5nZ9H@#}QyAfjijc(;U?KVKK?XlF&^>cL|B%F7_6tOxjz)Lvb#MCQxY5rUQ72HH8j z&GX;5mUG3Ve&XL5V08})2)_uHVC{u`yi{xYvPAilzq#1>i&nz?SFIRQSyet#*ESz( zb@I2T)`N4%54u+6M8roNGYtX5`+MCmMLYz1d)A%cl?2whj2ZO;7Yha;2J>n-8mWXulwS>x<%=0wAq<4&borN>f8NEVcmByXUKVs)}k zkL@K%G|&b{YQFp`lKnwxekWE-rv}Q4fKghOjCD*u<1>rm!>`FY(VjU{GJDI-!t$kr zZ>}|crC#5a%(Ksk^5e#DKE0MmDOa{>rr{TE*1dF$yxyHsr!IPJ!9u*&tbzma+ustf6H+f;;}q^$aHyo4sie$#?|eO}bP^(yG;aCO zIFwm+fH%-p(fw8Zs!)tGez4_!R7rIp=1=LJ0s%^z7|bZ0rvpy9#IV+2jsUmFs=~-; zHVRvNR?MERt^|zVthx7V{NDW9{_xu&h06qlglh_k&8aDo^jJ++z#HD&S<6`B3sssO zm1iTy-sHp^I{dI0Q^l%vO2n%vq2-h?F{rd1_`aTUEjynLiAb^Fq%ePC-@#45`H}2! z(fym#0ndAr;{Cu5!Qdcd+8;FQccys*=?MTijRwvp6I$N*6;VmfL$HGRAW&|S7K3{O zdTiz{%RboPDg9aG#tW(jx4P9>q^-MNG{RMyDN0Mpj85)WJ5Ck5Q~4^rjKKMrLnx>i zzxJ7+CQZ-H)BSsBI+Qs<%D`~#D;cPeo(TGmAiXbCDKpg2UL`0*JPadXwNSZo+N|9K ziEcRT$>UCmu5zWi%9Ny~?xGhiY~O5K@YgTg&n>5uRh5Fx%xI?S6Q^reaFreaYw3=otRf4C4;{8XqS#P`Ezt$~#xrt(BaUWbCVxuVG-P zz7%IF<2V&d-@3iJ^cm~vPUt%@jsa~`gO=~0-`Z>~ zY_EJr0B=x+;|qM7v9@{$>OOPCum9p17lSq*P^X&X5kYR_oc%4PI4rYlOVR?UN6O9gWEzYS)la>cCPi&ycG47kY#m|pPp*N-4f|fRx ztQS9jC(}vXu#-f_m#sQ>UFqy*Zyk0qAUvfdiY?tWpFE zi9Odd6DHy5kWDGOHvO75OeVY^!BBEZj6&6jVm4RP5XLN>J9(N|;khflCx_x=nqArN zjMyL5oI||+A7gJBR9CmOYX^c8+#P}i3+@g95}e@f?(Xg$0)d6QdvJFM!QI_;;Vx$; z``M>eZQz8RBEj`d-Ukh_ubbxTLVu{y57L~s_Y_h@-ukZ`JPT;9FVlLq>?%z ziQjsin$zO!E+|8F3=QdeN_$>)f49EMe0_{#5?$2$lCs7e@AJ~LbP$3UNI%vZ<(F8a zz=X81vnWF;{D>&BG(#{_@JtnRKy=~vHmDwOmvfr;rI1r~l~sG65j|JF+OaY;+7~e| z-adMV($<~Zl-;d$ux1iBjlG=xlJmJ{=6^k$6~b1d;;O%Y^k=fMGc&wN?_lq1d+x^B zKez3Wl;~}ogQuF+FCkGud)d%k%S%d3;moj`-!gsmXrbzyy`0xxAqKl#b44f3Ar#(o z5s`n3f-913$`qL5!I`hayR_r`>Y;((rJE_cH#&FNfTnFsW|e+sYH^icU?~ZVQhLb4 z67XNZix935)LnI8H7tWf)7Cj{&sIZ2Gd~S3WQ8_s)afM~yd}&fS7;#b1y&HFA`7FPn z;0+-X4kJQMpQib2?HB(>J@58wC$S!)Kt7Y=VhXN8^2FP$m!cwNMZ?xR)sTm%D?l|ORTQzb=Bfq!95#2v@niMvr+BbeeT(? zQX6{4mQ61V&N>Gga?z`=pBNIv)lGv6E|c2pKrdUb$=-NNGc^V$Xw6%3g@A>FHCQHgo;CrfcHV(_>~V;@LJ1%rQN*e0ov}#b* zZF=m@M8oSdLq4D-iZ0za>iUN304ScS5-s15Kxig%TMBi8#k+U6o+v6|iv8s&U16#G z?2Wz$KaOH1`TQRx=uj3A@)czJ4m3QDNp!QpuXon*E&>_%UrN@3X-9v>KP?}C&u$IU z@Pa2jl6^gF-a<6=*`AK)LaM?wzU~x)Lu#Ye``l2N&^Dr@6odKz{|+e)svpo$^8T_` z1UK8sIW7Rus15M=lhDWM3WCA!i;_)E}VL(hbJVQOvnl5TVET5q|rdiU<1*j#UyJ2Gq zF;prvf>-@vKd$PPSuZzFeKpV*HdWNhc}bLxJ|kV{{Akm@bwA?Cbfh_mDGqyBc85_G!^QR%z@~MT`yOh6i zq|@ss4w@LbXBNTcA^=J)prLh&r)y?$Qk22%GzEA0SVJQW@(;q49@4P85Vr*3QW!hH zeP}qL<*Y0BR3Khq_6?%>r~s+yl3*SnPW>csKFqUN$9ZQ0PKh4aXVvxW+Vs^__Bja_ z_b@p(nIT_ltP>f!%@}(Q*89Fp)75+X?0ZQ+lh$Ct%}bM8T&!LSh86T4`{M90lc_?( zK1O65L#i;)ip=JrgrYdAd=#79?Vxx4kn4asg0QLk(6$4+zcG->n%q>6Nh-$I;)clo zzF$N$Yc8?)xl8}p>y&`e0+v9DOX#ilbK+KsgoYMfgL$XG9GbtG*Ojl5VGZNoVFR z@?37vVCkO4E;cl{KL_5-1ICY>3&k#!Jh5d`9F_}^Jwz`TeK+{LC{?#U5HhvJ^G0(o zIj0-4g+8PAF*-6e`m@ohF=139)3zzsaE;>_1UJudl_XP}B^bh9zHUZCbv~HNJ><)l zI3l8fi9aiHP`O6O;N*quA_T*wKF|`A0{9CV@L)Vk&iRhe{ezq)1L-!(;f^7VC7>>i zv--`O>?1XyN40lNfHTvgl=o*t{b*Q-FsD;R9CYyQG)CP^+1sX{dxxLcJ{A)ebeU*g zIi9lW`>Rzo7)<|z#+th_Ngu=nUdWyQ+IO?^Xg=~VqHxb5w=Hme!8-<7KY(U48cSUB zyASY{3ZauSdCOCKY}r;84(wj#f$8_9D*0^q%kX7bF}^Y}SawzGE%HE-qPEG;S!|-y zz#wGypHN0CTaKO?n80^*jR*aB4MAebEP|_4vOIJZTqOw$GT%6oa4KAwf&qDL9l~ev zKji9qWx@WK(|ysW>`JJO^BV!MqmwwqX%xdBVhabiJP>tb7-%?b7}%+hJSyjGuTDx$ z{bJV$Op$Y%SYa$)Jg?oJ5o-23Ji2`hpJC#_1?quyf_GMl2mJn1#lNv@k10pX%9hV zp3+`8w3Z$sQpZ3~ZDjSC=L)}R-^%hPJY6e}n3_-`a9O3n7*M;gn*W);`fQeOEZSas zOwsnV$ikzp09;p!g4S$qP0lDGDS2>!lQ6Vm%sArHc2_@}4KQRuYcPP;GrKE?eLwGskcdgwHx=H;3E#B5!U@>;O z;uSh)tF>&U_x@tFF7mqe8R-0l1@@R^BkhU|2Qs!^gp8M26N&Qp6w;Q~iV8S?hemvE z@=}H`s1~PpKjNvk=3!Vgc?We@kp@V#MrH# zTlL7~psBsptW{_`Xwl2}7;Kqqgtbgc9Xk_vZ&@tztJIfz-x-MnU(?z=J@($w^uum_ z>D6Eg($(Ic&IIyXgYupRWH?9q-EG=z_!!YozyX=VTjF46Go{G;@EX{>$t%Gt86<7H zNke}HHVHkgHNJaRkYm7?5xjd${lHtg^NEJ7=k;ptS_IXaW9^U5VdBv! z8sZ!#`slG@QgdTk!ZdLy`04}8kihmkTm8A5q?JqnG(xi6wODnvvVwiNe&+kU1zr=m z+(TwJ)_yo89!a>mUsTCt_jNbN;P-fSJ6!2m)O|f~(}cXw9^`k}9fQb0XZ{i0>H6YQu`8;M^O<~vT-RI2>LPW-`1$sgp|EKRmIit*Q3Qi6 zN5WnL38m`wprwDZz`8t@?GEWlEsF7}J|(qsoVt6uEl!VCS4=0yRoQ!^SFAY>OO_}t z5$z2mYU4Vi*5PV-7_s;IM@0^+vzM2#vEVv1L|x~*_+klrG=2wP%X?9n8)`Vns&l=#!>4-eX|O1fc10`FqHss5qoB7M=`!B++fGtt28xs%vh-gtf#r znZ9prWD?SU6z3PCwATqOd>i~W)5_Ie%S*{tSD2a0TT zGvAh1d~b$xZDw-U$aHG19kW$VDwyM0ag1RWIaa%U*`exPw=kK5!YrrH)#q*qTrA2> z+b=W74O^| z6q(J|IG~Cp&U}M`e82G3!BXf$biL5feI$w;Hsl&vL4k1O)F|T>_{CtQ^|i30W#WB; zxW6LxRsPyjp0nbbQn<{zX@`!D$M=-QjEvW3QS59s^c7}Xu4fJOQ&G~EZH7`5dc|xG zf9sZ^BF92{58-K3z#?}KY;|UX%Jz$8ae80Y9eRuO`|y4H4OFeUEZ)JKHXPGqkhsYJ z=P;va%dH;WC!b*Eq7~5(NJ#qHAEJLjc(;s~%JI|6wwa_%g89{o@Ao*wsMPa@Prm<~ z1)$dr((%W}av6bN-ZHuSmeXl#DeqgV@__YJgo!zHUZFRDNm+?i5DN4*H!&fsbnrM2 za`i92quTstvt0aS9WNa-u*=}Z*KMoOikRQU%LX!)%QUhMA%k)BQ>gNj_~{auLD-G< z1Dh!0$fWhUc0NHNPP;Y3WwP-o_={~#9l~?6PjDx4I};-ZM@ql{L|J17wrb@7;^8{8 zBQX;NkvlW7GO50y4)qi9IYH1aC|$l;u96sVd!Ma^>tx`lcG!KvTxF&f6Dhft_(;vb zplp?)%P3(fve(mn0sW%azSL+Q)8uh{eN$0v*Ek6;lgu^Cfj6t?e!@lFZ>>pvHKzn^ z!_=qizUal});|r8J+n>o`%g6DPagh}5i$*H=klSdt+~3&BIeWK?!Dpi9`oP)r`u9nuwfN^ zKU@U5GTO!V*}{jG2Pl9u`ZjsHTRUXQxY*iYZ)-w7)1pwH#mj%pY-N4S5<)Ct_I(KSV+IOi!<>29HEgrh=nXCFx*=gO_Pm&WI18u_UK`h`R(+~5_q!C^lGJSgTbhArc7I@Y*L)E*Ob`D5Y%CLaa07Sw%EM(W(ujh5r03IX#x7gxTtdKM!Z$3z-N01Ze%XRC@rqjI_{2(SMXTQqPL&yZggLnoF%c}rM~EinUVAk$W5!M zwMxK6h)&RNc)IU2tBh=X$FRuuC~`P^lFe4Pkzi8xd63p#XAU~`nMyN%dvM*LIee0B z1NQ@PlUt*$R(^+?HLN@$lg&2yu)x5;nz|#N!;2Gzuvm$*%0FwcsHGh9QzT{@H7xX; zxQuRFRJv>Of>pee2J#GpK9$%xMOC(N5!boq$tg8@?#0`4!jz;{g0)K2)vhpCo6VLA581Gr=n*e2 z^c0M1v;Kq_%Srhhj-;W69IT^XAk0<%%n^rWd;SR4vj=MfCQ7o3OV| z6kf*w;^M{)NH{rM)+R;*3;U!Ge3j8+GxKzHy6V}PLlZ4cjt_-g*?A%-yoCa2xW|m7O_b5J2^F&@_EslT;1=ZVh&UA{)tNUHil=(q#&rY!; zmXD|8otw^=hkz~k5lgL+UGSmFrg0)v3kGOQP>?{v(|{6#k$@bBEUnIpP4in;J=k?N z?Hq4P@va|9oC#t#=r`Rc;uL5%K@?`*lJ$tjljmxzoP{Lgy$;+-ZNv3NC4%mP_*eL+ zsIimN7R1a+@(`CAPieu2Q?DgcAAS&7ZN_J4MES)yta26ZA8U(oQZ|M z=NeSo=hHr83GivkU=lM%J_-4i#2CdYZk+D9p#<9&#a77Q-*b~^iM-LMUc2v4QjXOM zzijqO|6&EV)?nt39YmUy^HUN7cLy#yN0x~9d&P5$=pFJ8fZA3EjqSDk@;ul6ff@xr zq+r>#<6do?Et|Oeit**l2X4*0uFg5d*Tm{ukDz(;yx!oi&vQ_RF1yVNuus_`7vT|3 zsI(G8tQ-S-qiF$~YytN+cPhby8h7Ac7}y{v_ilR9r|jR5=KPtuzywnhzlj06&Ny~M z|GVFkFbS=~A0W7s*e;+Qs%YfetU$3?e>Ak#31ao zmxndnRD))c`O4UnG)S2Yun+4K`vAi#7_D2g68<^jt@vFF>2T}O?y6!?9R(aV^XoV;hIh83X8h%KObUZ;V6ZXe_F{^HAa_XX0zlQzE*v&$}5xow7NbYZ^DkkF;5p zzW1!t^KM@}V@ol>c5X?5c($!cE~wzlEa_@wJ%(SukF!P?G>oEN&r@9+>~0OUzdDa_ zZKM+WPRO%4+U8x0kRn>5ksil(lCZRMGnuXnp|x{=?RyA!MsalEx-8HHYzs@>gL{Cy zK>vVc9qZzA<+%b^*ztD~uNDdFy9iVCV8UROpJdI|C-+T+0>qYj8*z{(zE06o-VB9| zzOb1t6Vvl?kW$KM4fv@%zN~=lhjQI!{&~r+=+YJGr>sUzJJuYspN|)fmPuCl+_fN= z)4sub?10RJF#<2W$|bG$0{g=Pn*sMp9~l}fN<@bQFtNhhIa6d98EO(SN}}qY_-42G zF>fx&3gi!f&>dGn>xB9EV$>~uk)1mrGvT&(2r`wil*G(^<94K?mi62h793U@+{p(T z=i3b6|NYCbpImP%Ppz__99iw^*v8k7MK;BDC`7e!US)qX@Y4oSXzm$K6e-h3TDTov zRyFup)9`@UAD6vhUBYx`=}QWY-Vr??FZ`ifey+sqRp*FLTWYm>4LAn6Ms4UBYDH^! z_4Y@&a@xhBKw*R;esJE9Cw*xpJym3sN761xx-{HhnEu}SK z{eCA|Ee!TzNk`sNtL|SdsIbm@_~V&p%gq{QyNb;0;ipj`Cc|hHo(&LwbY{PUN9L@| zJTR3zu`RKA-rd>^@+>9Q;MhXacVUt_t6^7JR#tf{{UgFJtzg$H@Jqa=?3g|)WHm9^ z#HNgF7F2K0%^W4f1!uMVGyU-JNJd?65);fer07be`g{J`%z0`vT{Tbik&jli7`$gGel z=0e1*e5n8HL;uG&5g}VIp#huDW#q;0eViyGyJxG;bTuJ1)(=#)o|V_5m~#AZrz4od zTdw9wFe8yK3w&8GLPXm&-6wrqM5FM6K3cjxD!rK)fqS0W;T19zLUgG}E&s zzvNRX8JR`hDN1`R0rt|-SvZoB!s5JItxJBYt+@0bVv1BzKXSR3VNlnZr(k%JQ-1*g z_xDuzzkc|kz%^2VUXJ5E_gAz{*%E!vSM5DES$p-EtIHZj||bJ z$!Ay@t#cR!*c9$|5ai@bD#*?aTIjr(Na*;FG=Sd2i2)6w$U&vIeluk|gGA@Mu)5!uIhV-s9hBJ z7* zg)Lo5)^5AuoRDI1x)~XrbRg(^kH(H@JyHJ|^@Gc4g0aPer7ajc#c-s6i(6hphg^cD zXQjnC4;gRwl31>zHd$6BO|`&wDDEfr6$jbpiqe)(Kk>i{JrNAvR9gk;9~d9?7}JY6 zQV%(HJ;kpgrQ3)sq;aq`P+Styqsn40c};#>jRR*MsWjs1Jt62HX;r=zg_m3S;4N(( zx~I+{KsVbYfzba-DV@zmKlMkYt{`A$dr2K2^zeHVi-#h#^ZxJT&wno4|GNUr;^9E~ ziD)9@c7wFjE#qdMWa@Tbfe;G&D zX~^%*Vdj^TH2v+wFiyi5AA&-|!rx4>o1gjY!3`f-hyBM<-}W+UBlT1p-o;*X{jLw8 zi>)7SI51Rd1>urD6mM0tUDu4QK82ez@qVdr7B`T&E%xv0XNHcXGlklO@{fu-W6Nm4 z!XD2>S(X7N4uqfDv8dDcSqb<|ykOEw0mRFnIjH~gG)TkI?8KOm_z{x3amId8&WHX@ z#!H6R{2OvSj17U5U9;)NPD9Uf;}~OQx!Rs`XisuOP$X@>?)Dh`ck_-{vyYxT1Mmbs z$E=DBt-gR$ZwgVWUVT#zFp|7p#Jl!byrR5pP>uNhtgZr@@#dr29l5U$Yq3^!h1qr9 z>u2V`u=Oa2-5lJ4voo;{0bOvzfBht`U$A&As6Jc9Vax>ea(=!LB0qHESZR}Q`>aRv zLw8CowC@evH^J*Z)8g|K- zSB{o?l&O) z)wQxTx@Y6Z)s@pmNWS+vBR~0k+Lpz&7Vh8u?!TlTiZ_rZsXg$WOlk?xQ)OL-Yxbxw z&v%av#BTGPBmgQxO+_ZhuJ%ab+ z+FDNL^Df5W9B(Gio~gNySGGE$jLv%QWGfP;UL%Cc+G9V`A4er?FM4`q+vkqwOO2CC zLmk*dvSFXSaLYDSquDgete^3FY+ajxg?8p)I6)oC=Sq-d+;a#8Do1&DAQKxF9uPr% zNs(-@S6KquujlC%wOXM}WQelhx14=cDtJrQEL7M3YJ_RpdZSnVYzk4p^r)1~mn@{% zj;kw7>(<44+n0gLQH~u%>|Y~;S%8g3qqgPssaxFRbcP{lP20nxgi3Ap(Y(D%$8Kv_ z__x8{2$9HRUp5qZ`HGTA4bF3--u-$6uGjeS9DKo$)X?zmj_K+-fuxjFcJbEZ_*93F zklf+@OzHV4%+`g*hIle?YWcPgH=NEL{E>s>tA%8Q-EAE0OD}u=PD*M#`EFd07rXS8 zO}OO?pd&lq_F_5SQ9aIXWyaEFfbjnNDg#DIN&MTWkIY809j93Z19zj3(S0C<9U6Z= z7NNiMq(S{QWD2_{y2~IwMh|UiViB0*dgZuWjdIhcib(yMAmctjvpWo zy1r$^mw9CvP8fnBbF1>$zOo5!!v))%QB`8uBn%oeL59Z&fipHnoWoajb_G44+=f5J zk|q5q2LH?A0DdWyLwa^I%6fHQsKL-6z?M(&e5F%2sZC++(3jv!fh7?yfz43_wKb8D z8Rd?{fcu4!|C}ir??C!dssGLH4`ctrG^)7n@f4p86cqEtqU^g6@qNaZXXHqdb-nv6 z{pZKbmGWn2unGGkA-yfj24Pvc8;8BL?eE{Ux2}iNQ4Km{`*ADg3zxcCz}|ITl%;B> z@Orj%xljG-WjXFt5;_iQSA42S`(CzTH4OXC=VAOw+8b<$nNf6KuCGcCi#bY`j~CZqGU++hC5P^QyJLFai#5v-TOUXXr$14yUimU6x! z0A=Kn{c;qwG35Q-o0m!Q_=Ed3!O#4WyfEJj^SfQs9A##O#QTtf0$K7A<9o#e)VK8# zvT4abWx(fp!jtXpnYT0LrmZn6sz2pbx3RabD{`UhlMZwGUL7VKSn7f*;H7G6L>?Ca zRU>CxEI6Je8#9@Bz(jqbV6gkDmS_YWe`e$Bn$CUg_E_wsVP>+(+Xoi~3$5G^Q`rN# z%GaHmtNp_MP)#w?=hw#xk%vr@Rbg!&Q`k)M5Bi1;BM#O)A^NH^_eP~PF(q?2$ zDFr|*DL=Uw|6i%~KSS!YFIt_0DB`Hqf9U5{_~*vNaa;hzZ{4aWenWc0Qgo>TX z&dY3aWv8DPttK|@v8eSZcxt$)Kj*^j6~C&;;BB5kzmOVn=qLwRHz9X)`$W6mZ3%cR zTfuZ>W%+SfFK(+UHWB~7(%FAj5~SH^pv$R|+E=BvVkDB+EiU{?h9OKGbJ9k9i@bb~ zG`Yn58XE8~2=ElEF5Kt^IU{Mf!wqVH{?mejZ?^Xl43^WNy^+XrbaUgoNvPM9*L}3r zoDGu>mtUXr<9eFSeQ7J~xeOXntEeHH!lfEde#}MKT{?RF|M_HD3}jhfO*+7-KY7^tdvTiyTESOy`TKjCF9{=fe_ppwP)r$hK*&dFY&&Ah{~ ze5&!etASD`5A7#5Az}f!*!RnSF_K#Ue9DXZ1kHZHI-j+>?9QWZpECJXvDh@$rn^sA z{Oq0m1)IWs48|P@Nu1OKZe0)8r-OaZf(F1T{j90^z)Kg;GC5_~kD)sJuWXAd2zSP- z^K=F6*!>^r7dQd`=yzI%FHnwp#2YAReg2es1PpBelnZ85y)Gw%D-6d22FTR<(>xRb zb(se+869iboD~|Gos$dS=rJW*>+=??DmdnZJ2L@^bs8;dy zyKsla54)+Xrei=`#{H{0!xxz^*>Y)E&&zEWdXQ-s}^1G(l#DWENVV%%}3W&BeL`AQqGU zq7p%R&Ehl(Fo5466{Fuk^j+3A?cbBgXAh{w3tpYP^d-CVoSl`biVA_rO|Hx#IK8p# z=H0a7*wM^N=O*lr|F`jQM&b`O0lLv-vp$8xmQFoi{}-Vt>P>QxeH1MDI5rxWvbzQi zX-OH6C@>oIfwJ$%Wq5^XYLQn9G87IjE7Yvbvg99*(nE{W=_jDYjS_TU0PT-xVEA-^ zqm%EbZ`HZGv%};CJnMC4NWM__cT6}W%#cqpILuD=KjQX4jfXKodQ@=(((f-bPy+*# zjR31+qnOZ%p#~*vOc{K|9Vn*E;?@B$2=UW5QT%=I-=QI9L9&S%SA|65+kwdoMmblt zY{rh`qc=7@AqunT?nXHmMDCHAj`guqJ&@&u_0cVWy8B&$ysfB;4zU1ByTjq6zba@l zIhlw}_ckKuZ=XZ!liwZ_PS!io?Ck8K4pZTq|H^v790SXS|879r`V0niQR(=B=z1gF2J_Uj zwU!?AF|;V5GE^?ddW}@xorkHfispyC)zi72oQHt2@Kvnwe3G&R%j7V%4}$a$Sq;b| z{`HYUnmz-TEp4L=fPLhjQJN^mLa-*PaALB3TpN1PMSt}Dc0QroGuwMu`4xT~16R}J^( zh{Xm=xj$k_>aKL`kqIU3nvEZV^-Wrum6XCbo3R1ZCcOVn>8z%f$-3Pbzw5GycFT6E z+6d69Mjiyc=l-WK8{b7W8v?soSJX6h0T240-+mka^SrP80;HQgwaD6jHnF_9$(?GQ z;`sij*^k^g%K6XV&Y^yrLx2fE^CMA$%WVL!N;!9PmPC9%Nzci$H1z4GAti!^POywUlW82AS2GL){To%D4)0{zoN+4h1pO-kj=LP*KeO zde}wA=Bj2&TiW!Q6~;N|$aCibraIN{uAfcypNF_k16cw+J%!M8)+n|P(7$SBKu4=B zukxRX3*|CUS&*pzCi^S~^6`1M!Ru<|K3vYHxeQxV=HjZ0uyRx ztvz@cchy$fIU4`{>J(6rfW0`(<|1m4PPM}9MUL#7fU-T089$S)_wz4(wj$rPtb_ON zZU9oOnbvKh*maE9Y3C4&$Du4CCNaUHJ!9jtNf(?k-R|va(d?BBxVXxbSNpw9_3;}3 z?&CDT0BD2TsmrU4N=Zpc7~B4}X(I+$S7;>YdQcB_vAy=E%d3P~*`wFy{xI2EbG};y zqZ|P#GbPr>tjme<FpzGZSiH;3z%(;Ywn}oPkgZjqCqHmK^QvvK9 zBYc5%0YBrhrpN8#@IOWf+Lz<_;%?zV<_;qh6^VwvJ4s1SCi97WhT9cLNBu3J?JWLe zGCKZ;#fA;~{&9aYxjEuIZ0#G7J|PKOhv=3qra3Sh$7byByDcgale}5x@3Lr%Wk@<3 z!p z#z|c7t?xooFJXu~GbgWRRH_&LcqK|#tq<2W)v3;yzKbL8q9;Ig+t}U2Os_1T*>xE{ z`a^Y-88!!45*tD}`OX+Pm}UdKhxUfr-wAL>2+5OnIKXC(^m|NM?-;;~D5+sBj0SB`V1i9lqw2=E#d&A_8OJfv4yu4bqiuFwX}d#pfQN) z;Vv?OjiFt1sZ>SUj z_ts*1!}X>@Qmqurh(6^Xpy!xVswEar_+|C`t-0= zB2myNaoQ2-5f+G7ofmKL;%2?@n1@b$rRrr#hi-C_<5f=!c=lT+~ikyQyzI2*RU%|=SCP9wYH612XK#|hc> zj(63r?=Hjy-`H$IQXRXwI%-c69Q70CChcH>t3f@j8B5V1G0)dc%ONY2lMUDBkxDOF z0!0#^pCKHPO~Q3!L9Quyo6nXJAXd?lB%4^gw&y>ye76ER#pfjhc$I)vybwT#%s}m5 zXgGtrq1mc@NemrM+O-I=>?}{*BFZ9=;sIr@UkNO23Oc4Zc~_w~E)jz01&~UnM#^oH@-6 zTQpyTt?31SoLuihMj zX$hQ5lObm1gI*XKoz}m>muc@iEydDYq9m+@Q|%usyq(t1CbHO_Ff?>g#bdF3=dG?% z6C-7)a$$bzyl&)D);gSsYeKB1AZfC-D|fTEy%VfJ1=0I>Cl^?osgek2sFp9~nJ@F( z-JZA%dPUztx|&VEkPiz3)UufuJ9&d<)M;LBRMfKJPwp_*eyVF;a%N({(UXN^PbB*C zq~&(vk)-k`78fpEuyKH(z~%dq4t*-&Qu<``=iQT_>XrtW#IHXqg>VLlpeL(vI1 z1@iZ;AXuYAToXJ8pUNKP&4$L~=be<@_V{MtB9J^HKK~msWK3tUidpIi%5od1Ckm;% zpPOSmn;kz2fIMcIC#*USc#U|7SJnEg2RyW?B|W7WwyyK5Wqx`bK5F9Ec_oF4ymgEl(_mR&zCN(unBQQ{8q= zIy@!{J*dmpYNJWrkZ;-$q&|avYQIzQa!HpYrM^?`m_=r+Zl++%PinolLCt2f=#aGR zyIl$YsaU%|n<`>4;29k`x8v#W%2NKNb716ts1mHixJuH7OQ%Hz6bYX~gk(Js(o1dY zvckUde5+#Py3DaA2c2X5oD_hg?5q#K2&rAFjO&)A=go4?hS<@ ziGq=0d%Nb$>wPnIJ3fCT-uvl8G|ap?-WFa^lM9rhFVi5FWk9kL$iYd3>ADWwsZafd zUv3Nf6!ilaC0vt$MpGgO?81|ahDvK;pv(Ct=W6(8!15>jfk@gBD#dVU^>IEt=|t=>LAxmq)j_?40A{C+GoJt znByzY@i05sKs&j9_DPdWa1NP~nPF;KpX*Wf0|VV$*$=3#0YHKMD5$pd-a9eM61~dK zQv2eFcjn}F;Pb%Sy+7@LTU0kE00^=^CJUCuD?Sa6j~6lIncw{Z&yUOm2_My|tbYhA zeq5>5Vxh0^ji$Qaw?B4WN<)-^h(safBdT7&QjssFPP%PHEf9ffQ90wpaJtAU{Y3-6 zr|!A#is9TthkHfYf)Vs{dxi@un3)+;fpmS_P%vVX+wvV+5o-Ijlj92N{Gx)3ib`bD zsX|YY;=7Uc!KdgJABt=6T&hahD(Ta2Z#ysxs%)&QZ1aGRvJ-qE;vKdq70fTLD4e{?34i5J}J6hfu;@iU{EUEY^4j7 zU^H&%UEmu`pZJqM^g#B^MD5_$Eu;KsO;9*}?W_VhHS%(q1+y!n68 z#H?bGypk)m8Wb2uold5iV=wwhhGo(R8k3$#4P`pTnTAsip%g3bMg|26P$XaSn0;Uo z93QhkFmm!Lq043TSVKs^l*0LpB0}w6ch@#Y!A~0MbQRnNRnHawq!uG&8$F&yxZ?;N z^nP+-j@^jXpaO*l6)ywJ(%560^VDWczrQ~rvjh&xF(~syhd%=TWs>Ze=Of!ucAp^y zq5Pun{s$zi%NA*$teC73QGC!ivzhyIMyIuE8=`~>SGAE2B~<1(zaEo&!Tbss^mEd{+%q*8l4 zSZE{YB5}$!!+1hJ+WK{{f!-ut*{##I<-Ss>b4_Cxg3TA?M@##W7SpgwJ`WAswfFG` zo@^P4J)$SN`n}*DZpWW6Wj__&x1=qbT5l=a{-my_GwJSEhfcUlM7v9$+}huFa`BAM zs#EsG=j`u%Xiv+j}Ay5y(o~nZP_^2jKA_mDC^j2HL+KCyR;jOzmS1q zGEWU;aUtj+lY&1q6l;4rw6)a6FN-^4VM@gd-R5di7P2MW=&-w0p`A zi{8e3)@jjE=p{=JOGm{0AKdze9E$kuYn!DvpYd5AW@{b!^UNaffPH%c0r{ zMJug;=(PVioXY5ErKSZNx&!38CLrfa*!Z&9AeRz!6b!BXtD@s=vVmKmj18QMae$ms zE?9){(EYrwfgabQ0f~@aj8MCWt3xuJY-ERksxSDw#*4od-|2`+wP$C`oQ#F7c`bPUQDJ?!-3}v+;j3=^Ak{a|fK=zrun7-eQ-Gn1tROdpNlMt>q zWv0W{mRvIh<}|e1AjH0nl`>4Bv(%NZ@^PkAK#cPkt6QR#AhkH%tuRB_Fi4D=yh~={ zFCq_l5!CbSDYyqxDDO`G^u4tbpS|f^d^)+ z{M|Q8Y(PN@sYJaxU38T+lf1@6YAWf=7IQN}Zh)h#6x$;EYN`74_M8f;mjTjHi%bYx zRc3z~y|OQysUmEkE!-`yL##@>-h|M5JBH=&?;g|wD*DVhH+6gq_$M*cHZ|YZ zqStEj`~UJS|9TQ(W_YKO&7knU!7auYQk9{|*dqKbC&R1MK}Fw@``ZJJEzO*(r$aZ) ziW%$m(tMe(tv7_Pa3bk*6^mk}aLgVViOm`r8@ddURLin$RNEnM2q3{q_STHzZz_rn z*4K$4t(Ps$EAmCPB1yGzToBuwt+TWVTn(7?q_ravmJN|D?=iNdTL$i0v8k07G&d_O z3JdLH6{neUi`U4@Ub|k^kDisY$((|3jw?|KD(?-*HJ&&=a=?>)VPy|@OPFZ)=B>U5 zi4gJR!RxCvqY^u#rid@a||49b@rGH?C;O{nh-@d!j(p~h5VgU&o=4tOFEsm99mL4@sWR`{IR|$yq;cI{6zEL&g_5zq+UtxMab@(&(T=dYK`*qAgCluDGxcsvx)auKbGk0S^$) z0!Vc-!sL8ayvE(#?QzjfrV((4$0)@IUP?B0<~@Qrtq%K%WG6%Ih#A5y7%&gAj!s+3 zQrkAS(!#!D83xL-cPK!_^1=wlf>&qq&aN0$%2PFjS+fQC8aG?75(1D8b5c~Z=H*XZ z(!HNju)XsB6f1|%IV=qVhQHl_e?EY{0ZTQETFNoH+jT1KQR`GxYw)&Ob*!XBxJJW* z)jxN*U#L%Ichhdz?3#TkKHxlM|EQA$rfeTi3CgVF(ar+Xs*bdeFj0NZn>n2O=wa2Zqn| zQdZ*g3W-eo#H1%iGNhQQoUDb?RXe6`m00P{a@7ienykGH$Zts&4u`QigwHG_n66Q>3x{*HABEn%USF5vODJFtssHIykU{=X#sN4R&>Ahp6yI`HV`cyN=u&tM zNmsZWZ#+JSrPaxVke{elK47N%xso@wGf#HLz%`JRQ_ygtoP~YG9 zPyWOYfQEe9aUISJzLbFTHJuAr>^L_TNS+Jn#j~3Co2ttVH23Ia@3|HcMPHm?>ifMd z2ubAV7^s3D{kon7J#ZijT3T)cXN?=tP)BWDcH>xDTN8C9v3Ygx?z!eERjbo0YPT)O z{e0q!>p_J|0_bEZ^7hS=Qx&~|$V4nb1&-BcOH1hwh?iY|31w*6EIVtYPwt@?+>Ev? zJ!2ncL-h!rN*=b@krzL*1b1V=zBT%8-|Yy}vtkf-|w-QQww#C~i zuX7edO^F3}1y{cuXq9eX@pJTbG@ITrY4dxIZ<;SD z#lZCz{G02d;NNw?g$&>7*>x?6xdkVyNa-gtHoYcDzkN>;F(;a&(S9B3^@hpcN=X)$ zs3*!J@c4fSdkdf{qqS{V5D5VZ38hg9 z>F$>9?nZJG(hY)?q;z*l*QUFb?%2epyBjv~KYHRl|2N-xznMJ@Gt3^IXRUQ#>$L`WIbPPb3v`<~KERO;>(ScnX+_G9C4fBB9&7Ztm4B^vh zxv)_opn_{z*R!_H8+vs`&?`UiJ&}#Z20zIQUEH#$2%`Iv$f+xv%);UADd{`53bnk; z%nwNZ+XG%qmP}{3y)tv zldn0%Yi|?;Uvs|gt84Ji1pIQQP_xGg z=Bsnw>9T**#|4o;_Zi*2Ou2BAts}|Lo~Y#WZ28%VER3J^6WSTA>Z0l{cT>T+{m&(a*94sC>lL zu+MtO{s75z$#kJ#bdY;p;55k-(vEhY4{{Db%%Ko-RAE&k;uXV{P?#v{-gG)s zO;Q8~MQo|&_R{9Js)g}GSWM!f)Vee7U)@&}1K9dnl2ZMc!Sk_!s{?`@|%N}_~ zs%U#xo_Z6nH*%MQ?Z=ucYjLivnFr-nNhXVHOXYI5eSgl@Z<7X7Q}aN9A6TWb6@B6N z&F~oVeJ|-y-K9~^EOGxr#ZKZKFtEDU>{R}3DBtm{BHGOfA08+;ad_a%2}#i1Ce(Ry zbC9wQaG>|ky|pe(S-C^^aAY@6kFbIe1k9zC?e=3Xwoy;%LU$ z%YJx!`%TSnqLsqQYnGX2-aRhTT2OSFYLpRudtEKG;T~>BJGmi0x6;EYq6x?74Uc6@q|PtH4i10@58OXvE~6%_T8zS9{Ux z{)s&M*-0Q$wUad#E}-q;FOtj@+;|KW{VHJi?#ah)F*MD5FXAfvL?%acCyl3UDKR^{ zFk0o&yov4*KK~h8Iyc?Dk=w)Zxkmi&vyN^iTL{fjb)`Wa-;ZAnCq%FBkXCvZNq`>r z%AZYNTptn^oBdt=iJOpb&+BQ()v3dcPx(^*$liXR%nGKJ!v>aQ=$OKs)4#X4h=vl+ zsN8Z=L zDKy}#kAJ)m;|$}_XtO8M<8)<1Pv+w*ZAnW4*v|phuU`XMe(Q)JUytH} zn3p_YOTj~f_Nc3}c{yixbzgH2?di*UUwxSpPNLkWn`L_D$&4VyU%MaW)d;y5JP=-y z@|L6It;Vvw0jj3ak@Hjn#o&NmcqCX~DZA^^J?Tr)@yx>e!wKYjmtM+J<`~ssQI}J2EV*Zb@j9J6JH&8K{G@<4`7Z z8Pa)^yO2AIMXuMLMwi_?w3PH8o~(~s;be3m6D1dsAz5UhbGdy#rKVI(jIp0mtd?=0 zA1IZeJtWOvL#Q={ggwl6TN!L-bMU5?W|UAO8U_`MR6-S5uR%?HxY(;WF~ zBD+be_`Dh1?%Im?e@0-5LpE`{W*bX73L4y316cBII+4KD-n>_=6O&HU37NC7jTuuG ztZ}@?bBSK@2o4R0YV8}+A>DiTC}!qO?fBk3&!l)@@E3Y%eBf}>g7b8Mo+IUa$F`@b zb8Sia&4=OSee6uOBjNXD+!ZeADo1VO#zg}(PnjARV?4tUh^H)*Aa9z8eXJo^F;Z`V z`rDH3LAe1p-k}d@JG@%JKrpuSpxJ?_1)2iWAbXHkiyRz+<+2D~2|oh{fWgVUn6xYB zmFbrOSy+#%ro>e|VH)D9mtB3onk#lujH*ST2^8|E{bC0j74#BnR>yHuy{&xv-r{CR zY07tvENXz975VGeYlQ0N4a}#AMt6VJ-+mFWp4#U0I>rTQ(?S_JwHA}nw@s@@8oL$k z$zniW3X>XQvVZ2YCu(Ep_`Qbefk*iTo`9?tZj%&9#(oj-HL2leR zmi3Xz|WvQcUbDYS=h`{^0=AZ_#e zLmlrZlr8h%SCPL%T9IDuI+$4^Ou9H$HT!HnE6ZRVXsU&&3&wiQyB=!tabNWxtFn#~ z=fcvi9>!zs-!US8723f>uD1ea?^H$ka2%8IUz?II-5%>*j0DxFt3`PDrwP3eoGcGP z^RU%>%kR?vl!7Nsl}9-dems%`rH7#|T_+w3Q4?>RKU>1^fw$)H9T~MPg!(+1-z|ql zP5hcS2{id|y}p(>O)7YaRfSkv@hutO)ZqGK+=7WiRneB9E` z>ZhIN>m@19;TAlRUBwI>at@B%sOsl;Tg^n9t0w(=1zkS^A^++HkSfK-Qri!i>mul8!mHbWEETT*EI$fpAE7}J}UQj>u z6tFT9jT}wces!HVVST#%@X4dZ1f*qH@bvW?8}A`9%?x75x#?DtuYxtCP9>3@msStg zoD;u!U*nbdLPAA`j5=TJ{j@8EAS>+gCf#bQ(PZmkeCXx}5`qxf1@cd5XbTbaceVE2 zmsG_-HH4XrI$Dbt=n#`;h*)dJYiiW6d|V~s-Ai|!-P_>I{LqOoTUx0B$pS)O2h<0n zUW47c<0VPU!zo5{TUsGR%x7>5C>JD4QAa*^Iv0g%qZ8%Rc`NOca*Y$X}i`TT8 z_a6c?$98MBRZq+KnVLB1b*c(#ry=fU@q(cql`b3|d7|o2@qWhd?M~H29GX-f) z9Ycs67xtSJ$q@KJoYvV}TV{GG^C61s49&z}I38KZ8?4rjLeP?InbcyY7lnwm_Q2nGC|`l%L{u05;`4!F0ZM3kp|UWV%^rc{U6kJ8%JOD z1WOkhVe%2>Vs!-(aS?SNHATZ+_p}B$nd`Ntf+qFd+<;5EWE#U?WXGLjhYb= z_Ryx;m2+Q!MQ+O+sgJ%@Xsq8#-Z*aaEY4$DDzI^<7!1^yZMVMqqzaza=bGB+X zwL$tQPV<@zH$!qi2Qt~^%b0)d`&2-5=)jxKp4WcED%WbqBi7OXQ$}49!B2>+BLTgD z{|oVh&pfNsAmdTq6wxgFUi*_dF8}p_cBV5iYU07CVL=B#ZI^%YIV8R_JS}7p47$Xx zLpp9Uz@`CV~L6eRxe9t+ZM@RJ? zib{#>9UdFD?0NrsV5XORuL!Gp+icZI$cv~!jdI+SH@r@VckhhXXW$`~`CnfV3D8&` zs-p@{NW+mGpEo3r24{Ma)0>T74N6YNpUbv;_>YuwgF4zCM?_NUEl`Mgl_c3~t=gv- z^)2V!i}-6G)4P@*Aq?Zu>r}h##|(Mz`{fnmF`xbNV;8v5t*lIxE4$-42|hwG5#LGS zCm(We=l6~733wYHmoT|^nYx>sKj`efu>D!3g2cV)MlmEwz_dmFQ^XE-nQn75;Qjl3 zu-*hoUd@lpVl&=QW0{p`N!Hl(+fLsmBnP~^z9sdR)(Pa`AI0ZpipuRr&2OxD2w9dw ze&BVnpfZoI>m2FTxPF7nM)>NWA$Z>1t{%>Bv>0qMA}Sg%8It|Y$wag10JOWL<4%9Z z?jdQx+Wvl-GaS6>6tDZ;5YAfJxP>Np-~0Mo)Na-ncSYzJ*%IpQ{M8*VY8u0{^=$^& zKs@x5DlX!#AMv)TII~9&ru%HwCTPEgQDYa6Fau{sVe5NfyZHz%_Zi|qI%8w@ z`5EvnN%+^tO(}g)j_(N}php$_*@w)hOI%a44`Vhpm!!Jz4u(5%p&iLJV&I3+Cku^H zXkL#fdBBm}f$yy3&w^qcB>mUag1$<0uiO@LG5ilE6Q9838Wv#adpwMrcL<<%tyn+7 zHR^2v)NclEoAd+CIral0={4J--)nU@I!IIEy0q|IPDzp41lq1w_#M={=|MepuT#F8 zhfU#yM!q!Km9a~UyX&#})x};Oz%CYy@zz2!Gd;RgO-*m|T|@jrXh_J9yQ2FnhC<8w zblzh2+)bw$%{QETG|yFr?u{NBF8bYXzj>B2sSFTwdkJiuPnSq=rjyB)>ogLNQ5jfF z*{Y%Ja6L>V*IZLa6Ot&)009M($zHTE1j-`)_#c02j5%cbJ{q}1ZrLDDf z44&PIh@Ac#bDVPQSzg8~mY08tjJqs=k_G~?rF+dw*Nnao^0k1wmq9z{4~!KihEK4N zEkS>#7i50lV8d&;a3-l>fF7YQi1*KX24$>&Jk zz6cp6DpZl)n zNf;5=VR-_&G52oxsM)p3)HWNAx&|}#%<52+hU?D91H}wK#M6Tfv104iprv75A5M-( z>E)00-!|5ywe%bRqEuqP@n+iTCF}3vwgvB}Gl0j^LP<9p{pkS)Ylx-kxdY$YwuN@c zrDf^fPlqL|vrNm0OC!i+uYOpSx+Yl{Eu`c|WD_=&!;Y#xF5zpiQl0O!jwIPt5?#~Z zFcd6``OI(S{f|`W_qL}A6Q`waPOiiI^>&!3MNO>*4$p81LOa&a(&8A|c7f{&La@ay z%cu4{hi#x!_xt2Mv)HAFfEhA)TjnmwZgB=JBO8Of1w_Q`!(K7`# zEV%L0Je@@~E$Y1nU*2`F>Hrh#MAmO09~o+D3eJc*M`PGxZ$AOU5n7lUZ8*^0&P5?! z40ERH4Q$JP-BqH{H#0MO;5=?fW|`D$p88Q@9tlh@$l+$Kh|d*q86@3H8jeFTp_aB_ zbSD@waB5p2Q*Aisj}VMZMY(TjKP=eSGy4ZBG$2?A=fqz#87wZ)zo~{6SAs^uo2)87 zFKgVcLg&e}23-$+8I8oCudf@M1crrwkBP_XMhpG5Um#a7HT79c*5VlAET2pODu#Nj zU-sNDy{LO+alJxwoxhK<-Q*W15dyr&4d$e!GhRBK`P;s-n%?H`Ad`Z>mYn=u&9RSk z^h3!VLro=Ue;-<;-Yy8~eQ^nxvYW(8@@VAb;ikrjCqwJi z`IBO-II{PS10hE=J+%3;$0B*Z7ipGYu03h`^+_x3hEHLO)v zW6Ew|e7`r6DH6-U3AGRup^lozsTw23^rbJ^k2f{djebX3;Z(n-N3DIX6tKVPL#^RT zcqx2@Y8NpPv377!zqELpb_O4==ZPVS%9&>QyLyHZ#CwV)c?Da9bZoEVo!Z9?^H=-o zUY#G&RZ0zClj3iyf*?Ny+M+6VRrSYW7_Np9IZ~HuJTZ7gLZv6OWiG}X{aj`rAKdCJoDZJVvu^KylXBe26T zVs@wMhf}5H%!s=V6LfsXPPNJu0DGBot4`i*k2LC^DYWY>E`U=ey!cOYf=hzXe&{;A zna+-=NkN^p+jf*%@GUuJZaZim8YuQmVmv;CoUa_7ZeCN5%7CDOd_|pm+}$?ko*b#> zdRLd)JQ}Jwo+GC%#v1P?3&LF+YEJS-y%eme=%ndxjRVJy_}+)swYWO)mzE8p9&lwg zLti*Fy8vB@;Dr!8jJD1`y}#I?V#20!qE2eV58*>MD(=z6ZMd4s%s zHt9$COOBWw(pzfR=d(+pd)~R8%{3f3ffNDZVAZ{eqURAVu{MlF2y|^R-UURgQ1!a4(nE1=U zpsfwT6u}=2FPEBJ5(MHd@hAwJNX8nERir_1h2Hnzgsrm3>PqQXLC1U)3oJGK`}YFXahIGEjQ!D$byMhq+BAEN z`)3aHAwh0qu_vw`lvQ@r%Jp7|Ob%s%@fg|UhtZ78!x`k>>nqU@1#pHw#KnCuI;HsJ zVYkZS(lW@kFGJR^<1W8xf9CnwUMl&5Ztc2fDh}rq&1zhc@gqHcm|l0D>nop8>tR^< zV!^Zy?GdxM_sbx2Bvk3!*4PKh*^u_W0`cGVY_P;F4KE+0P0)0z>&y3)M1R$RZDz2` z$jy236r}C^#v7~jK7Fs^TEgIq2qD=scn|nZwTP+X-_FGW-~dU;L-TlX!weBZejl6Q zvp8YTQrYe{J7jD8t>%*Mce}07_BCbdhYl!dL8r>0R^>`+bPg_{1lS=v5iWis@mZC) zxOaUL$2GMwi159g4OK@iXx)81pi9|7DpH4fXfg9+i*#xP-uI_@8TzA%4KD(4p$;No z2YwWIbi~#CVtMJ@O`5YAHC_(6k|2mmWUuokKKR__Kj<>AHIZ4-ziH!17k8~r3t8a! zRzw7fuN;eiDn;uYNq<2?%&S?Ct#3wnpzB8DvTX8M-o>V@A^3KG*%tq`!PMVfN+PGj zKglHat5DJywdGwlJ$MtmdRz5M%u`z65Sr&mW6f2|_xd#wlPOy!K3>h&>dn;C z7If-;^TWMIcgbRavMHD;Rd(1M4`7{M+kLv#EG)O=lpGyFs6zmLnBbPSeq$sisj#(5 zl*uM<-CoKB{?}xrdkm?+rGkZ4=Czv}OFC*)p)+Y|4!sNqm(5D{wy)#^C7M%8CE{`Lfrl+rjnWNPvLLG{lJD-5>p!#LtLTe5~k!YWF-b zfeX<)(hv- zxS*rsH!@iVM0{Sd*pj4t4nwAiknPXB2otNTh#1a^`O(PmfPMbg59R@ae_ubiU6_-T z6V{AsQ~mG_t7xJjUOP$}^IO5I{C$$-vhXf}KkvtdJmhnMcL!_rbs);5EzjfJw#&C0 zBHjx@?MYVopsIIT6pyK;JJFJb-dDL z#y!A$%ym7=r^H9C!+yO+3>p9UTk3I-0rCP40HXi;K_-JAM%=0_ENnt zfQb8xFu`0vN&0!$?Z3obQ=VTsgr?TQh;c>2^?@xfQ0royN{?f(t#aP?;%nP2!#O$D zzigAt2_*nUPA<*vQK8bOSS@Mye!upFsGf6Vq=fo$kyZ3?msx zu>KsYmGF+reTia($yC&rKn?07TZ1o>xtJH;s0b8T?fjfIWwq4C!-er-5fvWcq~0r2 zYCdmuY;lmcE(rD`C5BEwwo0gpk)o&HGS(jLzuU~;gGvK0ZkrMj7poS?$5$z8!b2`@ zASazBFw)(YJhFVyl~}MV{bQWX@9abq=sB{B9yYTxFmy9QCw;sk7+cr$iX0wL(^pW? zT3xtWAErQ4inA*_`#B1(Fg+&k@`eBb%&q2Hg5U0XJW?XNKdGU~eG84|?H_&!xJ#GS z9zJt1rK*sTkW$G3+WkU45Pkb!Q1TD{sKASdb{L6}SeRCddZfkUh+RioDpXY1+#32k zxj`+P-uhKzWA7>d74PX~IRAj9nuwP)yq@C|ni|ooKRePv>=*J2`44zH8AP8m=(R=p zT+XqdWsla=Ju}O8a+ZrIKD^IZvKScxsoA>_>o#bvGR^8~<+nj;7Csv|e5*)00wu=g z%l-pO{ufwJdWWf6(DGz%i$MUTjgO3dW)B7G+(q{;ZSUZCDKctzww-&`9;KL8-q!YjOlm;g;NO{QvCNh;x zOk#_V;JiI2hd@*%YX@9hE^Bcgek|8KWex5DAx|Rs-J~~*o_bv(fP449EbCVp_(U}2 zS{AKubR=S0In4C+Zc81q?z4qD7P2rLPTX0n!yAX@;_;J^FDYO8asIW6bbbORu-)o% zP0zV)|6Sk$o&>b$^8G=zZ%AY8vHiPiWuYtEb64q}6n+=Uj$#ZYhx276pUa!gugm`v z0{+WAJ%nw+@r8xhwv58AT9wJh#ZZ0wTOW8@G<|kfXccTaO)|kk>UfSIjakt0_pa|C zg#T_SHRWu`$y2u;l6&)KI7wj?*DD+7$GO_X1g|QQ*^wQrPnHvN(M@i&@1zGuRXTC~ zoNUrHFV{baN=TprNx3mXJgo1Rq!>j1Ux&~Y0c~Rf2eA`OLJ9OImOTh9LJP}j1-a}N zle%}m(Ia~I_n5tg=MUa4L6}@{$;cTf=8={SIn5)eIJ1Q`eM>0Q&7~cLw*=aa69e}$ zgQWy1m(!H#H1~XR3{#4?q=cww!%!HYuKaiW2L5!CAmo&%|B@Kr=FrvBLe|sM1!B=? zMV_?WZd74)(RiCqZTH3XU)!YuzZ7(VBbHDcYJ=Ogi;~+^v{ovhS!0Ci-RhEC4zY~w zpRDYBa6VlYXJca0dVrQ_=xr+_kaEr@J^qtg-uxrXSUHk7J!WrgjqX-BI z)77y^@m;leJ%mW5<)mCvFi`$^Cu9CX7)hhg0YVPNduxEI^b?qyUd%-V*n#G!uk^nj zV70D4v$Hq(Yb5F!*i80BK?FkdWQH*vkO&>iL6_D*{@k_t;l_+=R@U$sOr-w+&=qs| ziX~`b##8YR`M8;h8~q}uhYnd8PaI&7F+E(;*$j5oaYNQhK=~P5EaLteCfuLzW?&l% zPR{E&{@j`4mKa}xjq+TTE#F$-&Vrr$ZS`s$&Gck$`7cBq^u#%_V6RA`5hf%ecaKvC zasY^zg5MSH3TA`#USo_K>{RR28S} zaBalN`n6<@VuB@A=~qF?RO3`sDXfDz}abYE)z@Tlx05?LPz?{ZI zSrHh+8|CTy_pR80K$g_UcM{T(&2QD|^!Q7FBwoVo&r^I6`|3Sorv+I%zCEc@wz9G+ z2nqS3(VJFaDd%1z{@cGdbG)-#Le2E%`8W`x_*mcS`p6=m$-Rv!Swb^I^nr>(pJR5Z z9fN|B(n8t(Ke_7|9&#zPB_fU>7ch8C+2D@v+fV|DsFL<%FxozL6-B<^gUGD3azkSK zy!M;gGX6y*$joh@#lJ028(D~HsSDnd5}Anii*|fTzuF0IA-w-{z++O!vVgwV(0H8t z?Xb68d->Q8u-8DvI~5+Ev$*1j3owopb$Wbsc_^o2r6&{)(&osRPOS{51mraAFtvEpSKv=PUTokOlYbOcrS6OJWgrGxKA}JB(8>HCSB%iY zJ50bD$`FZU-C<@EB;5Ma=+5ZHeP@W5ip1-GQA7JvmiP5l(;Ah0x?QbdQ%<$=M4b9v z*m@Ae)Ld$-StTac>5wN3s0f^V>2e4>mu^K($7r#z2y4m{4|vce0sZu!cK(;Z0Kbz9 z0rkG?3=*A?srv^SzC~V;A_FbP*PdS?$e&6S?d_}J#(16J{^PyK6p<0~4Yg4*0JFvN zjGWIaXZcV-g#}9z{|mY$$ePOG&TUl8+sJh=Byy$N$U1b)}D=#?LMzNyxE<^Np72INX{Fv~`rokZbCnzQKi^sLJ2 zZcR`7r+?AJf52&088$LdI-qWn@^C{or?Aq%R9J#W9LT5%_``sRmLimZDWA#j(5H0R zXuPfcJ5pBOWh9L@Cy>-vzI3w`jlan`J2g-d5?<$@M@)|}aUcOv+sT;K!j{~=-r1rd zocKSmTO?d4|EATxyg`(`c=Z#~L?AA%s@C+GPC(!+rfdn*>VINd3vtdAJ0$?jbJ6K#kXE^D6 z&f?lQy{Ih z9IY-UM#0X;rmSuC($us-CHRUvj^T-DVa8S1ZdiDtISHw?GiJW#*q2^$Webga-z`!6RjTapx_<3=9>3b z>}>vP=iNA3jqC+L-ESms9xX}tP83c~XY>-_svv^Y4@PgrdmY$%t{T6|D<`(5Jn9r>16TyP|jql!%u8`W)_5yzZ8 z>5qk1pN||!y&E?@BUw)P3qaet_9HwKBm6Kxk3RA|yHSc<7lK$?Sy4U`t7_{rl**?k z(kV=bnIl2BW3jP&F5|?P&YLh(53V#UaIQm5)>4!aZyO84RgTUz2SzhpMa4oM&mWeo zR~kniyKh~Mw3^YcYZ{$>VhL9@%T)a*!FE)0&}O~@dH z**Qn0PND}X-(L@A0gM5aKmP*n`!(@i0yUcrweVlE4?ywxVHo28`AjoghjQJYCI}`( zkuV3Wt*@nR#lOCB9pgpC8w{Rsxto4n6owhp(th$72AKe&aWpm*_rel*~>}zZO@Kc8DR`thW&}&1ds5MY2tCr>B2(YLpHcRaesd<**+y zJHBv^H@;gVHt-H86Xzfh^x%phbdIOd7pPoMb69HQs&Ki~Gc$2%`xB|!^1Ue9Ymt2` zm?Eyvu30{?TPKS=CZJs{OBbV<$!P=ER#2YFksQHT#99r4KQMl)Rk)W=K3tAefSf*WQK=fAr)!LsJwi5de>QMuv- z>SWNaGJ8Hf%h_N=4=oCpc@szyp8?aG?0X{8khF4mc?iYdDnfbv^XBbIrjKX zLiKl)fR|uzk7&tS9Fm&aG#CUGhPbZ$IGWEIG6z5XdhUwa=`?tI+3(#kuz%5tkr>iz z(|7K_-4R9lm~gSE-R3}VgLNLV9m{dz+3$8rM@+xW4~9C`y8n-7ig|}X_mI+Nt?-tZ zp*VZh{Wlvp>{viji0Lr(dog=qYIN#uYS8#g02Wf;{eUp&u`i?Ir-SH9SoFDfrkhFS zk_BWE*^KANPCLEA8QQ91^#^GSgA}cO*$cm9TgrQp|^Fy`I*H2 zXq&EBt}HUnCQI95Ip%|gRl=Z6&DC15b86a45}=_L4L>X*Af=6`a8aY=4pdPFezml6J_kL1;u9Wh@bab3oT>pMhlQ-i z*8Hf6UCX_;T>_E6f)@bzT;v`8Vl{q(_r8r*#A4#&V?cHK5MJzA)vYaujumBP;||cP zMdet3DcuZOXA(7qU-%XWC9(F%h1M9}JaKrE2bC_*6>qMGEw%LsTA0WXRx$FC)B4iA zw&XZtByoDbey5=+Y=uKWuWdq)LyaeOME?W0WPVRBs8SYl&u{ouD(+2cBL)-W7gqt$ zONC6_ls>C#;p(Ctgw-`NnIT>dPDZ3UYD{FH+DW?oY{o18qjzJ}QC;Z^zNg&$ixvKQ zsrJOhO^gJmI(;VJ(?T9cm>FzT#PX0+rpC1F*LKu?+%YGJBdIyjZoVx{9Gw)t#6&&K zdg`xt0MHj<*$(VZCo!8`Y3gA<{rfu{gh^pO=I|zcAN^KsHts>kb|>-6cd!2NmpFu& znyD^mF~Ow-iK+hP<4TG6#83O!c%_L)oNNkyWeb_XRiLCgM(FOx*bfI>x6Ag=4vI&x_Q8;*YSQB_vJ9dF*Q*9XZja4u% zDE@}aVJ_=&N=}pi)?-*aJw8sqi$?lMfZuX`hFF!_%a8Uss$sZ4us!uQo9>%;2RqH2 z=e_A~ZI4P=Dl0D3_BlW162Xc-{*I%3ObNX73+tk?z6}hEWLsVheBH9Lo-jXKw06j6 z$Xd@=4+a`5SlvsVU(S9?ig(42i5*IQukkWp3lpl8^c4B}2yTuC6Z8Z0tpxUL8=d0_ zaysullz<=Sr}yA=FgAkKt34^*d0}5g9rD2KxyHDtlpM%wy(zK_)ze~!$!={U2^QYm zl9!X2R)a^k$~8)tS0>bA$`!v#0NJM9MT#ljD(7o>J_XfMj{1Ck*R6;~ekJNR8D{AF z;Z6_d5RGfN=JH7a<5L3I^p@g#aBl7BFaEBLMz2w$r*-0<2TCBE?6&%CCgvGzXPH(v z742Y_x5FI1>EGU_Ar%~;Z^qZkH3?)?I6V4^B{s%mjp*m;Xwiq#BD1k^ge6)BexVMh zW~P^le%}0t5gVfW<%NG%6RR~ojcp#wJyx4)y;juYxS|^N$)9i0>>O#05o!W2*5YZk zFY{CQIVH+BHr@ThM+LsVjGe^m8DunWby}5J>+X8UPxg3Ka_?DZv4SlpXXMvFPb!GdgX(y2K$gvKeA=So zCT(0&YNVmnl;7(g;P}o3>T^V=Ye9wKg`(N>zeulYNtd)%3~{`?e}k!>*J zwm*|GHQyAVdD#}q6UYFW%(S`r*KLgIrAo;#0jCYFbWAf3Ue|-H0njWQtg_WTVL4?K zs_!O7aX&CJ@Wn&E<&Y4ydEN>W7Nfzyx9W)w)6qZ#wqyj~dNJN4_S$_M4HVprYOVHr z@{45BmU@tU<6+?%Lk*g~Aj~ zv9E^a-$_lXzzBb-F*Z3L%5v(vM$pK(xpB$8VHH9CG_n(bMfa24$L&K51JtgIp^y*h#3S8@Btrk_hw4IGep&`4i0Qc~mN z=mom7z4kJS;KOVA`RUD)!@4ugp(=Me^e`tkcc)vS?MH4NeQL#ZyrK4VWfaC-CI@Ok zt>5s(`8#-h*&MWyqvzQ@59u$$;y=}W&V1oq`nDKNuh~hNM474z)BnY^bpFdL)OG&TT)WZmo75GzAeb=H^dAA)}a5jW!`l~ zPx#Zd-R}ajfcnALZ6xFMy^_PsMaeFZFZcd7i{p;kZg)lT?LYrl16hZUq*k-i+qtRh zTU6Wu>U!8~aSYd#LX9XsJp$A`u-`BV?XZL+52||5D`k*0?VvM0{Ar|!3KQn17R*vE z)YH>YC{GtlTzqG-i8{tY49u-rRVV4s{_;*zSc; zG64?^;`!a-b7^M@Fq9r{UHTcxg zWaCQHE||)lENl$HDaj4)+Z7>$ zj-~KF+ymezvKM?9s`Sn!yZO(`p*ep|kod|y$QNW)+gN&)-V|pn(2N zo#2<5Nm23d`kJtGA_ltr@gn zykXueeW-%9bH4BT~&~izvZfDghnq29|g=z_l5xozMvzdpy9PmA(LKzM`2b5MaYSg zmRb~a7hYhbNUm;&KZV2lUc9A~<#~mj!hR%BOc;O75$PK+i<;@;++y=H(I>8ku_x_7 z)~-a9`!D|Zkx`6IP00dN*Kzz8CVL1#__VY|IXR(oiEM&|6BEkR%BzX0Bn4DHsVXtL zPnvA|r04!SnH(qmZmylAB6zOe)vQbS+i@_vX?L~i%If2`B*x9{%OgFjfT$}V9!V3N zzcBmKdhWvE=Tm|l2PPkFXQicY4cuaS?3$gDv=QsJlUW9@LejNyu|{ zX^gMJ5hj7Ak8Kt_D+?=ubP&gZbO8{gO_y)f7f#pd zfzti5LTU+eO{Uv|AsiIpBmk(tgujngB`gLJeCTwltt6@zLe57GyAZLod3&2Ah*ufp z1km73>O>u%f=Z@FdE{7%>@!@W>h(FYVXh}rKX6UopDZ<#fVX`A@VWiQs5<<&<#b%f zwrqF< zscD%{TuvQ5X#6A#m3S{c$_cDlgS-O`3 zbI@X+Ain)$iNV!8D>7+B&UHeocN+o+o(EkoGxI*`qd)75s=7?DGP&VgZu?2!PhJ1 z6J5QTw?mmfU9c+ZyDQ3PP=BZPXf6hRdGDxw^zwyPQ^n}V^{TxV$h51$5WbIV4UK4k zgr=IB(2U8n*J@|rOV>6|G1OIX^UBa&j8bDa;$wbdneL~1b4nt7f1Ag{RjE|7J?F&! zU)Ig-2*pQtR#Hu{%&17S8%U$~h^(bOJR>!0jbZ7}yf^{9M|PWqs^G1mrWwYq`3Lv( z>RXS0>nwg_D*(Hj=f1TnGd7r9_=H5$`r)solf5ZNf8eYy=4wd zH@%O3y$!|lFD@OK$0<0~T8Me=30zx`@3VZ1afl+bTB%Yc8&WuMZ@Q(o%a?%Ocb5fc zu=grKCk8?P&|m=?uA%aN=J%y1yV2@aS^S`Hq7J>`lr8FN_7y(Dza%Q(9(Ck=Kyx zeym8XKx3@utu`u!olZ`ed8pZPqevJ(xZ1MMEneqP7>q5|FKsvuTLL%tDW0~)$p#-e zP^tHi(C1fv1ctYguHBMPt!?aONShVRWrhR9TLT=(yl0ROg2_&ttMVne3c*Lx zV!C$ggi1#!!^NSmPo}b?hz1wf{**_X^$Da>xSQ(h@nZs*$3tjQ%<}mq7d!=3*iOXL zb#bx5EI(TqkQ=Yo`=ZNl{AKA96*hb}NdMAr7_m=O0t>brx>d%tg@I%}@J~5{42;5t zoZSrwRnFfIrgI6=-;N~$qM>r~Yct1R)&dvnqis|1r_KS>E0kl7{VhAaQuG{`(~ zvJ!!Gyd|%l=-EIX@aTANbEsi<>{Y6N$$gt7$ZAT5`dqBp41#QWWY%{~r*M0V`gx;d zUdzY)^uQc4>u;I+qMDDy0MWpw-*HQ%d&zD7f~WagjGjn*LcGlA&N`Js*ozm1A>NCi zZa~QbAFp8Tr9wZYZd^66U&8^*%e>~w@IPp`gUO_fR-@fMNze(7u3YTveT0xkk0sUl zPH+<3E1Dgel!(mlD$@ukiRKrE7Ufk2>=j#4le0%-PsaRogqBa43h9@CQ}>JaccwER zQ;QkFHy6BB+g!u2Zk>yj8fcBx8#oPx@v0gpw?R0S2V)K!IVL>=9Y)JB5ZvaLYYq^Y zwgIm<6jTRE_jtvB0;2t<8)gH+aE+TU-a;BXX7$Jr%icCNxIKjW@WSYmZ({?0Y1A=` zBpqPV8DO!@s^~W;&6h+Pvo&s;X$VonL~ zsJFlOZNV+ige0uE0X$^G2`{<3U}Mnw8*^xTYUGdO@;Pr8AE!p)yy|_hyCcnmb`U;I zb*47AZUeQ~41xtR#gY0u4wm&pxrdmU%g=Vw9tM;^s)+e0qb(&eDzMqgT6Y0_p@ZonDHOk%}Ug~^cD;!2HC~Lu3;ko=WLB{*< zqYflVwC_gX{7zFheAfx=$@idxp|!#I1eeJ7KlW{{CxAJ6Mgy=s%vZh_2S{MoVTQzR zbMnib)V^^rQbpm-=h27RG@dWY6mbcJybZt8#?(imY%H+mYK?hY&*s{lzx}BT8UJCa zvAebzf=*hVee3HbA&}Q1ytG4Q$G`d~LWdk6QSALVt*W6c>c+~>&TBw7MmLl>Gduge z-NOpgx5dWFDhpJg{~jfrm|&33F!DQPZI%`huoZ({+dqH(aAT`G`jEx;_l3SvAcPA9 zVeF(HS08ovr=>o)XpOA_WltyTy~YD#9cBpl`J0<~*B>g2QMVke%%R@Q1``w!mioBR z>X$y(ZQs}`4ea*jvh@t*a5!6mDFT#rKgf_~%*<_M-^;0UN43{w_bf>VX|pbSwSR{8 zwEUV@o71ffyCUW~_|-)T);j0mm$Sp_i2l8nKDe!_XPJKU05-n#XtGVYR6ffSZM~%3 zyw|wyxm41%4rlnJ)#zwr$I)$*|76-|ukSUKGPT{C6H^iJMf>f5tW~CtJ86e=G2in! zq<;nWEzsfKc7~Wb)`qx?2nmwCZV;BkOFX!+GUXJ`GyZnqhj(dute^5qJ+ml zYEb_nZiYcq%=K|gpTH}^ykspmVC*-wXfMJhg&`YGcLUSu`kPKi_!6kWsxs4|dj$0= ziXU=>VanjE0e%r2Xcz;E&btTC043SaNlf46BHK&0?}H;V{y)OL0xFKR*)}93K=6=2 zfM6lGySsaEx1fUtcL<)~?(RCcI|K;sZb1ik28Y+2bIkW z&4tv#6p|=mEk@!wD}jMiOy(!@(nziL3-S^vY_d0|;EyI{3m2jegjGiYY+HYcYuZ%X zSowh4shS$f4rUY1PlGPu$C-Lbr92LUmU4h9x!^{v+>NiI+?#wQdO#@rjMiwf9E~~OLg?3UUmpg z4FAu(82HNs>z>JA`D3#G*GWyJuLYAYYqS_uLO@S7wpr=GgoU13YbJPkPm=HcvbXxs zS2@pbykE44PGxK$yp=z_?dk{P{DEhn4ju(rSqtro9+pN$ol()~rTnK385h@-0g$O?;j1(?;Tz0!BdS1YeC%rV5 z8JSs87eB+KiS(Oh5j@UEa`%XhON|FJ=-i>h6YJgD6F3mo=+x&1P%1h}zv@G~q7le_Hi0Mixqyv%d7s|iY=5m<4W}h+B<)X3B?>whi zNA??uH{F~Meae7MM;?T}w8p-sQi|{G>woFKLWfR^r-s)2q#@;Ym|`LBU>?~4WU{n1 zMCP02TkM(wYER7tpc+>$iP=s{QdGKq@;ugw?D_mO5697Trf0ROw(`Bx;>1gl1+x`5 zyx;beV6gruuKj^?bTH`AUcN5o*HJZ5qbHvdAxdE7j;|GE#U2#3_K?OplQCozO$Y5A~P?sh@jS5gOlLxx`BeLZ@{m*M1jrVH?v z;6=U@Fm)M+WZ)9CbW8jg{T>$dBP+(DMWXyC^UBRWGHVidugZv~fSsqSX1N2;ryE@C zTrcLSnPR@X1ndH+x_%zg>0hQ13kz^pKEe2%cCmRmd#%RSk(b$t6;faprSho$hE@Fe z*DQdNH@2oSrd7|rJ+dFeJsj~khgVx)dR_B0q9Zb?qh%s#`cur8zHIVu3K5r-kj#=i ziPWJ&Qrbezz1QQ-cf5RB*EByRmYSn33ryJrf(5H?BR9h z>wJ-g^O-w>@i+w?1e7YQ#H_PBY*Dw6F}HIZdwg3lG3p@fy*Qc(HkN5ia`M0t^lwrg z4T0^`ZqjPaZ!-5~@dHH1#jttJ<$(~hn2sYtrwfb&dbse{&dRJp(o{|1{EeSx3JSnI8 zP_o&a%GEfN0%=xRf!nW_h0tk8TA(Z`lLKtCD>>|&i$e*v!Gxm!;u}A`6#fVRC2_JU z$XX43Eg4H-jflf7bgRqD?ytP-EJEs!S2f~>mJ+vb+0|rrNGUGA)U}EpJfYk(pJ{n< zZ@X%}14qXvTKCXq4G@KwWWw7X-`*Q7wV94vO9;=JZ3DU-My;#jR^xA0$+mVnbV@}uCGL}Hi$vrNV|tin7}HB#_=b$P7IV~Ehv0vJW>c2oXYcQOT|?JS}G_(Ba@L`}$CzN4jfDv%FbR>T<-`oXSt zd5qUb(l&KoA{^AAV76XU`8{_ko~=-6Vk`#%O@+!>dD_b}WGkB_jcowo(F5c9#zdFl@Y^Qutm+(`!5y=AzBJ-h+)$jlYg!N z&1y}x*)8RSSqFvgrvvjE%NYy8_&nFpYu+k9C|JJC0vW5=y6t}dvcMx8rIGT>nt|yd z-C>500y;aMy{S4bOr^5%`R2uGA zp8(raCHEODH)(rkzO3NcfJFkgC$kZD!ZS9_U|2@dI#W?}1XvOA?TEZ^jC0Cl?8O+CoEnd_=<&c*-x1HzZE&>^1 z?~;Fgpj?zenli#a9gVGi%Ck_r&fOlOiY_mZCmcl z>Znmva%ki#fAXM{cs4nIlPc?p^*$jYi7PHw$h%%U3Wh9;6*R#l%gnSMo2#lMmBL*Q zVib_wb66Ez4ELVT5t}F>L*OTt0QQ`c&Ajh&_U*VL!J~KQ&v$5exTP{Gc=2hJL{%PA zBSZF@=~^xlne-KEGlT>Ru7OBF#&C5Xj}yY)eaJhlGE&l2qzldW2-n&jWtxfXl^@-Z z1<1ji4UhzSkY@Iak8y zsmr?J`KYN;NuZUM7PW7q@@vH59bjl_fzwoz?$~$r?9zgMdARhlsz5o#OnOBx*}-I3 zeU;P^bFq{782v-Is%*yYu*B=EUx!oRvSE;teayMO7zkPogEOI%%JdeP9RSoBj`?67 z#NB6Tsz5W#FRESdjZ^uB4vEYcDnzjnp|ibtiS2E{vxXH4*?p;BJII&FW#>GfeJmwLk>fRWDqldmt`(XkRM6y58qmDv=L5+pTE5%--uCzWy>9ueSZ!5s1Q+#3@jUN-@`Yo#cbg0x}KJGQFkEFAc3_$0Az z(_R%HzL<9`ZIq?s!>Flg;tQMRHd%`2NBIj6dh-)y+>z2<_L(Uz>t~@~EF>>R1upbL z@UqyZYv4bmr^Je($~ao;40Mrxm?Lo8XGmhRPFr_A{@qb+8o6vzK`a!p|F>q*_4dCs zivc|yUVL+6a=Z=gV{FR}*Cz|0)1{e$wZRW}hCjQMRF_VP{X7UCJ+GPYOX(e_hDUnt zz5yV!p?^)hY0v3$3`p5H!n6;#Sx?c6N{bu}T>z!j=OcNK?F-#zVPHs9f%MP5#5WVs z+hMv(LcU-}t6IrnD(m2|)NEBRs5Maxs}4iz6i4#m)kVvs)YZDD?XDb&e!bT8zH_T5 zM9TGXr*7lRI1~%flLkRtIa24tB!cMK1H^K^(mq>Jah5tJ&Hhp;>EfQ_ErmPA4VA^c zJ@He5h&M!LlJ{BB07E&o%>L*7n(xYIE4qsoi|*tB(+7eTn~gr)$xzQzm4TW)@z488 zF*-SQY3{L5_+S`~wjEp6R?Mx3I}{wo14@qmgXw+u)t7i9i`UgftxcY{GB`XY&6LC2 z?F5kXv10rz*Y36W-Zr+xC!tgJp@dDemAiLX8o6RF87b-b;Zpp)P0^dLe|dM$Pwfv0 z>I+~=3u3>`3{(B*aeNt6GI}!z1HvUgFtO1oHh;YBRA3eBKl7~=eDz!a-juBDkATND5^pC$qD1G{^Fy#pokKUy{o&m^ zKtsgIvvR$K7n26Cu9@80x0=+y{@OYzE;j#OhKKo?H26{w(;4nmYeSA{f1xfbOY)Ql z4;O>j@|yEY+s{QW=+55J-paNXER$3AN%aLgreu728@aX=_lh$^)cJ=uBZO)3)FCq8 zFPDSx&Nev|KWYh-yS;|K7`c!<)8a_I{yepL?-KaU=E$YFI zYv6>HYMkOLd%ER(xK&w=`Y{ zOk&B!E1E>t#7@G_6;;+t_^$U5x;KdV+Amu%bv(~sKFNkbtAUYaMn*1jTLYGWMvL0a zbwwA#ED1jzdztWx!Gs%m-o!tDGm_xH$ff&)?LHiUdh=@CT~;NpnA%gu049;Dbb2s^ zxeYop6AE|6aUrsjI)sHuipS|H=+Jm|PeamR6bUGee!?JHUDLgP5#FNh7pRNk13; zNLoD8vvcOUh_NSQXUXXzNVWtbjIuY zP3kO=#Sr{L7po(1q=-?0-Ve<1Iu*gtK)A~F=~|AL+M3=<4!D?8nr~NO#TN`Z4}c!# z@n`G9A6zdVBe*iFE4}hj)3UtW_prZs4ST=o8BQ%27o0sm;fxJv%fDYd*<^J+T<~(B zicy(vy19S%B#+Gt89#92)KzJ@j0K}Op+9ivYee6Nh7KJz7E8_WQrTnwO;7*|(uBX+ zfNdo);X{Qle6(0s2Q;qWpFaKXM=M!|^5_P>TIn;)n6|5Gw8iVzllwx%7EIjEb5D=K z8=L9z_b$H6AFNMK?9SVq?!Du<+rFFmkJhkc9?d?)(77v^KUQxgXimI}H}cN@Fgs%( z|KKv|boZg=eSSstXU07FIch)b$ z4CEZImaDg|5s`+Pdr7*b9y$|$K+}3GKP*+dQ83mEB{MJig?ZLdf&~ z!6xU6t}3_3@A+4>dIAyV2*+Hf042>d_P7Cz{Ne^WF8coxyPpUIVz5cbDSU_wvAG?L zipz1(eiyvpIP?8uqTlE~YnzUvYsq*SY|I{F8vQ1*soLgbO-8O1#XE0?|xRcW$| zbZoeW3(dY-p=7Pm-X29&u?W)BWVd7f*Ww0T;q#Cs$=`TgyzOrBa{tU|U2@JsGt*&Q#OHN_L6q^3(zD2&X6%0X>I;zxVL&mMK*H%B znp^~~h&!8FX^soC;1Yg0P{%ob^+Kkv?~B<=wlA^lvO2V10T^+WdRDbyF9?#|XQ@P8 zdK?rwAJmrPI=yj&tb0)_IS_qh^zl4Cl`9k)(Ng9nKuTs1e1lfyV$Q+*164H{vn(4M zb%{_+!bjf-U(6?-t7Pcm7?F=r+4Xk}QA|{l+WAhY{TRD&EFX7N7csWoz{@XmD@D!+%&)Uc;`+l{{m`bn$#Mj6S zpZ?Aeo@B;%XMAOfM(;G+8P$xB^4u_b!!iKIH(Nu*-@1*zm%JX9_a2a^}2)&7m4LY>>3ujzQ9%)Ub3W!Tlv<(nOk%ml>%< zyZQ8hZf9oZ-Y2f*2E>1@xP?;n?4~i$7#EsK3*V-2~)}1Pw5r%UAXX5Z5Tl4-*iuMj&l)&AZ@ZMR& zj|x_P=r)(tH_D<7!^qv)6*vQkYz!vBulw0bdkkgWk~q{Qs~Y2m zhW2CiZ0)oen>y+7sp8UJtTPg{;X)mQ`6|>DB!%0O(GHQnXrh0VJ#%5deBqvOwu(;` zxm>nk0ytOrM^fq=4F-cLv ziP$?av5B(OF~diZaNgC3U&Q}^10jA==_H&&51303gFl19`b+Bo{k&FSbLjV=7-_Qb zM$m_Y2Uj>606}rqyR@b3&sIXW-2l%!UhOD+0}Av&(?stdj@c<*Dds@U zHrNW43d(t7H5XTaUvQAsM$w8cm`VuH(|7?N_@CI0_6;^q`n?P=ZtX2Lx82?{R*R%=N=TtRQ1rY^j0xJ94H7KH*q^49Ym!Y&Ie#G+ehi^_> z_RGMXi}W^O4U7@DnZr208DRQ^66VEqt0TVa0!%5kHJFTXz6F81c>c0>#kRrh#o2~! z5}78Z_<$ftu757E?r0V!W|_#hUp8KHD22;40L5yYE`femNZG}omXnu47FYt`rNUH2 zlywWDn0GhN4uY@Wz(bVC5~kJOeHO5_C1kRT3Nii93V#L@j9`B_+p&ftkM~zc@6U4o zQxTCIeRRPD#b*rUkBlb0;5NSNhkaBIf3H+nX34;2A%=ST$|1jZ#VhJ2WGBBkt)@tb zp9wl z5n~LBu`7RiJ;yLt{XWfH5ZLECpy&I`kpGvHbPT{6pe|V4)}N_9`BPYrV7n@0rFBv) zw#U@r@#clzOf4=-U$B@R>`yv^MAdy6!&rpWkHJXEd~^IV$Y3n3m|8*hbbz>TAv;gU zrI;7Ygb_K-o0|1y(l}nV<(4GXDI3os?~uQb>2O-22uzM8QSjz1Tw;TnHzS$ltveG) z#(^qmbmnMx$j`(~Iu{7)z>^ORlD}2I6gALj`YzFjbNip=`@cN>R+2x|g5`h|$c-#C zhryCO0lL`BjHnpj2E}&ysH9z|_YPE3#kJ*B>om#>Zc$tT7hzKM(~bv(PZzCK!!5*H zj)^AnvNBd_6Yd%)j;Fn9#fOu|&!Oze`}Jt^s9TVAo<+&4hUyx+EyJOGK{}uvO2$j# zt+t?xjkAXfj8wtGxk2Lt_y(5hl>&*{^G7U2;k3htf2`7`Spm2XVLza?k_BJ+U#*qD z7vukTGArD!&~bh@ByxTm9zGt02Tf9+Hj}OrrFkTp70T%N(ZkGSZJ#|=d@u$?deAD< z(L@>8Bjv%i_lSh(qQSrf@4+qhx_DT1975+Qc*J)f$EY!XG}{zSOPfFpZtw~K-yS_c zku{orn`_^7Mj;>O$Nq!ti4%R8WZUq9zHa`LB|}A9d~j@!VaqosnQdj+R@Ru8i|w`;HELB)QTykH4B3S?*nv7rKUr z*BLegi!yMUTv&2V2O5Y>lTmRdJ2-m=9l@Sb9snFxHBPsa)gjj`hyk zL6BbPqLq;AXryJ}2buD+)jF>WPLI10RElCL6p}pB@$RTI!O!lCpCgXQO}_;C|GE~$ zi1#1sFF!5a{OL1M?YOhXEvzth@-a~2q(d=q?3;w^p%yO*mA}Ff! zPH=p@q}NFw!cWqy_2_kyRkrl9%8A_ZtSB47ZMR#+%Yyz_^Bo0q2B4EFME_$oOO<71 zzS~d&@uk3vtS(KIg31b3?nefLX$6zEDbdvcf?nL zFy|46#_p+iG12z98v*nkWTvXGVIFUq&ZIsN`Ta({JJ6Suo?+9VgAM5J9l@SHqOPp1 z47ifRRj@nfO{j4x;Z3`~Pu^~@epDvQF+>JEOuWK8SJ5pQ7ft3(gSKCAb}(aYbwBGS z3$nk6VdrqN?P0JZ8(n5;@`sjH{IEGJu6n?!A)CP^*JK^yvA_8`Uv3&Irk!*)TY5eM zmzFtSq@m4`?T=-f$G|3+L3@9;A+$6-aXoJ5%!@Z$FQ*z_4DXdTA%#TuE1>85(m z=BiTPAI|)*p7j>~y%a8wMOm$YwP@2g;sI;b5DrL`jnx$_V^Z5vn->0Z0)3b{k0mc~ zsiKIw%s##wow~IO z%i593R~b&BIO>~^l1qlgCo|E;r1++%hxBJ-Niqh{h4NvoU^I3$WOTcks(4KV9nZ~4 zMO5N8(yD~T&|x?)z4=1w%gQJakCB&9j;{}y@(6gKPz>hPFp5UjkdNGnbcGzCjZ%_l zUO+2g?!=XjIWIKro#+Sjzoj?-OQ<5$7mQ+qkH#FgvL7brE?Bx#ir?0^K>bd)3M+nL&d z&MB`$X9--(xs!AizCC$u>h`-Ca6AuXSE4%ZO@|3VWPfZrMRv4XxMC=f95V&VN^R9%bN?jkR!ljDOP!+_DE+N52e%v^ z7D~#>G6Zb-dH7M^xMkNp*KnMZlQb!7J)1;#Ix%Zx%w~(FCHFPWjWv+ix8y1pquSl}h#0*3!n_k9YG=bFv9P#(ur~f4`G!u=IYaI=*XnmrfTxMrZ zUdsx3r`l4OJfQY50Ta{RL4ZnW)W}rC@6SgMGbthuFhz@J;Ded-YGf+-8HwW`8EYii zF=5pd6CbO+Yb55wzZ5xyy(EkJ15IIhXx^TC)G`w?x1|Hm->_lG{- z?fzyhgGXOVA=A>@p;K>`%wI??M_rq!BG-{313cPHh`KuK<`5D0xHIyL+kTPqz;+V- z-gJ+zGb&S(4~cU#=K#^kbf*8gwr>dy&knfwX(E@m9?2;6RvRNTa}i5HENB>-$#4*Y zwD$L)i@z`b-%j{t`0te;Y?C69)U^AnhyjgU8q3_xqUh6Y*QFK9Bcp_Z$nu&2`j+re zQ$2*>hU*<3kCeld!&+IK+IHC#j)TfsC|`sB$1I;yTSwMUKkICOeOM=bwtGZJRE@BC zsOH&Kkp3#?|Ff{}`t*Yz{%EtGgja2v{FTPXF&y~_te!`4RBVF?o?MAL(<_Sqq& z14gt3Cp@=E*hcwBK=KS878VAuDZ`3R=9js8^Ww8WlY+;t_PRgD4$Oe8xU3SHelkjp z?_Z$i89D9q0p;UN0Qt6?=%mXb;N98f zgkL`mc^i_EDMB~*kS%_b`r$nfRf?g=pOn@A9eGuQo{L6l7qjGr9x%bNL!UNBdL}O8 z?saE#&+dBEf`uLv2LD}Lk@~w&P|M}=qhrNoP)in{{Lir2XXr0sKtbY1_usPdw5}n^ z#8UnH%T`fKEvFuhhQKamo6#1P@oYz-{j>bxJKHJM+!IRhu`~9x$P;5QBRUcpF&O26 zhr`*u_7rU&ov%LeKmUhg>{CS9*F-26BdCH|8t9~Q)=$tM@UKaEuQG%Vzllm&v_sWs z!STS3NsV}h5aasXg=5-HDhUVkpARNEG6@I(@qq#k2e`RD67{(>6g1-qUH)qMe@c!e2gn2_= z3VS-ZTog$7>FPuO0$UgJ`QAij<--psf+9AqX+ju^!x>A#T3WKXjz1?SXSwPdLho@g z{!IovikDM7RO7sA>Hla9-(V$Qc)K%bz6>RX&M+Pn>(75?m|qZ{(&%ky7{jlfyiAXX z+185?J47MPE!5kJXEMCGD%8h2bqUv@jx|?CD5j-~{6-J}0y7NlegBs$brNU1-qgRp$t}VSBRcg?5PY1xgU<=5l7w=%hn$}}yQ!5% zUW=tu|I=dlW1U2tzs>hG)2N(a-F~UOI?YelB34)mtb;E57`^01n&@r&nURrTV4`r? zP~z#Y<6G&LAg%iwnej?ZbE!nivti4!fgqQG=S@ijOA26*+c^H^6t%NU%Im|aGQ*P) zk*F57h30`stA$&5Z1-!4M)bJ~CUKaX$F;eUjr1Kv-$`=vfXK#MiWbB6ZlN3t|4O30 z9$QGqzVGRoTl8l5Nd5rnQ2htVC`H8_%Oj}mcul@Us3?n&@@6Xp%Md4TmUHGR3){4JH9Z4+jsZL)X8e-`pLiIc;1bv;|B(;tvVk2G+GkukfalW*>5?hH*a> zbftDv)cl6DZLGsBV@bb(7eg^7h-f1niF?^L7_DfWo|8gV`qY@MfE?0oyf}{O#q5j$M(a zyP3!Pj|6;Itt2t7^#&iQZ}P+4KfNnfjS71aHl2)IZ(4rNd9Nmh1?=pc98b#Gug?E` z^-cSQalH6UNLY2N2lv*$1rKKQZ#Fm69idr{zq-fiOHUK>^f@_azH``v`?*+E0(S6{ z8Sw!6molXq%8kcR`G-OB)Tp?uz+Lc6W$XU*H0J|k=4B;Um0k-d8>6K3?yPBLU=CGB z@2z^R`J;L%b8Y9wpuq&IaB|S!O&^?<@aFQ zQpz~_z!gq~w5jG?CoA54b?sbPWo(xbQ-RrypE+H@wZt2jaF3U@gZB?|QUq#ToC$-s zAaEBu3hEwWdb5@pTd@Hsz@fBlayldJbT!;NVkI=p^kaORb7-zk2_yYJv3s3l*!GP4 zaQ&V|ptnnU&vr57D-z$!Bs{N1Wc^Je<>&W6K%_gxy(La5Gp#4uC z$r~P?-GE7KlwC(zxhG&ztJq1q*`5ltPMUx;VWjmw;BKwLV%RGTtYNCtZ(KN5kf(Jy zO`}jKYSdDH_G4)?Y~(O{nrA^hWPCPJg;smm^S3*N2_jsp{+w%t8z-*89g|09i-izt?zULdl#q%Y<5hy5Zzc$smLmKI?ewt|-Ss>npdnMOVMU*e5;yY=(# zaf77H=KEG8pUZf)@A{mW=Ji|Vs}=cPjNMs=oAb)B0j5n#)&c+q9K@qKFQ4$@Xf z*U!1c`WQ^kX_RsW;#0}-!uCEnB+bxNgIC_6ak@u`ay9=}99Q&3sWIub>YJ~nD(92u z%AdlWgx~0rXxDUF^4c?$GmgVQ^%xmm(r~RaJQ|orh8j^Bi;H&KM#)nMMSxfGGP}8) zCi+98tPwX+=v=GxmQF(Fg}-;^7)Q@*!0#Q0RJyphR2p3vt3MuTsJGdtE;}6L)ztK1 zT%4OvePAJ+tY4V1j(#y;QS?c@DXe(96m$0F@&JT3c1Yv{rp2| zDIBs;qPm8IwfV5d_|)zUntNyDU}$2T8T=65e!?kBcDg$BaK1Qj2SsJ20P$$YxjzJ< zEy$xq=Xi(XZnk_XC{P}!zjP*~%RsiJHwyw!v+sPwv#}SKA`lsrpa|La>2@~*KUC8xB`7kj@8!Zhg_Ty%#u!Mw7SAraZr-Rp+YpvGudAyi7Fih|+yk?!y+a` zKQBv2MCz6?OT%Rk*#4X4@Ob}PIq%gG>-|n?@Pa1~FTtHcoE10k-Kt-<<)UAOW#krv zW!(JGuBlFfSVHhe*)H7k+=2AkYAV%+6r^Uug51Fr0db?IX1m_KO4l(-H=D@5u5a~b z9j}9V0e~8r4JT$rUTv@7!w8Rk-$^3Zz$p3Rg66#dTcLc5-~W-+XLaCGMoT4%wX zO$#s4x8(@oLH&4{cxvO zuY~>LSfyx6)Kk!wle)JYNmxQZLmWq3*43f`JSQ96IPW@+?ZeU+tr&GtzRdxL(vMH$ zeqm*~wBuU3i#v5ev#Td#RdQ(&C!)Be**{+WkmLk10%u8^nBvb-daIuKkv)*P)Yg>9 zV`a9PvliugIhAiQPdBq7PS^D~{hrhurpJOCS4;PR#NoN}iXkFDj93Rhzm4E8IKxXu z?EGzTaNZ-v8hcZUyZZs_j&Vc*YtQuY_Lk4*VwxD;jr_>aRa(p^+~MDV$1n8bDS954 zF-P@k&${X|RoO6eZ4xuMfzQkGLVMe4OkH@iitqE5N^V{2yM7mXBr&|xDS@xE#C*TN znL0YBrDTfJ)1{0UKXBSG0AR%lX5R%_Dcq7ox7{HV@mpRz>P@Kh!m^n7fK{mjak2}6 z;3u!Majes`rs&Ot$orh5x%*uAsn=VSHd04pVN0~id&*TX;z!}IL(LE@@vVxh8)&Wqv)P=2b|3Km+n?~6F zN}oblBf-*Rn$gle#kyzIa`p{^;N-&Q;3F5=XTPJhp(XI^jmU)C`*hb0&m%dZBKOHz^M0S>Bb{o*^m@K}OmOOXgd9G zkH`6$B<~xy9h-Ru#_h^z*nm#w*W3mN3N_#fG*Fph(rJ;M%k{v_Yr!-uqwC!U1k8)$ z`}(cWWZ(%m#@QM^;r;Me-^^#w{5H}qHj%LHuO&p~XEpr-#kOzFmz(Fl#E8P(4`ctP zmp0&mioe9*9gGs}s>X)kA17-tU-QB)rY&;!s%wC{N9lMC+xr@h&=wm zT$&7ixSaI$fPEICfxLjpt{0X}DxF-Yr!bI;PRg&IbBRh!$PVFccDzPY!Obl3mDmb6 zU^E%0PP`wImioS_cOkEnsUgxJdxmDYlx;a`23IrMtH^sNafs*Mbay=MF*B-(fRMw}9wax=w)_nvWg_j}LPOG*ujPjy5dQ(rm8=@8!t zGb`2HOp(-B6d4{3snq%kyfNQgJ6pIav|TochYvCHmuNITZnfP;K!Vsb*pFkBW%*9Tu# zz`sHZjjCBH=sZ?&`WEZ8*`f~S8x`63JL%)&lTEPC+V~!0#bn#$4imoW&b~4%SzmaV zB{s~%M7i#*BkSwg#HIGYNmNEu{e?kH5LLWVFVT=zCQpa6aQ<3lL^-6we!w&xWCr(un%^zZv3J0nc7oNYT3?k*71 z2fXj$o;S(!uAV&hZ8KoK?0+q#7kY4*k59ZLyF&sm^sufO4J&HZWT9E7%-j0NvotZ^ zN|)r~?6hFC;OexRd;n}q)JB^_K0b@?N?t7M;Bh$0}Rf}xL!&n#O zs)T5iY>H(i@4U0ZcOLdVaH+;9e*>}^*kaYhz+XMHGF;<-(o5`A70xgY%?2gh><3wNzow@XFhM2H+0TN(FVetCGvHA(q-JtleAP9N0 zyg5tZ!KlsJ#5$gl)$4xMQo`O;Cm*fr-HWKWNaLO7ulA6oO7&A|-uW2*PF*^jMX^0t zsQVR4ut)@8;jfc^Hh4XIYLX3y*q*n)$^{z?hK}b}hGX(aJd~DO zjYj^*L@@fuDrxZhR5^7fmhLs)t}x3OISUEH8j;dS(04BV7ykmi z$X~+7Su9d{5F``Vh3QhNwzjDayBlR5i5&#%YgF}d23F4?0sjJUl@CpX3uqxo+TuUs~C+jTZ1mNHocQk7t^1U`ec5Xsx-WTo- zs|i-+=>JGzx|mtGc+=LD>kwWWW?aE4ty-w4+^_Phe7j<`&7qQOf}krecl~43T$_pR z&!qOM+7b2pP4B+&0g@5zT$hf^C`lGdta;Q<53lpzxBk;-O@dRd4}%JlGGd?ne!Yz5 z%A#=LXb3x0R3t_{76=EyiDLd6@{Ve6Vq*OT<$1{}pp?+~Ns;uiMO%HwQh&qxW+`mL zfGf5_U{fEm9+BHvj6`vqAo@$UDJRasWR~&LZZ3Fcd%0(eG3Pej%g6nC5mxFXe#DdX z;1-{v$h=*FVK9Vp@-o0^wfQ4AMePTdY*UyZ!pZkm z(9QYjZ(;->X6Yt@KGuH$2WXhycy6d7pIN7d=ONTg2Vv~$)W}b^f5q))7J=Srz*$qm z^0qeF8z>pZ4Be!7G~eAlpaQ{4Meh-CZyVu1+9aGTWlZ>ljeZZ$Z!z;WVi)Tazfu6( zas)e_0vn?3s7aj-)6fKfLiijON5-wszm*HBHjSy)K(B^rxN=&V?I%)mWx$=u(^~qF zExfIgeEW+4mY3;iNvqiCFEC{j>C*X5_~xp|IXm<&vSR&#x~!-k9nq9It7AZ8Ai$)~ z72?6OmIswU6J*2Gc&0A7)?{b@p)2yoSCTL=${U94{5KR2ey1v?IdfmLNSUR@kV`Aq zE9Y@05TfX=U0lNML7U@vMxO85BiMdtb%Y(Zy4YmtrRwq-?(CCfk*DQed$c0MTQP~RzxF1n3 zBjgKHveRjb*|_>cd>kIZK&Dz6xxeAI2Mp2ugg66#)25qeo(u;piTF6mgm1?!UhQ_nzaDy`e zM~AeD*Mgxg$|Tvvo|+H8nX1F<8UXq*2GlqlN-hw6mnUubTzG%iJeI`KRgeC5EL>^^ zw}{t*(uVn3c_dai9bZu-oa`_Q%vw}Un$NYTgi16D)&$uaK4yO~AcDP^fP_ST4$u+H zrbkpF%7xt1YbGzE&4t7UC3!DceM@~@U(9&5bfo)8Dvi9Py3l6QmTkix`J#xeKpzB$ zsmt@&kW0N7V)f~0p+fNh~KE= zWKO?v<9(i;?cGTRGUR7W59j6?DmhPHMVu`&2?vY%D;>uygYUqv(Ez2mKZ4mV911H) zk306e5UZ29_)U+?SjzSwwC;juro*g9Z=AmoxHM0EZ!%kbotWYOwH4y0xp@+ax+`kt zXQ$#HyIf^Z=MF&O)3!j~^7ByX7L^M;qY85N>%0jO2e8VZCar;NlPX^ti&vTL`|->L zpd)Z2{0-nGDgc#uCTa)ewAIIlhwuqG*9ggJ;>(^($w3`LjSD2@8+nr^P#zUA~PGz?AjVL`Pu zW(5Csmi)DavNmN zDEfubcaKL1o7c8}D_Q8hB%K)fcF+`+ykYqC0qvaNr6szsS&S<2r&E*G)NAyWTv|yYXh2 zr&i>>lF##E6v>cNkc_g>sZrM>_ir!8&8$2aU#*v|d}wos`y$O)o<>F4@ZU;@ z@af;i4LI+%^ry~;=<%jad6gCXPxE@K<-YpbD+(}4L$)+#h-Jm4KfA8J+Uay)m=?o# z-=ufBgwD`-+?^j@j=L%GQRwFS=(H*_yJl5$Evj~up5({Lc|CVNd4=U15$~-HV zU(|vL5P9F|>ynS-p#QdIdxt7V>z&th8S71`;Q~kxYp9D$Zn^EY^c%zMGX5*m_tQhs z&juYH?z_xPS&ypn;k1~4>L-Zer4TD$9hVBcC-7P&ygsM`RVp$?Ep-1M!rnS6uC4hN zJpqDC2uW}Y5+F$95?n(dNN{VUg9Qt&4Z%aOpiOXs26t_=0|a+>8h3Yjn{(g2zwdkF zzH`RtKU#P1y;rSTRddd&WxdiaN@C)PyU%9GB3X%O$rp+Qc*hDSoJ^4D(mQ1vE~!4* zD(oVB|8vWlRA#%mon(qH9Uw2n>lyYndBQWOz_?EiM|5)~Xp?0MV09KVpN8?!4^GDiTQ&pJ*oelZbPFrIv1DaO@p|@5_oeg6Yw1fFV*!`&lCEq zPSEXEw1J_T%hHUQj$c}QbTwD-ICW^U%qD!=&Dt{S4~-XpSlYUn_piO%{JfV3gD$VN zU%8(kh~H)6zn^tk(rf6KYUpydTX-2B@i`YK4B_Jc5wyC`9{EK(b z!xS*+xQPtgP3q*4beVY`WYcOctpN*ase>#T%TH%?m__Y?%?f0}aLO9;)k=mkW~uo4BM4(8-WG6%wqTL>Xf=jab3#< zm!L_4QG#d{0|klv@`QgOXWsEXYAb_X|B`aV@mO~e?R_4((2P!I>q4-6! zb9Q`OL|cd0%|w35Tn!AgLn{I|8ow7@&%imk#cYSa$B|E|5>armq9*xO=Heu~>QFC1 zW_8*2ma4dAaCYU7+0}a1w$ezW^^og$wZRoh9a)kW+o*ecVqBI(9}5bHWq=SzH;NRu z=o2>ygg|)dy0#CwY!CMIW#CY?Uv*G5tm=0|>ZsI#?;)|3AYCPoTO6CA=5SS2WWiVo z6=o;ZH{$W^rscTpRiiVBsaoqsDpAT?bvqTQPy8k12QGF)b=k9bn~hf=9~WMSD?6}M z%SRVx_xV|^Eg7apTeF4nh1Crb-qOvw*(yb3#lIh!i7U_t6Cl9gnVe5n5}wc91`N-Z zdFy?lx5Lg?^#j+g`(w`SoY|ki%SPZTurn0z_;Uy1RODn!?Sxm$s>sc(n_}lNi2&cf%$R4dP+TxRZtOWRoVZB0PlIek zohqWH_4_M*(2lIB;-BYho|HJIdA4~po$B%))J-7fTJyB3*hQB-38-xiPCfuB?I7-N z-kz!!O-GX3c#YIjQ(& zWbJ17yjDNA{TLRJvlEB*HeG;7MP#FU7-W#F%U~T%Lzk4?J-SxkY2yUzDfyfM4u7@vzRb6Ycm$v*V+y*5kI0g?YHeQ)4~A`Je&LXZ71=Je5Uk^`cP_Fs^baA$*aW-EoEI z)YQQyTK|$Vi=&YFjzkn&E)AARxC`dUFev<(2;-%yo|re8Obyy3X&VokPD_E1f@m`) z`z=c;Ft~vO%0#vR=`P#5x>GNgnSL&Thxh*Jp@NNxYoRA|=j!hHqw;f-Ivb1?Mxi%x zp2^M`w0U#m)z#BOM5YLBmOa5I0GP=B3GKg4l!gq=#kRZBDJiRnb63_P!9^?eL!iyD5G)l zTUVT@v(t@56Yxs`kfr>;LYC5L;vrm$bg}#lLTJ%3PuVD`Cw;_8`e*zCNP#>g4bxh$ zVQg5f{mAm+EXi^Z4DBBObi0AUcmhS~cbgod9)S^(bk7in8M{WOuHj}kxK4n(^3>UP zM75(Hmby0DQqw*&g{pjwZr!6Lvu}RJ9;aJlM?DIV$@looL&Kz%Nc zR-NaauPoz*{TS)EtE}W%@D?*SHGTY&gzPq{Q+oQl z(RG3Pub=m9CZosYbG7!F#mc2+ZOp-<&so$De#)lr`rO~X3Ze^l-w}VzX3VaYZ8cuV zJeIG)&2OIqE;aI9Sy_1|#E1^sHtCmwF}_c%9dDie+!03VyWJ@qBw}L=S7bqO+; zu1>dCJE?u1>B~iZHBk0y4#FdUAy3?0U!YkueA;?05(grnu-De13qNC{KWN+l;OEV4;o4;~Xv8x*f9d3qP_UhKV0mIR= znETA4?|r%IFk@2_sHUM~-WfsZFl7qE0!zC+zbS>^p3RtE94w{Qw;g(&O_^T_A|hpv z3FPSPtU13&RZsX2-A!ASjQQw$sBR3&QOA`uU9lVlhj2!47VBWp+02t|PFGp=jAnqVyV9l!nn4D9hM_Vv0A*GtCDfaBomcNi7@iw;9F#O~fYVX&8jo!cM82iMzh zIG!nfKYZ!E>A$_c9Lf=Ya^l3XwZReIbboi> z<-AYIcL_E)+ssYd8o)^|QGq0${}>^`bpPCXJGgzk9L_(F>9x>@luqn8UIywwqcw|k z1N$U34&$`ILze+oBwXjl8(}tc_0Fg34O+-fB4uG?_a<_Pkv?#%`b^JwPU`mRw}q8+ zybfA1Rspv$+WcU0lfcYXjW~8j;PJn`zq`d@X}BS7CcuM+v4`1*vvogA`7JWN4abr9 z{9PfhQZ1gba0UX5)W4tuTm zT3-^eM8TrQ;r06c-Tu9w7Sl(9Fc9#MO^d5d)XN%a3<~jo+}BfHZI2fnOHm4L7gnq+ z1yj4s5%-U53}rs{;lHK@pSC)OiMs4guLcV_9Cf|wsTdI5t}mIYFz?Az&HM;lXYnAw6sVJ;xbv#@#Ol=-iY@fVnZ)l zpebbacJV4q(6p zVmE_DLRSlZ;MC4D-$Pd-@B8}a2d$UbXEjDyx62;h>nW};ZC>PwIj$+*Uhkb6_zltC z#dF^my>UA*Ql6+8vQu3Eg38eW6~dtslgBblbA5?g5ahIsv{XKz`Br&~X{}$@YER;9 zrQsACsHT!&PnRqTA9X*UbA?_Hp@|$s6N}-#aoO2;$97|J-ou3Vc#zucw90CtKTgyQ zYYUJ*`6C%{z=f1PH-{g}el~2cUaqIX?2#RHLN7e&;GEV}GOS>m zC^Ws8O#r69Ih(O74tK9jrS{mMm0PgI0og;QD=njwcf$TWRZ81$JWri-b!#wf`5ql*M8)gSL4NU=n>%7 zE;AxpMxHEaz=fTzuMZA8sn2)MX~XQ?_|7+mq!#gzvQ$8)z*g5@f~j)T&{>BbHiw8I_~G)kkB)4((DWZH$>>u z7$632G6Y_7itd$w*s1J)6WWdvZXjW*(}Ng6I5jcB2j?(DS<8*%Rt)NmAhjAF4#nK`pB+g{(sKH_E<v`{ZTN9LawYZOS>xl$2ru|8(9W`>3mK?KAeXi zkmTPyyHQ=k0y}gPxTbRHRLSe2bwP{yu34bkX z!r`Hqd}~e6YrkEV%2|aOqZCih6*F^ljO}sfI%270ST(R$hKa!GFkgbG?AL3fYpFO- zOY%ZP4ycDEIUe+er=IM}U9iJq1G$k@*Eu}g5l^g?UEadVca}Btb{jnkdU5Pbj@y)9tHB8UOoE13dgxF4COAmW zO{=+GAg0=%o2j<(-(|0aE5O(T$3wDE_5y7iUvWn0 zDK{AjEl=$+sC%c0Lj9a~^C_A;8#j=hiA`qh_8cci{FLya+h?-_aD7zFl&##wh6qe7 z+tsWXC5IXHsbaNUrBwL-BISPWqW9V=)Q{SVDk(D8alL?oU0HT%KM@b^G>CWRmx4%n6Z>osa8*ldNf#6Te&~<5GMnd zbDNCTD(uI%z)h14BkS<1a5COuDfb4vH{b5k34~uh;hNjFRtyM*A9j#u154J%bes{M zq+995s?KBRY>*A&(H8=EccYuc4+)w=M<8P8lOfEuzipMW26B&wwVw29QOMynre~On zj<@^IMFLX}4zscjs$xnGFrC?p!OkAC5^b?3;7~AJw)e`2^buSb_%`J0=dp)3AOxgb zS#LLy426slJ&Q>8k!N=$d`od*pSohu~9N&sk+cLIlTv3@c>0{fWQni-Vn`ryA{EJKA>RJ;#&- zYSM%U+wR{MaKK?b7tk-?7r{R$kGb4Y*Kw#SzJ{3q8Jz&B&ozl&Y{9TBDI@F8W)_Xv_b9UW z+WTb9yrsW=Ga`qYplu3bVh?`Y-vTlx*rz9$<7E0C>AD=X?i~$VNF`Lsgli(>0GpWV>g}1`&r@M&w~|)+c?9A4Bt#%k7!4sViAAZ5*l&2c!pI_wG!a z6H40M?e$EOO!7jdrJDBDvvJOoLjGlDrqFKP|0`!-PZ6breU_W#Uus>c1=`VvV@2kP)$d>nsy4OZD2Rmk>NCDwCe z^!y=hw1?9}xOj!do8z7)81DT&xV5*3p0Pp{vn%X6!#(-!h=8H(gbOd=N#m_CwWuBS zq|B}D?+<$*R58b4>w%o;9lXOGa!O7v9kD0{KC!eTOk7Q(wa8DqT zercUZ26#c52MN^uu%7Sgz%i6T_gnO{Dy%zi(%D;+9R`+2L!naY|GesGxt0WqPoi9L zyCd*s`YdP#quE|p*}&#P z?t|Sw@oV|k94~n0Vr@8~`odNMjyIA|O~^7(@i{X1^k1+^^ha^VLi78T5Q%YqMCF>% z$`c>IfH(DHIO-fX8n0C660>bFD{WQPxVm$+6Z>^amB?1ibh;D%hMW1(;`_kM7US75 z*9^4Fw@=m`yX}R5I=Fv=0n-UpWPCLn)z~ddy|@)S$#6w-An)Nu@XR-(6}ZBV{hxlt zrm;P*A_#my0^v^{N-(nB3)XnEl!~z9%inUz41oZ}<=cLm?f*jW{u9Xqo{M+Ur#n=$ zVFzuRCY^WbH5bF|*6hNu-4@0ZBEGrM3TdQJjP3Y6G58F_K@q9viDu@G$I!jwv&v4j zjcrg^W6+sM$5%J;dpHMd+R}$o+S%H<58IP_?^!S3a_6s0XF=V54PdE*H_HA|1{tf5 z!jfbLPbtc|Nn)lmG$8pRdb2@xG@BRi3#Sfwy2@FUXt0Pue`cQ`ZkLo{p|=M9A?O-D zIyU}8DQcsL*;^Ym2RMayFMXSH**;TfHbW90Jf0 zoBX`WmK9<5juou)r4#xqg1bq8EZ0i0KzQgVvo5d7e3T*6pKn@dq%TRmf})o|E&lD# z6)UUv`{l9R08{c%*QkBbBdjo1U-kN3O(vOc_shO#G6g(9 zbz1LGgI#Ka6v6iHg!WD_z^JVdB^;mP_DHzw&#FvU=bJJ+#)O>jlt)m$5gOea%<#_) zd;CwcN*pCS^h@^?-^1m)jC18zC5t4p04$a1@6Fy6>h{w9d>z#Mm-T(L6TQOM^g$&0z7A&Sjw$yVaCO**iD#EFk0USC$$+lkp z698rdpnJ^qUW0PI%Pv{YS)uGd&Ucj$Sb5foEHmb;kz$5~Ju{ZKr#Vq}L6&x(q~~>? zck7vUMZ(RiF@wcRio>%FtP13i+>qSO_e;EZsJwss0BkzYwvU)jK980DdnKV(PZB#P zvi)EIr`oHn0@JnL9}ifF;6D6VMf#uZfoT;@Jp+#1wayl15Y@|3h&9&bZ4k$b3T%2p zVso6BDq>dzKkCPoj~2FsHT1zaC$S#f2t%nytHb3n>~TCQ{GgCM9u8Af1u1$CrTvQ) zykqnQsDn=5U)>aOKebf~g>y7$J2WsBi8%l^P-Yk^W}fVo1yt@smTET`);Ngyh5EL1HVHtXhA&oG+HPP~%h?{89$vl^}AcDpRgOBBl* zds#c%fhvbDK4>qXoHE|m?oru@0`@J-F3-MaU8S$U1T3<<6%`exo5R^3s;zY_CrdNF zn})SGCOnn zB=x)or(-CTxHd=Fo7Vh8QyW?If0KW>iR;(VXH8c4sfI;hPB5l-{$jfCl&N@}^TyQs4W0yw7So&!! ze>hK-Zgak|AuA_GC?bzu7)F87j`AOo&;x_b@F=&r@?wXBZ5ddj=y+AnKCYwVqrEs8 z%?c2o-w)^}VmCDHKKC>_Tc;V!HCH3W8rU3I(>;Uctr8Itt+dWyboX=pDre6~HWho| zf3qW7;AP0w;4U0C4DaEV>fv0ukLc*0Z zq-$KhaI-@8mJ@M^{{tt8OUCo(XHPJ-H@aK*=YSiLVa{>o=$EiJ&Qjo-iWFnrEJZ; zEAsGYoBH*~zIp`M#o}!KRRx2UYqjw0U{|@;@#LRT+|g$bJx}C~kgjSUcN`aV@I`rf zy1*8Ve|Yj10$d6+ZJV%Na3Teh2Tb^LDNVo5sL);!LMKw zfFpF6FmT0UEDoRpnfP4q)*K-S#QvnX&NDwyBJ*MA3!gWhk1kbFiM}5H&#HW;L^Sog zW*NTNbLUR}?`FK+-LqqVV>!5D^|I=}07YcJ8{Hn<+M|4r6uPG|{(XL96uvy1EhsQQ{UHMmrIE>wZ>}12Zl6^PGl4F4Z z9J!?ssz0g^k6&9^`2$ctY;!b)9KZ^f?ndMWp!BPI4g1vr=LF|p@u`IK2xfo7jojkPk6smP%ay;2$#N%jAw z(%@oaxzN<8UII2S^hPv^icHgz&3$|R`h?x>9#1xU^}m)9l_Sh&uOm0Lk-4q3a`103X#;d^>tad(=C z++M>SW!G%kz)!^(g$+3;xfGoT5@b#~vhvga&7sMmp+@6eN%Y?p&FE{kQY>(#XBLyzvc#7ky zQxx#j!O`0HFklr^5ylE5<(38bL{n9&7+yj=^PR~u{7fE}cn(AGUh}A{GR77&)+jgJ2HO_-t&R*Ba`Y?AWkK!Kz=YQc6udOWsljAoqTUF9oU3nxbmb z=rkxZS)-BM)6$OQHhe5|#+bK&w>#NqZi3lNR}P8EOeyg5^J_ILOY8%%la^Q^;5l&i z*lO3lZ6Bl4!U<^+tj=4bQ61srNuej6B*o7wMvL_LSD?Jy+-lXfb7U6L$;mrH-z0g8 zNyYk^;eCgNs`9$JiK8WkVx>63cQ?n%B0NX&Y<*<|v!4cIm+c)bw}+lCa0#Sc zED8phzf9eI2td7nXeC$7QSx;2TvT7r(C4-)bLJs&0LXF*xw^*1M62Q`A|(Nd=9;N< z8Scmh`NP^yH<8@K`f+>qo7#DZJwfQ2GotA~^R1s!M(PuZT%M~-UB!R^FOG3+u8|l& zhZb+7^-t+6x9>!ns6lblt>dN1@XvH;k$yQ?bN(#x=49M;*y^L$rXbvPu>S8!Zfw}OJ?!5y)KAIch8yJ;Xzp?O$%98DKL;dfWtzI<vNMK?DPYsE@m!2>wNTM;B$G&+VN$ zvV;iMOEp-7_}f!^(NaTCqV8~~%OJt2{P${g4yNvaeqps18Nm(oKsdXeEhuSXY4kWf zy8W?EOe}nUb+$`BD5h;P^iyUSfMAUpP*;sC(wGP*-IrB~ju2B}JP%Qqkc=d@9y)z@? zyWW}1N}MuQHHbfc(;(Y1cO5Z1e=YRm;>2yL^rY1H=U_+S160S4VpWox~dnA}jzhpcf^o8h`8>=Kay}%89%97rc;6YR_9#zQ8KX zHYZBb5KQ5>2k8V1Aa5uTkvVJ^)oc-AwWSCcosf2&lF1un-dosa_H_(Wc0yO(?4h?qznortcb^) z3s|^oD)wvE`ZAK>VO?{5iIj zZhYT7HBDcV)1E?+bY;MG_3w`q>eKb zy^!GIx7E@xB?VSC3M_j0RG$7h=3D&YsPa(aF18vkdf42}uR@*m6ZEaY>Iko+4vWpH zY*j^GcI*Kx5}rj>z@*jYJOy(8TIQJqG&woVzpF~@@;*9sq5luw+GYuw@k zXmC_CP_DtAyH(#ij>KiCiC8!o+;{@%TSh&Chv4OY8r^ECMOEesP{r4JvP?ci?MF(3*nOpW`q?^2u7s0OQUT+5 zmUrv-r|olIqHV`-Pg;#|BdB8|E-2 zi#v5egv)^Xu^cya@CH<@CjAX5$eUJU0p4G;4|A&AGnT#oFAZm`Eeh_cLH&SFZ^Yfs zQp(C#zN*#v&L-H@@k9{s-td$HTG}C`Lp~;=FKd@`H(MjJdbrd8SvpGjNcOcA;B;2j)0OhJ^_w{@#y*#;uq%)1 znmHG}bk@`mNTi77`Dpp}UnMdDX-=;PM6U}WHeCS$Xo5$7=&}K7yPlBDiV$H%!m*@b zMJvm726sP387tGY3C5>HqxB_x@ie2c&+qUda1<%buM+U`f7nBUm!At zVkhw$KzC<*l+6qrJ6B&GSYFiNF9+q?{?ku9p_#LjFDv(f{FQY;9APmup zM%%wcYxklQ*G#1V+*scf#K|a5Laq3%)Kj1Qc1g%Lm^5qwvqCRHdb>e=vetVil8;4aAI2WD?dTy5`+$d7MQdOjJ59xTgkMNlN3kfKc#Dh7|ms++VMX$fkT^`PaE+ zMN5J$d(WRg->F1A5(=0`4um!3d7Sto{joj@D6_O)Edff{;fa_I`^ALTClh}$<`~Uc z7E)4KwMt_Kz<@?O8y!Fs>8}gxJem_i{RWlA=WeqIH@5#K6EFusecLoFt z33chY>zi54-BeihO@sZ39J#}p)&Vlf)ibNBuE9q(e&Zo$5pJG}h>K6K$2c!0L$UiH zDd6iKGYz^1V>yjZ{WCz2nNZ=&(}vcjy&=2UWy+`clOQ=bH`U|a&V1oPVG|IE`xg(8 zc3qI5qTh$I*Z(n5zXZftr(7gaxW3*%cuWk}jtE=~1i_io+?GRHJ`3a~xq%J>*BCG0|jyAdsVzdkUSK5xh%4YH^p#6f?4Y~Qwl3Bdw~VZ=Ph ze@zl(8{2S4xXt^W6!XQkKf~!YySN90Gg-#*Fb`@wuUfy_8i5@tI~W7J`KcqziHbmE z3)x1HO9Lt!J~OC&(c+tLRSA9T*h9~sbi0ig>a1;D)i>BRp2lahz97{n0DVh(90Q{A zzA)|->dEs2#Dcn#H%-U=Z?1l0d-D9{=yRwD`Oq3$Qz)p(wueygi-R9c$~ClX5*?bQyr`_;X%0UGI&&8vlLG zpQEEi7moGHrL{_qnP%}^Fnp{Ee>c;)u@gzvvHiDv{r?0H;_1?)Iei>*?jE7+zE>!0zpwcLuI=tDtw8PldC=9-9mkvY(H| zF=64S`MW5}Pe??|__AmZgj##-0}8|A!bmtJjsD{C%x-mj6>CgJ>ylBRvlwUw1Hh{7 zCE49o+hc76SrBi-&RoL;=jLBQmKBwD3*f~$Grohi1j=p@fQCUff$-BvGJqDOvRog` zsluR5FwK4zLU4AQFN}L|(IU>}abh938mo3n z;pg2SuXI`vPnXK{s;w>Uo{qr*pM{Z)fA<2Q zikKbjgtwoQ3}NmBxl$k>eDsR&>)U#>WoIR0%z_Wu->9+?*0CLefYhPEc>17;FO6G+ z?EFnKM}^VTXQ}>}KO&ur@UTvMLy3u{_&H`Qwr$TQ`A>dJlLmCK=bLz+kqHfhY`#A+ zgl_`SF^AFF$?>&I*fiv)H&^Eyl_Naj`C z>L!L#^zpzBt6f+UbD9t2R53()H5pZ~o}X7ZzwbkpEiceFGJe!TTMB4TQ|0ttu=M^5 zMXzX{k3;F$^x7wmCqr`mt@>SKZKbQ9W`z4t+4x^6FRZPuS!S#s(Nk0Rz5NRjZOb}H zaey6$emQ-1@aHk%lb&a8QI;-(BwHuSlBBnRhNQdgswl&WL--s~-4-*_Suszm@Sj6J zrI6nehq=13?tt)$YQ#<=e?_zW3bf>r`7|2y_!|Zbd8e<(+O01ZjfZ=yQl{d2t0#Wj`uCzcJ@H(Lxynjw_)|6SaU^Ov)y6+r$&~0f8aPyL zi&DrdypJBDi-^Y=*m_y@DV88tUVZ!bn=pQSCDQ~xa>h(-8l@BjWSaG0#ZOBM)EZd? zNH5+Td4xShzC5Z~oY51YH{a4@d_=d7d5URq zPOd%NEP_c|VC%DFn~)Iqa~|tS72CSi7j=Me95owQ>qih!a;gwt%52sp4Dy?KiGTnj zzqfx0ES$miF}tW05Il_135`M-9K{PNLNw8jns7#P-s4cDHb%BY2rwQ5vtQ(o_XZMM z0LTS0FU8TKx%h`$s@iOc(wki#5$ncV%tQT_EL>?^5IqHgr&izR_EeUr^DKZ!sgxu~ zRRWj+uZV+(319q{bPZUo#`%f+%oI_1smtY5qx==rt$?f+NL5SD)z;q!s#Gjy5q{zA z|Ix1yXuZ8NR0+Zo9BmklZVL(8jCNoQ-L$6|s;(BkUIf)i92eiXEgNzzZ3mWnk1PRO zvf^9K)ED^;h>bL!v{eHrUIr$O^gXlGbPp*)=&kHjtcnJtUfJ|n#4nU-BNobsicDR* z8f2@&XW&M~uVLbZ@`9w~Yi#F)NplW|c}GJ{3L#d+8np_Zl+DKnTh-+1OT0x#G=awB z)+>JOb3h4FIEnZ6b6X!5DR1E0q90}?D~RsIn-~_Ghm;6oL$rXi=^CaF5z@TUWnm@B7A_EaUPAxO*9F%gt-n@ zE6`-K1SqLe_S4d(vy^;uFuZQkIgFkfXO7bnH>R`KVKW>GoHw7eG(lHu*p|3Ot%rZE?xKSZKm#9RC{uE#XQ%6CqY zbfvZWSh12e2!}w8HIyO@|EaKTXFv!RP%(+ z=}k|Xb94qg4j`un8GTW0FO$ie5!EGG2da)}O9{4r+AcI15crcbjXqs_QYGtNYYZoh ze!5gg$WFHYNUxATLZugUqF$^&wZT;?lkz!C#uvSvhtP&^$!maf=#iC6qR8rZ- zPZrd_!bA=P@EFeiJkIx)tco0OA`rbuA1u}Y#|QF#lX%=*OgdKgbxg2bq}gVsTEbG~ zV$omX{PBz`3GwwYy|&ol%2UPM&pnWgH>pz{WJ^I{H?U%|fa?7h?(t{)!`Y!!WvDSb z!y_Z%K(Dbj)CKmhHxkS=Dij0EJdy3ER6ArL9U`og+N}Rm|LQyZw&iY*0QX(b%CN~c zXA$-Y(3z#{;_7U3O8vl>^~yD-H@QOBY{2j}&fPrC>xqx?Vg4(s-^DrU2J_wVHizmm zC}wUC&MTOx{kBMfy&ec{N7!wJbBqfg0>)hf!6R+r3m_$Qj_u5EvHr~4QK*_36op$K zsFh4CL9)I~Q%ltk;9Blmzj9HwUJBFo7tL*0pDT6jlY3HT61my57VsyclyF}6*fczv zCTW)NNB4`olwk>*&cGe92(4=A3$knDk7qh6<1qgQLjJ6eREmY!C|b7;T1 zKHwu}#kbftC%?vDEi0}DYt3j48`k<;S;jh${`ycRnW)|-cWj#~>X?19IiyYy(`-6p zdwU%XGY4nYe|aituT>OHTUEvTb98 zxh+jS?+qREQanfw&rLu?k$D6lyF#yj5;H4_P0EIm&^~0$EgPTy$WcRAKteNkBOv&d zX&9&@{G-(bh7I8`&!n?u+T10u3ton}srHB{6{$61WWOG6~(4ev& zB6mi+K&j-`q-mP^TAoTb_O}_q_RWy7)8hJPKkW(B!Vz5m!`=bQphd#jQg8Fkv99}K z&vW0@wZi`BYh~M~f38zh8V}B|9A@k&r_ul{0;Uj0H#0UQc6Ywl*y+``=q3JiCEcu$ zeUcG~&)DqdKV4Uqqw11YPr9mD5fKdG%_REPw`doUma%nMtyDp$9;5I1>k4%y^O4pL z?HQeAoas)E`+4J&cBl@3B9f}>mnhtKQT^XkUt&Wng=>F1C@QAERd^p?SxFB~Dz+IJ ziLUzmrR#&{O#E0_yJ?h?a+Cup^R!U^uPfdD3-;KFqt!~AKnKf2iCiPS46nrM@lQ)Z zK%oV}r=<^$r(IYfAy`)lgV|YM0e7xPa@nPUW(LR@wd`%ewIZ=Rk zZLOaP0@_*t`lu6?874LNV&kF#Ba}=Zrc9naB2d7~+1ey|y4DUh{8gDKyggzUx_8DI zSGQ&c%}ruGYQNuPmHfcjdvmcz+wEHhb{R+&X{Tr-2d4pCkb`10=Q)ujz+}%w@rBfS z3rr{q4_K=5ZT*f>`2A{noJO-A;@1V#jd5hkMUH1CDW(eRJTW!`5)r=eLY;u?4!aif zRQHa0x^BS$_f9*`)OU%;xC%KMCX8 z^jgAn>+-D? zbx|Qr3I~=h89i^a+btxkEHIq^mtxZyGo3uvAM(^wv;FJ;hp_jKr@H_D$8(}YMrDMI zN?DOTvrC!TayXn4vgaY&QE^pBGL9{K9vR0twnEv3Bb#K8BO{yN^K@PH?sI)}{r)+( z=;n2vujljed_3-tb$<}IE1I_292-eXyRM8|^5S2)d@=u9BgYNZ%(Qdc>>O4~XBSf^ zKVD`@VX75*>#j`OD|7+ZoFf>T>+ARqTJRtw6S`5vPnG&$_H?^%b>dsCR3*cc-Ke}} z&?nkLYhe63H-CkXlCDWAw%KYeguWJZ$p{vq%a*Z}<|w6R(rG(UwTb-~jw8#$K_I$) zXS7)P%sTt6vw74tgM9q=-;y(7$4F`fvQPERfxz2W(BWDt#`VcVtj|o#{EmW^*bXlJi%d3ddXZ7u~qn@*Z^Ua)CC4_$|k^HA0a6_O)-1 z({~G-S(7ki zPeE8&_1>XQoVh%+5!#VJp5pt&MWYH=znTZy*0A4(W^BOi*e1@eT*9_;KHJ7E?ultP zRt=fm^U$a^o`;{FpC9&S0Rq&P$I<;8rZYZ~l~UEpnAD5v>y38) ze9v)*Dr4cM=_ul7S zHrei3tai1jx2qjojEs_=6=Klq+9+7-gJY1hKI=1+1Nv9HSA)7^+sl%p@8_!(M=LxJ z5QRQmJFvvdjHPESiH5Qgr)^xg5tb4FBO-rxO4Sx#;9&dU-Mc#}ANx#F3NI;{J!Mq? zi_vC5-c(&TxSEIVx-Ct*;YM7JdvLD>gJh9IpYf_BU>n1bJ zy-A`#L!IxyDC7N*`+d6pJHBDiMLk>RT0Xcybo3ZX#%H@6)+*xz6#PXi^*rZ_F)y^j zgDTtV7XcHIxl#@!8NK0d4UbPS1WnoSN;`ddmuFx5LU#Y53(nBeJpL_jRwK5+`N<$l zkd-dJxqRU9=#C((xFbAaO{-!?VIEy+k6WjSkeN+;RfIEm`^&OPpQl2U0bgVbCq}GRJk&)8ZpGOFLq;cKll8s?_Bc zFUCnP*yTvy+8{i6D=C}*5y+2^k0+>6p&doGmfE}qBq{4J)lbK(3*tMRk(Tzo0h;fBa^}s$S0g?D&1qHLn9x!rEK|A@3oMH9Vp*!76cQ7@cAlBASXJmVqhLkcxr8G)k zrU99R%Qs@eH01Gc)zRAS5n69Ev6S|pOsYe1y%N--d>0ZMa#Zw80)6dV2;*{0iSOzn zm@_uJ*C150qiayt-UpTT9#P@Dx$YaB+WskBwR(8OzI;3o5q@(rg*^M-{Bkgtnq&xp$6pl;_v3(TICG6|0F?{kAL96ORcmytB$oh$@`7 zO!A2;G!7-%OQ-}jt3uonv&AU_1qIl-ocEsB9nGK4?TBfN3Q$yMYSr{jp3B^*ti-FG zk~=~~OilPmIkha!p`jA2LqVS_`o!qnrO6DvMT(0P;MnA1h#UM^TU`4=qg22I0<-^K zT%MNCw&=rUh6;U~Yb2bj>5g>*qcxKwQ{>mVYHL_S8^11wDN=r5^t~=`jEDi|_DitN zB8Y7=v`(6k9?u}8GE$@35!Ezs%FERorl3l4j2rL0DIRi==Oit%Z^u8oU32`4J*#P7 zOg%hxuv49uy#l0%hVEU-)T(Bkut?oCcI!7coG6{!NedImNOk_G?oRY{j7e^|F~|HX z$S$5ShOMvOJm>Uy796sYM2}HDUz%ClQ$)U2DLdcX(6qNKlnzR1%$2i^5QO8MUCHT!%r zMF0Bw9#a|qENykohH%RUcI8y_M&-&4g`b;B{h-_*zolk;FP^mnYHRQ35#_Oz>OpZe zYx<-fGl~*~x?0s&#f!-ZXOJ8WaUxaQ*}p6zoNTDn;0aTKT?*<3uWhpgq;7ex2Ng`n zi)$IQJE_+kM_bo>0cr*FYj1RybzyFxEF^(p;^s7Wbby`I!W+e=DUgXBy0$%&$}r!9 zSyN)Nizr0LlUuK!T?%#o{N^r+`Q7(n=PwU6T8h74l=|p7pC1is44`>|%@vf#-@KjH zrk@#4qcg)AfX{hP_yGVSL1O7xS_9Q`@wBXw7fwYlE1;WC9=+ZU@dF9jjOHVq)2xxt zE1N+7{{tYv6LdP=A6{|90qyY7*r}2B@@EP^M~><;LWaCMjPKc_X`Xd&U|HKh9w{CX z!{ksUz%!Ot_ZO7tST|_nZU?r`W^Kz!S$-E3Zqx@eOX2ccP&u?1JnIO9RBxeX(vQ2r z>}nX-I-7zHf2*A2MNu#|-zbkiqZdKyCdPWLrpLozVDHTJk_k$$dyt6K3Z&*TdyQDVDmDZSx+k?T>F48$F3uY0ss7!Gg=KbRdz6EHmbP z@3&Do%7EeiehIP2WZRgs6c6&Io}=h&geNr<%X9J3t+yQ7s>(^VSghX0vsiBa2}q*# zI4I>13%dOZlZOigwz31tYW%G`X}^c_p=ov+AI6+kjf`(&zVB)|_w=sTO=j~%P#3-C zR(+fXjypQzZPkVn7zx^4I;(e0t8c94INqMMx^~^4mcg48!%xZQnB-)j+ZkT~?xmIScX zxVCU*q6lfHvJaJfWYJ?!@jy}a@DtY~>bF3FS~}EaUb_7Lhj3l78ZvaYE!j*K$-|0Q zS9)K`@n@d{G%hely{4*~N3@Pq1}&O(l0&?OHS@IotCxIA*K5d*o$^|rGxt^$nOV(E zq}Ls1N2%;3KHW|@{b*1;c=U7w~uHJ-=}iouhv42}tmZ@p3N@mo!CJ25@V zJ74x>#AbVB{gRJZIa}s;Jt~szYqR*T;JQysy^9dU%kF=Bp(AhQepgTbqiSu?TJkNA zo>BiX{Hd~n493z4fNA+Q@#f2)Ij8hKF1|Z+yUmMxj~6$sTPvi;;uG|Kcu#r=6nqK! zzL=6U)E%u&-w);Lljd#of_>Mj9c^7_j+*y*>-o2ZAeafW9ya!&iI@8no8Vcx5tf~@ zRc0ew_q3E)E}IDUlsGWJ^2D%0=J$)Qk&n|kIc2G9CWgDv68;|P4H>Vc+WOAhD>{0% zy8=>}-tq`vHP<2jxuL!6_QM3M%{ro=ap{o?9}pGs!lS=`MV`}WC`RNsJ0_D z6%=i?-kT;@1MA%w;S=ydo;AHoP<*;|7ltlKlsgPHMeDQGbS0OxyvFs`*Oe?CfHaLZ zE%~F#lbt+{E+B$AWouZ`tsyl@p)&8it)3OMVvS%@Tj>x{D&=OFA*y7-1-=TP8W(U; z-7_ScmT%RnVr{*9rlWF*)oY<}*&1IIV8D^!8t!g{j?^kA?%b0-Se2au3R~M zAC2cF8OptsZI0zW@|cGx%lSQ^MiJy@&%Sw*LSk&EgUqjN-fOQ*k*a-r_rndMA%z-_ zRNv0)KDYHwo!?EMuEB4IlHyI=a z>^x2;I;Z#G(`fj;eG@v*5rvIGn}TX>KgIR9HobUq*4!f{-06>bGRpV7Ykh`F$Aok6 zquPGW5%Y;rU#j`=S)+K&3NxMeRlyhkU@}tOIDhVmz*l$==Z8t!S36AH)u8FHe_%I-Uw&nz@#fhm zfGCh@mcJmqG48Rpw9S;NI$aXCFY_7=m-0-y+fLAV89~E+^5!m9t@X&QAB6 z{BR`B(s(_~c&NH>M_!thlRrO^vw`>Suy2i(IQ8rxhsOMxhc=$HJ~=tyu{AUE!V|^O z*4n*A2OVUiH<6@IS$5wGk5v^RO1&;LYC|o2$*L<0z2%I}!hjKmy?P}!kwyml&Lrvx zt3*_bWZ@CIevrAlV_IqDsR>9a3|<{2~s1_);WzOEKxn$*i~fP@dk{p zXiES%&~_0ec}hZtL;E; zrYJPCTKi28w)?Wt5wMXVU6|^W3fAJye4DX{W3$bo96FqpgYh`8ydjsLDiT@&F|V&P0>@3?i`~8{yZv$_w%+H_>g47Xl7i`})Q7TB z1DAIt5FS*wdudCdwAO71J&O}BR9Qx3N1bW-##ec6p{r(HayC-29#p;uk2m_9(YU8I zKB>mjUb1v6uuMDyVvzR$js-4@j^as@&eynwxteX@Er^0zYj2!;^inz&hO^WiK1=_q zWj*ofT;egt)_WWl6C$kDo3)DZzdksUolLsq@N^3 z-4tzFYnraLZcwk{*tKBlF4WU8NPM4(=p6~!4Tz`y0V27HMMu3|R$ut-rhf5YmT_JmE20#cRi zNcIy{fRyJfk*y2y9Z$!t2ggf!ST|$=s>sBMQ&< zuIQ~ur*4pM2=63lS<_2D-U1XI!M+!>XlrV0*4_1z&hOdjT@u-|$m-m%r}60Xx}%uH z8J3IZqZog^Yjw?b&#ntk;}bNSj_3_v6VovN~cQb5j%*}}1wkCC%EOD#rbL^&cat>-HB z&*rqBi{Ie2N$|=Jls!QZA~s)$_4)2JP+Q>}@;WsxWLcwPS^os=qXly*M z7>&eF%!1Y?m5};;{q=fb*O{)Khq!t~+r|I0eLv$Aa*L!h5?1kmNqnY2S%*$=9)QbPopmWS_Gh>)u+Q4FW7M z$I7@3iq9imUhlq<>K6N3OH|;h&R`R(TQ^;xxtIp%DQsZNve%~KgV8mR1z@`kD`*59Trtvv_9H6pG3+yBHwVWi~3b zDDBqw>{2^$y^LBEBRf_u`Sig+&tmp@m6gS2(f{Gv-~W~6izGxt2@GRXAPFQawN!4# ztqJ?4y+<9WVtMJyvF^)7b*_>@ZR}7vDjY1g{kkH1`SSLKYQP2<+@zt$DB+wYvGw(p zdYq`;={$vxr*4J%b7y#i0{R=YcW5PjzZWOPfLKCm)|&}7f^iNoow(NC@g-qb0daS?Pm6e3UFzM8-}#98bJq& z@zM~8>wrrP5*d8lIs`=!4RecO;D^g;HS4u@te~Xa(PEX`_CtWnrX6?^2E~&bbn%h@ z7Lvb%c44=W?;+ibP+5&&4VTkyjEBz~I zLI_8Fxk7zbOEjVVFDA8{eUTf5vU64)9?RXXN`5sH0~sDVP5SDO_#eqB5YwP2LpgHg zB23tcsc!|=UG>{ib=e#binC~sHLvP{AwYsfeRZn+716g*?L?2Mm~4qiSOMssxeUn2 z=>G5i{#t+|l!UlM=zkk|O}2u{FS z6RUOx#n+xckw?m|Ehle83<;U8Ax|E%(Nwa-=qIVeVy1-CH^r{!Apr6@OH1Np9d zl?F})PMnfU&LJUHGgx1|xcVPU2-8Hqa-TKFEp@e3$(X-3bO5zCnS&dD83)w4eoc#~ zMt&vRgmlk!#66b#L;`xq>_Lxmtn0CsUoOtC0Q5g!LTv`7owMiSpc6SyP&4UAx8{0;3U#&gqW0# zI>x`CQ5ZS<{-J~xEtxU!^``(Atjgr>^Q$%dnMuA_LqIe~8}<0gpUn?JOLy@TLAohc zCms+}lal7$DQuXT)COt8U?Aq<+{+_gcyk&bK#@BI7?wWVX@>k+%@f{QnVRih#N$de zhEtrcj__u*61SV@DAo0u`O9#C8}nx=L*6I9w$`PX%_6$-*^H|DwPMh9I5kS+on1y4 zX~a#r{PaiiVwkA!BvUP~n9u=olO*2wgE7mr6$$p^haTc2o%nLy(RL8&dx>4Z^qJVO zJRBqH2JLIkmZ=MUsXX|iS1$6z)&~A`-54(iF2g-yQnkL$kkrGL2&FwM5fL%t1vxnOyyrQi=MbB`mwT3OfSwF&? z;R$*1_m%wCf$U@Y_hkhNkP2GF#MD{AN075Ej@r>1|FLgmn~Ybt7$tgB zG`^o%Jr=PNLp2>Mmcb`H51LRZ&kkXTEWc=iM=BAK3F$*RNeYw=pl2&$;d*l7$qYi{vF8GG|)`0y&(mm`5v$gT(n^csdFc0E~II4sool%c^$S+%gx+i8cZor2G!&}~12s;AT+ zs(Re$(WiUZQoLAtu2CI5iqcnUPUfiBzi-s9NB4yL7kmpZ*l34x7xSpzN6tD#n;(B2 z?VVKJ0Yh;?NP7~&F%!sXnH^X$_70Gb0b%9_JidzGjUFcI1jevxn34IHcJ7HhI zbtrmqp8|s68k=uPtaG&+u~7la{f`%d<^pveUALcyZ(zmP%MY^`;;U4zy5JIhKDBDK zbw#?xVIr%mjQ_k(mRSL{TbIYi;Vrz36B8p4SSuT)M} zFz`ub&xUlWiSg*8JMQ##JppDsgwfp698u!YUUS3-EL$H^L)TqXXPhUfZb8lY+8^Ic zjGR3k;9`A}x81$fz&y`pMgbg2pv7Z?GT$ayGiTgY(9`Uo zN9FE^YeR_1&MfP-k@_l;Ns)UUDvmwR{0n&SY45bbjJXucK2ZCGoNx_-~ zmxR)<7>|Kqo(QNb5!e`Vd12L?6Da{0NZUbP3A%aW^I?Z1mz+TiF6%^Bn;hR|PkZY9?NTyM8_;n0m^8?zB$4ytpNSW>$W^AHp4 z7jR*jF5I&WZC3xF#H9 zs#Urze!4*@JnRR7rBAJqv4)GYt(97Cm0ypUzd%

si z01tn>^#5o~IcJg!vi??Tor~9v&kjC+#VhE9U{(e=7mlgjm9t!_VAb zu9y6CX^`QEK#vRRFK>iWu@1Da;RhP31k7L0j;23hC^VeDCDif(tacw_&CASU7F*}K zi)WHJOga9ru`0aOhHxkYGXa?D&8fNg5>x+`2@u1Chv|FT|JYkLO)KPk>-v{+719I=ixA3qtU4djs##A-!R&b_ntTTX-uC zSib8cZe$^HTf_~#S*`Bo=*IwtUDk+yz*n`b09@*G#rGP?TtIJAEGQt!I7V}qa)JO)s*@hla&~%NH)c@Cx{_jWjn{Ys|H3d?^!oXz}Jmnh)Vq}cScZ>QP#nG1I0?GITDQia?bIDS*fk0=G7)A6d9PXhXGVW63B zrx)u*&38w-8=@*mAs?ud8(sfa-|!HTqJuUx6qB=j<_*VTA}*(Xf`Be(lS|0HgW(`? z;FWrH8C+;D?>zNDOB)j4M9=wTHe-J3%p>RI%O$%$8+SuWQ0Un~)$zlYnnFT%d-E}=BH=eEqUE`5#7|@#qEF;jh`rw?A2Z{@ zTsb{^w)-GaxU)*q13BAQ@kHW&F2=e)r0BfA|4lj5_ycyEB@>C+gjXr~@uP2K(UN>RH!}V`+9IZybyv80!bXPA(KXFXjbq2|`s^t=Ir(xpTPR(36-PK9bzau^ z6_5&hIv39eEqNdBz4BoSWS0aM2QaxE3SlT>uGr8<=(i?+~8|erI$pe+65JEC zlmj7ZWKxeIqa)StaXh|-wQ?FnHJOe9cT|-mJyJ)y>(Vb9(aY@!+RjlhxBf=J36%wX zE?2|D?g)C{V?pC&v4X*L(N7VQbA{ZN*}9)o1Ot^$sZv%%mMhX1zl%6?kLkX|udHOQ zwm)Wg-X16ojk!CId9k}&f(ng~ICCM)7yvRW?25*c7c%_dCQ$N=)n~x1)$w|ga{ZrY zhuuK}N)(RL1)8dT2q5&a*`u1e_45?jlpug$TU|t<9|XKxV{^+(Q8nL2yW20l&`>st zmri*hVKr?sJIQHrfyoT}l0z(4dj1Z8&lsL$x^B;kH1P3~`c{sAK@=0rgJXSIuHv^C zSr!EndGt6=NuHDBr&z;DBmo(jTTuD242L4*Ec}ymPvHpbgS$Fx)2Qlb+t%7z5*re~D@^In?ykjw{n!pla{xP@1E6j{7;m%Knb zL;o*ZQ$#WX>RMKJEAbW%oD;F&JEwU~5?}NCeIP3yP|9H1f^t|c7;Zw4vkmzKTsTM2 zLCkWt8u0|s0afc88BB)d>SGKzRetkN<&Y4cN_Vjx`&u8QS25GiH`6pkUgWDtD`I$K z@WKX;J;(nNF$oK8bEW|Hx(r<2hoh{{2EHJAKv0@LPyJuuO)j3qyV`o&b2Qk6VJ;3h zK6VNPv3EA7f~mKr{|-uns%J+wxX~I=S(i_lSIr#Xsy*hI&OKT_)rTSc8!3Z_VjZy6tH^29$ zV=DdOfhdqlyNxvvny5Ee9ho z*k#3}N(@w?g%AQf1Ipq6-Ivny*`isO^Z&7$zbfkh*^4!x@sbg;9}@3QSEY9Wb1<4N zwALiPk!H3K<18S~_zATifbM4{gAp>d`QK{N{`8`tqzGiq?bcMF0F|T-JGDHQJ3-RG zo55g)ef&9|@gLasYd?NpU)BXefY{FHzF3WYZ=MsaR+t_!^@A)q5OfWJs#lh{&T2!| zq)r}Y!+uDnT)+=ZES-1r*u=|0H?agGvG#ug5Fk;Rcb6t!ztt+e*G&!*(2eowQ+zA;c$YAVfSMsb+IT|o zX!u!9p8s4OEhJaE!nWsqaS<~2!hB05a%;>HScFqVBpK_@%1Eo~TYp@_FSzu9mW@AQ z$m{lpQ=CuzZ3GrImY`~t=Ks09M>U*@$+$*Uas5Zff%61CWE`To8Zf#s^}|*_x{UK zC7jedow2HyKJ9h(`E8FGMq|}+_I4mBCbANI`X3`>iv+}X3?!K_ZOvGLjD%y|?BrKW zYlejF282HlM=&2IhHOoc)$0t-E7FP_ zE~y{RJOnID?O=RVH;F7LrECLFk1|0|GQ?iRtqiBzlIq?{2Kg~1GxbVf+R3zmT}T1Q z^wq;YE}=ti&lvsYyy-z~;W$u#f`G!Z^Nv9-IRK*T9ol@v+?;=(x8rz&Ztg6Z#Z%Rb zngT>nHG|+p^237cg=bJ=4kr4EJ#|-4JVjT%a*XD>2r;MAj2^ZYSqz8#030d6pw8g8 zK{G{L_$Ad8StlY(;~H;|CX-P>%}Dp@;`P820L1!-FNMhcwQgbw+DOqW3c)-Atckq~B1r>b6$ zl=y~z@ySxsx^ii12xeHRalr{h(3&oD!qjT|>3`lN zQgY*sx}#IetOuX-dJD}So@pihN-GYiE8CNNH9 zDGbuv+gGMkBhy=@T1)TOG*2Ec$O#iZShlNMl03ZHVZpT04jSjfY$@W;xy8Vqh9za#B~xj zP&rbnm;1@YzxbDcog%+tTqXWR$Qhe<;^bNWI?#*V>}P( z)^o{y&k5oskLmzDot32P3>*p=C!z>i^pcdrNEHxcxayoWC%MP<e-}L9RcxuH-E|_AC-;sjdak#+m`*$rfXE$SEM&0zl2Ht8H-s zS^qP-sH1{_`+ku5Vwm*`N)hO@3uk`9a;Q463*r8rROR0XY+mW!Z#ksw3BWN$whtiR z3gWkSTS4~X)*CqxDb^YGkO~0y?HOTthyE#e-=}AQX}h5x6DML5^?NwxucviTn|$-0 zgfdlYfjyvlV?wu>SyUeoVa?N1&nt^y2M=Nvpw(BHRLv z9Q+9=BX#*xS;78L^;}gDJ&x?T2Zec_L806B#&Z7Sk-&`v#q(#Y!=q+eq^#WIS1xE; z5RoL)qTG~uGiXtle_iuGK7;)x8*qZ=EkcTWV#Puh)(odDfR@w^@`rQ-E%^xyW78h? z`2R(vk!S8KR=5&UAn0dToV_=@mKZ!F;Eo2}sXoQQ2oRP90WM(;1wsi&s?xz0Ssd`f z0D(3ZaAiBK@N)|M=ay+AfjG!`KM?CKRuo!ou_RYZ(r9nLT{;`3sr?M#Gq!5we0@KKgI$&$N|wAxw_f1^03m#R-g4!nAO!QJWpZ+8^gLbloT z>U)UPaF_9EU>Mk%FZ#B5CWFY}N8(3C`qFp3EB;@VC! zmD%IOSI8|)&aZU4C;Fjuxo%@v_R+59%zsaHrdcGnOtU3C`&myQyxdM~wVC))U1gQ27 z@ba2sFP{&iL7KNl#ej;SOX=l@CQs#(TZq38r#%0ds|(Pe>ZMi>kQGvtgL%wjnivnZ z24X z%+1X(t@X{;@0P4>0^>0d);UsmZQ~a%7pUoxgZ7o}1jg0ug)1;@kFm#O

Pd30D37 z{`leYZhi{j#3Fu|Q^eF^gj82>Q>?z&hzcP$38Fbdpjp$n{Nu)=#cn#}3Fck*M3;qt#@MPmrhB0dJqEctby?YE8) z!jU0QlYs!{dA>}gBSM+qn==+Bjx(g-T$>AmH^YMA3OSY_U#3<`q zB{BZ^OdBA9z8AUv_G$&Y{?>V+e|C@B5Bb)8xYtZY(7wG9a$Jk@>6QqqU#zOpSPdbS+p!2aeQlB`KIP11dQwmC2&aUD9TYX zOO~882yVVas@6Oq6Gl#a`g|A2FZM*+W@;p8=K}}6TfCc0(`FZT8iL`}GD?nE7w4E;wMX~^&XveuFUvwf@=iwD3g_{{nUTO5S$0?2{oY4 zqk%TB2YvvCyQ|!3A}>V`AcV%!%a?uyy0U1I_~@$zktv#?f@Igey*p!G#Tfm|D^THh zvssC6dzY=F>Q2-VZ!l(I#KndJO{&Mvur%D^qb9MM0fkIfL`LU-ksK9bAsD+CZ- zi|TmzGga_#{F7@$wvY-t;PM3Nr*bU7a99R1z*11X)uB)K0LW$2Hk5#up1Vg1o~Je= z3VGWL_@Yz4Wx%xMVv|d{;ZX3k8>QU$biKCsuKM0}1Ci~4=kojiM1I-MLv}vfuaK&7 ze6@%!MER-Ra(`~lKB@iB-fJo#$fxrqxwNUsBE1!Sd-T;* zOtegb18%lw=$t@_mxx4|0YMKRLCy%?bIE>2z6p|=Pw2X7gDxHYd7K3Ob|)D&kDIy~ zXQw}I?D#9ym&{KDb51s$RXUZWhdD^=V55SO>(N4s0w{#Kh|*NIFGMta4Y zTu)rnMV~+e-K#SN;29ep=mIIdaBEYNiKk^iwO%nHdhP9v63FO&3bsJSCnnU{^vF zUb+B`Q4azAAXWVqadvHqUQ1SBtd0Hm5ibWJ8(dz%i-NDAz403=*h_l@#2K&lDU)~q zpRM}a#sV1YbW*LLw7>FO|E%o&PaVTF(2mb{I!Qoy9~24Y+=PQ6*q2S`-~EF+VnJv{ z|Mu$~;mqgsS49bZWihgTzF1-q)2^vlZXj>xEgK3qp>K|{?(jM%{VIY3l>Yn&(9e=V z_KFHt!#_L=4(>Fnx@IQvj0DoH#pxXSnXC;I$bk?s3&~^w7(>M06>-eC-~zTUhvyrSkH8XY5cwF2kmj_ztP_GrV@DD;0V z5}*Nj-M4WHQGYx_!m+mS0jG4~tMMBsv5DUvq8J2QV+p1gT6Av5oAR7^*BX9QmM4kP zL2@ZOc9q3_5ZXBk9Hd0Ocf6&aFZ@^f+L{pj#Q447IuBt0zPYH#QemTe1pw}~jP~>CPhfa|A^4G;SP@4l=$Zo0e1M;|V$CzIc4tT3Y>0z!f zXThXq!U!4y7bqFaJ+_JK|50P0*$4qC-$t<)d40FXKwMvgL}~b=YSa}0Ks+V#isQB@ zz>@-mIq8EDv1sFpPw~zzTvJI0yO!Lj*us}jQb>dY%a)MV#3I%`Su+zn9&%tg;r7OZ zD8OuzZgS^PxW;Y-UjEdOpQ-nU<4pDG;v|rlWF@x!pKJk#!H@9h>mU|E0~jKbzzo+U zB6G0o;sPe8nCF?jz6 zC1Zf?kAdM4@sGy@sGX>M7Jd10t?5xj6o45%9`iLhW0-J5rCdE6oU7=9| zMiK}Cp6b+<4@Otp{eWO|O@Mh1abq@wvazijrb|^~Yl)Jtma7Q?R42c&H4`m|BcQ8Y zrP=|}xyEu1RNQ;Ax+mVt(8AKYz`h?A@Bp=1j3kt@yN)g(hnrUH`}O6`SP|Tvi`B+Y z861v&J$?Uio$Cor3G;RGc+a`)DKHx0Hpxqv6`_}2EH0MY9|Tz1f)Vzw9Y$-WfK00M zg7V<%DNw5*1{eh0L3Lo>@=_@dY7;|f z0<-{}&TIC4Dn9*tlUMydd`6pv%52XBBWg%U(m>wUy%FLg;aT20R^`Lf{tH8P2Tq87={mpS>{++PA0ldnI@yy%Ojxjsq2+97s}w zq0N>#SZPgBDZiT}b7RqTUdrswygTauN#JNQK`i5)0u9}V5$#|*qOD!;ByHZpsDQi& zPU>cjDt9tG7b;XaR>YS^L^GYhRWo`TP;_C zme>wiz~BO$jB96%69fuytAG#4cn+~tF>PVH${X=%*j*Y&aM6`dJ$$Q=Vl?L|qJ5Ro zui~1ry!pQ3fYMAkLjs%dIpg3GC&rK&LH*^yaSsQinTz=Myk3WY-WhV!aWSJ{@5GT zkr|Lfi6Kin*u(pEV)sESggWr2b;ruE|PqAC*gD2iE z#J~Kd8$G`+k&Z1#-+#89IK}-OyVOA+LCLZ)EqfrLC!qj~pH}HzO3XFRyP2$s?6iG? zvK;kjR*MyFZ!44*Zo?GZaJv;Gnh%ZXup5lkDE-t5AH1<2`si2;pWEgU5K7r)TTaO* z+HEv2-FtJP*;rNToC+QdQ+L?$QcdDI4WT%72jhv^3d+Az(X$^)mEltgV#;f}V<(p; z;tra=A6x5Ta=35DDBzl9WphBsX_e#zLg+bZj3ZjK##AFt&47)gh`VlN1MlZmW;OXe z&dvgdo+$I|c+~$l7?6Vy`?~BbP9+soM`?XByBR7x&5F?dpEs3U^mi5Edl|i4ix3_o zNpz^(QDTr#jBqLmXaYr08U;EVgVwL}()iH5jc)v0o8n+#M-O+%=(YF9xM@I(Wbd55 z1WMRq4RjPdC@rJ<5THzzg)aX>=AenqrwAaZ@F||1FatA#prd_+6c2&J=Y;ziv@;<- zm#sgr1hpQ{*YCbLxmAd+x(3{krLZ1dmhsg}X1C9GgL=d}?fcbx6%2+Sh_Q2x@!d52 zv^jG0IHg($$8d`bn{KWXLOS0oZK+3OsRa6VS~b1Ri)+aWT|~@t`a#|-u$-M)I}b7Y z2D$t2Pdn4$s(b=-o-zj97%%jE5ergW|4sdv$x*i|rkiK1a8;-+YtHVHdJP*UNL{*PZhr>17^O1H)7_?;bH ze`AM@VOcD$JcXW~l{kqkB7giS(^@`XE>jh73uVWNklhP1iskOg?P-t|lE^8*YHg4**I*L7vZ9g>k*TZsnIsu|b*} zo9$kmXnt{0a-~*>YKGsj@#@q!18I|gNOt~?Y z_}Dx}@S<0%4{9X(zF+1y$K|CIy?(}004fl_myY%}FJYp=e=LXfxV}}SifR1rbqR&T zRO?ftD4+2sslp4H-HoVh);-_rAGv1nj?|^WcP9oX3RhQ1rbZ(AOa@(xIt03!3iWj8|Wc+plrr|8eKr&6MH?ITE{iy#5lzW>ZM)YX!7(!s))!N=<%N99n zm+57PlXw$pNSvc8m!I_3jNSp*${Vx_@>QYV?G;)*~ASMzEz-R;Kt%XlarHCM? zkbBEV#@VMFiH?;SnUOP1M)Xt$j9ples_R`_E|K|IDxR)A+-+>$BfUM0_l+*}pcpK= zIpLB^?p`fxtj(_u>nd=tvE0!XsoZIKD@nX^;@Z7RU+Kj_reNT2h-Hgz@`A$Y@1^c1 zNV;#4em$evimDwX)2|WUpLo#@LMM}h5h%pmvn{1T$bre=d>cjWc4R+JXpK1VsfQGe zUKjI%!)EQ^@wIy0B(`N0qeA>7l?-X(Ws%y{J$0Cxc6w&2+o<$>82#ekacdB#?FpBN zuPLpr7Gd8C&76)2{L9%^7oy7;tP;LTK%nWT9( zw56hjkKbiV&1JwU(;N{}9mCw0k4j@3B`S@YA1F;%61R%CMl>?3jn|bldFz`5WCNTo zQ8|c{0Nu@FV@ugv0GEi_B_+N#J4^j0aWQui6z{T~$y-XT_hz z4Gf27Tze~xoOl6d8f~%+Ojmx{0c1!|p3_?{rqsC$%+fLq>4!<7dWzo->Ma|IqAaI= zxYa4AW|pkZE%#P=P)Ke?C)T-bbO^IqQ*$4`d3kJSQFm75QB8xERaa(%B+xT#jL7#; zwVSV)I|Ag)iD@3{!8b#tmiYY&Cz`UQCg9rCZdV7HZ3?;0?NYE5MCFm`V$ert5 z!09w3lbXmnA}Ik=R9@8Qn$?+yk<%a!s}WMW(6Qlnu;HgPO`5r3+ZRx*0PCOIYzpBK z3ax5V!IclyuN5clwQ+Xkd?QTkn-8|{d$gnHJmP*er7Cm02-HZlXCb`NY4%fZtxOXA zju_;6l?PwB7m?{koxHC%PG*;>?+^6Dx00` z#F#Bx8S!sHwGDEZ942jonBDm8PdY^3*1NpfEW7{G(+bpoY*qK_8>K@L4xrsO zDZW6)KZAL>$5PXy@2m|slAo=O_sH1NdMR!Hwe{j4b=D>1qHUccezCOGkUgYq(?&|S znRrlJBz}Kuz-9^wTn;^&6Aoq!$YlQ}KYcO0LtqnLH@PCKt|T*eadB~DhcU!EA%VmN z?&s3q5?^0e9Pp*SPMFtOu=)-mR2j}bMt)XIoPW_uV7R?oP)fNYLPUn3J36b`I6%0? z|8}xoag)WjBGy3B9_OCfpd2}^q?pz7)H*~6y}UxiuqsS$nq1`t$cWL_@s^5@JEcFD z9_hpC^TOv<<~~WGyr-Q8!E^^d;4OhbyJPh505zo=&k>Ua)F1=B^;iRNbgkVsg4?Gx zDnZqJsz41ey>zKNWwRM^v0xPC2f`dQ`{-wCJ{uQ+`nUqHFznHDio^o_jbrrjXV{g- zV#-DY@!s23`X02JEBk1kF}RYUzBUR@L&4OgQK!P6X56cOcewIk;Xh ze=f`5?xXyoc^3%Am@;#@yk~q7N{JU5#YO0~Rx2gq&Jl=3K9=`B2jk;%0BcRzGsa=wYlLD+9+aI_7u#YD^n{(pSEcRbbq|Ns9=AtaShgm$mU%HB#+_7=w~ z$==&>%uoqM_TKXlaqN}sy^e8k$aWk@*1^H~K2ERq`~Ca=KA(5re_ULQ=Q-yw@AG<_ zhU1kRE9vL~yD`!0J}%bS$5XE6kh@)yHpUo~lO-1nN?M)n)2$`*C5$!J1&OzvuwJQX zXPdEYh+bQRlpJQM=Bk2Fz%kzPg(`i4jIC&rCn(y`*fuCdsK2K>JS#lo+5K1 zvcx__@QM|5!h!9e@9K=s2LeOG)1lZ=W^D%9ov#)#Fkgz{EX(15P=;h(-IFe3dP1eH zudjBx98bOP?q)A6CBX0uW8$2Xcg*(U^F?T%&XK0@$l**J9$T4{-1SCU7K6f3!(x!w z34ZMj@8$d-LSbh61Sqbzs=NWWQ=fK&c}>Bpo(NmH(b&L5gS}`9A+TmPAqXlXbsM=2 zAs`UF!D%?;-k`79_@)+b_bkB_8sycz1WjNqUI{H}PzcM1|5a<$ZF@&WJ8h_HeIq;l zzJT=5-uVnX_wNitngm$`I_(h3Hzt+4I)bnrjNhz4Akjq|=_$WHNbyl#zmV?7K^qMH zMnxI2ZOHvWA+Q;(PyHnIibWm$`N+Z{69=NVrzxp)Sq)3d-dR()*_+Lzg6!REEON=O zc~HnFO7*2Gd7)wC(Ji&K-=#2KC`xPQ+

r|24LIGe6lD=5IrYIjgXvrm>dM-ze{ z+#jzdqwZaA`3lO3hL3>fshYY3lJi2;Zz_qnD(*St(4@Llvv4NZt<# zE8ny2W-~UC()5hCWNiaN_pt=fH)q$%dSd5OVrs6Bt{7QGz3MCBUQUG8_;W8^TZb^8 zk}RjAw0FHDSw)n)+OR!?sCPH;CSc#xB`+y5q1_-HYThU6^6_EVD`P=fjZ!C72h-E; z=Ie)vrV)bj8;Hbe5U%M}j*y6?);i}F4L#!;-sbbUd(77t(Fh>u(|zBNgh>BOtm4uLv9-I+~4ZX^l~>A zeGChxrLpT1HSQurZ)Z@=3{1|X^80MjJFo4Ing6{aDU7C&XHY71IV^;aFBc|V?9o*u z)LLso?muC;;bX;F_xE3VN*wLx6S3^tO;tz5wLQa#_8Z1tTVcXOg-}!qbO)A>+;CJl zY#3>Qky{6@lB1H{Z}_inwmeTZ&1`@vn?o$~K4syqR3y033GCKqnN*UYyv`OXWO3V? zquF<9&+(KtZxD#b_RbNk1?G`(-pdjt;=YE%l308WXsTqQM*+LXzvGM;%ADtL#yvWA z>OF(Wd5WCTmg!;QF?MZl&}EFPvd3d-J9=5zQ=TB~pe){_cH@56hgipaCH-gB+Be3T zu1rtqFhUAtLmsPqp1imA!F%-e3*N}-&kBKB%nz21zc+x@=W~iYQ}N;_w>vj@vA5&y zKAnAcE1Z{QTt|!BGHE&m;VN%TaCkm#FF)E<9&P(*=+j6yLX0Z6CZHHF*B=^B%qs96 z^wCchZ`1|)Te=3qH|j{n=ZGE2CbiqpHQ~MOk*v|nsRmjzYIBTg+JM+*4+w`v`oI8p zRxj2E^dYh^{6Wl1H+bIF+&iwsSx32(rNcCJW z%bM6*tQn)nC_-slvPl|A5i#_9W4vl7uj9fR54G1$1>pWd3wjgeD(tFZyt=v?W~Z~QZPgAV z=p?_m<6HM0jdTi~m1;TZ-EZ|FEKXOB;5l)v^mUHgu4B6~M)2MF0qiduRaN1+bDS1= z{ZU4h(!bswjNS-|OC9fbNQdWvq-UOV_nd_FH5@;2C$}hMaqf6(YZ>9X(adCE@)TEJ zDLNffG7vJmv9F;#x@BZ2F17WY=4$M~>6Of_b9D1=d#=kX7`7=aF44{&>gV8ZX*(v$ z5O+Qma&QiRepcMrV>$#u=|9*=Ff4F&@qnavQNZ!Q@ZD70 z^q#viSk@Za3x#)Eyc_0hwV2O^E*Rwunb@%)K!#|S`U$$`TFKbP;)rRWN zm(ukHPcUjwtH?8a2@lZVs%)`5+W$4DeYKnZt`aZx1Ij3#E6h_3AQ+O!md<@{G`VsJ zq`N=%T~%_l;9~7j^)O;M=kp4pZDtiq|&{55L;TG5-#)jyRUyGpUcuoXa`I4ew6+(BmEaQPdzipSh6#q)-;%> zI2L|LD8RfRx1mwCdgOqpH>%=9J&q{(;Y0liW+UxOeVZwvyd)r`S|fWyxA^F`<*^vz zPH_oCevMg)i$%A9B8u+AT5CztQH-(CZ>~Src>pfZ>#tz({V>EDGR7a}{qxy&5 zGpr=vHJ4B_NtNNgy$+u|U)%~h)qI0y5FBU=a-q6Vd^81Kg2aw%Pn>}C_EWf#p#qk=QmU1xg>vRCeQ8 z*-NML4e;eti`2$Eb{lg?(a-};@f14|*oBq5qIF@5U1vYy&*TUi6$A`tE39cCJ~4J5 z6#E9*hBc16T{O1Q;B6c0HE!6>6Z{oN)G_F)G&VrwYj;W;dGm4RA*arx1|P`l2GgvM zk5&g9N7orU^1)wAF(13HFFCONRLlQ5-mmfXoL7X@>yVrI_8g<;dUQTK-x@hF+kf>& zFB+FLrdY9V#BfS#V-_|yh1S?KP2VglmW(STyrgt(k@7g+-DZucQ058@kaSQ=5)%~M z*y*uvc#cEmg8H^X>tFNk$@c6X$YuLShu?|Gs}t4E9t_>Tlc#1ER^|>)Xo(GRk(m#T zJnBnYT^gEPvN?@nG9FsO?BY+jL!$J{+>0{tMIu@Yp`kCwONz~mug5C{##?DI_w1Me z+r&!Vuei>7*+?UjnSS6(Kg*<}e8IA1P>xbidLB6B$;hr|e3N6xkRlL>VWhrdsiV9cgwJ|<#Z&*yXrz$3@ zF)!kMD0F0zW6!%G?S0Mky;jBRSFSjS7OFZlu|hhu1dkbxer4ZUzd$Z-QyR2t9e6o) z{Fk9%N)nsb+o*tSQ=39e6yUMd-N$V$#PbiM1cS?K=yk3eJ%YG~c`r)h<5X=+uFkoH zvgRIm^v9thP)2`^C_GF;NS^F(a%BF3yR%7%Sl8RjRbHE~&Zk*0v36TLSn1~J7YSk; z5TGtPn2;jGeU=T$X$u;)3LAq?pig3=?a7CqPRwY3&yU1&CdJ<7E;K7Ik?ZubE-`dG z57U)+-L9v55;KuvEij4^C)C;^;A$c*6l{Vs_QiY){gqs~f_I%^4bB-S55#*C*X_GD zJ$fc{6j<-EI{00E1BqJ^m-RPhY5j%8WwN3GQ8!;YG;3I>^I5FYz&D-kkpLSABK&Gzc| zzJm_Bq;{@6%-}Q@hkqpWu9%QrS_}bYe}~1oiobAJg6m-9oj1kDkca(jajs+NB$><;%J2&Yfgn zrh&HK+iVDWV12Y9EOo?dJyxTyD6jWfm3X(%{zKqP*wzLIEd}RaH41?TK8e5GfB!Fe z0xwc)JNN*eN6mi>%b!PwMW>N``*T+xW~JvACd^YQ>Gr~JWvLBJf1hqOMIJb)HyE%F|@~ zShEfAQ%lA9zk3wqQY9=T`Bg##)5cbnzFr{?j-eY_s5xN-t5jLuQVr*-FhKUX4(suZYG2`$7Cc!E_ zo0?}Xt7)ls4{zSsbbFiU_&# z0Bp5JmAE{1*?KMtvp?(QO1?@qtY1>B7WkabB2?MS_@L$%deA4rnAmO1T(Qv?J0h3W zfd8k=E1G^Y>{K$aR;c;o)hP=0-zT5B7SZ&(6d zBcSqV73iS61o`Fp9VZr+g@M8f+O9m;g5q@Vef{;d>lF^4two}5;TA#6Z%fdnT5E&0 zS9dA`57;G?jPpjp`u(Y7xea_;&$eKUGq^Y5Nm=SZhN%ylAle`Rf#lP-QC>y^oJk{@axjgCrf+R(Xe2U{M+l^n6xTd6i_^++BENI7XTKPy zLBaelTkx^kJ!jn@Gsyl`5yvgx+U=x!7miDY?guf|g)9+{HHv|g@@j?rfytd{if^Cu z1mbMJGoP{2o+U{V{GC8H`kYFMzs;o$Zhp%D`wT|^m;H&A2)RTQTawS``9~-h6x^2g{slNR&1CmBfmaKEPghO9{rCqW31eRCbS~Fv z!Eo%sm?Uj%NqTp-lf6a5PoFv--sHlcPp7&y3J)|*#$tEqzH4N6rr4D+?2EptoZd2e zsO#?9U{|@F(qSjDm=?IT@vvhr=c3JXtlJ*B`s#0=sgjV@~|&>|-{ z)+mI!Rv6lJX^YCXT>A98=%_%pX^f!eP-Jlor#>%v`6#hyP9=aLwS^0QXO>W_js6fD zvQg*ESWx(PVnwNo$N{to z=aw=;xjA0rBe9yrP-e4JR?U(M5wH5>Olq#cyVeVJtQr>D61HruSH!|jGpe-M(nz3X zGiQBHL-8tM49o_(2RstXG&p*r;zDb&iUgx>3ia=n4Y|vqn^KN?I{5O$Wh$e+^`X}F zSM8fr4d+W4N)v)-=pCzoRmr6*CGZBF$_^W`@%{NeU#DsggD9UiXXk(Q#KTql0UkFo z{5KIm?&li#k^>0~2@!|i)@g1Ry8AyKy3S%S>MqaQn*T7AtS^Qdi~h<^z~=`tw88dv9aem(rql~vpmq{6T%-Gb2BrhQ*( zpR#N!=6j8EOZ)p*HH-m4-I6!<9Erm0Kgsy699PRE9dB<00AQ=r`PHH>g+s2sdZ@~` zLvr`iHgXv*QzKNgW)bQ64a<2rN7(xQdr;-#Pyn%Zy&>hgrj@BL*gOIVdRm*m(bFsx zC!a4(Swtx9ZExH5^IQU!=WNN5 zyZR`EQ)je7GJe)o(NT4g!RG$PHqBDH-b#8EVo@yT%SN-r=uZh#rFLR5qL=R^r|oZU z?JaVv%P!ovEgLIOw_II`dv$tv?Nafl7|-f}y`TmD^8^c<4e#f0&9)@ZHw}gwC=OWT zT8~NeMw(-Es>klA%f@b%Hr)8~740-3V=>>6ygK-cEmlaE?V8kyGU`xg+H_421GU?sK*Sk1-LlEvQ4jN<2ARTf9x_4N z^AlKl@TfekXbPBY=6|Qe*X>USQ zAL3GFk_}n{9Zs0hPg)$#vAGR<%_~4RES|b)ZN|rp-HjW9=*fdPhwZ}c`WXzc7LbGc zJYI`y)YHks%8JSE?ySOLhbh7B!@p-mXYa^~w*mMX%|KWaLda+51Wen_V2DRcr@awZ|^DO4vRrNYc zP*?2aY{ri{AHk{%e3H%ks#1Jj42f=R65~g&V+}t@7G)8v*LNGo8E5b)YM(h=xzVFJ6U`&u9875580P-D5V*cv^bntj>iq z+3u^b2}AIoL_&>uA1}W+s()SOV9GN&NzA;@v%jV%1!wpQ%a%-Vb$}ZXIQgEFK6(u`cb^B9H3^MCirlA+=omg0K@2wtoqf(8GMgA~BJaYelib08h z|2e=s)&v!JkY*ZA5%QcxS+U*WGV#~kQ9pXL{Edau>MQ+vawe9exPcoZah=WWYhH$? zk-=g@36+aNrMs4S(6G1toFlHS^Fp?c(3I6?Id;aU8lIccy06!KPv{3)-&h-_qKI79 zg;=#aA~xq;m0+lwt7u8={YQ|p2k5WO?wu-_70n2Vp-L{T&?Lu}kg>IbDSW_F?v=oe z>>ToMiE=G_`NGq;@e;(kk=?5ljp#My#07i%db3>qa4`kD#)V6$0>-f8>R?&3V7cE=I^m#5ch*|;aKm ztk#?PPg7%hW4?^dFup%)0N{G0s))Hu*W=E|7S%3GewB~Vr*rbzk(zbHCqL0pR%J?8 zsJ4tAkNb&UOeFrYbWIu~#(6-H#WWE8{W^V)l|D=aFI)+Cm`Z$K6_d>4dsKQfeXjBn zXLe~lx4^CszfyMBGmc5tN?%_1Sk5CC8D7>5C@vq?ttF$HIkF$@RaEyu4v zqoGS1N}~kc57*J7pNjwos7+Q(KCJFKX7Dc}75ugg@3g`Imv>?y<+)EKOg!gBLZ!aO z^Z7Qw2H3x5265Ia6;fhT?uqaYl)AELB7!Wa(z))sd{kFR^JBT|ZRDQ=FU^@x#l%^| zTqWM{hp|b#2}i;q>XD8a+>B?w#kpWMTWu{kMP5R{IB7~w`5a_ z7=+#I&{7X@IgOb&^DUZ=BHU-FsF+M!jTd$Bd)XodNZzAH3as1Rh=zYyFRji{nyE2W>D5k&x7V;LWl&e zrKFKtY)Lz>pKpPNxm3N@ar7%kRiPwnUW!zu?Vpc4 zEj?l`wQ?;IE!w!G=DxO&Njb5$2)74ukDpf2?^jSfhfa}%l(WEy8S_#zPIEy{8;6xSN#1Ugm3d=7?OF$;>Cr!buKTK)P}YA)-Wvt>5N zUMQ!-FSo+a&a{gC4gkfGKPK-u#0Q)}7mWETVk@ofau>^uVonIzjhH}U!vHM3kHwOK zXF5-*06#hOu=QNUFZ*HWGg)84D4wpqz<1X&@(YTKN6R1)SlfUv&mJWAj88WkP6ic*?l{c&LVS%T14Oh2 z89xtR{z8tHMNp+3vlmd{+?^$_1=Hx36uZ7>Mi_%C_X-5Te&E;jN`bPQABY%zQ@zu# zZQ&c`bxt$R)l?5kB)65!1=p7K(@m$|U3@t0VvIE|7squj;Vtvf?ed8_$U%J<@4dO> z!->_{+!PM|i)-n+#?iv&eMPQ{hkEo%W3OJG2E%#5&7bW?TL6-AK+LDyxAl#2sAc1P zg>DSj67*#r&7>oYa>H5-nqy$seYwzj1Mjxqq-VbUW}CZ9_A+jX7)*< zH5Y01Ixr*2&bCl+&}vxno|D!gsi-B~-yF|ndEMnZc3^Z|o4WHEo#NtWKOIKi-`Hc9 z^d7Y5{BA#&;W`zts(<@p^&^Od%ENi*U=F&*y@bnH6qA+iw@S;8WU6jn47b&bv3 z3rQYv$VSbM20GA-IG_2;y_~rFjl`yXQYe#$;}D9QDZ4Wp?4ylS+!Bbk2QkE* zt?o!;tpx~O#jMr>%}j1|?fI)BUnev-4}Mxs=L=l%?(cIiQ@QyaP~5|l6KxLRD`OC-CtN-l|78Is~dFlW#008V{=W?Lajqr5-2QHSZ#Ii|cy=h=mnRL(Ku zke;Tt+1|~P_>TI1X`V@KBU!oHA*j!!z=6xj(Wj6U`?gft-A&7!RgDcR%~0umNf+;Y z#T=#22RYE{*@t=V_M*ETw*rObf7G~9T{94KuTK*lw`f440 z?_@hw%(v~bv+m1uo5|8!uM8pE-=~B|9DjTqd z;3GeKL>-D(l-Jcgk3|%w$rL{%BF{v8F21*l{$?K4*9us^U~_JqU8!na&Cfq|&srhW zk~R1iK-kh|C)*XBRj2HT7i1?#zxpum+9~t(aGYA@$T7l-EpT56>^n4a^K_ zOxS)Ofwwsh_a!?eQFW9<`H2}^R~ro*rhKDp3Qg-~h~`oyY>%d|-Mw?S>kjS5Q`a1v zzFne99eYH{_P4%cbE8&b>T`HU+M^6NmOMIv%0%vK72Rg8L&XghWv#=)=yhUQm7yET z;RU*3t2gq_sg#JilN#)adV4%F<{9%-j5#0rsHcIrJJzuAb+_61{Z;G9#;KigQ=!Wv zjB1n0xo$KmsQf764R~1*vG)Yr&eccFl-BjB6qR&!JCR4b#>B{|wvQKS-JuD5+^%(- zm*~USzEU1K=QdU%<5q_>RX%s#OH#u~d@g$gSt=sT2Kse-K77)D`SVm5z6aM3)qS#a zuai%jlhOriRr_W%l^rgUZfa?D7S3>g=i4B<1f4*wA|037sg_>;_}oJxLjCvIGx;>F z_NrBLV^H~ta+wz}i&|Ik0b1McZy9NmT@7Bc8a)a1L7y$2($^K9@v76Q8&L>vmTeJX zFk>fR?P0j=c5AtbZX%BOm;&+HKwYP#zSwE$E!Eb~5O!qEy<^lOovP#=q98hGje0Nn zgLii?TJWSbzr%g>s`)rhhLxZH*!`P9grBCR8iXi*N1u~+%{O*(5o#LhV?5l2N^eM2 zxBOcA;abRFWP`v)WmQgP90O1hF2*abYx*TyC2kvd>)bn zt-UzS*>WwDG^$XqxLK!f=+i+hm@C~?*|=5=ggebtS=ugOwYyjLq<(2mF)!m~B!zF- zRr_7YHdL;*Y1D+jD#^C(Q^H}Nc-iDz;bu8Vb%Ut$o6RMCOeV#-Pf_jR@Cr!{p^xim zwA80TUTLkt;yer{T%Ud64CS!bQD8~kr8Sodgrr%(>TCx5RsypUMs)IUdng3b++1!H zE9E!Rq?xN`*0P697iwP=s-&&>Dd)fok0vi~g{!1hGQt7C30@>Fvv_Wh7 z4`-jE9LzHDjgGLid`(5KOwhFHl+F0GP`>eSF3ZfR#=iX?xO4&XqoQ;8fYi6N9<&FXODocR1AO= z%jhKPuLGIgx}#qazAp3SNw<{9uuaD>S~dej(p1P;q+Kx-f+R+V)PopRUzA z8G^}>DWb6$xMpw!QL~D;aw0dDY%)m{S&lo@SHkek6?shPL_bf4+OHdM4y9P95H0Z2 z&5LtmMp3EZEKE)htR#h|7!ER8dB@>*FrbEJBy-3ljnW#VOU202e>zr%u$g{bMIJox zWCCvS^;5K26_|}g<*^)+B+BjhjgoRLOWTDYnnl%|>;+$?SRIC4iL8%ls5y-c85zhk zO#c_R8zna?w)vUJW7#TW^KSfr<`mX&_Tt{nmd+%6&@k?{VntzhYjFs}5~0O9$^@N# z`x$7-IqY!meIDy5Q}KA6{{BR}+V=CTlMDEWaziYbw1iFip5=EV?bRNCHS9MFPo0HMaqbd$>Xfg9> z^x4YK|LTzF$wQlj;SB7J1KkZ%_aY=-wP(O__)wB^_DZ-YM^Zo4cE%FF+_vcTnP-)X)?)UTv< zy|&tavL{?kjBWgKsDEcQ5Ix);QQ1iE^OLnNJv-Xy2*;V!UEf>PDyC9>f`_iJ3=RqN zl&BMFF}TOcH|v!IBv(gt!!1pGH?3u1#mdJA_FcbLbF!WD@aGzeXI1@9GD!jM5kCC;#>b_iU zWPMk89}+ZiLaJW|u5&ynhhK5&(XXeV&1Ui1zq}^7dqdHXPX940u2EUmQ#dqOUrak5 z{SCDR%$XtN#t{; zX5gO*Cqmp#2&2$}jmKk&Xl-jDdxY%8^@-Xs#O0xo_xhe$5eGP390U?A?=|SS9@t)H z-ziA6`F>_F|Bf7|fS^`>rzkNKT*DCt?#ai=HE!LE6PCfsfAbTCq7%`DhxRci(DMEx zpJ)83VJGVtoc8hYB-am|?!Tinz#vJ#F1l~;Kf60y*he!c=&{LHv!PYz?KUakX?y5G zz(MooT@p`}Gti#gLTK1dx|d-#7e|y(_AA{@S7R~D$U4xLUN$K~L(NU}z)w>^85}vb zdU;#wB#3h%Wk_%NA>7;aniqI!Ul_x|2_9CJ-2GA{oE{~m#4V6MK$C;IOIujNPj>|PZ&P>2IudXC$t;|; z@XO-zY0(oH7`z`7ieMY1w5OHy4y6c09MvBCmw6VoU6gg>Bu z-{Y2w9FN!BS+K1I*{!`Fylr2O4-wqwceR*ZWPigs1kN4}p34>ZAtC;v&ZG32P4FkN z-0&H3eS*;rPVsY-dbS`Z-|^cjxBF*?i`AWOg3YFL3N<-6qBO!?|E;XdMsXFm5fcvj z?eRVU(`x$+KDQIcXPU7qRBiX~7HN{#`wu-N&Bz~7;Gu_;lbciF*pdTc0z-E+EeGyA zQwfy0cGiOJVnh&M&ipUU@#*nbA<{~NM@zVidGU&g>CHA^zW9ZbF{`+*4#^c)FEN8={s;+h!<9KLDzK>NYH4)< z?A1*mz8nKM5;JZ@>`#a@$G_+Pe?Fe{S<^<(Tzb5-XgKd=JX7qtIBmsIcM%NRzy~s_ zC1FPeg%3$?la@{dY-KlT`l0x)E`@kdt45NG0Oid*yc^}J!T+IC{kIpmuI6VVHV`NX zUJI-z4UI^pINnDab=xW={Ybg?bfkv1_8sf7;^+`|pK%p23PJKeP67DEySx5-`6*K^ z)k|4!RFO$nGOkuQ%kVFq8Sa&tzXN7Fl?=Fs9z&p5HRNauT(e0@*J#uY4Q(Ncjez^I z1zO6~m-%*Wf* zD^UKM8>6Os4#S<49ZhztWhTc)SCO|geSBW%qNOSs&*n@=E=Yh6CQK%_;o{|0lLYh~ zhX7vj0EIre3!oUK_!2`Xk5S!=je5M19>ONxz+*0GC{x~qE3?Y~=TmE{m8ykFE{Q)6 zs{b>35ttU!;C+Lj;<}c9kZ?=n0w&i0zzo{}k+q^Ntgry|{!05;-=leUCx8g)290Ly z+l*5t$7Iq+n}JC{%zmiR*H;waZ(jpVhuucM`ah5DkGQ{@T|UCnrp|-NbFp8zxa8}O z3#2Y%lUh8|uAyRNWaOErXSGtQ)O7gsW6%`-&TUuuz3QpsgBMi}>GcPIyV(gO(%Dik z{CNsxrl^7EgO%?OyXEHxA&s1!U9o}B!J-rBTkZtUToqZ*Gr6je<>Uluj6%YLN?o0+*RT@7Yg@k(DRdv6|2PX`#P+?Z36(MG0T?ISP zEGd4n)0ZrcI`XRa+dMm9B_>gv^d}^(%yq9b2DCZL#av$+eaar4Y)^-;T%1)P-4MJq z7Jp9wL4)e`vThp;iN3c#qh5=BQE8V<(xc3nfA#=KuvA6_jb8-#>2>dzDXKp#5kHyj zGm#1SvdNOeu{x~Nad$UN{0&!stb8B)0>m8XMO}Djev0;PR8*AN_qS9-^-y{=U^^ zn<^NG`(X{DvsmNAZ^}Zr`!xT$aB*al75FCq(~ESz`+?XQCRJwL!($|WvJ|^hqbLZ|LS+@#+-~<P}pEb;K8z|6l|E=f1Yc z@Uw1qp-0EIkY=fh>6wB;BrqCK4u_z}^9#T<>Cs~+Ycnz)FZCsJR!%vEN`TJ0GRxtd zD*T5xe*w_il4^wY|3#Gk{~z-XvVJC$v~s_(&H|b;kRj3@($+OJJG}n6l;J% zw6g7Nf}(l5)&-G<{F|@+Usl2A>zh-{iHnFz9Rl~_A-z{OJa=aoURx>?YAJkKltl5I z4e$%+p9a@LH6U9p0-#gu=UeiJeFgB`F*pP`*IVycpW&6 z&HDOvV9d)4SgAr;4FF)Y@R7go6SrZ_EYloWy1o*c>+HI_V_wJTIE zNo`%!9=bnWFA~ooPME-}`w_j>^{4Co+$*;onBs0hUYP}5VX zg9ZADO8Zm`YX~d!5Fo%R;JNoyQy!24OAAX&ZE)S&Xpv@@+rDB3|IyL;pZ1O4>Oh4K zUs#SIS9zRUbW{}S&1}%zzH?{dR}QXvnXMXVZ@&gm>jq)4LI5o|M*4VFwLE1n|H;hv z@lw;%+*V*SMvb+Hz7y4U&UO-X|#IB@Bdtt71DMo0%}R$KtjjRNMs3W)`) zx>}xsEX{2^+ET0r+-W}J#8*S`^e-dT4heWS_j zwhD-spZyWP@CIU=V4G}!Y^M(T4R0?r{t1oiqXTT`ceh--AUF>zNS0_uJCNArOzEt0 z|MPN~RGRG*$9(}L?v9~s#l_h`x)@DW$%@|M>PUIWMuawi|ZCQ99oI4Ft&gTIkMQCVPC!F}EhCYwt(&m^Nfn4TwEt0qBjbdvo2NR69rm`y8xTaS#R2Y-V^YH7fCFVMZlAc2HZ-~+W0@2OQiRFjry$x zFLvwf@Xg{o?M&?POq&;zVHUcp@zKox+K~Rpiv!5v`1p9hPt5HA^rV}y22Sr678b^Z z$4YZKIXGT72ax~18QD)ZDM^fRjKLM$A5B8=$s7UkTX%-(T8eq#3Xd+sv(zMkLDzo* z9w=%sgpUF#Nai><{XLa0?QBpbV62Hl)kjB1HD0{PhF`Whgbd#S~&Jb6UALf>5jTR1r(fdf3dh*k3tV4}YODvP3!{frGySy`Ee zPSJ}Sz#VDqNfOm0A?!qf5p`>sf|OtV&l&MQzkBC(Y7E@iUQ~9!U&7J<$B!qgsLGNl zATYND^lm#p%d_2Q*0%9m?1JLz{=KCBY;@#8rtR8UY* z!^nsm=u}0ai6?j`u&29#meH$4Wc8o42T4P6_R@aq{>CW?H`|z@SXMQQ(mnFJ)11Gz zmg6mO?~!aupsGz;&LmbniLF)!WQeT6ukh;=w@6Kg3j|zaY-iNHZdKnyP2e59XSro} z89e&Ge?Wha3sp|9R96!d?>dh$b~V{ub=rutD@F@7mt(R>A{!Upai3x}uK~=}40uim zEz8}K`n`lY!bWWw*hgTVqNAhB0Lro&a8nn+-53o(@jBq%r#BS7dP5hYp>cC{b@if? zpaI37EX}{j2;?fvDYwLe3V~5t^;(n%{N)8RvggmAKXjOt0ewS|g}D&+7?QcRkycZq z-__Oi8c2^h0iI>YX*!APAiBMcHBa~P|Ni|k%J%Tn4cP_Ifd94G_qfD_^sL{<@BjW% zdI#h;AgTA;_V5iVH&3!h^E86~q* zt=%!Iu+nZ4()q*Ok|xg#D5!=ns>VWpJCWBvefo4AtO?)`|4p*qZGRCKqEx%U7BlHD zn1(VjF?EAr(BF!&+|j@+?N$N)noV=9&~f(U*8Y5ibPQ>`0K4^`G-0BDq3JZ8&Efm+ z`_vWZxS2OK|M(_;s?LItSPZG+;t?h5ZICarnEDHNoW1hBVWe^Dr61U+2O+ww=TWc6 zzrNf87iyR9$u>JGyE3H|95l#cLrxOp(QI#h!s%c!w)@9>`a4`)a8LX3AG*EZ^vO|4 zwjy2}v!1SpRvJLD6^K@MdQ}EQ3sp|b)9&|wvf%SBk2% zMDpYw0-->rlIuJ90`F!jtYc|AcHe;`5*;22n+sx6@bdCnVXD~D30T#kSjPT)y!_8z z9yoEbx^zkIlID}i_-efwb)-t9f>l2A4T@~*BDl=PJL_NSR(%Wy^~g$#VPFj9YCalHVLkMa|>Ag>(am0KLH{YNw)AJDbgWq-p}R<&P1D7G+@<-v!;si@bK7pgPS-S1OU1L zoO>N=p>_<8=yz`Si{W**(l02ABof8*D%Kr0>;IKvruwz0-J)r|hf}Gm z5aac+PtYZXijLCzob<7QZue4lQ`+pr$7apjotG^8C*g+Tfm(OPpyb?2# zvfLnHvK`c~^YN*defuUCXwDjVSOHSw_!+3G+xqxRv|KHoWEx5wZML}CtN}cTM!4A8 z--Bu>tP|jagr+nW6)7=t8(2W>OAM+tjJ$uzLBnbq;WW)I5MQ!7lFd#!A)iG0QB<_v z&7Ri1-_WVY!`Fxmkf5aziiSoKZ_PpcG`JM$X_ac58%wydL0~9l>chiuc#FSgCw$Jo^ zOSyivrnff5j?8g7Wy7$4v)DY&)eu>|(Z9nDML>~MjmfG3pn2fmQ(k^HuB37!nnwrw zxkQHv*h4O&%U_E(lH>S)yLL~K{DR?=^>~1ZT|}VlqQH4I1RF9wUf1kDk>9U6POthU5{DvXc@Uo*LZ@osJ`V}a~)xrayoz0@HF{F7GHY2|BwH@OkraJ235~6cd3P-V)F zA>s&%ic+EoNGmCgAV`CBcX!Ruse*z6(nv`6&=oZ!MO~ zyS&W2?-M8X+2?G|%8jy{b>W^x4k5n|Fzc6E>-f*>LSMst{PVN3vst}O|BsKmm#zJ^r&y(pZi^K3Z%PS*VhZ4p48ICyj!X#+dhMHPGX z47M%rGP}>uId~rJ^h4`Udb(?Q`Qugu_@4!hT6Z9RdG<3gG&19U%`#~!x2d8ZF(!{5 zV|LveTG*X3E_wP=qer0U#G<;(c9bmXXs5zr%e&jFdRd$0A#?P*!3Wg^lI5`y^BWr5 z^|p${YP&I~h&|#{;}0z5Ym)=LVTjdLJwMb{nu=iwk>z06Xl#%=*OAO~HMjWHfo#P< z=h03M#=q~Wd^EtVw()gcns=Qh7T#Bm&&)}9rn%26nG!joqb=Xtw|ec=8YhF8tHeP- zM1_Wd%pT=2#qMB)!`9&XJ0b-9GpGs8Myv9wz143;spzb!*apeN+gzrAa<`D5KqLD= z7KfU9l#j#wrSKyp&O#0K`MN)Tko36_M-H3Dr+u#*>S_G+J^27EWK_ot(+FuxyHQf9a7cK56UR3TjV#E_-x5ESHXYIw(^`{Y@P zT*yErh2cn-7e7s(d!B!fTMdyAuf7wX*DbhxkE}Rc9~OB>v!;JrcIMTvvdL(PjEGfF zaW_MkeRoZ9yA>BkdG|{39$Dhf@i;@x5kptfk1DwlhR)47?WVAydqcmDi@Y`2apq*B z_jx4~A#|i8eihfR<_YgKco83_Dm-heMJRcz| ze1J6&$7lrXdH5^Q5A3K6o?VW&t z!U#;4{R!KfKsy*I=E|?IMRaI$l!z{_sOowuZ*FcbAC)!Xg7%PGOEtgc+CR|h6*IT7 z;TkZO(Ahv9Ox5B!FnSH6{bTvb+?jf{u`1imr@ zDP*9}U3RNP)_GTcFGp6xQzW-<3=8;^D?l@CUF~i2lbYBCa zH~Uq4!?y;L!5RT+01u?QrOWrQ6pVE+K-@aln%ve62$vzs5dKpvHS2!lwi=p_<*<7( zHrDnN1i&4EBMy>YmG){GmzvyCF<5M+e^<9wIjSll&Q3kLjKh)rmi0=UiQmj%#Dg`y zJ}x;K*)XLN7N(|UE&v{~4TP-}`}b7I(3+|@8d*!6n@C>R{JfynUJe|=(ZD>guJIC; zls(9PzfT2eJgcm*w9HUY+A<0?tsc!ljkVV>1rb@;nj2)nnF=_vx`!Lau)ft9moT-E z8ld=SyGrD$@BVdRO94>uxitHdMsCd7)H!YH$t@6ha##(kbb|Dz4C(H@{uvho=?+w$ zG$_3G^#L0Nj4HTCVuFep>S8)fq>Z=pTOKQuqVWD@kFH#~HYhNm$NGaTUtC2c3Wzc$ zRc6`_qb_B{H8lsFuf7=+W2G6Ei5Kha^3p_m!W`2i!Iq8bLYe3mv+aJKz7xCC^^B+X zAMqjvx+-cGY4~7K(0u)3?Y-^?mbv-m1@+(B0lv5Jo*kC_G}a6>?j92cDIxQ7f~D^%?vNb= z&}1f0Yx*1BJS2d_*7!lFTM5#@S&%>=e2aiZGnC^+y7$DgwZPWUv+FxR{oN+SHeUx+ zp3$lj#a^JRM!{{e&eKGzlAd=CT5f}!X$Kp659-)6IXEJE82vtDf&j>kfjdn}D1%AD zY1x}M02{d6Q?lGCvu@R_fYkqDf~+gQ&$=evNDGN#s|hdB($b)-twwP9Iu?(9GTV)- zgd`KyJ}t$Il(95M8;;)F7;@zYZ~O11K7Jk`Af{<;8B;6w(!y5FdY-10nJzaJ!n-Re zrV$2^hJyMQ_TCB8-9@!2_={47z9>%)lYVE*eSY4UTf-#smh#A^SKnZxSl_^ixzAA5 z5}7?HNBDuV^ch@&&aP8brV_AuXO@Vg4)RZz4BgriGWWL7(EOtn|NHZ2`L~&TL%sXj zJrVy_kfpN#s+I$MwRY0$$QtMz_@W(v!(6PXB!FN^`JA&Z@;I+D3Z5SH7&rPuHi7QK zs0hF+&LFFk|OU!9Q~7P0T@(%Hw9m zQ39IPmBaEh1Fu@|m~am;jyI^6TdwDgFKd-sPA#MR+68o1fyQczV#8uxjiSkb`{#an z&5J9Z={FNAPm7&3i@IfEw5|NCc)3tNm__H90y}#lmO&&}i^??ZAz3RCtJ<6xFfJk; z%TQ#Obd_Alzx0t0H0v{+G8ob9&^fWnlUEaAkw0|emnUK|nInHg*~0i{s^eo82LKt3 zY9tW+4h|R4WFFr^)TR7M{|-8IXMjdRcn5Q}G>vThLomuE7g3;Jh|>$KVK5h3K5-0x ztj4vZ2292Zi;H7GD;g{WIk{S9UC`1D{F9s}A%yJHY4s)ygiRnH%Rh#y&nHTBRv|p{ zuaO6p!h>)gk+7R)&`P=cRO~%I5q5mUet|&MzUro~x|;W!1RD(%OYxQ1XPbUe2|Ln>-~CakKWlzR zaD>8o&h{&k-mm9)Z+>>V#K?IbzlQjLxC{iPvjDU$MY^L^52C@0@MhS`iqjgu?F1jN zM`*o>>PZT22cvKsg^NEx5wlg4Slg&^5eR0Etu}h^|G-6Kn=38@0b~y1;oYP0FTvbf zG7cRy$U#XMgvldN0!zqA-cnf+8|s8=H=3AQblmFZp>RCfzQINP~IalTyO)){ib{%{JsG6-Bj)GoL`b&>RTfm+d*Iu+0zNKapEC}J7o(F}<7%oPMr zrhIUER>+c=N}*aP?P}O)5ggbRi$+`P_X!CJ){To`I66;2fU)c&@>?phXF8pP`j`i^ z9L1xrI^$%fmZf?8Gi?SgXq3m^6Vt3MWz%=WC3@uCx&Er1oK?YUAnGP$ovjVP zXh5)KrNut}EkNkv17O`YAnd%r#K)D2UJ$jEtEPY>0zjF~uS9aK%cCW;#P$2H$v{I~ zKbQw0rfrVq`+Q`V06>{x^;H%I1|mpp8U6+k`D_Jug_(y2TG0;NJQ%I7UN?X`4!-^P zOL;fud2soGS&t;TjfRNPmVF?@K(4i>4XgVo+VWZTlucKdK~>+X*%xEe{v`Eie#e2I z>e01KiJDtgE-Yse!ME$Zl2Sa-4vYOdD>FDeU&c;5>qi-09&}ZGw-Z|Vy=nEUluSAc z`Ds<%V=zP;q|Mb+%q%R?K)3nZYAvmF4CRQH<9l`l zgAD+%L9LaD-0%<>m{R$srl!r^aoY}G&=G^yV)?f21!f7iquE11a^yoz4PWI(GpV)Z zN@49i+H)PXq_>zRtq(`1APHPCQ?9A8R)pG&obQICdf>`oo#7k7ce4BG*hEVs$C7l^ zVWT*`CUL6+v!yo%p1nH<`Gv6Xo`?uK^>pX$_=~SZvgGoqMHpUPfpWR@jpFYQ?YL-E`H{95QL6U!>5y*y77i~2k(?;+#CGckDyn85 z{Bx(CM29)v`J|hK^VICi{0>iSgZezlGfULGU9{3b83K|3i*#o&a8=C-?G?Ft(R5_<84-ssVF=6S46Fn(Mf*qBwcA4 z)Yews(+97R;8BNX4GP)0Kk|nWxqfNEaSasM`+flV_75+-jXCBk?0Q7rKPEraJ(<&e z3z!pG_|pu}w*+sWw-8dMz~iAF3m`t|0F$vFpo=$<6lnZnf4>4Ly(QNqnl|>`zJ(mt zxd4#*;b2AW=uan&1=BK?BV#}q3AEu<3IbI;fDsq{a9W$FMY(aK$LNA^-;B+de(QO( ze>bW#ZOfdY@j zDDPcd`Pmhy;ic(R!VBJmF!Jw$t>hCDg4IKx3qJ1h}BzCpf+q;uDOCW zPAExlYPj~u1)dJ8fvBFhd>mJa1?SHLYsx&#So5|N#h&lI-t$Xp!(&xE^;v`d!#uhr z=|bD~wq74_Ml~MS97I?2C?#r|G?x$4DcSoid6DxyEz95I=s50-XZ|Acw&+Y;o^lM^ zrv4aqht;U%$sYHgSpaVs0TJ;k&jKZy3V8o#)b^;5oq3zH&y?e0l{+{iB_+R=>4~vR z%N&6s>CB*_+|3n-BT$?wN2}4Ht71pYB+RzRsv%g%F{t*n&a{OCN;6aPODP?lL^BsS zQ>GwbLR4257RsOO6nIMjCD`R}L!S%uf7cbcu(m_0IB2YNEQRMeCI~g6O|3byoo)vq zVBMVXxDuWW30QA2!tGD(7&MBQ&6bdD9Sdbvwc9}8G9X-v{#a?MKEg3*BVkYf!xO== ztxhvnMg)O#WC_U6PtbiW+3Ka$`z=m??O~FL+R_aP>ZcZ`D3U4-^{s-BdQ}Io8qoO- z*tPGmO5NAIjt|?;I^_V@8Ep$$ffUGSwR1s%`x*$C+QEo#DlptCS%_4{YALo&AmJgvQ^M;$9y5QzY>8>Sf~nIm)< zT7FtM5F{lhQ%n%S;W3Qc!Q&jo;gZqh7%nKRbC%YCKXfqW-9^0@l{%hjYFP@^R?A;R zDU8l0psJXoW9$nx6$IEK_xwaGS6W3yENHV<*R+SZ4K>Q2#f_ZTln8au6F-6%-TO)i-g=4m_J1p?u$R^J|tzb_BIQwwy!-8n(B4=&9Y*xGUUr(a)# zk&i)LZL->2o?rh9)+i{NKr%=2)Ts`6AlYam2vp)rAO)T*dcQxH()aQ}t7|0QN7dL~ z*Rt3G=A7Mb%e1Wf3wp#V(D?c9vLt+&z91@9oHvYDBl@WA->awcpavjAQ)1Mc(OgdYPHLOEu+$oamXz(h2$a=)r{&frdQV zm4X6_U##cjR0!X=`}S(Pj+~SusO16Sy#Ds2h>eH!>opBD>AVUmy#X&dUtpmpSnOKe zot+Uxa(sRMt!QaOa;`(rNmpu{2>=dfB$~4chOC!608ZZcM!KV;q>f?@g&b+HPTwIh2oa)>66^J$s?F!s?ri5^OAq>Ju# zhQpL|wZ-y->ra#d5rr^e3y}5i2(dvWvyo79$BDR+Kj`-{dX&k0$O?_8L;nnF2)mz) zOQxF)S$5)V^1H~%eA#Rd6i@b|a`USsOYv=1g580`Pni}QiTTV27zkD!iy-No^zQ~B z`L*X{`qg`~a&qN{k_7-04pxV^sH&*61MZep^bej}WfUUkEC5g6gmjMLFRyw?gbNtg zyMYW{$BK!)hJYn1MeDQ;t54!B`>g%IN5``i1{mR)%Vz1-tD~{7%XRJT^h6rtQA5<6 zr4#yomXHD{{X=E~ftnX6nAuImqV?3I)UAGgjGZLe#7h4mp4p6RYPc_o&O`J@%;b|H z$362mZvbZFb#?mCxmfXvH_dmFGfR(!{{-m{#uiEX0~**T?~_XOoc?IQ9Ujj`<9&dl zYJ9Lc`*X6^89+H!s*_{zJum=!d86tVkOd#VHINzaAc-}}9eq9`TFF=2n8t_mbBTX% z2Zs;mfI~^%=Tc2-ChE2UxUs3+NR)P`mWUZ7RWba|ZYz^%_gN|h4qFjjQsmBNG28MD zj{xJF8P4pG?7fI@mCS43N_v>1qg1i)PE1v)2oIX?A{s=hWAxlY@v=Ayhs_I}xp_GT zQ;!-(iq92tg-Y>Wt+tONtL->qV0-;fe#0#1bGDdsQ5 zJ6K=n&mIAV*9pL(VBCbLD4OSB5GOemDEi5jYT9rG2H!=Nn95cD3}i3@a-f`g!$qn1 zmoEcAl0~~p3C*tu9Sd?ZU;uu&VKBECY-P;2(VRR}LsQ)w#Cn_kQY;c^{0+;;f* z0fHz4?h<>F>AVkHMBGN6wVt)`(#pY+QJnusmDSwPkjzo{LTTZNm1TpbxFDPV4k3|> zdj^a(tt3-wy&+W%U1J9c3M)3Ng2moyRM?CS%e_^;gAi%K{L5$m;)h+zyHXFlCo2#Q z?PIQ(beV?qQC0%#HX?Fs{YRsHy+CFC1E93EfpV5kE`Wea zOgjQ^GN9de*+8smA{`Kp%!3q_y+-eOcsSLFjyt?IaZU@r;Ozd|J=ZH=^w+#-mJk(l z!EG_of>bzKx%}6!wv3Ke8xp=aiFw#)ssHeWkN+}}2x~FvO!!cV8IfhOpG6xNtq+ba zqO3`@;v`xsZ6wt#n+t}`$HY7%Go?0^w%&PX;pGOvY)okqXj>nZB#Jsu$t2IJo%tgb z!Y8&?Tn9FuAJ!6NDLzmgI}CWDY#6~jkmOeGGdJ;(vCtTF%>md(N#&z}icEG^R-xmH zjXQw55ZEO+Lp;#632nedyhoy7sfX@XAiX|L%)WUFJUg0n3Isr{bLRmKw8j6&3DOy;6|IN5teKqjfV4Ra7?mX;YwiTSFq_yzBq{`CPvm)=uP$eFxYnL(y|nRMiB#1wuioy($WIq#0%B(h{^l62OHF@j)z@d8M zJT=Wb9jmB4wh(Y6 zPEo7?%>lnjKt4jXm}rgX?_e$|!qS?2-gV<=6T06DTmm`x2zI~_H8nMw?FQod@bXD7 z%~B30F)@EKy9SK9)q0M+)yevLr-k0+;R65|JbzC@O~AMAgQlwi!U#QyxMiPx`rgVo zEl@-Lu)8uYWnz+XgP3@HN@H>hXp$D*+Xw4onNXb%L6gqWDowjkuJ3j)wLoqFC2xnJ zZ~7dzWS3+4U1@wX(r1>pt)t46orp*hU$HO6d@vB6mtIUXq=BT=-g63DOpN>YhM_xS zWbsiSaOVzXvyomP2o!C!V6)!98F_uw?%Bd#DsMz@>{XIFCVdn#DDUJ9S0b1H%EAe8(W|qy7kUNyZJsaSHX5THa1nEoelbqVn1x1*9qh~04r*jPRO6*Z zLTQ=9i@p2h4n=v#s5US~1YtLmj-F)GS}z|Hd=n1cM>WI@bK#0iBc@9Z&IF?X4owEk2_* zUjQ|GtXFCjErj^2wn2c+;3ZDaWZZ>A+yq$MMv_eQcP6@(>(^_3ln@D{_(XWp*gA!! zv|O2fpCk}MY+;wO8Wk|Ftju^v-6xW^RI4?P_8)30c8#X38J66CPp9FkSTE)MNtZg2 z7C%vSv0@0<*!_zIBoE4+P+0XijqJ=p_xb~#)t@$)4Q5a-m6ec4^bYPH7@%fg=m@5T zV-(^Hk2N7+(<@j*JR__1WulIDeIpAqHz2vrtKaE>yz$=ap#&aiv3@a|4p=jEXa#b@ zK!gzm=+NQ=E@`ugSofQKfyi-k)(8iHtQW`1a^AS^w=90RVn7r#5?&2yxFWVX}h>YZBDRG2ELN8y+o-tm4|Pkfh3ilZj@8L zZ=wG$enj5z8Cd^k*$w_bNz@BFq%%afF^PmwHD8?ukl$0LH+MR_*uxQiHn!1aQ1*YY zaj<4CH+us=^1NS&qoO4rYhg8GSd!u0`Ij`OCmhJu`7KE{|8l#+#3lOSt5_>A6B2}g_p<`G>0WnE#@-0}gyPVuHN|79SiNKo=qOWQ*x%rI7B&Ti;>*^%vneo=?umJxn*gfv&QKJDXcuD_fSQ zIwBXZz5xSo7H=Ousyr*I`3m{U>zBd0kbuG(%WfXF(p_Cs^Bp#{BB-+i!pCRzSJ)9| zpHos&Lh32x4Zl2u=~i_6w{oe zubT8zjeM>E~{@-C`>I)HFW*{GCn@4h5(X@u8568 z_W^KoF&KX+(zyZStcghf#y69ysS{YH?9e70FPY<=jjV*M1ud@A(E6w}Y+@qVdnDd* zgXAA81kTH)dZD9jqdWb)9kSikE6nsa!TZIi453zP#OditPe}CxvO5m}OuKtG_ufu& zB}ny|atYYBY!*v%VDV>|(GK1#Hytt-NbIg)Y=K*;dk!F7}x z(QFFZM~cbW(TyAD$KrQ+A`r7ynp!EL8G^EzgzB27|Ej3%^&b?p87Qohm+Z(Tskslo zQJqsLsiQf~&E8X*L^0kZO`_s#PD|BUF?3D{*}rACVm>g@KX&>kQe}Bs&3@1U`OYjbfx24bCq<^}$`nTsp+**i;y-!8ZSr)D%1-v~ZYs8u$*Ws-~Jbmoxj*or?SwX>p`%b1ZYibCWISclE@eMnpJ{`Jq45e5Lz( zY^_1a3{>(%xc_`buprP2uFhfPtcbxCy*(CbC0_ClUY_v3{zwxQP5;*9s# z5@MqmabY8rRpF2{f`zeUCYzUgXNlTbaRi)@C%43b1-a-BLxeXLdbxjqWRKX;6>k0P zKeR;N`iMMX%5hX6nh_re(`a1UKz1JRvWk@%c95YhOF%>3%>##%LD4$JPh5yNKDQIf z$tqHOtO1*c4p+~2Wa%1ZOg?Ua_NA}M4ZuNcR^p}9f1ATg-A5E*v69tmHoa|Z zV$d~dNDfDuY-p;qd_uIWyf(qzYV^^k!RO<*bZoHH`BWm4&y9$_vK)1-u$GzuL{NRx zW#?CrUD!vj>Y6y2pU@Tjh>sH}fk~B>xTs}w4MOB+tiEFzLb$$D5A#Yo0aJ?Me&*}5 zmiFHdND0G^x6f$_S_;|Y- z_Wcf^{6z_+39xie&fn4atXo}}w##`6q{C@qY;tb@<`12L!M#2WUywtr;L{eTm=F@_i@k(``f;p>8m$f{HxIS}3Hu ze-f*YXJ_@x?LMe{(O+8$u8~@U`3uj8a?!Hj7uF(6OzuWQ8JO95Io16IC)k`FCo*%~ zV$}x%AR2l;Pu=>ANM%0ln#$MpkgMhXLW6kEEP@t?eVmwLY~BaW@}IB&8!_J(sbXy% zEcIk|i$Iq5lA0;Ff+=!(tKAl)VyA}ck<0M<``++4RP;z6k=Wj{m8RGTj}N;0_pcX< z$A4m|beA_LI#SJ8;|0iQEe0smO{*{p3amIHUr#R}?RL&)s5+jp`edtSU;koOimTC%oW0LC5sE;K%#2=_mhO1@tBh zVV7poB^XKHrK2l|h(y!THRgGDBL;)g8Z%D3gL0M!n@SS}nz063oa90ctINZ?=}I&5 zi9-^A9pS=E_9EpxSyY{lak)SIgf{rRsXa*)6Acld8|Qo%uB#a|Gae0M^|5ts5A9Y} zzsIj*fFyFfq8RKT5}Suf5(vD4{^x-};pTE$Tz#e3Zbw;oqQUd0Zy{N#HiJn#y1;2I z@oimIY}@~Y%ITcpo@vcru@dr($IA|YUA@^c=t>VmZ$j7gH-q^#faa4I5>EUZhevNs zuLp+pHGZTtiFg5%g-5OYib*$4Uu|)k_nvH5{e|BXXaCv5-Uq(S8LWk}&#$6vbTbVV z0V}~agTGk+`HR~5!+s^<;l-2J_iE#0@;_w<5~j=D$iFPeO)D2=ScNduEI9|J*QeN!Bmlh#2`sc^KYB6UwRhLUek9 zg^Ec6Cyvl96sM1$M*?A+N#3c8?B;o``Ck;&LFqKgQoi<{?rvnP zS1szHrS$9!hp6)AU?BZs45LF0UY1;;Vv5rv6xy>lKzz|1aR5VIp}%K@wCPxq@U$f`d?r8N?XtM z>3a_&zl9JP%Hx&p=GT2IW7IK}JevMwp~e@@HADlB4Sz!Zvv|IRm!H71%6n(2?MamLvbeU#9APD_ml z^5dPo{%;y|_M0E+lbEtY$R?1ok;+4y@j6vr?I4Ul-Z0P~vAA8sVldt{P<@bq4I8vXUG&~R>I-{E_d=V}N% zdb*8Enrm#VKV2u_+|wPA>{)CagO_?V2YqLM&*q%Q{LjgxR1;#L4q1pcq(zp)ERz|J z?aLV--`g2RHhm5_8b%|;KY!7?8oh|TvJ^on2>vBaXJux}ik|x@R1Ius7NPhhJVS{nxli)?v2K_)j@ClyJ&=$q8t2>9IV!ck+C_(%2bVJyyWVovWKC z4Xc>85;!kqIH%s&a{@|Hl+|qfmNDa^F#VS#6o2V-AXi$v-FZ9BnpN1sLLJ@68LHBJxR+5cWs%sTk|d3fW8I{ZODj2PVjfF+m;^fR%1aqx+9&fcqxwDiIM*pL z{_oG5D4~;evPuux$PMnvA*%7K!yO0_Q=?oj+;Aq6Knp36@?S@{K1k7*%&3&5HD`KM zL`yzYqm!mZf6ytOOH<71mO7hEQ|al*d+M-1F9}}zt3=ZC(*@5Z@en?E`vP%DfT~z( z6?b^MQ9gaz{E{NQV7i zTfFxfnoZiQEkuu3H%7xm$)W5EvC&xK4YZi96kFzv4#ghy$HI@5q>lo_Uw`=fVMO$X zvUOaS3cZx*JQ2Je5^OEe?KN%kV$mJzS{h;jw9RDyd5E)(eoJ$KdMC_0^_|>v({d|* zJ26Do&7FKI$kSY}r^Lq7TT|mme}_Y4PFkHoME&LDq{ywaZ+z&xr=vv2VzZ_zUE<_W zw018(J7x|p_SeyQl!Ucgw%tbHywj=y9q^Pl6b}ghm}3wtSouX>oOo{bRY&*q|31!x zM|xPB4gu{=Y+|IJhIeI^<#WR{ZVnyEvAZL(;ZifINAB->W>W4sRMKY(a`0I0 zzx3?`yR90BgO$wr_KuwR{M#QG_H?yZ+@WBVf+Mj1-m+Izz3CH`r!HB~Ptu0_a)%^& z;@>f`L>zF;`U_WCeOYq;Cx> zWeHaaH!I(`N+G^X_dnJ^sF~;?OLzDze#@OtRGFF(?468b)3k$S*Pgoi{Gh6hgC$<1 z!~9pPW1}|F*+F^tFJ}+eem%*6$Cw2|dcJ+3bXbWlKk*LQb4TrTED!; zHy$lEj_nuSeTgM}-E3(&yoF%5{qCT!o28+dDpFlt#3-3Qg;+Y9@M}oM|5}7lp$wwi zy6O-1zq>6OjoMc{<;=dh)9eZDADKUac&ZS)3g+Az-0x%{zrUtTy`!Cz#@wBryCz*P z_f7D6D;B1bwCNXRqr2yuqVTZ+P*k;6#>*^;Zw@fiOSiIPTFUOVuPNUEOc*N_&JN&GIff_N9|My&hk712#c%)tG+1B~EdETf| zUF2!))0;yw);6k>wH9jcCx7gBl5#(r5aiaJ5Tw)G%*JS6j(b%rWr?X-ZWAdJ$JvIS zEdy%*GN`&|11?vk`g#p?{00@Em=pEYgZth?=gEfyl-`aj3~ z{6A~(-Z3+U#zp;Pdx0R&iFV$NkAma&xR^|&1bru4U5ct!e(G6HpY>xCeE~yx zu+EsnVC>LsvtB^K7CitnXpS1ZXl&&lC$y{xW(O^hmc{uEX< zcU3jB$6N{iX>H}PfNF4i7{RbnM|I0US2_L-B5}P7mwqig7c3q+1k@N^fxXI}w%+8} z0z}s@iZQ^n{rl~IQp5Ab z$;qEpw49yGOUuf@bceT0{6x-KG2LHt+Xc?edD;yO4e89mSv|v_l!eMjUw%ZvbFWI> zXzhT|{*)gMecLaZaRnpj{y&Er7(%K?io`Fc9fLGpH2S$A^TYcWt_2;wVP^K0_8ypa zJ`Tt<>fpB!<$rFdpoo5ZN_8v`W$A*^5k?e2nYfH-;&;}1K6Ot${=YvR@4@=!TOC)W zoBjJ&sRvr#M+BL^i#i|?`FLh~|NcMGe-G8dq=MS-?*z4we(E|xDLwU+gx$4t;=ruy zS?9g`3DsiK_FU;{ZB3v}opw@;5GLOJM=_9C4@GG~1VV+v!$WuKI?`P<(qlUKV}rnd z4rqT+gDnSan`Qg?(wzU)YE^)VX`6zw7)t`TMo#pmgHqQFR{1?J+1RzbSaoBNpgDQVP%1+u~mTiU)L`OYog%mV;i{m3C^RB>Y>V& zYZaV$@-%SEBlnAnz<&MO}5y`wUAMO*0t7tzh)~*N|M7PWs{T>6epS$S>*0{;q`23~{W8 zXUxjR>_{lC=~t@%yFchHvcU46zZ#i4uS)t!W@*lSC0JwBJ7sjJv(egfvw*aEw!YlC zqb1Xr^wWpRcOom8O<30{ePb^5#&gqH@xA^1)%R&1#i%xj%D&~s4QzP3E8wbjvtM4) z`SqoHwPqIS$)SntZeN8sB6t4wzW=PpzL1`8)qeB5kabCN(nLi;x3Wblv&QKzg5qEz znBw3SeU>0M7{E3E;_2^i&$=Z4nEWQQ3C%y3(6^J)ZpP}uyDmmV_v)VBs>3x?wAfs(}Bn=0{iB*1R@z4`i#`%rka0OHgpavDfA@y zB<5FT)$&yu>_*3#%~3-oI&JE2|37C44kfd1dzeS%@g_;7O`nSh_i;0VsNbt@d`RC# zt8sps#QX5%&ZsbU8U8gQamxFc*-QG4;~SinsMB$Jrg>(*Q*W1}Qa-oI9KXgzk$ z(*ovgnK9q%YV+I0f8XYx6aYBM%UcdcO<{P&9zUL_!`sb0K$=lqb;zc@98AENC7617 z>G46lh}}~IOuSc=p*dSkA^F*mhv|L4qMg}WbNFQ(W=q2eYu~Phf-AL1iOCz+%ONv$t0 z(X1HErN6UH1o9fC6^F~d&EK-!d(lntt zw+3{iq`sdNPgk?Xe|{5kx96|p96{xCvttt3M+F2WPZ!mIJyGq-%@F39%_k#$hWP`p zA#!5C@~ZgSq_FS}3$p!?$tRl4s<&PX#&WX;H!zTfkUYLXwfK$JHlE6`{qp^`cE0~? z=+u3?w!FuFJsQgL^{dHAtG%Kdr>2(Wzl60tylqqv9>Pcuv_t3*@+trFeNsUF@d}Gv z)SrN!855L99n-rKct;|ev97;~K6BpnFsF`06WiBL=6!aRZcod!G6WycJ$NLKHP!=d zn;_~!EVU@#xV+k9X4K)!W>7mI73AMm5!k8HeIGGCV0jeTKQ)xWce;KkaL})fpZd9_ zhVEHXZnf&}2xrHSaHjsTK65R_w9nTm@Cn+*u0-FNmcrI2j_-FsD6YSKHr8{5@?r|( zi4DM##)Qg{JEZ)`t0dj;e!-PoLKA_r+i1PLz1z0ixU*Sa#+^7?GT51#W)~yC>l$`9 z{rTI3=Ok&0pXyzVer8=$&KraDm-&Z~w0+Kkjjm{2tzRI|icN#U0{<`ijDc$#N2bwuP`KiG!H z)-KF{i{)HYPJgFpK{|%j{?W?J@S(X#Qu0nbU;Rm;-Um_VkiC6t*c@G1*z^6@n#{2` zBCq%GeVV*qkOMtX*KXC9e7$M(5IGYYpNT<%-@>|{G=|OXXB^gp)j+9a-UN8hB{L%@Ru+=hCp&2w8d*1oJD5fef^X&PU4s)i!|F6P^D zaYb19{n+Z!=2t8jZ@EGTUW`mu6nt8~=^^llTeFUjRx@X_GKtgB76-;?fSC=Xx9o9$ zwu(17k*&MX{~JkMx#4>{e7k?L|C=u-TkY=D=ReHV1kx34o#4$pzCf zL=f#*`?|^IU^`j5n||FxQ4nCayxX9}oHln>oVpf`x5 zJGFC_yD+5B%P~i@OqPk;)Ys4$Q0s2GeLZw$He#&CUr+K>U^3QdZ zXj7Pn`8Joc8REP1v{fv-+p7jQ5@w5NX;SL7N`t_gxaEp54MDbd@vUG)Q5#n}v66a;UxjbLMbw&54L_bv7arB@8_o%yrb*p14$0Tx}`b? z4)8La%(3EtLgPD2sLsVL(i_)r=V7hZ9ocA(fEk9ebaZq@>5#9%BPq^Rn-p&tKJ)y$ zCVD=2WHzYq)84zba^xGqB0R~(;(GjCA~#8HTo;F84h&V@VV=kjrhRiy}r~vU#EM8TW|Qa8ehGj!s>ZRcYA``4tVA9=A>*L267zRx8$=54ic?RFT!2{tlio{GOJ@LpDM9N9hY9J^TTAXQI{ z$$0g!xrL=<_Q%yf2VO|fH>Vi0Q3UT+R&CZjtKdYN6~QdcH-g6pT-CE>V>$=zHOZ{` zKJ_+MJ{92`SOQzIJ>pPkS)Ojaj)jHAOzPcZGs$h&H!ad4pL#x;Pe$3h#X2}Xlz4me z=KVi7p|~Ta;I|>FQ^e@FaFq~~s48P@(nfr9nk&obug}l)Qs}Au=S{|Mbh-yuPK&ue z9z}hyeN?dlE1t4tqWv&9>8){7({9%KrdG>rP&4Ut#Yp?MN7;nODaQb|A%BEHbT2kWLfv{RtbInKXWxXN%|TZ{uK>hyp)%VinO&_{jg(N15Yc6 z*GzAJ_txK!6^9Z)90FabzxfB|B-8nv8hz9u$Wq0s?QhH#> zfG4Km`q5O|t+sxioljna$lNMHe}kGdy_k23wio3Mo>AFRua=H+_Vf&Wvsj3uqh0E0 z*MTa!YryxqVNQ2$6{U*Jbv%Q%ykFDvgE_<81&_KNdjnEx$V?r+9?3_2sUoy)!X3f5 z{6dZUU7s8XKyVA}`jG<+-PkxoIk)|I>-8{$n%TPMF#`fj2tJtK%QvM=HBK$ssnPdz z-Svo$A9|a=>$tzlx}S_ZXmogYGL!3WY1NacW>Fzpe&fS>mm$=8OsqQc5`FEDpKn2+ zTM@c5O;N8LX^IAsz6E|sV@SvSS=H<)lYBI}@`I#G1l|Mpdu-bBqnA13gLjobMMKQZ zw|b*b4N=_OqrEnKQkf(@B$+?d@A4N5iAB#zsoU#(vy6H7m?rFG{{!wuJlqfawqnbz z4t8={c;W>;8R%9&Xi%@SO0DadSCX)9RCgV?HeRdJQY{@#0rA$?2>CQzL~h6&u_@|G zf+-||JC*pV_<}x0%C-6lQWq20-Lh(jf^2$Dt!gUro=w8qH-&EkO!4@FT)6UwM6q3T zp^NUBR?iXzGO1b9b54<#EBGfi3&k~ZU3=G;n*A=Rt#Kg$&tZoNpJJLmFioRHWBsk@ zFN2p=FA>_L^~dtH4MEauWBJDnEEG2H*oee#F2WY-)BScj3LZ0Fv%cF}M1K5szV^mD zt^ovIgno{w0^= z&CipUyaGXLkfJ}@zoxY;OcXHhaA9N1posBLqTu^kWNLEIdT-D98}epcZ9qS2414GW zgKg*5fedE0kY00zLYx7K^_XML<1_=3@I{rqf+x6vRbL1tN8cZz{vTau9Tepk_I)L! z8wpA24pF*Px+RyCM!LH}y1PN7ySrOjK)Pc=8kSg^=jPYX^UgcNA3MV=%bxq3E5D!X zeCR(9n1qp{r2AN*0{?nLB`7W^Yq318b7h=%Bd+@;Q~YzA`(|VJvSBGa@W)RN<$XA) zc~lE)@nnK6()eB{)1=x7$_TU8sL4X4uh+nerfsx7=gwOaQKBJ%~!7P)3QQT|=VX}h4Ayj!I=#!wu-jDb`u*&^0#8|Uo zb+Yfep>2sVD=WdW5n4~O1-J@fMMRfT`88{4A?mx&=~y$D6T<^P9^D|q>RlkT;sgzDUMv08a%u%7EIt*xkAomb>;-=ni5rYo< zRfZP+8~bTdgiU@xBvM&Ef37 z@%t0on%90XO!ey!)abDv;(}y*v^^={GTC*}%rCTl7Kh=iX+JNAa7fB=)(>woLg#K? z6?>mNgE5C_n`NWsow!_duTu}YSqrCF)!A?^`rX2+I(6YuwpE`NYVjL5{17FhLz%sg z*>NVon>C)-Sq0s}VNgKJlV4u8#V~P>?max1EI84{_#-{E3p1(RjXFFs+xr{UVZny<)v&%&rl{I&DPcRfXQeP^S#&k4?B1g$eu>)4C*Q_~RM_)cx>T))9`CubmRgZK9ZoG4m+81bD$ z7cuxwy}9vXY1#3CcDyNmw|nu;<8{cTqS$#pH8XMOi@PqeWm;8;U}!O1xCp&vCIT2z z@p+!PK-rf75Z4g+dy!1C_Yhv{YFxkvyybrpPS0HFNESLFf~+OT2agw%LZu+}xI@M% zQW{@>NrJae@EHreOFH&q1u@UTQ}Shb7u+$;*Dgd_n^m;oZ3(#RwJxgk1cmCkPEW3- zDsc)MSo8{tbt-iO5XAy%xpft(JQS)}dcW;T#fAx2Nq61VlJsN4G@*R2;oaq+c7@ZR zcOe%KGH`+*h~(j+N;lp2>|6Y=N7^fBMrTJEUV6~|w}=pfGpDf1_<8XsGts`DwO&#+ zRu4E)nr3)O(&!R)iP9I9oXy})ViArp{#|e%(TK({VF$sHI4xY&!JE@?CgSq6kElir zrE%=G%jFj?>INx0!g`uHGafnqy*`s99)j~%{wG(Hn0D*5Hf zyfMmq>r-!`>b!}+yMa^EIr^obnZ+4-LpLzB)_EJcaQyoqQEl2Wle3IT>+e|rNu9-* zL*Hj2<>a9&3R$t(RIGF z;ijiYB{jht`U0YHLt0tA>KugKkmFkh8(@uPt@6^5b?DXapBsj=mrLP9KSNO~*(wb3 z3CR(Mi~W`>c3`6PO}SS;MMQw8GSW$ks=Op)Cl~U7Q*sHfZS%FA!U2Azu0X~2@S#Mh zEJWqgqe&ge>$tk8-iqe7?YSuYgV@z@bC21)g>zKYj(b(}yqD*&?d&aAY~L5vFmoo= z)VIx`Wk-ydv*ut#l&#JI1_d~40hj(@drMiahPc80o7OegZkP_33su+Rs*kNV59yKY zXEX`s=~gv$m&YOzdY2k32Yri|hLML}9Z9V&Y(NR@TUIBccIlI!>l8F_WcR&mdeK!BT{U%PP50mm0|d8){WYxBwf6L^#RzL#4= zXmQ)xPdmWTrNNdNT6WvbE!1C3*@2|NP5M4tYS7c4XHZyjqt9P=n!S!g8UKMbpzyuIC~c(KouS6rE^MZN}o z*XNfI)i!*8=W)1WZVA0lu7*M46cll@RJ9VcqJxxPXWzL)jowxF>RtPDZ_2mXb{ea< zZQA%_X;$SI&uM(c@?VgV_(folnDxUNl&@g~A)tG9OB;`6;+6*{vC+Zb?DS;O|9!)p zEK|eKui*7bC$ia7PDaMOW06eK)g|kF)gG=3UDNH}u72D2iCo`>tA5=wyg+=0QpwU? z-1SN)$JeVIh5QYH`(Lka8@&f!D#0DAxe(d1pz_@|h#JVf(78=2T)fg`O)+#&df#;G zZ&Kfk=cC6y3$zV&`Xt`WC!+}yJo^gEhOY|PoYKLJ6^%;r7oNy?EdrkmlIP_rjzNo= zBUQ`C&{%wge4w7wh@_j|_r1$xg`eEKes+Y?axHAiDZ$&yYeNiK^h zEa@w0Q^LI&lRBU)xuccGCm`ctdZtxj-HV&SX7*Lr@3y949ACyo1039Z9DXr?&*YF@ zD{3KLqWB8Tn#MJw5m80Lf?mJ>l^a`^q|R&~NeyY1sW35({P89Q#rU1=zIvfR}3 z779Wu&QL8b%5U*vukd!u8o&7M^KQ zylO%=CEBdx&H8v8u?#8Va_F)PI!n^4*7)HjL4Jm`#j`E;b90^N(lyP6{z_A$wQT+R zUoXOjh}i1!UCqk1NIH{m`-G#|bD;MkX|)ApEQ@%oG*W5!h> zoNK&V0v!t$k%vT5J^@Ng?5k$`x68*1auO$zYN!y{Iw!R~B&!d<`g#OPl|mu*>5~3V zLcpVAQEhXR_6ev^NgN|)#&PVDW=pA1jof5FwhY?$r;D&i;dfGW3};Vk-Y;C%c(tqW zLW5e3HY=kQJ3CpdbXlmw@rOoT6^|Y7Y6sZ`xUwolq11aw-IF>&0CyZPCR{hoJMG%N`;b~ODi;X^o3>f`5X;Rry(kXKsFFLA)s_u%z9E0F!4$wtl zHooZp;O~Nq=T(ny&E9X(pwxwmqTMJDL;^C$d=t15<>q^A8@2-}73-q8?_;K=exZ`gM$FD0+1v6HN#vXRiM-$BkV6aR`mx%VAv#P1N ziXf}>gT}BZ#|kJ6t=%Fnu7|=xu>yhD^G-XCC+jkETWv#El{8(K%BjkVdpMzBjNY<)62RpZYap+N<>yo1fLWaNWwi zuqTfjXr+vS;80}E`1Ku9jg~z}9qc)FS9PQ++4HQ1t;~DGO1{t1>wvk>sPw&?88X*r zmVtWBA{<$af1tY(=(*g4Ih*Xjtjkgx?|z8U>Mjkb(c8i_gCTerAnu*8V}&-FIF8I& zs(;yZjK~;A$JIn6!W`9-2MpeCj{X_XfkUgPsxQ6L;pafA8H9Oz1>sStuT|jd@Gwfk zZJp?7%>Rh*^$jh13mq*9o9W1^@J#W7ApS%SsldZ=8>9ZX)g*@X>3Ml2n2j9T_`%D` z**=i?Z4MgyyR2tZ@iE#BBoC7I+*!EOz3?KIo`uGed72PXMTf#A0(D-T3+r2MVa%Awh^9n?$S8 zPd9`rNO|umWyMx_fEg2WmPQ-^btYGJRr$a{vGc<<1f-^i(hTqLeDhjn4pWB<#xT^jPyh1mu#b;tm9 zAr#aa+-ClgPO{7(*cDak!&jV2hFCpIk_i1~H9}qi`ed-XC#zz`OP$xqt&n`_h24B7 zHKk#(t)@wRNeL)4ux5OH#q~#&bLLsQMK~~`U+em`RcZ8UVN9^EI_rRaWW0W0a`nrY zb)mXabUU||ELH)Uy6Cqa9m*608)A% zYq+J3Xivsecp!xMQ#Rn!3Cuoa6nZ#~f@SkM@hAI3lm^`Q>HF3SXs6R&h)?3L9AJmd z>g3b}>`g-2hVzFkqChq&(*6RJ&#hr>xSEZ{E!T9*G}<#&jH-5Z+>Y5>0+^O;x+ys0 zh3C)0@SQ`aGAs)1?|5G%)fF-5un4Tvu%?qosT6&^3bQFls)&Q0A~K2QU^QaiZpqpk zg0~GIH;J2A%nXGpNeA9qb@DSu-1KCc>>Rr*^vL{2LH(!=$&uU5`J3}zmc^>&XuJ^U!JGJ_9HB=8+I_%w=}y`5A?)|HrjtA3DQ5v&%-gQ z($FzVO+RZSyVnSgGdqrOsUYD8iBc%YY2x#*9b*?zrOD=6I@eE`3(XrOXIVn`qOweP zZ*(AW&xa{|m8pF|7t`PX$ko1k#A=u^pEKvhpe3vGkh2Pp=l<32hw?YWYg~Wie<$>^ zq~gXAj~u-Oezf|{=i?UiI&=}?dU%6`3#rO zSDKYw@|DjT5m#&GDo3JZg4>|d5SiNh(`;pS{A9Aj*Gbq_FBH7Y+~Us3NqS4HMkeR& z=iV|41B|Lh&bdZiQnw2Mn_DGNEhe&duY86S25M|k5-rb12v{3tf5;G7D};}Em}uIQ z19v-xVM1rtp(Vf3iZV<;Oz7Bgv}%s#l_Z$`#XaJ?OR-6Ly1eZ4%a1nJT^s{;Bpjrz z?jfBdg^xMyWXus2;V}V6w-bzkHcWN73+|MOg1vT;O7q8cn2Z_os9OYh!xs{lJiAvq zQk;}C;xMqHx?{O-uJ7FO2=v@bEx9YkG>&GklO88r37*nrGn`;=n%8x=12g|@&*)Ac z%&Q&eW^hc~5t)dM>~-d_`HNiH(s($-KXQQE1p%GI+%Li=^lI9n(<2aAp>&(DYlmx5L(-?OCbpUShxRX2X~r#Y#_% zXELH(8bWCVqf;M}o@z*|(C-A<732vxEN3skgRkxe!Tku=a_{Pgz1ku{KYXlN_v-;U zs1eYGuhx|kE&T2&5a^_u@bbq5Z*ONidE>E;g4$PL+YZY}T{+8J|4wf4;v}EhPaQEg zb0i^=US*VCs+u>6WKBTI3H{E19%k@en=Ed~U?gQ5y=Y3l6$b3oShZ3LYt=*>wF~|J zTzWkwpa-tJ6l<0f3nhA_Sqa?oz4x%5Xto&fyUNY9RZmm9IekObgE%2L8xqn*1R6-zS2#IVR@9e5Gi18<*u?HwT*cUSfE;Q?J3^Q-Ne9e&{L72v|P+~lQo|Xz@DIgV{64dczQzjli2wAhsHN$196Vp z_pSMI&ga2tVer5tlgh$#1-NdNb0G57{Djtgs4c#1)oS2dDgAN^d;LRuQ}gdHnJkKg zBO7_iD_)H{Yljp%D7S_kT-cr@w-mweY^o;?C%T8bxijVshX}TzbEHQPXJ>7=uqFkoqL2bc>fyd!kCjxh{e z)_^-Vz7_HF;rLC{&H>TiUAkvTS2Ev))XF49HX~unU2)8cB|{6k)y8Sd$j`4^X*SOU z;%~E=XtUqGPu~+kWKz(R^}M!#>VLbz^DafqF@-8>6RhVw^MpjeP99-3yiKicgW%ye zGT*+M<`8CzWte@pV!qc3&DIhsQ@yN`3eb?@tJn?nECBy3N(69HfA{yIm0{`*J?)^MTec#Uoc4u5 zl)7)6X1@N!%apWaVx;uNrD9v|5#Yj2N?xlARTJDl)m^j^Cz1Ik;bt;kq#MM+Kw@Q? z5!d*8EzbN>+N42KvQ_WcG>y6i$zX2|+n(bDt)WRylq6nR=g4r~>})zh9CnAsENAc| zAs;_ivrD^IEgHL3foe+%$&CB4W|TE%2fkQ1-4DliM6=kT#b2nW%gsw?YqgH$3;5e+ zth6`zNzH`R41USm@*^k&^WQCT`VL=p(IMtMyk!gSK#RMU}5QO6jcFD7!IdzZo%z@uG#aVVM%$jRdh9uAb(96V-p;n}6?E zzG?6dH5jlLviae9UQvf*=EQ?urN?;^L&l^)Kj{bJND9b!q(v92Mn!e^Zshj!etg{C z0w7N=68qLP|KQBAFgr6t_CP3?4@v^+LBWkOb^3Urj$%-G^3g*=Q)^s>>x)En^&Kd6 z8kE=u4_gBje%*__bX=Uad{>(DiBrqp8yX_}KEc^$6Wx4J2gA9f!kUMLk&BFDNw;T+ z$TqjEYB1ZQ>!RDy^8l<}?Y4+y!qGQe{WF37ITGSdbWV7>pUa4X@xxJtyWvzjZ;>Up zdCPH??lY-)+Lbw0jC(U}5oFKC0%v4AME;vexg3vgeFA9ksJJrXQpp2=A9zvr-+Acb(ylaocsNN$$00P2D1Gw0qny!JnJV6zNaH@ zOU)7NU&y!6+sLp!FETV1WPSFQc-WUu)7nJof4)>f>)490U&m(#Kklo9eZ68clTdn} z)zR1!EE)Qc21DBTu>1xgU7ScNmI zbQr(yhD8QGl*#hSpS{nh;IW&zlV)vqbKwNs`W>7HTJGiE*bCPkn-Uq41l9)D(%=q9 zCf|QcQHtrET{D8$C;QrV!X0IRU?Rn1OPd3ph3QF<9fRdF>6)S8oU##)BcQywO`&Sd zO{kwR1NnRwsjHs&Py2KPQcv507&vO^qvE!z zZ-Qf5(7yQ+QRBA4*C#+{!$uw3u2oT6Xw_-g@Orb@GZS2~>M3>ND?I-x_cV0e*|7yr z_TZDzE}*n>P1K|5p6109)w2yyxg<7^w-#FYQaLm>RK5IISr7b?*KU#&73S6DyRoJT zkfDA`mdr|xf$nF)*JJf8P9Xj|MM0Y~XbnohkBe7IH$d|-{A~2`1ive?2dy8$6cimT zWKrlw{d@YR1$LYZ&&A_FyJlo6{8%K4y?)4J+fGuK4X5)M;Vb5Kz(Z!`gLZX^`i`Zl(_qf9o#BVqJ6~R$3B*cy zlKGKUqVLwo>2pUKO_sYiIfb{~o(^r=jmy^$d$+rYkh)kTc^rn^vnTdW{YR%EfT-Jd zD6ZO+Fd626eY-+r3{TTImBVF0jfBeYySw?Jr8Q^+btY{cPAa@)yD0%ifwRl#jbM=S zx?V>7fOYqpuXXH|c;clFCGWCamt2oicgK#wuj6MM2TGwrW1?D6-PppXz!F*XQ5Ihw zc}A3y#New6eMIjKrj`vcvV?sX%4HiI)c9;(u1lY2|4e)Jz?gIwv|dU(bCwS7t?Ag8 z$T@UqOd?eyY?>w5ne%hUCB>zS!Es57>{{o)ShL*rco==<7XcgMVV&$qyH9jXs&vw7 zaEe>H^$QwZC{=C6IEQXUMKEK*ctPsGI%>RBZbQX$-BU zvpke{tRu|{IFXTZVV*POdp06RmDX2;KJFIdOkx>Q^akyL;Qoz&Q0aV|?LF-(9t!6x zgse)RC=Ey#d7{dU#}c6NwZZWjRx4msGcT7|a!I?kRxHtJTnSgUQsxo2JCo3jR80_wz5o5=Xrb?W zc1uOdwv@aR44$wDWSd7ZFY&$#5m!o1`b&k~wj2Yr*5iCAtQJot{e_Y$^sJ*qm?t%xo`cGwRSYv@8>~bt@w+PBlMCfw2_Fr!M;m13$ap!DOIy zqw?5+t|eo~W>xMPN)226YO(5K6)2UfFqSi!YOf8ev|S|6Acm;>P=o@1oPZfxWJj3} zoDmslHbeH3j3c?#?|T5ipkNX`2Q74` zz5^{y?{0q7RMs@_6WKSh$}sO87cDgdiMS%|`DXeSUg=v+{d5(|7`0%GZ zn9_<;)r9&NB2{rj+YNJHRC>MFBLg5ynh0o&5I2=rp8dXegqo^!Tsr*kXk$u6C8?w~hIyO^PZp zz)6`_aL4QHP*I{ug?DtA$5_mQ=7^xPNcv~>uDwom^?4s|-+at^&h)de9MSqB;`a9@ z*9hlE;e)8b2(R|0=5XL*kF5qxDZ&NbdAx|$tl?)Rwyuq@)KjdDH9;4yY>bjP9{>@U zJ;xSQ-|?lO=0Hpm7cu+BFA6b7Zbsv9ltOQioY1+%0J4dJzQDM;Tl1O{UR$18+o1|0*#@_?*&*1s0rygLuws8;YlBphnGe^9~RAXxEME24h%xWM!^ zdHhkN+5=gARouPrPzkMGs@lE)x{*m6Un=S>XPlE9i4H5>WZ944WosVZ9gSF9wlqcD z98Fi9rqm&XcVML(Ji}8`jvR?&+zC&};}B0P2)MsYi*Q+rTF>D8Fi{MhQ{g>-H$`9NYn$=~8s3%K_CL+fGZj$3WbhK@Th z>oAb&xN2%J951;aw?XzwDqf^s{Sdm3(`xzhN!Pp5(nHxms>7lChyouERHPfiHCGOQ zD-9Ea*@;2gxcl|p=P)mf>A7YFxCo|KGzzh)li5|D_2^-t)<@&fg-nklZ>DyMhQ2M_ zI6X(8L(-iHn@c-I9O9pzV=+-_3bH&2G-@Us=EB?NMv&^5xn$|-##e2>QZaAFWK{q= z>ghzT5tVQlJji7%MpKqMCA1^rlvG9`#*v z0vv&?oOpzs2*j=1LuWeIK-Vp&rK;nPauyrM|DmOfcJyqlYif z>LcX_2gmT}#uSX_gV#9&9GJjo#Mr^{o$yK1B#H&0UNU|gDZ>-aB)F%I$3P{J&_Fg9 zoN|qQy23Vmxk+?f?R__Bruk{g(RQ5HI@KWeJ-zm#5Bs;NV0LZwSnJl?PHG)VN6wgY zR_y^cq5ex>KHy_B})GKuHpXZoqL9%>&hOEHw8+Zf(-217$Iw;?N6C&2aD`)OPV#jZehWe6av zzW0U}TF%Rk)xgX!O%>p-scM;w)aw_l?dmen`58JL-QB#?hELmn3ritAaTMy@FR~9lw>+Gf|cdqeMiJX{8%o4KAs3LM6#x z!GI26@)!lZq$pO1yy{uC)Iqr~r0+!}nroiNIgs?+5iBC?wZ#X!#~1-=Ww3{EPgxgb z_l&;@47U#V3L{4QGV`j|Mp`J70wu+Ey(GP!_Hw_9HJzr`GbpF22q^uV_4+pw z`BRJCDYce!oJy}$ELAG`ggswBpx+WlF~d4bko&?Zs<3+&=P8vx&b+uDBD)+Ey7Q`7 zy2?rmkCylPE>Wt#FrHZbkG>Dcy@X+e78fa3m&~F#c*K^Mysny4VCNYAEA{!eeF&gQ zn30`96H3 z^bdGu_|~_L*%M-&b||&$5i#IA4k7eS&L>3k$@mjd{ikm9*9+gbGzx6|;)1zn+&%U+ zmTgyoW6%TZyxY%2xJOj!!o?ag4Ni^tX0_=x<01Vh1AxLdYrwu_J@r4rndnP6Ott3a zQ!WC1+=y4EpH0&)NsRs&ckqv}{ht@)ACUk>5SzZ7X3OjXwVx{)@XdUJ|KuSs7y;1h z^sV2{zeJsIG4_==r{i#QvySGORhwb#2=Qk#>CL}4^FQDE@26~rlPr)eW2M78IP-%` z9y;_9UEZ}Gb?LS5`wyMND|0<8JS?x2Wd;JGXn_LU)0>1{U{LT`9E>z7b$uBlkc!0D z@^IQ9J=#wAoM8njLn>z_N)=TWSbuM$fA-T~?*_pdf3f+kSs1PHawWCg49)b%9gi6hJ4M?_amZ( z3m)*UXVIM0+DokIUjO8>hCFNgpHLae#F?q1?!bDL)%OB=#s zTlB#L9v)tEF)^n_G9xn%&i?|Afq$)6bnVfihA~e(Vk9XoNiMWRCR66`TGT%{;GZv| zHQ=6h8qjv1e|5n;&K{8WhUBX)GGm?M* zt>|jF2VnNX3b)Z`LiedY9Hsg;3xfaO?m2+7`sJGSxwVl)p>vzLfX^<0Hr0wC5jn{)8N710RxzpFzY11AVI zZYaBgfA{hYAW>zG;QQ}0`|s9b}3vaHV5d@=iCe$IYmPspR z!3g3-LS=f+>Tkf>v*n_(o2K8)J#OWZkykc8oR{YB!h$~j4-hZG;u{2=^S03vY>@)= z<+Kf0rT>RZ{MVcm!2x9S)}*?Dg(L4yhy_B^M;Jm0a-{< zJ}*uf8S7+nZ^1-*)4FtdgeMYmq{*pj98Mn{A+BMxQ0Sq~9#loJ**U26PjnwEvo?#) zwIXd2nFwI+k^mF>*U$hwU!ZnuS6VJ>7ic_~J3&choop=>6*mn(gY(>XWks;7&gzSR ziPk>Wb(3Op5@XNqIx~#h$eUy&n{eV0wJgu4F)qGc(%)LSz{wqUQKr%;>{O2)qdU5kXO?p1lK!qng>}iQljJ) z0_Cf5k|uD~XCXSxB|Z!i?ff8c|YP%G(f{lXKH*McXeY@YS-MrdkDZ#c|uGa<1i!%vqT+;!2r6U30vXKRTuk`md%)cbZ>|Hm<5pOs>V@GP7_0 zF`MVfJpSN@8Ry-AXU^0}vGp_z`tz)_U~)U<%dV}=)7PO{1{zs9CCC^H2oULw!s zK=-r9t&OtOE{w_x@Z<)Y(;7dA0a7l6ASLAIdR0#SLurY*dlXmmBrrzd=~JTlVb07C z&ORVMZ^$X~KB9AF){}jj$I!r996j`XNjNVLp2zN%!W+%y3p+_Cu=D>8a=Ike-bK7L zOqj`skZ{PVxwmDKGnq@brKrjeUn1azUp&-U}cKvx>PcRkHgSz#$?+$-Zm zqU-m5P1&HObkT806t>(diItcY=OQ)gRSN4z$43v7sJA6yOGi4lx2ukCfSDZ8*b-p& zH%hF70Gg?&qKf%j$HcjUn0NQEpD#Lu=+DnZqGkhlz>!o120nByxV^5siBjxi(Q;pz zC34@lv}|S~<@~pO@Q@bAg17vNS*^~4bVZ~*BeOc6*bf=y@c!nOyO10l9DaZOtpHr) zhLjm19xK;bl$^<4{&v?=H{j0?8l;Tod#icWVq1S_J1xkQt!k#e*X(}z^$o!0r*6HY zkdUohApP{SgLipEl`UOpSr$)c)mzaV)s ztP$*vQhzE!FxNyCttA}Ot6(U8Dq;gau=1TeLbYA&-fvHs4N)Y>1(4YK9o%%d>QnzT z5)p{}4_BMd^WLW33q76*K^Dv_@;+g_X($5JFu7CTlSqs3loP03Qx^3u!~b|krv?jyzkguaBAtK&bhJocUD0A$uMfb9x$felT`K%HvSH>Ils=P?ixNioAY0 z&VbW?!l%sKC{WJD<)!EB7gC51FrJoS=2O~+<$&)4Ml$_hEg{)+Nxp9n6RsTeex>28s5ZXFO@gkiNXs^aIc*3^3@@R zQ#*&im&Ll%MU3g;m}XT|vBf%Xa9z5ctr0oWXJOG5~qJ|YI=&% zh!SFW(}-Y6h!_^3;cH|f^S{W~YzuP_PN@yWp@2?Yyh#6dqB&7)* zUOk~cH`j0s0loa`u0p7tlbN<~ejL7VQ!l(c=HUzbo^$fTOXin=zkBZx;oL|^mmhgN z(2TlqR(@T$10OSpmO6 zd^y#M^iS~kg4x%T`5wpF$7fLBNdjU42So8L3{L8Y<_r*tiX{&7aSQx1cGJ%$9)qbGI_DR$!<*Bv$ab8!T@j~};<$F;Aq`f2` zSsz23LIHFK=w%d-vr9R~5f=YO)8d7Un`fOX>pvo0oP4e)LbjQa{kgr-6t!R8YIF9~u=Au;bE@?R;hMZoUCck#9|EM_6L)TDIztk!33B+#ey`yf zh)fAuizz*2#IkI-^h{!bxdIRK`LjISoTUZ9mG84=yb}XDT0w^kUxA)9yBJNFJ4F-d zn%0aclcch;T=enC=RjcPODoYgQ2oP)E`ej^(m*pM-j%43Rul9rvxKZ&_jv{vgw5kP zUC7-{PqpYHdhO8caho#4?r`e&ATec`<@!@c&<;trwO?OL>5_LlX2iiOPM8Vq@_3cC zqT`|YDfs|yh$NQAGIfo-k4R0_*3J1VnkBD8IxhmJD(Px}03wT^#e>;=3bFlm-(w_A zVVEM>XI4(9>Mc2{s@8m1wN=T<9zcAdq6>W`6|PG-d&o;I-(DqDNU`KbS~>vSw#Ww~ zdg7Ogy#IQZlj#*1(7n?u+u|jlc|PMw`CTFKz3ugnGrlLQ{;sk$pzo}W(KvvKEN32o z&j#)HJI4jw5RH?U*%_o%c6IHXe!E z2c&*xy3-S13Sy}|5MJz+fLHcth{f$^N3F**O@zM99g%1FcmemO7lKB7BWDY6Z_w8Q zSC=^aQA%y3w+;pH@mNR%mW`PNeZaN!SA)U)6B7ALQC_PY6KvtM*5CVoxGePA1J7cE zQehntx}7c_TH%l>kc?Cc+=(D~o{a&Ix#YBH328g_WUb5^B#L2f88?LaSI*M;joRir z$<+A$n`$~CgJXSjjk)XGnfzKNUC@;%K zRnDpORZ9ijk~~7KFZ~n>QXhFpiU3Z6=?x{;mcNQ7U0?4XXxR!sfhxUa8HSa8$Kx;K zo|nb%1eQ3NUaW1u!S*$m17#t^8QNpapbJgvB}{K+Q(Z6+4Y`26FRXsh;lW8tEHt6F zWxiyYHz#?qR4~9W=#f}zd2SwFo0t-837SKQOLAbBV)NzG2Kl{+Gp-wpRL11oP$xM) zDGye?BkmY!#+i2F&jGws)v5(7r1gAE&z=k1e!rC|sg)nK?~D=g^Erqu5SmXv4Hi|z z^mTs5A=J(R_Bubytb$0MI|0ip(vSxJL!!ktSry#Cd6o5{3uPS(`-J@DcTUuTVwTs9 z>3sY05x6JuwZGFB_Kf%VaoT^5^^uT$8UZ)D@25J=w=En7`5pyMZEiCdmLP9}+n$X# zN8Pr{nRJR6M9nKu;XL9asoe6@+(UZBS}V25M9ES+B}%PtVx`(pjxS4Q#N4eQ)`L7N z4QQHYLHRw?qE`WpQ*vrE$tMBR^8?n&hmOsqfRLeb?~*t8GIGLYIv6dSpn#`&H3!^f zhG{99elQd>(Wg}3j1;8kKKWbUaaKTNtfbqhgMm3)rU4nZOA{lLUB5|Vs%m*+p1Gp9 zM2q#k`?^RQ%&63@4Z%NOC-s}*%+sX zy1s&)mbT9Na%7W)E^S;up6OgTXkhXnacLWsqOYSQWB9JHWcQvnHR5V(+t!0phe<17 zZ1$>k5Dx^R0Zgsa=ettYm~q87{SrdzUm5lx^D1uu$nF||zTC(bd+0|}lxf1=!FBCx zMz*ZLX5$)DqUeDIdqH%t6@&_g1|ApHW1NeDecbO}sp{4?Up+y#9V5Ir6HYD}F0%N< zu7lhWjyzt=@?O)ho6VJS2wy;Nr@~dJ5EJVTW&AHN;$W3$h`Ns z{6d{^EN=e!zS*SIh{l``%fPUDVayHilS^=+ki#>Pi=B~Aiy2Auz?f6t{QQT$FWat` zmX{r6qvtZW`TD$5<#e|n|E>pNvir`k++ra2cI5#Mw^!_$ZMtk%?q0{eTN}Vqr&ZmE z{l>Tj^jWlEyxD7D$jGeCsF^_5FoiRCd(#iptJ-CMdo*ljI9MxXUfr5ku9M}q=`&jG zHn9HU<9m>jip6kPhJneF(6fntsFn_;lJaQ`$@_FewQMXZt?%sbt*7Vu70Al<7{>6#gy4E*`OY5+t)>Ox1w3YEa^$UGpx0Q*?7srLen`emu zGzvOGRc2WpVypBMp=6s$>LwHm?UMK=>M!F9QIJRY97aT$OiR|5FB{#kZ3pTbwjnEW zfDdPd8rq)bgS{E_ftGIOwN)p8_64(&l;xh?|DK+8IVP-lgOie2$o?kA(s&+76|#{U zH-`DPV?Jb!=1^g#`=}+0vu@_EQRg1V%-w6co?nAW&>&^_=)+kJaF5S_+rEnXE21LL zL2|kOw0b~-ErP**zxD#>`PYTZmPwjY zj%K-V2fE$`R|jovZ&L0gIL7mHJ4fq0uER{#EGNkoWO;+>gE^)O+f$>74c zOTUCUPskDrD%NGM8xxkd(rc!!C#fcV4&i3IOq2_$%$H$(95{QXe2sZ zlw6iH%ZB)RRnCw2D6+2(v#_T$>OU`@+Knr-D73pFwbto&sZmgA9}T!1r-)RdsO38n zJ8ajHO7{}%mPS2E&mJ&s1S_06+#h~*DM|oyGEQXK@26BQRc?fIT(#WNYufxHSb$dY zpV>E6^dy`eu{YAGby9r0mW38@fy~k;*1^fox7?g+o0w^2GS!-S@0Z{68HEJJj;A6_ zHBAj90Hb{X59y!OOYo@5;_ z-8*h9Re{FlmRO1=$~LG7F+nJ0;BGB5M9T(^X3r0%RVe+9KrnS3$LG@9A}^sE+wtC^ zz&~o0J_*9|QQ9vL)5WW|!JJ4UA3KM?3=TKar@on@uCVsS)g-R+Yd-fqJUEJLUFt18 zlE&|}p#8g-AiRf7_k)naGBd!S3j?0nrWA{# zv77s)sIAn*h#&SUyLp(7MiBPbz-mUx*1f!V?^x!ovpOZ`>;3>tr?VKe0b>}+vz==2 zum7>m)%C1^_0Ujlw)#le^4>s_48XgXx=qdY-)>A%CcfR^qd@Id2;!4Hf0sHp%EGtY^fzGNw?Y2?f3u5vWI=P?IV6rB#A%;;$M0u6mAjuk+1J zr2e4Yrws_WZF}v@4xzIk#nqQ+LfdF^KB2p778HUTMgnW;2O*7Y##Db!X7oj+SgTtY z-H(bw-LqS8Fv<;l&u@w3?xwm~idS;0_0+q3gu@Jbo4JahBbD<6Avu0zS6_UBkVq0u z)JDn7dv7ij<+o?0Y|4o^?W^^k9mKr z9f2RkqG=3tkLR4F*-SsUm@Ft3-7VkN*VxXoJLtLHb#Nn9e%U(pvDu)@%CWM*k>~8a zow?V=LWHcww{Vy#pGpMA$66149CikD%Bs0TdE}w z&^R0=EV-5QgBdbdY|r*0qM=n|U~1DjU{HYhWF2`>Wv}AwfdilKG6krIr-p?T?7=Ph zR6M#yG)C;9Ekjpio%wyLeO-MqznZ-^!K2VE40g6)PP|eR{aJ%$&RWmuB~_PX{|V17 z3-V$n!2c!HVQW8aOFR?Vq*nHEw>?e0tlRE?hWr1LF@Ci`GBZEt@c0uxr1(kwyBw9t z$~$CtCIAQPX)^;6?uM^rI~l+Jf@4p$#?0332J#D_A?arQJWwD^7mbytY5L|IKr?lk zSMq%Ax|P_tKKPQ&Y{`ADelZ=2b$3Vc#TTr5C7zAFA((#0ATiBuJ`-`K5RWz}mGXVC zPlL0g*vuT0wR)*nmScoW+G!If;jR0^pan5NlP-f7CDTD9!!fmx-(<3;CN%NiTUMT0 z4w5`Wlw$N2sM$({pB}_rUqFu*qX0i9Pv(`AGjc-b&QL zleQ1FoEdU}$v?+V8Jg|fF{TT9iWMB#LBA-$wbA6gwtl>(7DMl!!Eo=vc>?8&!GXUj zGWhMADVzGh(`<413b~0vCmb~W?L#igefWC?V2%JopgjKVZ$(LTci)W(>Zi7t2rtuU z6B;!;rg5eUYSxt~4$_8hnMXDe+}e0RS>6_KjN(29`n%=8!nEaOKxwmA;qo2L1Z8WN z(oGM#o;KcFoYmRb^mxx@cfywcQMBF)`e%Z-YQ&F-T%-M^5m`pQOdaT@`K^QM+sHe~ z54pp*k9f9--@Q^W-BWN!lSui>ab`VddHxDU7}7}P{Q4u^_EYX`-y_Ja<5km>u?b5b$>%lZT&DwT!Hc$r13RGc-5@7M|T*p2;~O|Q z&EVegQ(s#hLs;9OSl%@9I;8E%-z?w_fH?FNh5^1ebZQ`hQGnUXF0{chOlgWv5C2(L;TDv5xD*#b1PtBpZzF&4t$y6* zu2)2wJw;cO!J~Yv<~F>zB!l@g`<9oV)x+Rxo4iRJ`qeKT{p{RsMYwAFz~p3jJy31x z?|kn2Y}fu{qx3OD#M>Kj^ehjdyk33VI#rEvQ4*vGa8FUkqgesh&0lC}d2IQ(v9LFM z;PI34990zGHrBTXpYu{Ds|O`$!viq0_EW-r%JOC~#Se4g8oHbryfO1ruZkk|gN*PO zBEQUZIlmp&6ePmbb8R2G`7pg>?~Hwk{aB#?543K zf2Y^7Z!|7$c5u~_Eniaao=aBU%$n3vOmgm8xw$%5Be&f$PaWAIH&C6DJ+58u9n8X9 zgLA*AgK_b*8hwglw@0g&A%+T@L5CDS;S)EwD0@9kwyOwi^1_vX8@uvDXu3 z15{}M$X5ANr2-Jy%Zp=C*gOUYn8*rji*ki38#waBmr|PUag`SFWE!j$kM-Y!#={-( z3sbkbf3%=lYsbOQ?}T+9T($iupn$d?y|S&SM75!K>wm~kmnGg9jD+nd>$D;oO7Nq# zm8gKmo`=+v-h7nL<1N_|>R3@UknX$5zc^W9Ue}oWumu49y%beR2KNjZDVU?dJvkh2 zg(|++NgtMz+=Q-7mxnL4E{n0Iqx)EXO)j6^Nqf=OW0D{es%N-01LObL?{!XBb02{F zZrWh&8oPbnYqnXC$B25XUN_m`jd_#!rE-z3x+rO=S1&aEcbB$23$T))aBQ-oKHqMd zBze6eWdQ;ey$vXyiOYPFV!=nQO8%RaJiyqlLC0LMXw=*E2e-Yw!bYQ1fvQa;0zKELx$fEt}4pdX{x-+_LGtGBWDY9=xf zBYAjzXFT9&YMy&)8>oOG)lrJPY|5dXB8xR2RVWr$90e57F)pk+?qS4ULbxSMk}>0$ zsE;y^sMr$Rzm_&OP{SNu*;ZJuVmR;0XGKq%@p?t#NR;U(zP0BY{hVg27J8mtcIxO2 z-RXvX`QWrYRpk|7Z#UE34bCQxjyb@LB}*E&caBeR)+PW88camU%fajiUjVnvmRNWO zJh^09&W`!mAyU4YL=MbuWmqf}bYw6{qF<8Ddo zYPGkmHI&ZzDjCP$ur=+9{?=GY ztqMY#OsnrlaF{d^zrBflWQHc!)B=){z4i0~Jq_j1rahlyKP?p(ooBlHNifIP+qad9 zK|niAkp@+JgM(weiPZJmP!)(cLrGmzS@qx(vSZF+3f6G##MjT%8)#z>Zi`nb?M zg!AxD_+#`r&_h(WHzr2KEfASz&;QLz47V>zrv7a9-DJ7Fam(rWrXTkW$CcCZcDuw} zx3Y4VCtQL0Es%#uIR|d}ZWGRpWo}Np(=cwkx$=PhAL5_dV@cLmn@^6Izmf5LSxh?Q z+Q^%7h9(^vc0I1SO~D@o5p)i6g@eXzdOUA_oupTHZ6O{gcL{yT9i znzUghLUgq^nhCXKbfJqvp;pf^$V9LGby@c(E-Yys8oAZ08$KAWp)AP;Ab0w9s-Hlf zAr?;zbxIt#)u~zqWC6bPuwTwp1mg7i2k+*%wYJ8#_zMnPwuom?Qlx>FRP=apQ^%bl z!0=IsvVc3af@q%f@kyn};MW30W5)4yRU4*p)lUdQjxyCfyuq))h~L4%_ic|!30OMS zZRT%wGtGuOVtvt<&p{euML@tJ77rhm^*nAQkX06H+*=O&QN_INnbTm3`UUcDmT*Kc z!80&0D0;@jM=fi99FCP*2~w) zE5NP^Q_ z)_F3p>{ky=nI1{^zzVGX`H^{P?TJWC#5d;!Aa1`-5N_-y8TQe$jdqg^TD<$&L1M#; z;PzKT1#7dAC(LB|2S8@yu942FPMg;K+Y1F9_ejX|Dtd_v;zFqRttZ{J*4K|!x98kv z%o6z44I_Qc#e&tkd+D{eXl7DuwJ&)^aX6bcwrg-mgq5fzcu{H0ebF}cqnC5zRiX;PL z*z-=t^aQcDyX)EFj3iGxNLcIG>b@8+2l*JlY<)1^KCzmu9<5Vu&v;XunFr;0Q_!OH z^qC87mU*=47mY}mU}ri1qlW5ZG;&79K+HekfrueHXZY0c*en033P;M;k+1d)O4s2~MS)KJfy#_&E8Xg_a^T&yu%@Hu8@<05eI z@VNi^8=7van=2OEvz!`dV6G)cdBNrQ`y1idNL_19LRKdDkhpD~@4-nedASW(W^r%m z#gqPg8~u8s!Mz(q;u4OBHuc@SPk2aoc%5I9({usJ)S0uhdI!g+jrf{GdNqcMs^vPb zCUqJixrm(%QX@;!`IYWwukbk?NauQ<$C9$w9uu(it!n8 z*!{ojK-t)L(eJ(l7`L=Q{O8od*}TKj1hAYaDq=*`;RQ|Uw6^`o z*aO1xwr)#&#;+PpBQ09*$egj$=na`K2OYHBPaf(gYNrb(bi3E42^~I;|BpbD2KX$FzQ4 zedNv%JF3aqn{V`}rWT`}-0Qc(62nt9O4r}{qwk;5G@sKnor+KWFPwp%2I`AVi%KARu(y!IsHt);{>Q&HcUPyhj=#qUTw426sMtj$L zUWw^V#{k}em5VA`|6zp>=4oO(K9-g-P2fWu`_@Bf7T1vQ2F>N-%Aj=?ldp)yzB^P& z37jvQOsj@v1I}yeaw%F$n2g%woyF}Fqw$m@e($ON2TPrz;S3F1tL&lMcQ|hy_I*}9 z-7*78CuZ6ZpXTXl`qtLp0*0xT8IHArhg=&B%dZ8x-mche9N-`>S$bix zW_{=;{kZY02w}epzlZQ-q#pIS8-?t>Ix0CD-x~-ar`y}MP`~j z8+$M!T-wOZyN<=%J5ZbEG6T3&x}`K>G=kBiY@uK6Y@jQ*df3NLF>&7uNquKx*eqgS zr;MGAgs`A+{Kb)_wMdWhH?aDeCEHy@jxZ~R(&EKEZTS+Lc$R}f(m1hUv|Gj@8T@D!s&{x zDJ-iQ&C%fv5FgF(%$ds-A09a@TUF1vVL_R~l9Rf#;g{nQvC+mVVqe9r7@wJMb{Kxp zvVAw^T^UqmVg~u~X&D<+a#OO12ZzRvX4PC!-Mm~%u)Hr%FZQ6)IKF<617~KBapVXa zCS79msMYM0mfV{NkPk^;oQ%;8sla)kLoZTVtUZznwJ|8G`bNT5(^~tuu;nIwQUu*! z1t~d;=j-U@u3BraiCR9MFC1GxX(0J`A#mWqZBNu;>{vFx5VpIy?D08SR^UFbHBDmI z9C(g=W#}cTB!d9f@QzqMfAzt!M?x;!JSLGe_gL!AbDzLC8`CDvyt7X^E!!V2u+q?( zs@x=3^bF)9`#c1|t)(~p3DIq=xAI0QoZ0?cDGO&dM)Y2stO5QlLhdWK?bdBr^)1*- zf}swU$AUyhT@5a;0KO^fP{|uR;?2*16i>f11bODG4{Wfd=5wQpVvj_EkeaCxfs@0_Hk#*`A zw7iC>tRC&8!sphf$Ev%gq^B}%tqntKrL{8H`8N#!NAe*1bEG8UKsh6EO81ILxXXPJ z>?hAVQiAaj+oF=3%}F@pnN8l=rOA4n5c;jS+I@x9lSAGqRWDAdX{tZw-K@OhbeZ$G zONZ_lno;REtB_@9_)@UU-IiA?jzjL>wW~=VIyQE^r9_%A=Iq1QAfg_Xs~ZIy!LIrR zDC?ASfd-YuC~;sHwKHLjD_XcV3Oi$sRm;cBK3Y+Ms)1|;FE9BGj8y{q#Z`%;r`-Zv zdphG5j+Z~a9o)P2dxLh=b1aY2zUZo!N)Ea^Gzms+uAX)`O#izbye>V-OoC_B3VQg7 zgj0j@E?R>TM>8z;7Y)qc=>AgTaiCGFr*`RcEF-OC9+TnFpCr9}*QNvta?dqID(``G z$T1bS|A^KTr7V4EI7ly%DFE|5>*Bl7f*y}8l}6tQ3;(&-$N$;tO~MlxG)H5dc>dl4 z1SyaKS|8qVe&!1EOns$K3h*QUAAU5pz;>sxi$h?jgNQAL-zhi;TRaYwe1>PX)o+z; z&>>2l9bUlq;*mIufWYr{+5ZH(QvUj>5Gm&ADsN~*V{H1| zZErj3T1NX)uf!zz1o@cE2?OhZyNt35161jJcZ^2@haiIqFs(ex*gH~G-?6c{v2YrH zP=KkL23%(NmUXblN!2Uw|K*LC4kPeDw1=3;Qu}H`gyHcFuR~umj~+a;2M_oNN!S}4 z9HWM_AbCx5+TX%U6PP+=bRS(I))ULs2;DHl^z63u-(3AX?-g7K;TA>Ck zk$C&uM&06^n%b(vbDcoHxm*`I#F3EB+H{JCoYmc+kLvk@2KTQsn%>cP60%KS zw%0r5PIL@Gu#}DYyfi3EYqy99pkw-9e$h3anL}wKpKzf=6;14!cEX%CJ_Bbq3bgdd zh(Tb2N_NjzYUyUwXbnyH+AO~=kOH-$^a0MnzR053F{miBU^yg#E4<&tI|zIH@tT?C z@8AB&GcrHVB`Yht0_lj!BmWgnB#g4_tZ1Q8%|G%}p^a+PHCTR1^cns?k}dgHh}0C9 zK8-|@K^Vuow&)|| z!cK0a&+FjJFvyNEj`-Js^{b-{Kv&>37M{W%B!**n0fxg!@4 zdIOL8aG)9%eThT(7b~kfJx33YlU=<3hsgiRNg$R7sE)^R?Cu4t-%fSY{Tl73(>4RZ zHt0Kez?1%oHu(L?N6~Ne@cfn}9o?kJ!NXNuge?Kda`r|)@I|5pZ}In?4b$CB>B0VU z*Z=-F@IXmtg46Bfdb5q_f~;5!AT})?*1No!SrkdYGm;2^XT-&(|JJr7+>f3-Ars)` z2(N$CsT$+^jjC3mP|*T6z|DK}6FF+ywkPIvKv?`=WAiWJ7^-pT{MM4h&er=F-u7>p z^=}A-PQ_5wkdfncJU;xQv`U{V@=@3!Mly8DmAj zg?}TW4wl#0I3s-8dbGd+0s?L_Cw5-{fh7|VufHep``_EWPY6{_MErJ~z)}gP?cheW z+bECz$V?Fi$hL!gCgGaG|7F5JJ`ztaqgH?immVA{ILo#b4SN*t7mYq5vUgH2{CX?k zmvq2GFx74sxk|5Tqy_cNgd%V-VPfjaR@*u28l{T~f8GN45GLHC%lFhAhnXGX<%Wy)?T*c!M2wy1cDbCb6)-c^|l)0PO@apRK!5B~lk^7JZu z!!Z5Y0(QfV0sr5}muI_H&qHBvB%!vMBc#)4UMc4tB6EJ4zZ6;rnsD^C@|3A7{Va|D z_i>U8vN&|(V`~Pjru?fDnz#FoODqe2PCCy1W(%`~HyVowO|=t_jrCqpN>T+&Xd$C8 z1d5oc08XXW?wp>@OnNlaXNi{CrB?Yl@zLt^5>fW5Dbm5F5KLOxR84XIfp zu9*2BEI5l}r&st|)u$gkLppev32N4{EtAH3e*~;o2yk;oRZsFeUr(K}5aoj^(o{um zhTO#`cjTy9)ofsU&3dQjde4I-s9964)>hTcpR?XU^g}l|P5zpE5_yimTX1?o;d{!V znp|{alF5A)?oMzmKe@@g{&({cuRuV?nd*Xfd7k zT=ghyN~JfFZk|cA^Cs=zU;PUi01p9t$&lK*re?aCm3s;S_Pp7y+j%qUoH$Q}rHKN( z?*7_}C_lOK7|4GDmgJ z2>y7#Q%|2@zM7-{HA{`}IxZgytHPb?8I$M|LvVhRr|Ve;#bQ!BIX()Y$0 z=S7M}RG201n)Z~oFub!6wnhztF(Mm%^Gnl=l7Eh`{<`|A_5DXee4Lnw`Wps=rzvDp zn<;)A{59^1rg{(uHD@uMmlv>!vC+a?|G@G~93)fA?!?)Ho4%$%e$vw<1c^o7UfZzA zWRlgDt@#8o&}Z=yj;-Iayl}2_J@^h^w-=t7U^(P681fn_=8Tz%?|#x;EMMGDD62P; zR3YkF{K6Y^eSwTnTa<@YFJZW3cerw88?Mvj@1qX2Ec>&+Q$OAj_@n@-H8 zqSre#^m_pK;yN=T#Gc&%z#L~}m?^DG+MiM>l0iaM=}Sy)U`JND4iPl0TH!~*01$?J<`$Mn{u~G{y*h@T*wvu)%y1nTCjr#l$GZfzv6BcYuXi#<>ZfYsg1N+3g zXt?WrWX_R4mtSVK7;qZ(&E0; zd54~tHi8dAk<-lK7V%A7sNwK6%XxPZb%5wVf6Dd8L06L0!HlJ1Q*C|g=TXDR&grb( z@p>IivU*$xk3%+VHy_{~+rLYgBM@w~yEmmH8sT;#ZM_)izq?(uN?72>+VeZGyoUYt zkJ80kj1ey4e57)LcLLXFM1>9$^{PaCd^hg?;s}IENQ06Yx4ZRyc-1B|M0=@k7=p1& zW!7)`x9W-1zH+TGs4HkPCx6>p*G_db(LbEwWjpsscsB`CgK@$~GpJ|p!j~q4?g9HF$huvu_Pg)AAyu^^Km}gh;{i#AyEtJDxc0fj%O#S^;tC2=`;?Cb}%=Qeq_O_q? z(rDCBRL*Soc7LGi_~}aPu|3=cY+RaJqJ5z~;_mRMduBL+GYe>nsZm5cPlfviUoQf> zJ3VVx4uCqaiDb*y2ax(^+^$J;9Ftc5Gh6VUKy74gIG9@R5Z3PbBg%KRsrEHyv7z)C ztQFlhvsi{OrWcu~s~IZ6y_e=vlk83uUSqbW4^D;6))0fMVdbVhdX}qc8r4B)Qe;@B zgj+|E{TxC_^F@7a0Qj0;@QrEvHJsAfH6yFTMYyK4VksyRdgG*tW%3ExF74oe2c0!R zPJJRrDG{=-=|hazl*aO*PJJ*yVLDqEEe4CtrxkgYzr|`X5Db7Y+CUB@?@r|ircZSB}+qK;bP!RR?H;Olh?1u3Nr&D z$1JxpA{(Qr(rh&7bV{-9x#dq=pu_c{ivI>M9d}} z>FN4;TWRT}@E{xQG)4~Ht$wv{SXG}Mz?X_gTwJQmyXQWP804M0UXqggC={YYzK#;S zIMyE_yz5dNzJ}iVIzL#*)F3cShn<5%{WwPGh>f6h5|lvK^gv~yo>R=%ABYirWlOedaQj_<8+Yc| z4*2@6D%wx0JVt|%y^umduV`~|TZqoJ2+jG!quhv^QURv0kIxEm<*4a|Q1#1rK+^?W85$epSJY7DPe(j^Ls>dGSy_EobMw6Xu zz)is=VEdcf5dTa|(}(+wD+$%+<0#!!r4nw#%pTACEGVF(jDLREhdrfd-@nS?QqKe? zlU7iXT!8$PmnjtjBr{7+mp8?p!jl^*Xjq7`p6i*+=HnM=C$2_3e?%MV^OgIL^Ue`C z9B{s0W!gjR_3KCI8qmi2qQrK^B!|2LcgIMARj#duU+$^VYBjsU#aW{0+6MQn++0xt z+uzYSiRiVGC!#f?p6m@a+aqhyN`ltU*V_WKvowas3%DqU*$GpqgeskM$^g6wKP_Lw z_XzuioJi$s>rd*m?mti5DBh8sh!t?xsBCVq(Dp@rx@X;$(- z`gZ8tJwv#vj+|w#t}RBoZZ98teJs?-0aP}4{J5ns=RcMGzP;l*U^_2#Yv8`jWL zC|0%~{``@68LzSFZwm#yk2FrH4H5(S9?!FLC(dDqnESE&Q`r6~G}xinoO+}G#RgBG zDkC@bmCe43t!cgMD@jil;?9~X((N5BZGw2J7iQp-ZId7JIXVssc4dRl-;!ra0 z&sp#hhFOKN=}xlx(1%jgHiJjQL%b=#Md5R(mI6 z-}c$bk9F#|Qg$}PDF#=(Kr@0@9vRSVC$m1BDax$ULN_-Mj+75*_{NkuBbm0gZ=7_3 zCAjEY?R(U`IN5I6vKCFnrOPb1;A#4mhqK0APBc4@^KrT(oAO^QibVc(VEb$m6V-%y z|6XoWdh=xp|F{^%*hCw5=l@4DKnDC#`{0LFib zGee3Ly1^k(6H*U1FrYNdxql;6q0gXe3$V%`W&ELtrc0|V`=3=N(F;tMOIf8Frl+h& zN?uyk{ZyCLv6&hLYCSh}f>j_9wLmW*!Y*D*fw)}?D(y%=q)SMc1W)~Jh2Iaa(jl$~ zAizqfv=*)d^M^{6Q_udzC4gx2IL*%M^hE4<_rq=-)0SRSdR_dEpGmvxA#vW+I~V>d z#mCD{0@kq8tUc_p*Jc$=Yt~k{e=`6`nDt%aEEh+tkax92Kbx1@&ZQQA)`ZfjxJC*p z5Hj7WEcA1Jxsx^3ZQkMt`fk9ww%LdB`pBw$9}rxF@w1kmEe}bt!9VwkB6PKRA$%lO zsnT{-Vo;uaAa`)wYdW!Bsp9bp=-jlZ#KpnF>O*q|%EG=@ zRTkstR~4*&^u_Ec=f7~hg*<~+W|5xH^F(-RN2VWGmH1s;hX3nfYp)g6Tz$xwjyir9 zi=U-JH?R1r311t&ZY3I-;N7;ojpN)G;#D=yc6Xi`Q}Z1`?3X3<(o)0kVmDeGg)8U~ zH{w`ZRb!m~kXO6Aa7Pfj;`8kR^Vy`Pl0s z-m=dGJRJMS7qG&fZN}v4h+ECma3!}mS$`UbVUwG3omj?}y;HPmJS0*8>nOskOibyuJ2fE+MtKjtV|7A)&oJO2s%1EH+q>Q|{P zz$b?O$V!Q!cAy9Dm{=`2|`w$(k@k-UmG%7U3+Lihe(mM_KBuS0!DC~jz8}e77;t}a_54Rq zGmcdYY$&w0|2&jvKFHeEXSBHaqimm+V6J{*@d&JGj{Z(F7&O}Q>R2qqP};v-4#rJi zVvyCCJLSc^q4|N?6*dAFNlYXfTdS5xU%ZyB45SGGe>o5G#xK(A7m#V2D_1Y4YRnf= zJa;yYuq(-ptKlHwB~4Q}3_U5wVEbk3^X#gaHnn)a=y2dxY{?RlE;Ik6>|B>N9B?)rG8=7Egw8I zu)SuvDCr2MS=kZ$j)NxYR8Ti+5IB`REI{v)wb$yJi14lYtVssblsiPCdf~O+U@y<- zsD*RJvK^G_9CuBN5Xf$k>x4|d#(+J5P8w3%#Mz+WMmSAf6Y{3WH5sK7iT3Dt#W{0@ zDKQ*kXG}ZIdD;H}93;i#fU;pxx`_?Cw_`;f4#F!3oIFVic-##D{=q5@@wrKKu&EHPQ7oY;q^;P#z*_C!v%MZ#;l=q`wjhvH z8k!g?O=#eo+#AdA_F9cMeIiD)B`1CV5(S$Bu;dYU`dptd#&}R<_CNe5)fBTcM`gf! zqIDt<{^vd2u-#kSo}C{rXsv$PxYgKpr?KgNzt_+n=A4y>NbEy$?E8{ughDz>{VC0ul&+2DpzsrlZrNmj%;ICW1X$S9~v zSNNGLE3gJd%|LgF1|kWwszm*v^rzQ`IvibTmzqab8?tlx@ubI zj2Af;>cC+G<(!m0*K;asBAaoQMkMS}+ ztz?#fHVL`%4C-;`So=^DIxIXHHjdAdL2e6M>0d+^4?yvB!5$~GUS24O zPsl2V1F^!9_I~J45n>c{Ub`ZfOAyXJAD$X-ZvY;5vV=-enhuG z!RL?@Mt|yEUp@VFr`_+uOR7YdZA+ObLt2zjhO3kKX1w&&-V2^oy&YnK`Lr3D}(Q?MP ziJ!3t8YSYj7+->A$ztalaU(U+>L-S}?Bq#}=*$Eg$k#QJ7xwgJ!{VEQpgUei*}`a&|G^B z0dx1X;xw-I(j=SUimNlH40%ST;GnC?LAa#=CZF1=`m}E@D<8j*%`xj#`pQ#fQTriy zi?8-Dq`Y{8#^7+gjaJD`bP-)iXPA35`E@x=q*>2Cy9PX7%2JpM-!`w?6X9Sc^-kME zXKrT7WlCAU%$RfBAzRGb8o`2h070BhN{2W!(@k{Y6{tSiUXZ-~`nJf=t8A2-R@Mt! zqT?b7k6$QpSPxOY!_)@la4T1&s)(BduVn@!mZA;e8kpKPs2qK^F-)A2v4j2Pz!hhP z?k`#7vm`I`T`LJR$7}pvx*ulGO6JEMGU9m7f65-`Cp@dhJ^90oEXI@ii{5(#g|GI^ zwA^9&9C=Np##NK2{bjm8;jXE*2S0tdjf8)2SJf)){;g!JeDo-#{AJ03@)S2u50giW zOKc0`l*MFa)O}FbMk2+&Rr8G28s8S?(A@Tr(e6?>ZDNzsM zHhAULY3Xnm^38i=ovYjr101`_2X`$S=!2&6|CJ+_J4`#*w6$T`-#NA3M|IdJ>((}T z_%!q%tUm5t2YOvDqm!~V6s{I`)Wva^_i>p^4n{(3=1>-`aA@Z9TKJD2q{KclGsDKh zLZ0ZI?8;jDrb7tpG9~9jweg*aHFT|{ImFDz)G(i7>ZRC7)T6i z`AP2_>EdZ|WssS*jaRDfaT(}w0-&K0afVIolJoF=O^tmZ5+Kz>uK693W|qY#aE>nd zw5W~Si&n9R3-UlH2=nJ>;cl>R>pJAOZK<^SnIJUrem#2KvqBlAuL^PMpmomvJJapf z>iABGU+VKib)U2>rPlLxb9{3XL@x5{c>G?$_UgzQq2)(1=?03L@_IkW%N+Xc4kPvJ zH8bYDz6(QTc&c^ck%Tsi>0@8*lf6WI>HHp`O}e2PZm3_jJ!`vg_ldz!47i-#V{z_xj+u^|Kh2r3`6l)Zn~yRr~x|iJ_EXgnrW(tW7VeTl=v{ z1tF%Zkje^&bD&n>yoryY+KDi~Y|zBWr$K!7S#^9+jZJGOhO^HdpASq;_(8M@H%+Zd zQw^H&;8!sT4|KOF?~#_>oqnp!(vuxt@VxiHWffqeexw|#W}X+=0jA)Odt+==B!BZn z)6F$D;Wn^#X65a$yMyzgp|0qhJE*vyApaGz+f}q0;sAHv#@x7Z*tRZ&fm~JTF(Jm_ z`&K+B>2{lAHfxx?dWYV#j!ns464sw8Ouhsl**MEc!RMc$23pZ{{r z4IB2BvbcBx@#;f2Zq z(RTgos$1}H*xNY402Wn9pGu9X)Sf9E-xxKVO@bi&_2{>_H5V!+coxse4n;dWd_%5c zlrZw2P_i+VR{yVTEMWf02DkkWRjWt*QFV2Bm!%QiP6zbz)jtKx&3Lw$N|KDnXRRQq_R?#WpbFm ztV7;ykt?`*mH++2Fo{zt5Lvh=_Ja@KqIRvptrkZI6;>^V@|hjKEJ%n3iZkSLxNf||ZF8Tp&4`X4}}YS6gIZ?o^bw`)z614f$~owL!k zv(Fy>-!!IKgk-P)6>0z&+L4^j+SB^8_>|KNIc9f!a6&VxsR`HtV*)G$0)gtdeoLs< zX2E-_EG9AWF25|PlFEJ=LG!<_Xy5X>UNRoyTVSyzq31kLxf8!=|mv}_^s*!`JWq4^=?a_50;b-ah|Utb`KkOOyv>6w zq1w{oDRsuOp&t7`B_NU~c-taloWnaJEIttekv=c03HhU-k7l33X%yrmx4N*f?^85v zWcZu{rnlO1n644ohFLvB1_b0a%g-F*6{w@3PRm!+D!MPJu~6h(_yj3dhFz<%2R`%Q zPpdn3wDgp0)w>X?^_Kz{+3E4>0rmIp>#v=Cp<5@gV0C0-{qb>J!o8bM}v!N zFlKS|(FvO3b|f8D7!ID2R90>-lO0FNtSGri5L~IDU!UoccO{?V;1!U6AO6C0y8^^l zc7Y6*Mzx+vqn;^xO8RXph0^%sNYnWrzfWuTjzCxE*F5B%Z4Zlzp8Q`fy@aIXbpz2m z{$7TiKQ}xwDXHlpFx7Y#df)_gEZAc!cOTOS!tu<^&EJ7D*lO*+@g?~$)$&#yxz^hm zzGBxZFDUM*P@&+x`q3w9fAUQqyME-vDqq&mnw;@O-s%sqkvw~K<~S`}=XNOiqXuVU zBZFUYNYplhGeuT(;2T7$uGpUrgtnK|de4@fSKO|^~t!s`Epe82x2(>BJm-Kb=5 zzLjB-;|#8nE0Kx$a>|v zu)|xej`Bk~<_Oi{nBuXZU5ey?{0XNgfJyr}{A#I)PYS$}OUWB}4e?12l7`+;XIdmS)bJ^u z?`+dtYKi0Q5TMPmSW%>G%5KsZg!=xn(5OA{ne?Vkl?CaGmb{ss!2YW;$p5ZP{z+hZ zXzFUd-yv!}yWsiowhI6hp7o2VNs^DnYNEc{|D6S(TZskKCNm^u6l}ilN4FTQ1byYa zqRq$KjvS8>=rtxLr-q+#4nI?B_D!wmk^8o3K``=f}_5nGw@aRHXCP9uzSyp6fvXZB9$b8 z$t0JThQ8_bV92~6k~Frs#quOi7>LI%>087s83upVq?cyPhc#tQJpo19(?&P*XPrAB zL=6bk1nUlaA%-H`2h<7B_qEck?oMuep(Y0ZFW->dp{P(Bg&heB@dT^9RZHPOW#rR& zLC}`*>S>PmIT9Ej#bWi*DS;Xx8pM3A{Ry)9RZddYO4cT1N|6n>lZTc)^pzZLi%MTf zsKK@R`uVvw-@F|P=k;Xnil+ODrS~e@7vfglj#H|(L$^^lFS89n(?1X5b)CJd6S5Xs zT9hp`D9uF}ry~DX5d$yS@fPclYaaFCl2`Pe!^4^eCv{LeEOe}KLxUMsFA{*N>MFbj zacSkQ#&;{GW*;dRh~)DUq+-s+ND8ZuhX)tBI6dkph_%1L?u&6^U*%=01q zoB81U0L&nMkhK=qpS)*O9+tgB^Gg$v=MI#G6ug1^2a(2G7w6kFjar>>pIg9=ij}yW zg@7E5(u=-v+!}1RGP|)T@3qqpN$G^5e+q>E)kX~Q*l1dZXsugmPA6Zof8eY5nQA1Q zCfpcnubpPnbQ1(jFmt!0k*8r8&l>Qg>V598S>^Elb8!kVcWc)F>Ztgaz8 z%BX#Xp#8_*ZzR7HrrB-AOf85oL97|(=SD?;9NxCB+UF+9aWF{Z+L%xkxuQ_iTP*u8 zHF`e^k|hr95-n2ZZJwqLd37@7LtdgC2Y*2TO~{4VXe`e1q-d~H^XDxz)Mx_pcfWcJ(EPt_ucn$+R-)qG zGWipGOpaRQL2FRBg?T?$cvZS;N?k7Nnh6eej3$FlFq{)#r9iGL|5KZF?>FWocTu+b z*vxNURcyZx#>U}ze!S7o6e+e0x_FT)@Z9M!i{u%@N3%4^47t@-MRDc!da#Ut0pp=c z!`inIYq~Rj;4dmQF>9i1x3gx^!87`SAbx1zjq}+FM`!&Qx>BpXo7(>m)*u zuNPZJn2;lqCsi{OO@hf({iO2z*Mi{I3yynbDHCcKT%PD%dB{I9Wqn22Enzp?_6bm7ITj^%doyd#IUgEPj%&g zG<&bGLCsxFNuHhl2rt*xtQ+_gd?R0FPb1 z(6FOz!AzO(20ZPChL?(5B=~QJ^&0jv`=fRtMSmn`%aJ|P(;F4!>V+;a zm2lwY^6tlWYFJivz!uNlkD>K#EQg1O#L_^zf6jM*5TDhlga}p!-G--G0d<@8Jwd1J z=|kYZ#fpIM6sQ`=f6{Z3@&T~H-qDHFp=(t4R4q~GsJ!M&-URCgT=U-HQcGE{VUY*NokuNAqgBc{qV zc`d8av$HMiWHMRma$~0@PT9l>9z`4o(cY(8I{523w;sEEWqXP)h)UI@XbHCp`x+tH zea)Hu21nlCPFd%vzmPmrQ&ekBT_<{3V=U?Ag}cwB*%BiXKYE@L7=56Gi%^=9j+PX42mHVEag8 zow>XHNyv;dPj@xqKu9j#uw0k{Ad`wM9`Uy0jH+nHwqqxZyagVRA+`I}=D5DOT~2zJ zNAKq6yoNiHeht`yt6#{M%eF+_-jo=+osyXO9FiV3H=QrHTtFr&dxz)h9ff;G;Me~V z4b>{)8oEXW5>h{EWNEj33pLrg1)F~*=YX4FW&BDnwq4wGl{xBCkfteQdpe-XI%WV_ zV!>Mxu-W=y6C7B&p)c$t1%Y*a6z4X*8#vm|v?4il?=@h+4(4*=lX_*CTW=gSxGl7o z3i`%ABbWEBxv+AOTHM|Im5EjZ+v|Gi0G5g|`H#Jo^!o%hsPP56`?tu3(TLCbxfQXh zh^gpw`(|sY6 z>FAEtl$z-oi6Z{=@BG;z>dK?I9VCylB3cMSM~Ydf4ipZ262(8d6>c7l*H(knwuMPJ zd`hxQ;UKjO#Bb|jzGd|idUfyuJ*E~*jPdJZEfpO#@P|y2w%`Wsp{0gAuhfAhe{Tz< zOtP4zdLzOfCqS5J=O{MZHmnc&mZoQ5207Su1-CC z930a8Ec_$;!DHf+Ai^3!Wu`eb=-E>lxGTCOk4*(1m3FLJj@;Kx!knHVYGh!1h`e`f zDsbtR{^zn&y@^sSeguj?OQZJ$NbH6dRG5mZZR%MYs4IRU-%Vj`I4i7r5#|Y0s-^z5 zEo&FXE4PbZ;pLS$Wu2Gcpfw{LbuaR@zK9Q6$t0Kgm(hXepDntdW}3G%-$K&U$8oiy zEoRe-FohPh$zvK!eU`U_=9JEu4MeAb-M3tFE09bQyB17Z6(-zl(_s)WnFF4k6?+xr%ymcbJ zpfYulL0Nw8jbGNHXm?Dl1zQXvX7l8%pTk47lsCu}3uLV}$Sk4!OZrcy2DfDRPvt(z zw;vh)&Dg+|PO%lNIX|5Y$Y{>aGkn4ka*wDihD(CM7jb3t4h6}Z*UwJu<&K{J80ZVG zPwn}t7XJW+6={+fgFW_x%ZeAA@bpLTd32-hqsb$iKVl34B zXP50iS|EJ6jur&QyQR9Nr(P?3iJn9UV?eC4s)pjD$!8r6Z1EZbg=#MdtOq!B_XD5( zRr7y?L&8^~PrY2nF{?TzX(}y8B=t}UnCS$v2U!_0a<+1g>ApvBF42FdB#g`O6A`&h zaYlX>OWZ;{DrKelY20_$+?kIPUG|6eA?g&%7cYczT4+>~0fB|wB1lhbd}|j-xiMHG^KA-56UtF>e(WAf7?~!K#0Yz z|7E_;@n>xo#us5_IjpI(85u^V*EV$+&q8Busoft7kJZ$JIYG(*Pcp*TxU*+O?r3~v z-Wg^dU0)b^U1ih4Pm6JcV;HwxU3$a=!gESV-m<%?#ooEJT!`WtxGp`3JsuGQxTWDV z0k=fibax%^HHYfq9{pUoVY%S_f3mWea+5MV*{ytR)G4Lr>E3=Ox zVZu>dcbA3Zw3mm^i3^M1)nbNLd76M&vgJ0H_Z~{EWq_lSq_;qsbKv}!oGQ+GP~zOK zrX^+^#Bg(^_IR+1Cdxg5K~tW%=)ut~+MalpeTxzBX!zLtCB_8JgPH78So z8*c`bUDnu@Uh*RjK5T1LPA(IkuA|=Ot+_V|3cdfKr>1yln6uvN2RKv zv~4|y39Plh5dd}fhmOA93n2ZwaH^`leUP>1aA!+cc{njl90K`z8D^xjoz(SWd1{H2 zASJpJxHgKWYv;|->Ky}|bbcnHP^8^U17jZ(MtS6Y30dYg?O6Qo{i4Qw%>uFJm!aF< zY|jw4H(Svh+>eLOuB1Ry>)#Tv`Ic9HG@4y<667=y;s6i{v?c{4Tzp}nqIuCb{FKyG z@#X;E1?7d&Th%c$`}HOeFlGp0H8`Uz69b`2RO9CCLqpwchMwbhbe(Bf>IE0E^Cb(s z@9bCJT{r3+S~3xj@aRoFqM06RCoJanmHN`JGM+uP9lq7t+cP0oGmG1n)H|jpYYV;K zSC^4l`*z6`d3PC6!jMv*Bn;&E%a!E>@cmwSq#w$8v; zo1xyePINna=bbwFo>o{F4|ptNIJm;3l1jYK6O&&Va2R5A7<4ie*HpYtD}9^7=iD@Q zUv3M75#Icm14!~(GuXwYW?{-%A*To4Jb;&BcSadJIxev}z(uzlo zT=jRuSAZafILQieKlOrq3HyBYdABm~}&9ita4MNpi3> z5HbQiea{Mi);o7LY$9#1_!!@Y?WP%yYU%n#6vnGDDQQFz38)>zS}GJcCKp9QMGo3m z;NmDC2l_}=<{20Ido8F}o|nZ_uSmpIiSESb9GC18T=81d>S9{C$0!C(NSTleu5IVm zf0r8dRUrGs0pNh!5@)x!T11)z$Uan;Is&E6FCe9W24^TYbdeFpvU>>Bd^Dm6n(#b+ zyKOFbWzX2P3tchzupGHY;a?iXKc={`2&Er|O|+~^vtl`V$PI9O({NaQBJ8ZFFT_(| zrr02AWiZBBODV#CbFBKTlX8{GwCWbEAg9Z^-uCoy@@;zNR)1%0XbM9z{Tn=<2zfEp zPPfYs4*59IyWq+&b8h=;)l-{FLP@Lwfep!xOZaUh4=f69t9WzgldxS7M}X!or_aTF zT4uAbm+gkg&_PSLT9TvzuY?M0+8g;}O%E-em3eAK0KQL+PF2MaOyw37(tqa)&PlVo zk1&NGk#*z>R5Q=xB@1d<7#X#ZGs`C6FcNTjc%4g(*s z@%Qv^d9bX)Sp0T=*i%`a7X1X}T;Zw-#!h9ay2;K9@4TYUT_*=5*aKjirI%J6)(Aak zW09P$8ehcsY3eIYz@eai;M+Kzh!3eGgBj%~e!NDdG{?W-HF(xZSVGRif-FdpE!|Dn zn@MxFDl=D=N@tdTaCo`gQvt6CuVcxSF}iT8$IIjm0rb+!R&%2x3~ZQq0YU`}SfiGA|r3bIap=OlV1e^f(OV-lcSxfWDR@5iZo zjL?UNoW&6w6lpf;18J_K`JB%Y3)P6IoAY19g@Pwb6ABp@xZumxx9&~e)Xl{dTik{p z`LK-3^@hPZ?TEj7?$tNCduP(&wxnP@7~YE-C~A0WuD)zwt%!nS+OB!WeO^L2ZPB+U+?x#SA68n)Sz?FsHhd>A9xdji@L@noS3)n z04-Vbn#F1|1n2{y`PE<`(Aqc+E7V1`r^!?*Vyi@XU1|+r zTI^pFJe}?oCMq$cN2#h`wXx+Y&82Hs&pizFZc`R^+%)#Z?2@=7 z%~i_0^3U_Z({=VZ08vdSTyUwe6D{p@yL!m3yp@X)>3b^BGea27=&}0h43V(XN-I}) z0pcV1Jznj6{8n-{|JRS~Z#SO}-8)l6d`@ zA*<>$#VJyAZQDf$Le)zFowVG{LB3Fu|4O3J(&q1MFaL%$LKr;-xe5Tt&#>M==Buql zVFZG>_vCxz2rA`8pA^3;0Y5Lbtv(_>?_674!uwel42=v7l*!5KOH0~?JRDbH=I86w zG0R#9-#;%6m;A@l1;xl4UO~pXkXOlm&BH^qoFtiuFwJ{8i<}^pF@7+GCe`6JQR(q( zq)YO6Cy32crj{?=}x z!h@Yv7kanbRLp1^rkyX}!arbcuN{=$HdT~S_}%H1S1X3mg`+9Gv1;^IQYTZIH(!zx$=fxDIXZrq3r8I{SA73rIoqqpkU1-$ zr2ZalQI!0n>j}a+#zHs0EBPMetga_PL>wwQ3SZ;3W+6K#K3Oio<%@D6H%GuI{58`+Giik>`lc*r!&jX(zj%+dzMj@-r3`OPpQ#PEwrGxc@ zD-m3`yUqBGS&~Lwe$ZUFs7ji*tMJwKZ$22a|NKb)t^u1Y-b zUAv6V&<=rX?fEJ*J-FH-tIZ^%e#_H>_jBzll8`r#_Nsu3KgjTBfv zag`kuwVERFJK@BkOmV&YU)FrhuFcPNb-is~eXALO-}VZMloFd% zD%hUUO>0j@1W!9ujFIuF&J)Cuo`Y;XzqqF?pOjZGeT=-TE58WLWi-(0+b z0=b=m<{HLFemjdjI0t|m;|3xsufn*Dw*#qZ(#f3}Vkiky+0)%5YKGclXHd7*0 z+>6Tj&6eJNRHIGL1Y>?61O(O*3Iq3)eAzsxYj`Uk6IVKPGK$d_k%~YfIXzz6t()O1 z$iBvFX<+c6BzzAolL~i@-OpCREsqv*BzjjVhA&0{G$6duqfINLLL8v(JsW&!KWxDw zTgZF7sZ%TP#8uJ`yl9^SdFauC^Kzz2Fc}F&DqF8>?Ojx5=JB$T{Ks^*-)z3lewglT zsVS;1KSh|H#*oj%NKEeBzw`cd9|5+217lyl0AJ`Qnupg+G1yw%gJVSDFpmDAq)*GR3^UssmH;8MkZ#vB=K%AbaOBT`HKrcirx;m{D z=Z(5+q_0uU2$41~`0Ym?KViAv6A%6$YO0}Q%Y)MtE@Zu}X?F{}x{en(TbiPJaZJf6 z+tKAWIbIo^aggS)<3@&C2#b{q#%Jbdm#}HNkdr)TjN4zbqqu;^pPdOXAP9%Sw!Y*u zz7z@0(o6|5VdN+uS7fk7SP{gT-rT-?#u`gi`N!fg73W_Emdf5@XN;eI?EQrc*6}Hd z2u6B`&C^IjH4cnR{yEGm#5c6$AE>yyiQY6-T_{pGx+9Kyp1p?$WB-&&<|i-$7Ov*jhbD`N1TA-Te+e%<&KLui(~^&VG=@t{BO`k$h|P;lO$ zNrWXWWim2JE4dQ<7Da$T7oGZ2)wMubq+&GttdmTR>e4K9GkJCPSZ3)(*Uc@{-VPbZy(pS4h{N$$I*5ghYfGU*Z6M)s1Q z@ofr(gTbK{#jLHQL?x+;CL6sg=PMQd^|OSTjM)OcHwR;-RE=QoU7Qm_)7j(#3eKi% zJ4Oa(uHtv$J=S+cQeDk;;oS9^jEd6c_X zCi=?`Xc^#{Qr5X5 z&&%;r+eN+Wli6C-nJ=<1O33@!kxI+;Xd3zx&iHDko8cE8o@M;DxFxoCQ9hYGntE{j zW&$=>#Mhq9%7x3fKkxg+^iWCQ4g88!cs|2YvAE)jZ z@@QJ;r+(*_7k-?mZU0EXz5F$E?YTXeAxAPfM{fIZ9B>W8XJSej%~tW7Ic8eXp;I#n zoN3lB4?PQ0&Jx7J*o%}-s5c7MvD(pd5n)w^mFfGOjUfoWcz)C%4WNMx8H~%)(hkqE z&Y~{p?YAcQDO`AX;>++Q6WCWXq-r-mos+p@M+fABoslIefP<~>%@~&6qY+M({A?>SLC*fVL}uPMZ$B2 z=yUE^wGVUut(!Tih{$qB+x{np;ha1S!#)e2o-^#XJ$QvDkPUs^VI-?AJ!|^LvlDO8 zr(wKi*<>T^tgMthwg|(cv=U@Xm7;@#C(3bb_Mz^bqb_vFT)qZ$4KF)EvNR9UtCLIv zL@-};N8Zz9>Aq8&K6!dlXsfQb46(F4dENm)_3YIZ{azA#AG|M~PCOmARlP}cCdM4A zOWDS;!0_v>Rza?K?L7Qu{?*5U7Re*M`JVH_^t7&BM|#BtTWl565+Wx^d>Y$goaT&< zK!w5$wew&7G!RE{&Yx7PC|61U7c(DR@k4*3+kIP+He{#OTi|;1R{sP}(4GA@3R9=n z#noOp;K78hCPnx=-fTfBgMaDX8=1X-?FDluS;N%;Thj)Qf6cf&6-wN=$rLsO0 z5HS|5O7*$UbIJQiC)Qa5*|}nemwszjiZG0qU2S!Rgs-p2V$#?2OFw7KZV^p#xO@nW zP5zCh9x|quHY(oKC(S`szeM{R4le53jYe$nf5iZ&Vx!4olzeF2 z%iVowcV|t%%8Sr?I-_uNV!xXRq~z47DSQ+(6qOPjWG6Caf~iy)6QnaHBdn%@S;;rTXh!wT*w1Dp-+?nb6X+E1xp;wJUYR?k`@h#$ zL&FUs^U~wO&AwjUhR*C|T~^`dtqZJVdVNJrh1%}xf~fP+_uok%X;$77h@M(xi0>Nw zIOVEoG#kluIg-gXXw1l^w+9sU(R!9Vj8c%;qgSq{d34IOp#Mr1mcq?e1(P6c;NqmCoEz&eaop5b}M6?wUC(DfZA79XvMn`|+E&D4!DY%cK}L zaOFJii{-3-VHH@>N`PI)_Mm^sBcZ7W1Zi7llN1)GFIu!OI0k+q;;+#lG*7o6Qomi7 zFVAicFF={fQ1CF+F0D^om~`fVQagT|Z0+q!#2}XH^VwQGZk#JdGW}`P)>(K>9QjTp zOnt7qMHYjTGQp}D=j-|dry8V5IJ!p-#R|L{3wYAoVp*jpR8P#3XVprK(|_y#X+!04 z-PRo36)D};wI^)(DgAz|x$hQ6xXcx&SZ~S-NCP4tm9yX5BY*&ueIl62e#TS&j_?;{ z3^^X!pRXrm&q}(isIw9zhqG1MdvOdVjexz&?g^!Ai*;~P-EGu}_7oVjmyM&3o#mBC zyVnnPk1!1uYX|r0+;J*jP@A_V$c@zJY3z35h_vXNZoOxXDGe&~*w)B~6iW+@lpnPS znF&W|6-z(Wv-p}GyEvI2C<9PfnUs7c<KkTn-)05p?qGrd9mAC zr#5%KeTAB}=Ia@ZQ4ump)9F8%5vh$B9rJB!XlOH5tqCQ)Jr^xlQZ6+y#Z$}^FLZ?h z4}hcucS}*%D!1@1OGmThkz<2)W^U0I#jNHH433Dw(ntqaQGD;=evJ>O`%k;He7-j> z4trAz{w6Ps^=jbAs=sGw7|Q8tpJ-QXxTuL@wOqaYD;TjPot&9Dt=ih#OUA)WVF+3e zQq14#eujak{%n{s=ahP0WAih#`?P*PF2g@LhzA)1ZK6-ihJ2J(hDjb5nivh3RJEZw z2(lxM&kn7>H1yIBLO@MW3h#N9c?hb~3po(FJ;t%-7Lbe&g`U0T*}c31JQ)oPmCY0s zDx&5ng|ITj`_@p4CD$bwxSFJ@x*FMF!*({$iBjEwrXJm{TB*@|v7J*qq9qFf=i^Eq z;kdlbcV;sPosU;rzHy8PmRKMne#$0t1EiJBh34})_3RU&QC_VipazajMyiA0CaaSO z!OB?F9YX(Umj{@Z5;wa&dQ39D6Cp>+RTv_!s8s2b_ww!0AbV;#fSY?~eg0(#K1$Ge zF8{JAI`E~ULak>5efzA<#}zErBrZ7e!L~!sY(HFjK(PMgl26R58A!4KVIpIUIf~(n z*X&)PHsi=7a2Z>NJ_UTFB;92bFX5OpEz^iXK(_1)xu<2BMVGaIL>hpS9%RkE*|u1D zDW@bcZ5m*LHC7wq-w4OAxP+s-f*C-oDVq?UqzaL~8G`(wwjek+Qp~gJ)=E zI`KI2$2i7MAFGCXu``>=-|f3Y;8Ej+Ff(&jQe4h39Z|SY0OgEFJ_L2iqQX|??!7@< zUF7Y=PG^66mZ^rWMb$%GaJO#B(TTmRok7Z%uY$Tw*9LlU` za4$)dC25F)4>fWo0p_yRwv==-5q9x3miMv2{^x;k`^=`1qUciTF4Vq|)Pf zSnSRfl{&q&)e_Mfq!9UzWZ~;iSg&&Qx-8J%^@95d~e%A+|;y#P;7w1B&Zrr3} zIO`J8mgE_;WlM$7s9>EVSGj3CUqlhkgv3iPS>_|O$!J=1orU)EE=x^JFT;_g3BrOR z%0in4pTB#kmAJiP$jq=eITNCIoeb9yZ8#*bTRzAOJ(|ZVy{0`Pl@*)29r2||zU@%H z1oUFSMTOdA6+gxrbOX7{j8AMImVF0th_Fce2#)bpt#6TqdPZ84P*P5()V0Rlp+VS@ z+DBcxRPLGaq$cM~PQY8KPsU!mB1|0b;D7ZAo$yT(nMdo?FX-Hz&pv|OY=86mYRocG zZ19njjCl8)rRR293|FI&Jp;v~b52pEeSI)NGJ6AigD7w-mqmkX^hBt8KQs}!>!vXr z?6TYJdl4b&<|cNY?1QS<6ds4r?AuO)#RlsJcFiBseT>;ci|96c9L|YB$GlJ;7X)FC z!L`|nV-*Ei%Ej*UqfvFvgX1*;b}J#fhJ^*Yg=-JG?zl+nKkSS$NXyGn!#+4Fq9uy-j{Ezb`LIt?;41}h3gX+I#E3y$u7Hm4rx z(m0T$vZNCK(xkYipFgLdY+d6E8NSMNkE$*0T0`@IY@bx$1-qg7?+3erTsq$;SO39LFJ zIK;QpF|yHFUa@q^pzqt$#ifI;vL~*zk{vyV%6O#G$g>HTRa+K%w(oWd2=rD$4Cv-f z+~qBQ>>G~OPFyoe)D7+nb`qq#Dk>U|@xgxX^^&_ESNqLxB}J5lE9*nFalY_(Q~HTS zy1aB+9t&P76_|c*+C;?bTie2lJ+HFQ!^|?v%=w_D%fn15VJqv8=qG3B$=_-L9>$L|B;T5ii14c1o zE<;8xx+43}leyy0{*IGBk>GjAp*-Zz?9LtpOI3+sF?F1hBHrke9a$1Hmfx*3)eIvQ z2%}_tOXq|3eN(1-x&M~R=nOWQ4`;gK->Dn+$(EGM>KV41@c#7nUx{&5`A;F-YavlQ zF4}cxrK43#()AJ)n4Z54G1pg5$EpaV^CWh)Q(Z1uRv!B5VU7iI)ju$Vi8L%MtX)*c ze*E>@kmgf#^xxZsv@x_ecu^oDb}ahQHm`q0EWZ{Ci!hDEuF$inGYL6$8(eoJ&-H3H zM}G~<<`dU(K56%z4f`B<`r9TQ$!~RsjN)F22j%>>KKnRz?{_hac=g;Yk#B-ISOUgZ zSfniCZaxg`whTsJ%%8#J-5@b92`)QN#*XK;2(T6oI;WwBDk`{l)<>GMwW~_jADtE0 zx08owxwm|fgp(Zxr^hJrYHZ(qvQ3D!;lr-Ur+?Pb((We9epn&>_flVx8NeAE&@EwO zjqJSJvv&HyrTTCe$*sk_-w5n~y$?J{?jkU8?8RN#q-h&7s{#y?^Y= zp&w()a5IssDsk>3HGZ6Z(89)5c-r;7I%6qvQwKUEzO|^eUA8V3xBYp9(A=6;pu$Y%<^CdqABfYR@Ual z3}?>$L-Jp%_xtbk>14v&mWpPaC9{zCEb?leAH{R^wkXi~z{mQZqy7EcV>kVQNSzf~ zZ3e|aVZ4ha^Mq@Oy*K_N%{e|s~t zOu(B^bKfueJoWsE4bg8-V0ocK+w<>zS2>ql=lGsQH+25$IaIl|*`ZQ`gVp@)liYL2 z{G1()NPC=s1`qdOl=w%zqyPU6fhZDxvFPx4a=V+Xqi;sQ6*QWqW?i8s&)Yx$^OsfF zdo!e)^dftFh12(UZ^j4PG(*byry9?yW*AMGk6|=4%5||WzW$8hkJbpcO*Dlvzun>7<8SME zMTP|DdSb&c8}VMFa!A4{UrPRxudO#EL$m}oMgyMcH~ubRKi`abiVPa3U$>;a<#x^9 z#k<$ZCg*4%lo5@@_E~Qg)ON#aQ z9{13y3lLP1>`0eYhFyry&rAz=1>F9eqsxpEt8x81`(Brgxm91* z=G)zx?l6H$31(!+Oz}k2#)f$06&Cdr!=RBpQ$$Aq+_t_I*@<6aC7lG^z2f^l zL-VtJSNW)77bYSv&#@vI#+GvBW=3UZ!7GI{YL+G-;^X?FSN8sQARTZnnHggvh38t> zYknCRTXy(O`7f*qX_4244?J)(A3T)RFz9keM~bXOko+)Crmy0#@>s~;YSll8iH(iD zPBO=$e(xjBFM9_kBDu(67SiJ~kK`>F>d)y-Rn>Oj!QtmUq#&-ulbHic264( zJ(5z?Ho!(eJN<8n>-+3WPM;;x9L@lxz_oP2HP&HN6c7R*QbW~p>w$iaeo1u zcL|lvyz^!X%KCx1zOP~qiO=K-xM^l_we5qfvDaJ(M8_jW{zPDtx2I}Y7sZUdU8`6} z2Cn;IcTO$?(#4;M4t-pJDZKt~6--sTl2*g!gXSosGiXwk%UxYAPG}HU(Al>n7nmSA zB_E}4d;iTG<=^%AWJ{i%tI$+J;edvg<{|X4tHLp1=TU8QECVE# zU0A7zA;T3FSFBPVzxY$y{}?sAi{PD$j&n1pw3}!MczdkZq631;Q+>xPM7wp3Wni=W zq$!pyZoL5n8#=6`|7*K`e^Sga26XVl=I-Tf3CA^sh~)(=S+48~u&zb&eO#V>L^@V> zU(AZrz;{f&N@?(s6fy4})SD~;+E_YQd<{bb|E9_NKgRImg6G1PY|aDJN3rE{-Y`5o zI-K|wwYUzC3foV(cj_Ea!n?EqLVv@Lx74(}W?9tL5}R(SE0jr@PQ+bLWaW`0=aDR2 z{U>{`Vhh+OVDCCu8?63j(cEyx`gM7DIVwgPdvAgFo7OO3^(CekkBp2={KVAMaR19V zy_y5b|8r7wF7TOA3kA-vuLa+wtFm+6Kg}qvQ^GQG1Eriy(ISHFoP5GIs`5?)gKn3q z1@g%e#21~HPn-z-d}9jqJjC(iR)Y`)0VS9;1>(&ZOFddLd?Y5rTC;Zgi9LFxUa z4)bsIBXeRnuCWdfv#?UoutZ*ve$XHJO(gPve0`sPQa%9!yP?aj-#_5xU~ujp;0qjT zsz0_>O{!04F;w64HyHkFQHhgpz?YhNTYN8bxBFgVtv1W8yD=U<^us{;vKzD-8AhHv zhn~3v1sF_$?~;{^SFOSSe_UfH{#grS^1hbFl5lD8K|tWi)t4OgGJ|gNdH)yG#C;-v zO=j@XITW%qcrH=8g)TCu;oD{6{X*zpZ=dm(<@(tR;NRdyrVU5^Dhu*+YeFN!lu_35 zD$#(e%>Nbs$%F$CX5`EH&LQ8tiE~%$*ivCqO?VwEX-H(3YL{8Q_|W1%G`IdG3s`Bf;z&7HD_c`}p)BpFBJs^B= zGTC0klD1C`k^v#F9WuZCj9v$P4No-Rpnx-(%wu(JC$b#UJ1|^)HoJp3tk%o^KQXe)4=DlnPwCOanBWv zEH;3)3Y)Pu_&Y?(0E~)aPCqdd$CAY~TxeLQxmr_$_#&ecb?A{nR+H-w8GFPT0sdVPTl480MA*F1=JIcPya zfdRN_I)!Gxh+|LgT61vv%<^6-o1`S{{U@Z$8ixp_lb(r~-9rtxjj?ge!127e6xvC4W8%0_%BWBrx<8SXcF_f6g)d zhv~y;B(eNHDx_HP&EBufzXj21d*9wB1m1d-Hc04?Rs8iM4(_`Mb2EmSjkJyQcsmID zVEEwTkNd1nT{wBDl7_GXg~?fD^e`cSQUpEL@~zq>Az@CKX~9j-*G{oK)G6uJz`MMU z$^T`cW$0v@eb7}|ZYaLRRYkGNiT{w^l1N2F@k>v?5Ho(++E~4e|}3|3yg~~`kg&{|Mz7_&PzWo(y*v^p!iPK zuR@Zr+auEn&}Z7_vd#B@4G$~dkVJ@r4v)=eA`LE6HzBcCs0juJU6v!Y`?t^~35+dXzA|`1_Q_p&CDnaDqxqV4j`#KsR&9N!+SM7zwMBw%E zYE$ad&0FJN=lIJ~9&CJ!8t&#YT9#N`T|F`-%db7UNUH)GSvTbv(RsHxW6N$DGm3%b zFc(Q)9m>DNKL?5ikEqo4rci)beL5?0ktT_^G~Yp|WJz{Dl2LN|V2*$J1UY{4$9jz* zhmbcBJ7}%OF`ZeO$vqkKlqY%ov!W~pH$4aB6_6VGDBmJd_jta&o;APQ&_6ax4qhp+ zHKV(n6(J6-_WwetEM(|{PXXX-OGyv*aPq;}|Kn54Y1})D?KgBZdhd%#uAwzb!Ug!> z+g39@m4%%6%OnoS8;CwZhu&$Q0pgqnTjK{kRW=iZL9B5K)4@ZJNBpSDdSZU?o&Wzn z_*TsK=2gv7BYL3$x4|abh zH87hYmsS~!N(|+taxYUk6FqWJY@Q5(`x2pU=y)5SLtR)8`{s{TTq6f~x4?SLz;w)k zvp%O4IwBv%i5HyXwKP}jLRC-gneK}engZI3lG3NO_gsrD46gf{yPEIE_o|I! z%Ek$SfT`jOh&KF1;Qh~$tMa|@OYzjPoyNn2*&qHbRqAUK=MWn+a{28g9^2%N-tKaR3pLV*S*<@)mGe#kx)F$-IA7Z3z32*}d;&uRUMF&v z4EE3dZ4f`70dynz7W}4c;=9Z9%V-oJEPHXzp#tq6a8qJTKRowrOpR?%#3`J=ISjyP z0%r1h5SE!lTdvvIs!jM@;oSH zaOrK^_=>dQ4LMT5A34Pn@n;RT12^Ts%{thuVC6903m3_>3zvnQm z>tbaMI)ohMFd0KlDrn5xQ}QF(hgS9XXWf#&BJPJycd-f{%q@LWKc>q>Oe1BzYz-hce3w z>r3Qq8nZaBTk6LXA^tSsIq>;Lwx)|KfZ)X)*9H{fD^%<(D;IVKK=q zg=YJGw~<>ZOscL2C`p1m(9tRs6FW!PG;;^K7F{O)o5h9D?9)Q(!NhgH|NY*1MShnK8=l}Z?9Vr|Xe)ONd%+<=b z(jjY5(5b*GLItx`ex?9G;%n}A4q)??Mb91%*Z-v#d)wcNhil)XPa5YwCI@idBNs^# zkjzdaa8UR2T^1Mh}#Ur6s+;p18G~OrMgiDb1rB9P?L` zePCSZWZ_}HgEcu`OCT3N=pFz!s0!Q~-v~mhTM!MJ$+$(G{EO5d-kU{|+FA+n^C>m9 z03@zOvGw;a_uprAxcjun=F_4xPSNGzuwWDRs|ikHBbN*YE^LKtKOpqAKYcwh z?$=2iaXb=66}{ywm$kVB>0I1@%kKVT;kV8Lv~Y{N?egBWu8E|7*u@$;6c7`1uH7>qcTwc%XNT(m5e@ zb@ktP#XrU^7hPFoHPU!qnok>_kW8<9;gy9P9+*WcSJP!|{j&%Gh9!%y-*gtRmyr28 zPZo;l-K0+hs!@JqTv_Y05VZM#M=Do_Bjlv`1d7I^mY=E{K{voU%>8>Cz#!D8Pq>k;lqbwlCNI9qG`AVy_+AdE)|~F+W!M+ zulu8?Juah1@}_e6>6>1RVBuR#C(hn*N;+NB|5lJC7YN>}5H-iWMBYeAT47Qx85Ur+ zkW?eUz81~*@w9TeBK=zBe#j&j1X1ua2FYyYiIBLzAw*Z)TxZFxnWQu*#!mBUM!l03 zC(x>&@f=5x(%P{T?B~v}gaMW(j1tx6*mL$TO4zE8mkTJKR$UOpJ9m`{)6fcF3X=8{ zeKGg&a=jK4YivEE%!mm7qB=%T!R$D+a&GOvfZ{+F)Dbez-?+~&_?V> zqA~oKK`e)uOIu-Yb^UWgG3JF3ZY?cYA!vnga&BbHtIy?cd_B!XPRI`UqMRJIC{U%Q z&#=EGD!5?OfIjTBUi~X`1yBeEgiXJy8&5VJtUvBP+C?8`v!==_NOk8d4GYD&BYpdc zMjs486&Mo50S&Vfn|{!jE3qq=n}}-RzQ#z-Wf_C+$_Q8eXPL{O@nXPhc6uUtnX-QQ z)G^@mBa{K{Jn%lI2H1`pC=p(Hb&!LPR@L^r&4bX7r6n$-^?LXxOUq(qx?9xMoYHRAJR}S-8EQO3lORl_lgn zti^VIZ~!{Bl5De*<-Xm}7%eDRflRC7pYmDGS7kv;W<_Jj_>EGBmMy+P_uXwVzPWjR zbrI7<`J}FZmNii0kyn2!ZT;Dsp>Ys!-#Nqg9kRG7);&LEMNc9)5mdUeyve=#MYjn= z-;r*qO(;-af^?dU^?utetq`f)&>mPcp{Qa^V>9eS8&=SzLceVro|IKGb!%zfeV2FF zS!%wrGf*RlJ!8c-3KQq&HS!x49De7 ztE`E1rT#mLh2heI^%Ek3n{Rp?M_!>`_VK}b^n*iz9WDI2-8!phwNh%=ALp`dRy-DD z{-%2bk5sMIB>J=bmFNIzfgVgo^rb4)rv>dg0D8QI3P|*G6+GD)l;-+|n>UN-OAJE5 z7T-!R885gN*jto%#J9+P&I9!wh4y)kE16)2KGAMFJu1`b!b=;rTi!``MD&`Z4JY-o zxwi?E(#(5rx>!6!GG>hIlzW|_8cf`LsknoR-qBmW6tv@Fks<}|wYO=eU7p_tyJEVt z)bKe>-mojz*7d~%kOWS0^Piki@!rRd!dFE1~W!j=~aj-k!nm_!PzUaddL z8Y+wyWTuF0K<;a7ckt!(%|Vf_C6#f3p>8$Snc<{zi^(me&6R-Hu|+$}uFfj%^Mc@l z*|+m|(X)L#8S`W>Xt%ey)uo+Pwxn8;k|O=wygmPc1v?CJwr^P&mn4MruXVa@Z+bq{ z#S~4AZn-XON%KV6;-SaT#;~ZVEgK^zz#B9IKp5e^9Y1Z3JAPJfYPcrKdT;(FfEDg5!oFda;4o z0%!?8C;9R^;4o;jqK6&(rBDqhl=EQI^h(^&MV9(-i}K=66>GKwpHX4Ke$C3hi=fWR zpgIm#I^wXB&sH&$+YpuztkIi6vb?@A5nXuE&%XchX|(uCaVNOjVfpI3@pcob`|i!x zcX>7Y?`(8*bvf7g54Nh$H>I{@1h1%d0Kp)ud7FrfU_tyO@-5Th+)n)l00= zDb2CU>Ylmd5ZG-voe0g`oHrcbo!iLPIy1DpSwG&$?@hv>yLTHMNIjgD(UA~bM2$^r z?v`u-B?;FUWzTrl!|c8x0WQ99tub|h#>Y>y)Lhn$a+%|vhsAStc!ovq{+K`uL&My5 zx7Pw*9v3gKrsF|qZcix=mO6uqhbAyr^F+zeVWi)(*YV4i9^AV>mQXKy*|s}mr9&;; zxB5cG8aZZ@jrC~~14S1YON`n%VfEk|B)hH3s7 zCjDHx64mW_mfd85QTz@j@+uDxHX|kTdwUODq+wp?1|v-`TdW^PCq=Ig^nY5tbX&{g zQ5@Y6z9tysP$NcH=0|Q0%8;&H@SFd<{bfRew7cVblA5b6wL})xkX|FWXT}tmkaQjU z`3(Ws=Jg5Fl!TzgUXhtKs?XP`h6wy~GzCkFvbyu;@hZN!Yc??C`L;RS;@mqhbR@Q9&bRxPX9CM5;hQK|nf)(i28dS_nulAz%fm5s=<{2kBB0bRhKJLXY$s z0)zk|1m5G^JOA62gphJs!R>R z2w>ykl&zS-qibsNqz|}%iA_Ia@#q#Q|ED1QN}gPIDBf#>;vWWwv{j zkW4wx>XO{Y+H%uE9s4{^U}r%>`|B-m)_v+WFac=!rrf;7mhqhrvQSUzCQU^#C#teJ zy`=!d9=#hrrOQ~;17zB_(eVJL$07;xdvYzckOW4_{MUn#X<5Mvr;NT7hcZyO>sN zy+TqX4x}mL&$r53~3|Q zxq7SW#h(~Cgf1g*3`qEw*8dq)>Pj3pF50r>$)d~UkN_NJcAT_RdUcLBJyu=)))`3a zAW8)9Oqq{KFjBL{ZUAy*Eb!c?G1EMODBuX46W}xHC}tj$ZhaB`L8@Tt;OuDigzJD7 zCc(-fhi8gjr2YfN493zfWu+%*hbEYQ^b9rr9$Ke9c+sl-d*Ljw?&N)q)M!=mw%i&>-TC1;DIz$@Qti}&=>6b-2CTd)|0D9#dx?^M!He6_YKgE2W zMW+>k7*>nB{Z#RT&RT;9vYMW9STL0(+L5Ui`5Ls+ba>G3vG`#(Wn^=S^p81-YY`P) zGCkkv0=M6BEH(e zgGXc7OIy2HDqE%B?V3U2A0?PpAX`^UK0^s2b7YrIEGr+QQtGJwK35Cc zjmMnE68pVpE@0+Seg~Gli{`aZWQ@XWh*$`we4m9~N05_M=`E(o!SFuHIH~Qry8c(F z8-V8^hB{ULm5}*#uQ0kXnEO%L*XkzC<8@aRF|F@-MjO=l-^9CZq@*h^(P_+*#h;FfBS)SWzz*wSD} zq8c`h16dmE{guK(z7Pyi3=m3aMPe1M?6LdT-h|wg%>g%^J?8F#BiUkxEvh4YG8uNDlYE!7;qmg_WV3{ zHR|mFhdm0PW9PGA`q3;1qPR24fxD!*b4^TbJrUp`tlbztxav&E&1xSU>8SSEU(UfK zRDDhY=}@D0K0{fZLqxA%l#cvh$)ILew;#-@1d`01Hpz+}3+ZlQoIde_&+I#g;Ys5^ z`GBtZFa3{e$BYDuLB!A1^yn_pt_`$g+w#6sTo1f9oBU`wR=i>|S1+RWg*;nCx;caqBY-8{8-Y~>ta^|)vxFn6iazd*A zNv@~jx>E^GN0Sp6)EHFh5k^~-1DS+R?;z+LoACAS#!z+utGqoP~x+uPf9@02WUt;P@i;`?7YW= zMp?rN|4d9Q+59rusJm-2Kz9bRG_U~yJ?gI19A<*-T1vrWXo zzNSi6Spn=lK}xGEBDgzOgm9b7zP4QEa}^?tI$2y;GuVb}QgLxKM`bZWo3;0f&eA(h zM!!N&S4=ip@WP{Di<)mo^K zp_+(>PQgX>R@C9dRHz=)Nwoh29#I=srFkp)182A}2Y3anx0$($+xi|w_hbbp>u1D% z2o77O*-AHV;}QEc+pY$h4sqVW%fCMiNuQo8Q{j8zCX{!+C{x)F^~ow0)QnyG_Auj4wBM z7^-?F%`5}!8RJs#%!_ocI-PX%EIdzGaOIiO+x1m_lprTPgOclv%M<;0LnohQ%60{` zM-+3^nGCM`Mvaw)E=Cl)IQ1;kC@^zs{CFpIv!Dc{j#22~gLr5R2w{P+k|%z9s*qTK zB`Pd?_Mc9GCMOT$_p34FqV0U>fVYSbu&CVv)&T7AxM6(Ex-px{_M~ou$i!-mgy?6P zfz=gqA(dLSQ1^0NesvQF6NTTm6h#)_0v9b#)*}Y*&>Rev#(JTovOb~*9#?Lm*SD(5 z81S+1oIq`IswvxN()>=8{0^{?*sWX7DIl>t68C9XcPrUlbVZL@dtsMna^>yNHwPh591KLA$0^9(H3bH%84;`X{=Lo=s$eX!iDo&i^e z;9$MY+-cV(Ykz;fqeX?IxCxWn1(?F;SW!E6YzC>a@w}|faNSb!>hfxVgQ~HcutOZ^ zQ(l_wk_79dgPxx`w#1q>2GJR$+*31ex1F}yA}X&BN#a7)LH6>!$dT9R>O8-FylB;w z|CERt;|g3pZt!zj^`F_pS1XtIhLua=4dsP3HSvD^XT+QF1_ z^vaGU7Y<6g1J^a!jqL^*%1NbK$vn&c+w%ah_Q|i@BJeWjukxD%mSd`+@IQO>*Tw(i z)2nlra=Vt06Bh);0P@xEP(ikTRSWvh?BNf|yl>pPk$ul=>j9>AYlL;vn#-E~f0O+F z<$JQ{x){*8v7AG=kR~I<4XV=e&!}GT+p=bo4_CrV_1r60aZNqaQT+~ z9~#*ICB5~}>}2&lxQdm`umh9snN9nuZBAVi%q0MWucB$8TOH2zP<(z+e z`mtBSg1Q4X8a4;|@JHS_dUHT|(xc^G+20TDJi&*$eS+%2G$}FuGaBm;ziL;fSNuv| z_`{hdpGPJfDLP5k4hW}5tT|jh|3xRR9tU&`Vv9mI1D_i-mqkSV6fgN`e$tha zreFRGkj;~R5aHPNZzbq|_|^XppavYf!h73x-^Zn=z&xH~(yo&$q2#v}7%ZKQGjN$d zWbNlL*q_5Iw*Zlw9*Zgv_w(K6fnHGP&e0_Dz=>tzT zxOe35&6O+uDx|(bpnzk*7+np!Nf3PTFK*7i+u_fy{qd&m(a{ga_6s^@w6&kAW`6}x zc%&@p|8|t6&YZNcBGj*@vK&s=R$xRYuzsWOG`+>v<0WABV2PJ$Ib&&tj_I$%?z{ zsHYFEIh@buK5ZB&X9vxyvQ=azbfrY&)zZnStN6 zq?lW!z{7aY@*6jp_sEn3KUhU2wW^f69^rA-z0H=lw9-pa>&o*X{h~HW&*Y62tRFiX z4&M0}zvAbQa{1hna3?*>ibH2;TQ820Q43X7U7A$UHmOqHs$D>Gau}1(PEL64lH?}I zSv{qWJm;}?*Q4uF3ZI8RwAKUpD8<6)@rB6HjvI*(&Gni>*HnuzJ%CDZT`axk8(vZX z{dOy+MM3I2e8+!w)pQ5rTH$q=v5ZMzwXdf19xjAkl5FQIp-;A-@0>Plu;uWg78n$x z_lC}oJJXQ@PWTJ?X7S2V2Tbu&Qu4A-&%%T#%a|Ruq))W`g-pl7+PNs!^5!g*k3}(% zE!Hcm|Mglj&({Rz8F%W0OHJqJ!(+~Y&RLx<1N+=E0Bxx2YXYN*CS1dJ#dkb++EFrj z*@b0c)%EP&jEsw$R7)zl&TW6xpC!Pd^!2DM`gx4mvc<6HD#>`kQ3oY8U{&WCFXZet zFR2ZB{CU5!^^{@3(Nf>x>S$5-TKVQNG3OTz6Q`1oXmbTM3t=%gco~Hv?CdYN?`j!n zp|m=-efoh)Dj5_ryElj6)vi|w=gXYl;fD{`4KcU9UEFuxZQ2O4o4c^J?Mq$-@02dW zsBSaj5B`AbcYGv5N@crg3iFYILa`ztB5@j^l|s=^F;=fTGM7mSpmcHL5%5A71l7e+ zV-&pTvYI=sTU|S`^Yxd~6F(jHBCnWkbg32Zx|ft_F~_R62=K-5+|@^gO4j6_FQV%i zrHix{zKuo<#6FcDA%k=6=zL{(QZ_IT8mOOoegT8Y5}H{KqL@aeS)Oj05|2IhU0#Vw?p0 zjw297=~q_xI&brdb?z1;>d4mxpLPXI#oDjS+P)q2lxTRi@gPBWg1Ck26i?tm=^bwE zSucFmSBal4*ddQiwnQ$*Lqxo>nsHb7I3yl=_DVijo{md2C^-U)T!TZ8T(fbTmmMqS zK^nY_dO)Y~4C3=;kM?H<0MJYB8G+K+J4sh@^?*XXK8SG&D3c}9ByU?V;5 zkfNdlmck?I>1GgGghlc}8}DjD2OB6+DFbQ8I^ESE2>JW$@U8l9tS0 zUV)fKl5PyNS-PKilL@5`F7BXT6oxMMs20QA8SJl;`afS=XsNEZj$V1<*l>AAeMT3& zY-DEN)j^1uIEp0W6QpHr_nf-rVX;`aguQMehPliM!#`>6UDG)6)|#N^9p$R8?j%i7=KC{Dw6F?SGBgleRd@lpS};(Fe!h zR+Ep0=FGY;-bd9K^_R|fhesDw=e29hW{+!2RxX@Fz{%HF19;7&T=oMAYdLp5neQg* zFzXBZo5^)KNwh3=KV82IKgvilkjNoQK8T!}2VSk?;ij2qixS;N7h{YVj;%7;L$G{N zDi$p4HB=9m#bNn-{KSRY%nKJNpnkX9H^n~Sv&wC|PLe4XA3ZB-2!!LdI{d&AR(2H! zF=0x$vTCSHUZzrJj}vXu{bfkpM+1hM2I&-=bJb@X)M2~0oPZ+-mQxJs#Z2!(lGF}w zZ)e@1eCvO3Um#h;^G+OP@P@e`t%r%Q!HeuN*l8Tesi%w<0x{iqKz+3xB2+~*)m*Ba zwBKP{)ul}4JjlUy{hlq->kan**a+u78@iX~vYJ$^%`RXB}>^dOM9JTJciHP=f^b}x=75L%f$l0NH2Y;S`x z870ULeo^*}B^TQ8tR%x&6v#NY^l3;LAWdB;a}20F11407F6>)3x14E7EF=9!8gMLT zsVM2N4<5yjavt7ncd0u?uhw2CKr9N_cJ{b`Bj$0B1W)gdLTPEDo#i=!eNEaZTuY$> z-so>OSp&`%IW=$jNhiy4;FR5$?5%Mm?zSo&l#LPtpU7e9w%dy|+bgB^NHFQ5XV0bcUR%7BFQL6`;s&7PZ2|mAmN8*YHCTLY>VNzX7Y+ufRn;p&B%wIciD1}J# zsLAe`k2Z@~8PJh@QcOGxEcjJwZWlnHum;uDTXDkv*%~TCZ3DKlLb3LE#P*c$+eKoc zYf}91J5@vyPH?<>Lq@|1jUBJ?Btvt9H$bH~RcdHhwKk98=hVy1<(bGj&zx@K5B15G z*bMx&eOXhCBWcx3tu!+wMbzV|GnagYRLJ-ud{~tg8)LgAK&hEE==2;E;etGPGBbM& zQBZdki}p2>;R8T7b9$b*Ox{)7O^nW&)z5%Q zG=UK*X;yA^s1J9Vcg`PR*)1ddvtj2S$rZ1=CUsf7E1fgofFZLxp}z@BdkU53S?92L zFxa&goxcN`)@8r&xuXZovd zuJCWVjDMEPJea_^iz`}%q??9AotMV$*U4=6IL01f#CUBu9V{UV2RZ2EZ{sgb{4(kw z+jd*tu2crEL2|}v9rifaW1FKW-_V@xU)sLw*pst5*+cWqngJp7u%vQE~U8e!XJ~ zMF_V6OG>MigrFO0YL``C>P#As?_u&_=BtHQtu+r^%?ie+ulaPeYXp!NI`TbU!jy7a<;esp4OsrupXje*}u!7|6esI^oxvmql}ytf@#^r1YaO zCrWG_q%%pZkIt7@Qz(l-S%9uJSZfadS2T*<8y1H4EZUl8eVVe1Fq0cQSme8o;Ctgn zy*{blhP3Xpc6m5T{@t)s$^C6grJSmIuSd-!;hhBTn_$J)FcmSc@2T()D(M^iwIlTj zaHFiW9gB}GQdsz|ROvkj|FpDX@cVYv7k}%OSIhGL?BlHM~g32*$3*WB`B3xTK+8k?gc zE3X{*$}vk`2e}hIrP7eZQ5MLB=Xjy6O|VsHxEvD%V)Xxrw&$Mx@E{8GY~2x`GF11N zSNvi+5wkneqmK`h+?OW;Z@iqNwp;mA%kk z^Lu82kbylQ!gIW+GGw$zEIh{u2q~`gXzUO*s0Ztkx4t`ioXX(Ce;CAB9BD@hDzxn| z>DmBCWL+zPtk=IhAELcF%d^1lZ<^+tx;Qn~75X8wR-cauJ zDD?{UJfaY}s+?}%xc@36O;96DLDV+iyfkw$a@k4*;?vv9#5&}-M zi22}uEJb^hb22Myw)X-VGmjk4z2WZVabQnu+P+@JrMR{ggl=7>HcDD?(|eeyr|*UR znl{{88Z7)^??yIfIr!Ry4NK|Fel^;YkPvH-!5#B4>>`Sp4y5veaOD(&ur6O$q@j)= zq}p(!8}i@P6F1yz&n!grE7$GBKhc>VU&G4a{gu?~0cy|cv5%iVk4lF13u*ZT`E!Lt z0rR(=XRfI-B`hK}ZSMPcL-`{gGcn_nOK@PaS4C#UOdq_tLdTpF@+=H!EvGa3;8iEm z?~${nJ6DaInUJkybdtPz`IiOb&Q(Pnk^0$+)@{rDKiMmg*y zV22MA8)jfbPb+n1rPTK|iSJ=hioz10j{rXD-qIkr_ zrku4xqtB&*G{!3shGkWnKM>lt9E>}QzB@oJvFMj-JQQhFb{9_q)&-@x<84^sHbS?Y zN3DvinF52F!HT!zS~_eDqm1JcM$YMv>xpaYm`&PkUPySeA=HjtqJf90=_=+V8a&Oi zm%0>(I4^Nz?WmQT4$Ek-gIztG>kB)c0ZXp*);yfgCh%{!@xx72@6zLbg>akfLC1Az zun}#>_LRfUu$#GQyF*JY*z}9G@s>_bBciByix;IfC(8nNJj#kAmqvW9&kH2G2JXa^ zy!5?bYm?;qQX}3#WTKpaW@yh^dwL{IE-?#Qx4gCrjTQ}CmfFE&G*pRvtD!ebuNSzM ztld8z)UW1>G`m6CQE)I(x*{|ndoYj4V_)BVCd&hncfvmoxY<55WM)0=Q0lqT_)p5AjN+LenQOR|G;DJfD*1;t?ggUU zrxejzPZklcctz(8a0YEJDo;Dgw+gbq=>BM9$i!H=HUJKel({pbhYU=D+*fa_FpZXz zjQ8C+%n2CtEb~VU?i2}dpV}4kQ?R1V&D7Hli4j+faQ3MP=C0-I#bg{PB#?c*)7ag`E39qz&HOqJyY{2m37@{KFkTch zu2lMhk0A=lvv?es7G0CH=$zRhkR~?19u`4yjsYy?t7B~GeS)=j-niObmKX)``uHQlzX zNerrMfpo@-H+^;tL|%kA*|Z|@%wunE7Cuanxy6Hq82TJeHlXUrUE-V&)58 z!cFtVGMy$3-D9Umc%P+kwe-#cKa$rxVW`jj0a|(Vi+Zw?w4u_ z`@HTCJ-vGY~CuQ9~T>4lIw`G+&ttV%>a z&tFXKIH$)EnmQ@(vCH+&9ntxCJ zu7iSe1QR#s?OS=) zG2RKG=js)rWLd~|y~;&Awp^F67AdZ7wWy`-JK*J+LMrLLv3Q}-fJ53=D2gbT$d`z3`q_=bF^BT~^jJb{(b&W;ein@JrZqNX!#?z%E z!kX~9G;)z=HL(V#TfWu*6xfEPo_Ni@)dKB1$ggevbqH^tnpHEC?6RnIgP4vBzi<0hpvmJdidWon5SIhF^MWMo} zDtXy4PmAbO&o#PE_8!!Al*$;1{mMGGY<#=>BKqTmqhz#RYZ1%f8IPP+`6JpWKhgC1 z5>caMF?L>$>b-;JO_+)ydpo=^KRA9uAC+-5;^~7Z%WFxW>7%-aq;q}alo{IhoC=x| zL)?0Rgf`iIOYPA|LigNH_3lC^p|fwknEFeb^5(CDZQpLP4{0>ixKYCm6#~UB*L%OW zweb#(UN&{bj|Jcl+C50G3F0#|KqXAwPlZKdd=nemcYK^pt2<20xnT#sM-t`9cH}Y_ zA}0)?=i9$R4mNzO*nx&5TF73Y4NkRyGi@rm%_Vrd2RBu0Wro5J%4H4ilIngI2BV@Evt&-3SHs zcTB#!Rvjw5cC+T%9Zlc;mxFtj+3da=RQ)qu>LS3Ko7@DrvOXw^PyOV^a zSxfwJ2@Rk6*z6szT+Xn0XeIFg)a!cdjG^OHrfs@sO@S$En0UT+Vfe?v4vR9hQAQfh zOf5xBsH0`L;&noG4^QAPWCJi9H}o3u4z~T?1>VR+x!kO8R9fB^q;>WP%Vg=Mu)W9v5ko}ztIP2k(_1QH-8VkEVY;WRZN7?1IQH!e@S6KMwVcil zVAsp*&AI#9l{V%=603<5jkMxp9rdlo$G2fShH=o^g(V%U8;k{mVv1nl$Ngm1!6Z4u z`lg+|Pu%tVHL1Z%X=sp{k_hCC5QmlkyCJj9fL{%;a*+SqBM8HplA=5U;&js{l7zK-ptCyJ!222~!f2 zgvD}M^V(l0$I_qR%ToNjAE!ShrO`|Qa<{iPEz#aw0pygAB}H8{_YQ~lkk%G)$~)vs z1zcIobn8?_;jUgb-%`WPdXTW)QY(pxZ2`o2@zn;GtDNN*SNVT1r?OWsoY>-Co89PEKVKi z-MQ0z0j)6ZJy9x!o4l+sAZC&bwVv6gwC!?ErgN*LtB1;x5A#o%P_Ejaq1eiX7dLpx zNM(idZ+nN4X{u-$GJB6iuD+0(;5DZ?i%x8gZPBVXEnvbx*yEJ*(GY=os6!_u5L^0L zCh`GPKhZ>$H?Zr_8dLeCUuaJ9c@22suz-khS$lZgo2h*Wzdho5Ak0FS5R5B|nWFz^ zL8qZ#znAgrzSHtEapqsZhim%FBSr-oQJeg~3grAdqvi)D&yoyKVymvZ<{$sshvJ8? z{_~tv4WIz)dPGiW>ff^CUKMi#-Ca$*^^gB4SM$>g*Jc8`OJT3EGk@-E`qP#d{(onW e{lBy95ofCB^HXr-^~cA6KQ(3T2gQoNz5YLwBCXv3 literal 0 HcmV?d00001 diff --git a/doc/source/_static/images/release-train.svg b/doc/source/_static/images/release-train.svg new file mode 100644 index 000000000..e270699d6 --- /dev/null +++ b/doc/source/_static/images/release-train.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 000000000..09bc98eb9 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,159 @@ +# Copyright (c) 2017 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -- StackHPC Kayobe configuration -------------------------------------- +# Variables to override + +current_series = "2024.1" +previous_series = "2023.1" +branch = f"stackhpc/{current_series}" +ceph_series = "reef" + +# Substitutions loader +rst_prolog = """ +.. |current_release| replace:: {current_release} +.. |current_release_git_branch_name| replace:: {current_release_git_branch_name} +.. |previous_release| replace:: {previous_release} +.. |ceph_series| replace:: {ceph_series} +""".format( # noqa: E501 + current_release_git_branch_name=branch, + current_release=current_series, + previous_release=previous_series, + ceph_series=ceph_series, +) + +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx_immaterial', + 'reno.sphinxext', + #'sphinx.ext.autodoc', + 'sphinx.ext.extlinks', + #'sphinx.ext.intersphinx', + 'sphinxcontrib.rsvgconverter', + 'sphinx_substitution_extensions', +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'stackhpc-kayobe-config' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = [] +html_theme = 'sphinx_immaterial' +# html_static_path = ['static'] + +# Add any paths that contain "extra" files, such as .htaccess or +# robots.txt. +# html_extra_path = ['_extra'] + +html_theme_options = { + "palette": [ + + { + "media": "(prefers-color-scheme: light)", + "scheme": "default", + "toggle": { + "icon": "material/weather-sunny", + "name": "Switch to dark mode", + } + }, + { + "media": "(prefers-color-scheme: dark)", + "scheme": "slate", + "toggle": { + "icon": "material/weather-night", + "name": "Switch to system preference", + } + }, + ], + "features": [ + "navigation.expand", + "navigation.top", + "navigation.footer", + "search.suggest", + "content.code.copy", + "toc.follow", + "toc.sticky", + ] +} + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +# latex_documents = [ +# ('index', +# 'doc-%s.tex' % project, +# '%s Documentation' % project, +# 'OpenStack Foundation', 'manual'), +# ] + +# Disable usage of xindy https://bugzilla.redhat.com/show_bug.cgi?id=1643664 +latex_use_xindy = False + +extlinks_projects = { + "kayobe", + "kolla", + "kolla-ansible", +} + +extlinks = { + f"{project}-doc": (f"https://docs.openstack.org/{project}/{current_series}/%s", "%s documentation") + for project in extlinks_projects +} +extlinks["skc-doc"] = (f"https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-{current_series}/%s", "%s documentation") +extlinks["kayobe-renos"] = (f"https://docs.openstack.org/releasenotes/kayobe/{current_series}.html%s", "%s release notes") +extlinks["kolla-ansible-renos"] = (f"https://docs.openstack.org/releasenotes/kolla-ansible/{current_series}.html%s", "%s release notes") +extlinks["ceph-doc"] = (f"https://docs.ceph.com/en/{ceph_series}/%s", "%s documentation") diff --git a/doc/source/configuration/cephadm.rst b/doc/source/configuration/cephadm.rst new file mode 100644 index 000000000..fb5d37639 --- /dev/null +++ b/doc/source/configuration/cephadm.rst @@ -0,0 +1,554 @@ +.. _cephadm-kayobe: + +================ +Cephadm & Kayobe +================ + +This section describes how to use the Cephadm integration included in StackHPC +Kayobe configuration to deploy Ceph. + +The Cephadm integration takes the form of custom playbooks that wrap +around the Ansible `stackhpc.cephadm collection +`_ and provide a means to +create or modify Ceph cluster deployments. Supported features are: + +- creating a new cluster from scratch (RedHat/Debian family distros + supported) +- creating pools, users, CRUSH rules and EC profiles +- modifying the OSD spec after initial deployment +- destroying the cluster + +Resources +========= + +- https://docs.ceph.com/en/reef/cephadm/index.html +- https://docs.ceph.com/en/reef/ +- https://github.com/stackhpc/ansible-collection-cephadm + +Configuration +============= + +Inventory +--------- + +The collection assumes a set of group entries in Ansible’s inventory. +The following groups are defined in the Kayobe base configuration +inventory ``groups`` file: + +- ``ceph`` (parent for all Ceph nodes) +- ``mons`` +- ``mgrs`` +- ``osds`` +- ``rgws`` (optional) + +Ceph hosts should be added to these groups as appropriate. Typically at +least the ``mons``, ``mgrs``, and ``osds`` groups should be populated. + +Example: Separate monitors +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a system with separate monitor hosts, the following could be added +to ``etc/kayobe/environments//inventory/groups``, to define two +top-level Ceph host groups: + +.. code:: ini + + [ceph-mons] + [ceph-osds] + + [mons:children] + ceph-mons + + [mgrs:children] + ceph-mons + + [osds:children] + ceph-osds + +Then, populate the ``ceph-mons`` and ``ceph-osds`` groups with the +necessary hosts, e.g. in +``etc/kayobe/environments//inventory/hosts``. + +Example: Colocated monitors +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For a system with only colocated monitor and OSD hosts, the following +might be appropriate: + +NOTE: we are using ``storage`` rather than ``ceph``, since ``ceph`` +is already required by the cephadm collection, and redefining it would +introduce a circular dependency between groups. + +.. code:: ini + + [storage] + + [mons:children] + storage + + [mgrs:children] + storage + + [osds:children] + storage + +Then populate the ``storage`` group with the necessary hosts, +e.g. in ``etc/kayobe/environments//inventory/hosts``. + +Ceph deployment configuration +----------------------------- + +Default variables for configuring Ceph are provided in +``etc/kayobe/cephadm.yml``. Many of these defaults will be sufficient, +but you will likely need to set ``cephadm_osd_spec`` to define the OSD +specification. + +Ceph release +~~~~~~~~~~~~ + +The Ceph release series is not strictly dependent upon the StackHPC OpenStack +release, however this configuration does define a default Ceph release series +and container image tag. The default release series is currently |ceph_series|. + +If you wish to use a different Ceph release series, set +``cephadm_ceph_release``. + +If you wish to use different Ceph container image tags, set the following +variables: + +* ``cephadm_image_tag`` +* ``cephadm_haproxy_image_tag`` +* ``cephadm_keepalived_image_tag`` + +OSD specification +~~~~~~~~~~~~~~~~~ + +The following example is a basic OSD spec that adds OSDs for all +available disks with encryption at rest: + +.. code:: yaml + + cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: "*" + data_devices: + all: true + encrypted: true + +More information about OSD service placement is available +`here `__. + +Container image +~~~~~~~~~~~~~~~ + +The container image to be deployed by Cephadm is defined by +``cephadm_image``, and the tag by ``cephadm_image_tag``. The StackHPC +Kayobe configuration provides defaults for both of these. + +Firewalld +~~~~~~~~~ + +If the Ceph storage hosts are running firewalld, it may be helpful to +set ``cephadm_enable_firewalld`` to ``true`` to enable configuration of +firewall rules for Ceph services. + +Ceph post-deployment configuration +---------------------------------- + +The ``stackhpc.cephadm`` collection also provides roles for +post-deployment configuration of pools, users, CRUSH rules and EC +profiles. + +EC profiles +~~~~~~~~~~~ + +An Erasure Coding (EC) profile is required in order to use Erasure Coded +storage pools. Example EC profile: + +.. code:: yaml + + # List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role + # for format. + cephadm_ec_profiles: + - name: ec_4_2_hdd + k: 4 + m: 2 + crush_device_class: hdd + +CRUSH rules +~~~~~~~~~~~ + +CRUSH rules may not be required in a simple setup with a homogeneous +pool of storage. They are useful when there are different tiers of +storage. The following example CRUSH rules define separate tiers for +Hard Disk Drives (HDDs) and Solid State Drives (SSDs). + +.. code:: yaml + + # List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. + cephadm_crush_rules: + - name: replicated_hdd + bucket_root: default + bucket_type: host + device_class: hdd + rule_type: replicated + state: present + - name: replicated_ssd + bucket_root: default + bucket_type: host + device_class: ssd + rule_type: replicated + state: present + +Pools +~~~~~ + +The following example pools should be sufficient to work with the +default :kolla-ansible-doc:`external Ceph +configuration ` +for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. + +.. code:: yaml + + # List of Ceph pools. See stackhpc.cephadm.pools role for format. + cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: vms + application: rbd + state: present + +If a pool needs to use a particular CRUSH rule, this can be defined via +``rule_name: ``. + +Keys +~~~~ + +The following example keys should be sufficient to work with the default +:kolla-ansible-doc:`external Ceph +configuration ` +for Cinder, Cinder backup, Glance, and Nova in Kolla Ansible. + +.. code:: yaml + + # List of Cephx keys. See stackhpc.cephadm.keys role for format. + cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present + +Ceph Commands +~~~~~~~~~~~~~ + +It is possible to run an arbitrary list of commands against the cluster after +deployment by setting the ``cephadm_commands_pre`` and ``cephadm_commands_post`` +variables. Each should be a list of commands to pass to ``cephadm shell -- +ceph``. For example: + +.. code:: yaml + + # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands + # for format. + cephadm_commands_pre: + # Configure Prometheus exporter to listen on a specific interface. The default + # is to listen on all interfaces. + - "config set mgr mgr/prometheus/server_addr 10.0.0.1" + +Both variables have the same format, however commands in the +``cephadm_commands_pre`` list are executed before the rest of the Ceph +post-deployment configuration is applied. Commands in the +``cephadm_commands_post`` list are executed after the rest of the Ceph +post-deployment configuration is applied. + +Messenger v2 encryption in transit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Messenger v2 is the default on-wire protocol since the Nautilus release. It +supports `encryption of data in transit +`_, +but this is not used by default. It may be enabled as follows: + +.. code:: yaml + + # A list of commands to pass to cephadm shell -- ceph. See stackhpc.cephadm.commands + # for format. + cephadm_commands_pre: + # Enable messenger v2 encryption in transit. + - "config set global ms_cluster_mode secure" + - "config set global ms_service_mode secure" + - "config set global ms_client_mode secure" + +Manila & CephFS +~~~~~~~~~~~~~~~ + +Using Manila with the CephFS backend requires the configuration of additional +resources. + +A Manila key should be added to cephadm_keys: + +.. code:: yaml + + # Append the following to cephadm_keys: + - name: client.manila + caps: + mon: "allow r" + mgr: "allow rw" + state: present + +A CephFS filesystem requires two pools, one for metadata and one for data: + +.. code:: yaml + + # Append the following to cephadm_pools: + - name: cephfs_data + application: cephfs + state: present + - name: cephfs_metadata + application: cephfs + state: present + +Finally, the CephFS filesystem itself should be created: + +.. code:: yaml + + # Append the following to cephadm_commands_post: + - "fs new manila-cephfs cephfs_metadata cephfs_data" + - "orch apply mds manila-cephfs" + +In this example, the filesystem is named ``manila-cephfs``. This name +should be used in the Kolla Manila configuration e.g.: + +.. code:: yaml + + manila_cephfs_filesystem_name: manila-cephfs + +.. _RGWs-With-Ceph: + +RADOS Gateways +-------------- + +RADOS Gateway integration is described in the :kolla-ansible-doc:`Kolla Ansible +documentation +`. + +RADOS Gateways (RGWs) are defined with the following: + +.. code:: yaml + + cephadm_radosgw_services: + - id: myrgw + count_per_host: 1 + spec: + rgw_frontend_port: 8100 + +The port chosen must not conflict with any other processes running on the Ceph +hosts. Port 8100 does not conflict with our default suite of services. + +Ceph RGWs require additional configuration to: + + * Support both S3 and Swift APIs. + + * Authenticate user access via Keystone. + + * Allow cross-project and public object access. + +The set of commands below configure all of these. + +.. code:: yaml + + # Append the following to cephadm_commands_post: + - "config set client.rgw rgw_content_length_compat true" + - "config set client.rgw rgw_enable_apis 's3, swift, swift_auth, admin'" + - "config set client.rgw rgw_enforce_swift_acls true" + - "config set client.rgw rgw_keystone_accepted_admin_roles 'admin'" + - "config set client.rgw rgw_keystone_accepted_roles 'member, admin'" + - "config set client.rgw rgw_keystone_admin_domain Default" + - "config set client.rgw rgw_keystone_admin_password {{ secrets_ceph_rgw_keystone_password }}" + - "config set client.rgw rgw_keystone_admin_project service" + - "config set client.rgw rgw_keystone_admin_user 'ceph_rgw'" + - "config set client.rgw rgw_keystone_api_version '3'" + - "config set client.rgw rgw_keystone_token_cache_size '10000'" + - "config set client.rgw rgw_keystone_url {{ 'https' if kolla_enable_tls_internal | bool else 'http' }}://{{ kolla_internal_fqdn }}:5000" + - "config set client.rgw rgw_keystone_verify_ssl false" + - "config set client.rgw rgw_max_attr_name_len '1000'" + - "config set client.rgw rgw_max_attr_size '1000'" + - "config set client.rgw rgw_max_attrs_num_in_req '1000'" + - "config set client.rgw rgw_s3_auth_use_keystone true" + - "config set client.rgw rgw_swift_account_in_url true" + - "config set client.rgw rgw_swift_versioning_enabled true" + +Enable the Kolla Ansible RADOS Gateway integration in ``kolla.yml``: + +.. code:: yaml + + kolla_enable_ceph_rgw: true + +As we have configured Ceph to respond to Swift APIs, you will need to tell +Kolla to account for this when registering Swift endpoints with Keystone. Also, +when ``rgw_swift_account_in_url`` is set, the equivalent Kolla variable should +be set in Kolla ``globals.yml`` too: + +.. code:: yaml + + ceph_rgw_swift_compatibility: false + ceph_rgw_swift_account_in_url: true + +``secrets_ceph_rgw_keystone_password`` should be stored in the Kayobe +``secrets.yml``, and set to the same value as ``ceph_rgw_keystone_password`` in +the Kolla ``passwords.yml``. As such, you will need to configure Keystone +before deploying the RADOS gateways. If you are using the Kolla load balancer +(see :ref:`RGWs-with-hyper-converged-Ceph` for more info), you can specify the +``haproxy`` and ``loadbalancer`` tags here too. + +.. code:: yaml + + kayobe overcloud service deploy -kt ceph-rgw,keystone,haproxy,loadbalancer + +There are two options for load balancing RADOS Gateway: + +1. HA with Ceph Ingress services +2. RGWs with hyper-converged Ceph (using the Kolla Ansible deployed HAProxy + load balancer) + +.. _RGWs-with-hyper-converged-Ceph: + +RGWs with hyper-converged Ceph +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using a hyper-converged Ceph setup (i.e. your OpenStack controllers +and Ceph storage nodes share the same hosts), you should double-check that +``rgw_frontend_port`` does not conflict with any processes on the controllers. +For example, port 80 (and 443) will be bound to the Kolla-deployed haproxy. You +should choose a custom port that does not conflict with any OpenStack endpoints +too (``openstack endpoint list``). + +You may also want to use the Kolla-deployed haproxy to load balance your RGWs. +This means you will not need to define any Ceph ingress services. Instead, you +add definitions of your Ceph hosts to Kolla ``globals.yml``: + +.. code:: yaml + + ceph_rgw_hosts: + - host: controller1 + ip: + port: 8100 + - host: controller2 + ip: + port: 8100 + - host: controller3 + ip: + port: 8100 + +HA with Ingress services +~~~~~~~~~~~~~~~~~~~~~~~~ + +Ingress services are defined with the following. ``id`` should match the name +(not id) of the RGW service to which ingress will point to. ``spec`` is a +service specification required by Cephadm to deploy the ingress (haproxy + +keepalived pair). + +Note that the ``virtual_ip`` here must be different than the Kolla VIP. The +choice of subnet will be dependent on your deployment, and can be outside +of any Ceph networks. + +.. code:: yaml + + cephadm_ingress_services: + - id: rgw.myrgw + spec: + frontend_port: 443 + monitor_port: 1967 + virtual_ip: 10.66.0.1/24 + ssl_cert: {example_certificate_chain} + +When using ingress services, you will need to stop Kolla from configuring your +RGWs to use the Kolla-deployed haproxy. Set the following in Kolla +``globals.yml``: + +.. code:: yaml + + enable_ceph_rgw_loadbalancer: false + +Deployment +========== + +Host configuration +------------------ + +Configure the Ceph hosts: + +.. code:: bash + + kayobe overcloud host configure --limit storage + +Ceph deployment +--------------- + +.. + **FIXME**: Wait for Ceph to come up, so that we can just run cephadm.yml + +Deploy the Ceph services: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml + +You can check the status of Ceph via Cephadm on the storage nodes: + +.. code:: bash + + sudo cephadm shell -- ceph -s + +Once the Ceph cluster has finished initialising, run the full +cephadm.yml playbook to perform post-deployment configuration: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml + +The ``cephadm.yml`` playbook imports various other playbooks, which may also be +run individually to perform specific tasks. Note that if you want to deploy +additional services (such as RGWs or ingress) after an initial deployment, you +will need to set ``cephadm_bootstrap`` to true. For example: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml -e cephadm_bootstrap=true + +Configuration generation +------------------------ + +Generate keys and configuration for Kolla Ansible: + +.. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml + +This will generate Ceph keys and configuration under +``etc/kayobe/environments//kolla/config/``, which should be +committed to the configuration. + +This configuration will be used during +``kayobe overcloud service deploy``. diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst new file mode 100644 index 000000000..7865272ff --- /dev/null +++ b/doc/source/configuration/ci-cd.rst @@ -0,0 +1,394 @@ +===== +CI/CD +===== + +Concepts +======== + +The CI/CD system developed for managing Kayobe based OpenStack clouds is composed of four main components; workflows, runners, OpenBao and kayobe automation. + +Firstly, the workflows are files which describe a series of tasks to be performed in relation to the deployed cloud. +These workflows are executed on request, on schedule or in response to an event such as a pull request being opened. + +The workflows are designed to carry out various day-to-day activites such as; running Tempest tests, configuring running services or displaying the change to configuration files if a pull request is merged. +Secondly, in order for the workflows to run against a cloud we would need private runners present within the cloud positioned in such a way they can reach the internal network and public API. +Deployment of private runners is supported by all major providers with the use of community developed Ansible roles. + +Thirdly, OpenBao is used to store secrets on the same virtual machine the runners are hosted within. +This provides a secure way of storing secrets and variables which can be accessed by the runners when executing workflows and ensures that secrets never have to leave the cloud. + +Finally, due to the requirement that we support various different platforms tooling in the form of `Kayobe automation `__ was developed. +This tooling is not tied to any single CI/CD platform as all tasks are a series of shell script and Ansible playbooks which are designed to run in a purpose build kayobe container. + +This is complemented by the use of an Ansible collection known as `stackhpc.kayobe_workflows `__ which aims to provide users with a quick and easy way of customising all workflows to fit within a customer's cloud. + +Currently we support the creation and deployment of workflows for GitHub with Gitlab support being actively worked upon. + +Kayobe Automation +----------------- + +`Kayobe automation `__ is a collection of scripts and tools that automate kayobe operations. +It is deployed and controlled by CI/CD platforms such as GitHub actions and GitLab pipelines. +Kayobe automation provides users with an easy process to perform tasks such as: overcloud service deploy, config-diff, tempest testing, and many more. +With it being integrated into platforms such as GitHub or GitLab it builds a close relationship between the contents of the deployments kayobe configuration and what is currently deployed. +This is because operations such as opening a pull request will trigger a config diff to be generated providing insight on what impact it might have on services or a tempest test that could be scheduled to run daily providing knowledge of faults earlier than before. + +Workflows +--------- + +Kayobe automation has been designed to be independent of any CI/CD platform with all tasks running inside of a purpose built Kayobe container. +However, platform specific workflows need to be deployed to bridge the gap between the contents of Kayobe Automation and these CI/CD platforms. +Workflows are templated for each Kayobe configuration repository, ensuring appropriate workflow input parameters are defined, and any necessary customisations can be applied. +The templating of workflows is offered through the `stackhpc.kayobe_workflows `__ collection which currently supports GitHub workflows. + +Runners +------- + +Runners are purpose built services tied to a particular service vendor such as GitHub Actions or GitLab CI. +These services will listen for jobs which have been tagged appropriately and dispatched to these specific runners. +The runners will need to be deployed using existing roles and playbooks whereby the binary/package is downloaded and registered using a special token. +In some deployments runner hosts can be shared between environments however this is not always true and dedicated hosts will need to be used for each environment you intend to deploy kayobe automation within. + +OpenBao +------- + +OpenBao is recommended when deploying kayobe automation to achieve a simple and secure way of storing secrets. +OpenBao can easily be configured to hold the secrets for all environments and only permit access to the runners which require them utilising different authorisation mechanisms such as GitLab's JWT (JSON Web Token). + +GitHub Actions +================= + +To enable CI/CD where GitHub Actions is used please follow the steps described below starting with the deployment of the runners. + +Runner Deployment +----------------- + +1. Identify a suitable host for hosting the runners. + GitHub runners need to be deployed on a host which has not had Docker deployed using kolla. + This is because GitHub runners cannot provide `network options when running in a container `__. + + Ideally an Infra VM could be used here or failing that the control host. + Wherever it is deployed the host will need access to the :code:`admin_network`, :code:`public_network` and the :code:`pulp registry` on the seed. + +2. Edit the environment's :code:`$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms` + +.. code-block:: ini + + [infra-vms:children] + github-runners + +3. Edit the environment's :code:`$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/inventory/hosts` to define the host(s) that will host the runners. + +.. code-block:: ini + + [github-runners] + prod-runner-01 + +4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/github-runners` + * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined + * `network-interfaces` + * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set + +5. Edit the ``$KAYOBE_CONFIG_PATH/inventory/group_vars/github-runners/runners.yml`` file which will contain the variables required to deploy a series of runners. + Below is a core set of variables that will require consideration and modification for successful deployment of the runners. + The number of runners deployed can be configured by removing and extending the dict :code:`github-runners`. + As for how many runners present three is suitable number as this would prevent situations where long running jobs could halt progress other tasks whilst waiting for a free runner. + You might want to increase the number of runners if usage demands it or new workflows make use of multiple parallel jobs. + + Note :code:`github_registry` and the elements of the dict control the registry settings for pulling and pushing container images used by the workflows. + In the example below the registry settings have been adapted to demonstrate what a shared registry between environments might look like. + This values maybe suitable for your deployment providing all environments can reach the same registry. + If the all of the environments use their own registry and nothing is shared between them then :code:`github_registry` can omitted from the file and the template will expect environment specific secrets and variables to be added to the repository settings. + This is discussed further in the next section. + +.. code-block:: yaml + + --- + runner_user: VM_USER_NAME_HERE + github_account: ORG_NAME_HERE + github_repo: KAYOBE_CONFIG_REPO_NAME_HERE + access_token: "{{ secrets_github_access_token }}" + + default_runner_labels: + - kayobe + - openstack + - "{{ kayobe_environment | default(omit) }}" + + github_registry: + url: pulp.example.com + username: admin + password: ${{ secrets.REGISTRY_PASSWORD }} + share: true + + github_runners: + runner_01: {} + runner_02: {} + runner_03: {} + +6. Obtain a personal access token that would enable the registration of GitHub runners against the `github_account` and `github_repo` defined above. + This token ideally should be `fine-grained personal access token `__ which may require the organisation to enable such tokens beforehand. + Steps can be found `here `__. + The repository permissions for a fine-grained personal access token should be; :code:`Actions: R/W, Administration: R/W, Metadata: R` + Once the key has been obtained, add it to :code:`secrets.yml` under :code:`secrets_github_access_token` + +7. If the host is an actual Infra VM then please refer to upstream :kayobe-doc:`Infrastructure VMs ` documentation for additional configuration and steps. + +8. Run :code:`kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-github-runner.yml` + +9. Check runners have registered properly by visiting the repository's :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` + +10. Repeat the above steps for each environment you intend to deploy runners within. + You can share the fine-grained access token between environments. + +Workflow Deployment +------------------- + +1. Edit :code:`$KAYOBE_CONFIG_PATH/inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github `__. + +2. Run :code:`kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/write-github-workflows.yml` + +3. Add all required secrets and variables to repository either via the GitHub UI or GitHub CLI (may require repository owner) + ++----------------------------------------------------------------------------------+ +| Secrets | ++===================================+==============================================+ +| Single Environment | Multiple Environments | ++-----------------------------------+----------------------------------------------+ +| KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | _KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | ++-----------------------------------+----------------------------------------------+ +| KAYOBE_VAULT_PASSWORD | _KAYOBE_VAULT_PASSWORD | ++-----------------------------------+----------------------------------------------+ +| REGISTRY_PASSWORD | _REGISTRY_PASSWORD | ++-----------------------------------+----------------------------------------------+ +| TEMPEST_OPENRC | _TEMPEST_OPENRC | ++-----------------------------------+----------------------------------------------+ + + +-------------------------------------------------------+ + | VARIABLES | + +====================+==================================+ + | Single Environment | Multiple Environments | + +--------------------+----------------------------------+ + | REGISTRY_URL | _REGISTRY_URL | + +--------------------+----------------------------------+ + | REGISTRY_USERNAME | _REGISTRY_USERNAME | + +--------------------+----------------------------------+ + +Note the above tables shows the secrets and variables one may need to add to GitHub for a successful deployment. +When adding secrets and variables make sure to adhere to the naming standards and ensure the :code:`` is replaced with all supported kayobe environments in uppercase. + +4. Commit and push all newly generated workflows found under :code:`.github/workflows` + +Final Steps +----------- + +Some final steps include the following: running config-diff will require that :code:`.automation.conf/config.sh` contains a list :code:`KAYOBE_CONFIG_VAULTED_FILES_PATHS_EXTRA` of all vaulted files contained within the config. +All such files can be found with :code:`grep -r "$ANSIBLE_VAULT;1.1;AES256" .` though make sure NOT to include `kolla/passwords.yml` and `secrets.yml` +Also make sure tempest has been configured appropriately in :code:`.automation.conf/config.sh` to meet the limitations of a given deployment such as not using a too high of :code:`TEMPEST_CONCURRENCY` value and that overrides and load/skips lists are correct. +Finally, once all the workflows and configuration has been pushed and reviewed you can build a kayobe image using the `Build Kayobe Docker Image` workflow. Once it is successfully built and pushed to a container registry, other workflows can be used. + +Sometimes the kayobe docker image must be rebuilt the reasons for this include but are not limited to the following; + + * Change :code:`$KAYOBE_CONFIG_PATH/ansible/requirements.yml` + * Change to requirements.txt + * Update Kayobe + * Update kolla-ansible + * UID/GID collision when deploying workflows to a new environment + * Prior to deployment of new a OpenStack release + +GitLab Pipelines +================ + +To enable CI/CD where GitLab Pipelines is used please follow the steps described below starting with the deployment of the runners. + +Runner Deployment +----------------- + +1. Identify a suitable host for hosting the runners. + Ideally an infra-vm would be deployed to allow for easily compartmentalising the runners from the rest of the environment. + 8 VCPUs and 16GB of RAM is recommended for the guest machine however this may need to be adjusted for larger deployments. + Whether the host is in an infra-vm or not it will need access to the :code:`admin_network` or :code:`provision_oc_network`, :code:`public_network` and the :code:`pulp registry` on the seed. + The steps will assume that an infra-vm will be used for the purpose of hosting the runners. + +2. Edit the environment's :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/hosts` to define the host(s) that will host the runners. + +.. code-block:: ini + + [gitlab-runners] + gitlab-runner-01 + +4. Provide all the relevant Kayobe :code:`group_vars` for :code:`gitlab-runners` under :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/gitlab-runners` + * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined + * `network-interfaces` + * `allocated IPs` + +5. Edit the ``${KAYOBE_CONFIG_PATH}/inventory/group_vars/gitlab-runners/runners.yml`` file which will contain the variables required to deploy a series of runners. + Below is an example of how GitLab runners can be configured for deployment. + In this example we have two runners, one for production and one for staging and will both be deployed on the same host. + This might not be possible for all deployments as multiple environments may require different runners as no single runner can serve all environments. + Note a GitLab runner can run multiple jobs concurrently so deploying a single runner per environment is recommended. + +.. code-block:: yaml + + --- + gitlab_runner_coordinator_url: "https://gitlab.example.com" + gitlab_runner_runners: + - name: "Kayobe Automation Runner [Production] #1" + executor: docker + docker_image: 'alpine' + token: "{{ secrets_gitlab_production_runner_token }}" + env_vars: + - "GIT_CONFIG_COUNT=1" + - "GIT_CONFIG_KEY_0=safe.directory" + - "GIT_CONFIG_VALUE_0=*" + tags: + - kayobe + - openstack + - production + docker_volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - "/opt/.docker/config.json:/root/.docker/config.json:ro" + - "/cache" + extra_configs: + runners.docker: + network_mode: host + - name: "Kayobe Automation Runner [Staging] #1" + executor: docker + docker_image: 'alpine' + token: "{{ secrets_gitlab_staging_runner_token }}" + env_vars: + - "GIT_CONFIG_COUNT=1" + - "GIT_CONFIG_KEY_0=safe.directory" + - "GIT_CONFIG_VALUE_0=*" + tags: + - kayobe + - openstack + - staging + docker_volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - "/opt/.docker/config.json:/root/.docker/config.json:ro" + - "/cache" + extra_configs: + runners.docker: + network_mode: host + +6. Obtain a runner token for each runner that is required for deployment. + This token can be obtained by visiting the GitLab project -> Settings -> CI/CD -> Runners -> New project runner -> Complete the form including any tags used by the runners such as kayobe, openstack and environment_name. + Once the token has been obtained, add it to :code:`secrets.yml` under :code:`secrets_gitlab_production_runner_token` and :code:`secrets_gitlab_staging_runner_token` + +7. Deploy the infra-vm + +.. code-block:: bash + + kayobe infra vm provision --limit gitlab-runner-01 + +8. Perform a host configure against the infra-vm + +.. code-block:: bash + + kayobe infra vm host configure --limit gitlab-runner-01 + +9. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-gitlab-runner.yml` + +10. Check runners have registered properly by visiting the repository's :code:`CI/CD` tab -> :code:`Runners` + +11. The contents of :code:`/opt/.docker/config.json` on the runner should be added to GitLab CI/CD settings as a sercret variable if GitLab version permits otherwise variable is fine. + This is required to allow the runners to pull images from the registry. + Visit the GitLab project -> Settings -> CI/CD -> Variables -> Add a new variable with the key :code:`DOCKER_AUTH_CONFIG` and the value of the contents of :code:`/opt/.docker/config.json` + +OpenBao Deployment +------------------ + +OpenBao must be installed on the same host as the runners. +If you have multiple environments that each have the own runners then OpenBao must be installed on each host. +However, if you have a single host that is shared between environments then OpenBao only needs to be installed once and can be achieved by running the following playbook. + +.. code-block:: bash + + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-openbao-kayobe-automation.yml + +.. note:: + + If you are sharing OpenBao between environments then you will need to rerun the playbook under each environment to ensure that the correct secrets are available to the runners. + You may use :code:`--tags add_secrets` to skip the deployment within other environments. + For this to work you will need to copy :code:`vault/kayobe-automation-keys.json` from the first environment to the other environments in addition to copying the host definition of the gitlab runner add network IP. + +Once the above playbook has been applied you need to grab the root token from :code:`vault/kayobe-automation-keys.json` as you will need this to enable JWT support. +This would also be an opportune time to encrypt the :code:`vault/kayobe-automation-keys.json` to protect the contents. + +.. code-block:: bash + + ansible-vault encrypt vault/kayobe-automation-keys.json --vault-password-file ~/.vault.password + +In order to enable JWT support the following steps must be carried out within the openbao container on the runner host. + +1. SSH into the runner host + +2. Run :code:`sudo docker exec -it bao sh` + +3. Run :code:`export BAO_ADDR=http://127.0.0.1:8200` + +4. Run :code:`bao login` and use root token + +5. Run the following to enable and configure JWT support + +.. note:: + + The following steps are an example and should be adapted to suit your deployment. + For example project_id within the gitlab role will need ID of the project that the runners are registered against. + This can acquired by visiting the project -> Settings -> General -> General project settings -> Project ID. + +.. code-block:: bash + + bao auth enable jwt + bao policy write kayobe-automation - <`__. +Following the instructions in the documentation will allow you to customise the workflows to fit within your deployment. +If using multiple environments ensure that :code:`gitlab_kayobe_environments` is updated to reflect all environments present in the deployment. +Also consider the impact runbooks might have as the runbooks are designed with a particular cloud in mind and may not be suitable for all deployments such as hyperconverged deployments with Ceph on hypervisors. + +2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-gitlab-pipelines.yml` + +3. Commit and push all newly generated pipelines found under root of the repository. + +Things to consider +================== + +- Adjust General Pipeline settings by visiting the project -> Settings -> CI/CD -> General pipelines + - Disable :code:`Public Pipelines` + - Disable :code:`Auto-cancel redundant pipelines` + - Disable :code:`Prevent outdated deployment jobs` + - Increase :code:`Timeout` to :code:`12h` + +- Disable Auto DevOps in the GitLab project settings by visiting the project -> Settings -> CI/CD -> Auto DevOps -> Disable Auto DevOps + +Sometimes the kayobe docker image must be rebuilt. The reasons for this include but are not limited to the following; + + * Change :code:`$KAYOBE_CONFIG_PATH/ansible/requirements.yml` + * Change to requirements.txt + * Update Kayobe + * Update kolla-ansible + * Prior to deployment of new a OpenStack release diff --git a/doc/source/configuration/cloudkitty.rst b/doc/source/configuration/cloudkitty.rst new file mode 100644 index 000000000..084fe45e4 --- /dev/null +++ b/doc/source/configuration/cloudkitty.rst @@ -0,0 +1,131 @@ +========== +CloudKitty +========== + +Configuring in kayobe-config +============================ + +By default, CloudKitty uses Gnocchi and Ceilometer as the collector and fetcher +backends. Unless the system has a specific reason not to, we recommend instead +using Prometheus as the backend for both. The following instructions explain +how to do this. Also, see the :kolla-ansible-doc:`Kolla Ansible docs on CloudKitty +` +for more details. + +Enable CloudKitty and disable InfluxDB, as we are using OpenSearch as the +storage backend. Set the following in ``kolla.yml``: + +.. code-block:: yaml + + kolla_enable_cloudkitty: true + # Explicitly disable influxdb as we are using OpenSearch as the CloudKitty backend + kolla_enable_influxdb: false + +Set Prometheus as the backend for both the collector and fetcher, and +Elasticsearch as the storage backend. Note that our fork of CloudKitty is +patched so that the CloudKitty Elasticsearch V2 storage backend will also work +with an OpenSearch cluster. Proper support for the V2 OpenSearch storage +backend is still pending in Kolla-Ansible `here +`__. Set the +following in ``kolla/globals.yml``: + +.. code-block:: yaml + + cloudkitty_collector_backend: prometheus + cloudkitty_fetcher_backend: prometheus + cloudkitty_storage_backend: elasticsearch + +The default collection period is one hour, which is likely too long for most +systems as CloudKitty charges by the **entire** collection period if any usage +is seen within this timeframe. This is regardless of actual usage, meaning that +even one minute will be charged as a full hour's usage. As a result, it is +recommended to adjust the collection interval, ``period`` (in units of +seconds), appropriately (e.g. ten minutes). Furthermore, when using Prometheus +as the collector, you need to change the ``scope_key`` to match the metrics +provided by the Prometheus OpenStack Exporter. Both of these can be achieved by +setting the following in ``kolla/config/cloudkitty.conf``: + +.. code-block:: console + + [collect] + scope_key = tenant_id + period = 600 + +You will need to configure which metrics CloudKitty should track. The following +example, set in ``kolla/config/cloudkitty/metrics.yml``, will track for VM flavors and +the total utilised volume. + +.. code-block:: yaml + + metrics: + openstack_nova_server_status: + alt_name: instance + groupby: + - uuid + - user_id + - tenant_id + metadata: + - flavor_id + - name + mutate: MAP + mutate_map: + 0.0: 1.0 # ACTIVE + 11.0: 1.0 # SHUTOFF + 12.0: 1.0 # SUSPENDED + 16.0: 1.0 # PAUSED + unit: instance + openstack_cinder_limits_volume_used_gb: + alt_name: storage + unit: GiB + groupby: + - tenant_id + +If your system had Monasca deployed in the past, you likely have some +relabelled attributes in the Prometheus OpenStack exporter. To account for +this, you should either remove the custom relabelling (in +``kolla/config/prometheus.yml``) or change your ``metrics.yml`` to use the +correct attributes. + +Post-configuration with openstack-config +======================================== + +This is an example `openstack-config +`__ setup to create mappings for +the metrics configured above. Note that the costs are scaled for the ten minute +collection period, e.g. a flavor with 1 VCPU will cost 1 unit per hour. + +.. code-block:: yaml + + # Map flavors based on VCPUs + openstack_ratings_hashmap_field_mappings: + - service: instance + name: flavor_id + mappings: + - value: '1' # tiny compute flavor (1 vcpu) with an OpenStack flavor ID of 1 + cost: 0.1666666666666666 + type: flat + - value: '2' # small compute flavor (2 vcpus) with an OpenStack flavor ID of 2 + cost: 0.3333333333333333 + type: flat + - value: '3' # medium compute flavor (3 vcpus) with an OpenStack flavor ID of 3 + cost: 0.5 + type: flat + - value: '4' # large compute flavor (4 vcpus) with an OpenStack flavor ID of 4 + cost: 0.6666666666666666 + type: flat + - value: '5' # xlarge compute flavor (8 vcpus) with an OpenStack flavor ID of 5 + cost: 1.3333333333333333 + type: flat + - value: '6' # tiny 2 compute flavor (2 vcpus) with an OpenStack flavor ID of 6 + cost: 0.3333333333333333 + type: flat + + # Map volumes based on GB + openstack_ratings_hashmap_service_mappings: + - service: storage + cost: 0.16666666666666666 + type: flat + +See the `OpenStack CloudKitty Ratings role +`__ +for more details. diff --git a/doc/source/configuration/firewall.rst b/doc/source/configuration/firewall.rst new file mode 100644 index 000000000..37eb857dd --- /dev/null +++ b/doc/source/configuration/firewall.rst @@ -0,0 +1,462 @@ +.. _firewall: + +======== +Firewall +======== + +StackHPC Kayobe configuration provides a standardised firewalld configuration. +The configuration uses the :kayobe-doc:`firewall +` host configuration +functionality of Kayobe. + +The firewall configuration is provided in +``etc/kayobe/inventory/group_vars/all/firewall``. + +Enabling StackHPC firewalld rules +================================= + +The standardised firewalld configuration is not enabled by default and must be +actively opted into. To do so, make the following changes in +``etc/kayobe/.yml`` (or +``etc/kayobe/environments//.yml`` if environments are being +used). + +Controller firewalld Configuration +---------------------------------- + +.. code-block:: yaml + :caption: ``controllers.yml`` + + ############################################################################### + # Controller node firewalld configuration. + + # Whether to install and enable firewalld. + controller_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + controller_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + controller_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Compute firewalld Configuration +------------------------------- + +.. code-block:: yaml + :caption: ``compute.yml`` + + ############################################################################### + # Compute node firewalld configuration. + + # Whether to install and enable firewalld. + compute_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + compute_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + compute_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Storage firewalld Configuration +------------------------------- + +.. code-block:: yaml + :caption: ``storage.yml`` + + ############################################################################### + # Storage node firewalld configuration. + + # Whether to install and enable firewalld. + storage_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + storage_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + storage_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + storage_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +If using RADOS Gateway, you can customise ``stackhpc_ceph_firewalld_radosgw_port`` to match +the ``rgw_frontend_port`` as documented in :ref:`RGWs-with-Ceph`. + +Monitoring firewalld Configuration +---------------------------------- + +.. code-block:: yaml + :caption: ``monitoring.yml`` + + ############################################################################### + # Monitoring node firewalld configuration. + + # Whether to install and enable firewalld. + monitoring_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + monitoring_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + monitoring_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + monitoring_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Infrastructure VM firewalld Configuration +----------------------------------------- + +The standard firewalld configuration has rules for wazuh-manager and Ansible +control host Infrastructure VMs. + +.. code-block:: yaml + :caption: ``infra-vms.yml`` + + ############################################################################### + # Infrastructure VM node firewalld configuration + + # Whether to install and enable firewalld. + infra_vm_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + infra_vm_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + infra_vm_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + infra_vm_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +When configuring wazuh-manager, remember to set ``wazuh_dashboard_net_name`` if you have customised +the network where the Wazuh dashboard is exposed. + +Seed firewalld Configuration +---------------------------- + +.. code-block:: yaml + :caption: ``seed.yml`` + + ############################################################################### + # Seed node firewalld configuration. + + # Whether to install and enable firewalld. + seed_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + seed_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + seed_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + seed_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +Seed Hypervisor firewalld Configuration +--------------------------------------- + +.. code-block:: yaml + :caption: ``seed_hypervisor.yml`` + + ############################################################################### + # Seed hypervisor node firewalld configuration. + + # Whether to install and enable firewalld. + seed_hypervisor_firewalld_enabled: true + + # A list of zones to create. Each item is a dict containing a 'zone' item. + seed_hypervisor_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + + # A firewalld zone to set as the default. Default is unset, in which case + # the default zone will not be changed. + # Predefined zones are listed here: + # https://firewalld.org/documentation/zone/predefined-zones.html + seed_hypervisor_firewalld_default_zone: trusted + + # A list of firewall rules to apply. Each item is a dict containing + # arguments to pass to the firewalld module. Arguments are omitted if not + # provided, with the following exceptions: + # - offline: true + # - permanent: true + # - state: enabled + seed_hypervisor_firewalld_rules: "{{ stackhpc_firewalld_rules }}" + +The following workaround is needed to prevent VM network traffic from being blocked: + +.. code-block:: yaml + :caption: ``seed_hypervisor.yml`` + + seed_hypervisor_sysctl_parameters: + # By default this is 1, which causes layer 2 traffic flowing through Linux + # bridges to pass through iptables. This blocks traffic from VMs (seed, wazuh) to + # the Internet. + net.bridge.bridge-nf-call-iptables: 0 + +The hope is that in the future this can be replaced by some additional firewalld configuration. + +Kolla-Ansible configuration +--------------------------- + +Ensure Kolla Ansible opens up ports in firewalld for services on the public +API network: + +.. code-block:: yaml + :caption: ``etc/kayobe/kolla/globals.yml`` + + enable_external_api_firewalld: true + external_api_firewalld_zone: "{{ public_net_name | net_zone }}" + +Network configuration +--------------------- + +Ensure every network in ``networks.yml`` has a zone defined. The standard +configuration is to set the internal network zone to ``trusted`` and every +other zone to the name of the network. See +``etc/kayobe/environments/ci-multinode/networks.yml`` for a practical example. + +Custom rules +------------ + +Custom firewalld rules can be added to ``stackhpc_firewalld_rules_extra`` + +The variable is a list of firewall rules to apply. Each item is a dictionary +containing arguments to pass to the firewalld module. The variable can be +defined as a group var or host var in the kayobe inventory. + +The structure of custom rules is different from the default rules. Custom rules +use the firewalld Ansible module format. Arguments are omitted if not provided, +with the following exceptions: + +* ``offline: true`` +* ``permanent: true`` +* ``state: enabled`` + +The main differences are that the ``zone`` argument is mandatory, and the +``network`` argument is not. + +The example below would enable SSH in the ``provision_oc`` zone, and disable +UDP port 1000 in the ``admin_oc`` zone for the Wazuh manager Infrastructure +VM: + +.. code-block:: yaml + :caption: ``etc/kayobe/inventory/group_vars/wazuh_manager/firewall`` + + stackhpc_firewalld_rules_extra: + - service: ssh + zone: "{{ provision_oc_net_name | net_zone }}" + state: enabled + - port: 1000/udp + zone: "{{ admin_oc_net_name | net_zone }}" + state: disabled + +Extra rules have higher precedence than the default rules but are not +validated before being applied. Use with caution. If you need to add a custom +rule, consider adding it to the default rule list with an appropriate boolean +condition, and where possible merge your changes back into upstream SKC. + +Validation +---------- + +The ``kayobe configuration dump`` command can be used to view all the rules +that will be applied to a host. + +.. code-block:: bash + + kayobe configuration dump --var-name stackhpc_firewalld_rules --limit + +A shorter version, ``stackhpc_firewalld_rules_debug`` prints the rules in a +simplified format: + +.. code-block:: bash + + kayobe configuration dump --var-name stackhpc_firewalld_rules_debug --limit + +If the commands above print a template, rather than a list of rules, the +configuration may be invalid. The ``kayobe configuration dump`` command can be +used on other variables such as ``stackhpc_firewalld_rules_default`` or +``stackhpc_*_firewalld_rules_template`` to debug the configuration. See the +`How it works`_ section for more details. + +It can be useful to print the active ports on each type of host, to create +rules for running services. The internal network is currently left open. The +below command will print all other open ports: + +.. code-block:: bash + + ss -lntpu | grep --invert-match '' + +It is strongly recommended that you dry-run the changes using ``--diff`` and +``--check`` before applying to a production system: + +.. code-block:: bash + :caption: ``Overcloud diff example`` + + kayobe overcloud host configure -t firewall --diff --check + +Baseline checks +^^^^^^^^^^^^^^^ + +Before applying, it is a good idea to take note of any actively firing alerts +and run Tempest to gather a baseline. See the :doc:`Tempest +` page for more details. + +Applying changes +---------------- + +Before applying these changes, you should be completely sure you are not going +to lock yourself out of any hosts. If you are deploying these changes to a test +environment, consider setting a password on the stack user so that you can +access the host through a BMC or other virtual console. + +The following Kayobe command can be used to set a password on all overcloud +hosts: + +.. code-block:: bash + + kayobe overcloud host command run --command "echo 'stack:super-secret-password' | sudo chpasswd" --show-output + +The ``firewalld-watchdog.yml`` playbook can be used to set up a timer that +disables the firewalld service after a period of time (default 600s). It should +be used as follows: + +.. code-block:: bash + + # Enable the watchdog BEFORE applying the firewall configuration + kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l + + # Disable the watchdog after applying the firewall configuration + kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l -e firewalld_watchdog_state=absent + +If the firewall rules block connectivity, the second playbook run (disabling +the watchdog) will fail. You will still be able to get in after the watchdog +triggers. Remember to disable the watchdog when you are finished, otherwise the +firewall will be disabled! + +Changes should be applied to controllers one at a time to ensure connectivity +is not lost. + +Once you are sure you know what you are doing, use the ``kayobe * host +configure`` commands to apply the firewall changes: + +.. code-block:: bash + + # For Seed Hypervisor hosts + kayobe seed hypervisor host configure -t network,firewall + # For Seed hosts + kayobe seed host configure -t network,firewall + # For Infrastructure VM hosts + kayobe infra vm host configure -t network,firewall + # For the First Controller + kayobe overcloud host configure -t network,firewall -l controllers[0] + # For the Second Controller + kayobe overcloud host configure -t network,firewall -l controllers[1] + # For the Third Controller + kayobe overcloud host configure -t network,firewall -l controllers[2] + # For the rest of the Overcloud hosts + kayobe overcloud host configure -t network,firewall + +Debugging +--------- + +To test the changes, first check for any firing alerts, then try simple smoke +tests (create a VM, list OpenStack endpoints etc.), then run Tempest. + +If the firewall configuration is causing errors, it is often useful to log +blocked packets. + +.. code-block:: bash + + sudo sed -i s/LogDenied=off/LogDenied=all/g /etc/firewalld/firewalld.conf + sudo systemctl restart firewalld + +Dropped packets will be logged to ``dmesg``. + +How it works +============ + +The standard firewall rule configuration is stored in +``etc/kayobe/inventory/group_vars/all/firewall``. + +The file contains sections for different host groups. There are sections for: + +* Common (all hosts) +* Controllers +* Compute +* Storage +* Monitoring +* Wazuh Manager Infrastructure VM +* Ansible Control host Infrastructure VM +* Seed +* Seed Hypervisor + +Each of these sections contains a template. The template is made of sets of +rules. The rules can then be enabled and disabled in sets, based on properties +of the cloud. For example, if ``kolla_enable_designate`` is true, a set of +rules will be enabled in ``stackhpc_controller_firewalld_rules_template``. + +The templates are combined into a single list, +``stackhpc_firewalld_rules_template``. Templates are selected according to the +host's group membership, as well as a set of common rules, which is enabled for +all hosts. + +The rules are then formatted into a single list of the enabled default rules: +``stackhpc_firewalld_rules_default``. The Rules are manipulated to reduce +duplication. When no zone is specified in a rule template, it is inferred from +the network. They are also validated. Conflicting rules will result in an +error. Non-applicable rules are dropped. + +The default rules are combined with any extra rules defined for the deployment. +The complete set of controller firewalld rules is +``stackhpc_firewalld_rules``. diff --git a/doc/source/configuration/host-images.rst b/doc/source/configuration/host-images.rst new file mode 100644 index 000000000..530068c4d --- /dev/null +++ b/doc/source/configuration/host-images.rst @@ -0,0 +1,102 @@ +.. _host-images: + +=========== +Host Images +=========== + +Pulling host images +=================== + +StackHPC provides pre-built overcloud host images through Ark, which can be +consumed using the configuration provided by this repository. + +When configured, an image will be downloaded to the seed during the +``kayobe seed service deploy`` step, and subsequently deployed using bifrost +with ``kayobe overcloud provision``. + +To use these images, set ``stackhpc_download_overcloud_host_images`` to true +in ``etc/kayobe/stackhpc-overcloud-host-images.yml``. + +Currently, images exist for the following operating systems: + +* Rocky Linux 9 +* Ubuntu Jammy 22.04 + +The image to download is selected automatically using the ``os_distribution`` +and ``os_release`` variables. These images are versioned and a variable for +each OS is stored in ``pulp-host-image-versions.yml``. + +This content requires the same set of credentials as is used for other +release train content. + +The Ark pulp credentials issued by StackHPC should be configured in +``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password: + +.. code-block:: yaml + + stackhpc_release_pulp_username: + stackhpc_release_pulp_password: + +Building host images +==================== + +StackHPC Kayobe configuration provides configuration for some standard +overcloud host images, built using the :kayobe-doc:`overcloud DIB +` functionality of Kayobe. + +The overcloud DIB configuration is provided in +``etc/kayobe/stackhpc-overcloud-dib.yml``. It is not used by default, and must +be actively opted into. This can be done as follows: + +.. code-block:: yaml + :caption: ``etc/kayobe/overcloud-dib.yml`` + + overcloud_dib_build_host_images: true + + overcloud_dib_host_images: + - "{{ stackhpc_overcloud_dib_host_image }}" + +The image name is configured via ``stackhpc_overcloud_dib_name``, and is +``deployment_image`` by default. + +The list of DIB elements is configured via ``stackhpc_overcloud_dib_elements``. +The default value depends on the ``os_distribution`` variable. See the YAML +file for details. + +The DIB environment variables are configured via +``stackhpc_overcloud_dib_env_vars``. See the YAML file for details. + +A list of packages to install is configured via +``stackhpc_overcloud_dib_packages``. + +By default, a UEFI-compatible image is built that uses separate LVM volumes for +different mount points. This is done to pass Centre for Internet Security (CIS) +partition benchmarks. The block device YAML configuration is configured via +``stackhpc_overcloud_dib_block_device_config_uefi_lvm``. + +The 3 partitions are: + +* p0: EFI ESP bootloader +* p1: EFI BSP +* p2: LVM PV (``rootpv``) + +The LVM Logical Volumes are: + +============== ================== ========= +LV Mount point Size (GB) +============== ================== ========= +``lv_root`` ``/`` 5G +``lv_tmp`` ``/tmp`` 1G +``lv_var`` ``/var`` 1G +``lv_var_tmp`` ``/var/tmp`` 1G +``lv_log`` ``/var/log`` 1G +``lv_audit`` ``/var/log/audit`` 128M +``lv_home`` ``/home`` 128M +============== ================== ========= + +A compatible LVM configuration is provided, and covered in :ref:`lvm`. +The Logical Volumes in the image are defined with small sizes, with the +intention that they will be grown after provisioning. + +For RedHat family distributions, Dracut modules are configured via +``stackhpc_overcloud_dib_dracut_enabled_modules_default_config``. diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst new file mode 100644 index 000000000..576dfe63d --- /dev/null +++ b/doc/source/configuration/index.rst @@ -0,0 +1,25 @@ +=================== +Configuration Guide +=================== + +This configuration guide describes how to use this configuration, and covers +the various features provided. + +.. toctree:: + :maxdepth: 1 + + release-train + host-images + lvm + cephadm + monitoring + openbao + vault + wazuh + walled-garden + security-hardening + firewall + magnum-capi + ci-cd + cloudkitty + ipa diff --git a/doc/source/configuration/ipa.rst b/doc/source/configuration/ipa.rst new file mode 100644 index 000000000..5ff239030 --- /dev/null +++ b/doc/source/configuration/ipa.rst @@ -0,0 +1,35 @@ +.. _ipa: + +========================= +Ironic Python Agent (IPA) +========================= + +Release Train IPA +================= + +StackHPC provides prebuilt Ironic Python Agent (IPA) images in Release Train +through Ark. + +These images are built in CI using a GitHub workflow and are configured in this +repository. See :kayobe-doc:`Kayobe documentation +` for more details on IPA. + +Release Train IPA images are used by Bifrost and Overcloud Ironic by default in +the StackHPC Kayobe Configuration, and are pulled in to the Ironic Inspector +when running ``kayobe seed service deploy`` for Bifrost or ``kayobe overcloud +post configure`` for Overcloud Ironic. This behaviour can be disabled in +`stackhpc-ipa-images.yml`: + +.. code-block:: yaml + + stackhpc_ipa_image_bifrost_enabled: false + stackhpc_ipa_image_overcloud_enabled: false + +You can also override the distribution version pulled in during deployment, +to do this you can change ``stackhpc_ipa_image_version`` to be the opposite +distribution. For example, the case of switching to Ubuntu 22.04 on a Rocky 9 +cloud: + +.. code-block:: yaml + + stackhpc_ipa_image_version: "{{ stackhpc_ubuntu_jammy_ipa_image_version }}" diff --git a/doc/source/configuration/lvm.rst b/doc/source/configuration/lvm.rst new file mode 100644 index 000000000..bb2b7862c --- /dev/null +++ b/doc/source/configuration/lvm.rst @@ -0,0 +1,98 @@ +.. _lvm: + +=== +LVM +=== + +StackHPC Kayobe configuration provides Logical Volume Manager (LVM) +configuration that is compatible with the included :ref:`host-images` +configuration. The configuration uses the :kayobe-doc:`LVM +` host configuration functionality of +Kayobe. + +The LVM configuration is provided in +``etc/kayobe/inventory/group_vars/all/stackhpc/lvm``. This allows configuration +variables to be overridden on a per-group or per-host basis (which would not be +possible for an "extra variable" in ``etc/kayobe/*.yml``). This configuration +is not used by default, and must be actively opted into. This can be done as +follows: + +.. code-block:: yaml + + controller_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +This will configure the standard set of logical volumes for the ``rootvg`` +volume group on controller hosts. + +The disks in this volume group are configured via +``stackhpc_lvm_group_rootvg_disks``, and by default this contains a single +disk, matched by a partition label of ``root`` (as used in the standard +:ref:`host-images`). + +The size of each LV is configurable via the following variables: + +.. code-block:: yaml + + # StackHPC LVM lv_swap LV size. + stackhpc_lvm_lv_swap_size: 16g + + # StackHPC LVM lv_root LV size. + stackhpc_lvm_lv_root_size: 50g + + # StackHPC LVM lv_tmp LV size. + stackhpc_lvm_lv_tmp_size: 10g + + # StackHPC LVM lv_var LV size. + stackhpc_lvm_lv_var_size: 20g + + # StackHPC LVM lv_var_tmp LV size. + stackhpc_lvm_lv_var_tmp_size: 2g + + # StackHPC LVM lv_log LV size. + stackhpc_lvm_lv_log_size: 20g + + # StackHPC LVM lv_audit LV size. + stackhpc_lvm_lv_audit_size: 10g + + # StackHPC LVM lv_home LV size. + stackhpc_lvm_lv_home_size: 10g + +Additional LVs may be configured via ``stackhpc_lvm_group_rootvg_lvs_extra``. A +common requirement is to have ``/var/lib/docker/`` mounted on a separate LV, +so this has been made convenient to achieve: + +.. code-block:: yaml + + stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + + # StackHPC LVM lv_docker LV size. + stackhpc_lvm_lv_docker_size: 100%FREE + +It may be desirable to use a lower percentage of the free space, in case +another LV needs to be grown at a later date. + +Growroot playbook +================= + +A ``growroot.yml`` custom playbook is provided that can be used to grow the +partition and LVM Physical Volume (PV) of the root Volume Group (VG). This +allows for expansion of Logical Volumes (LVs) in that VG. + +The following variables may be used to configure the playbook: + +``growroot_group`` + Host pattern against which to target the playbook. Default is ``overcloud``. +``growroot_vg`` + Name of the VG containing the PV to grow. Default is ``rootvg`` to match the + standard :ref:`host image configuration `. + +This playbook may be used as a host configure pre hook, e.g. for overcloud +hosts: + +.. code-block:: console + + mkdir -p $KAYOBE_CONFIG_PATH/hooks/overcloud-host-configure/pre.d + cd $KAYOBE_CONFIG_PATH/hooks/overcloud-host-configure/pre.d + ln -s ../../../ansible/growroot.yml 30-growroot.yml diff --git a/doc/source/configuration/magnum-capi.rst b/doc/source/configuration/magnum-capi.rst new file mode 100644 index 000000000..7885fed6a --- /dev/null +++ b/doc/source/configuration/magnum-capi.rst @@ -0,0 +1,117 @@ +========================= +Magnum Cluster API Driver +========================= + +A new driver for Magnum has been written which is an alternative to Heat (as Heat gets phased out due to maintenance burden) and instead uses the Kubernetes `Cluster API project `_ to manage the OpenStack infrastructure required by Magnum clusters. The idea behind the Cluster API (CAPI) project is that infrastructure is managed using Kubernetes-style declarative APIs, which in practice means a set of Custom Resource Definitions (CRDs) and Kubernetes `operators `_ to translate instances of those custom Kubernetes resources into the required OpenStack API resources. These same operators also handle resource reconciliation (i.e. when the Kubernetes custom resource is modified, the operator will make the required OpenStack API calls to reflect those changes). + +The new CAPI driver and the old Heat driver are compatible and can both be active on the same deployment, and the decision of which driver is used for a given template depends on certain parameters inferred from the Magnum cluster template. For the new driver, these parameters are ``{'server_type': 'vm', 'os': 'ubuntu', 'coe': kubernetes'}``. Drivers can be enabled and disabled using the ``disabled_drivers`` parameter in the ``[drivers]`` section of ``magnum.conf``. + +Deployment Prerequisites +======================== + +The Cluster API architecture relies on a CAPI management cluster in order to run the aforementioned Kubernetes operators which interact directly with the OpenStack APIs. The two requirements for this management cluster are: + +1. It must be capable of reaching the public OpenStack APIs. + +2. It must be reachable from the control plane nodes (either controllers or dedicated network hosts) on which the Magnum containers are running (so that the Magnum can reach the IP listed in the management cluster's ``kubeconfig`` file). + +For testing purposes, a simple `k3s `_ cluster would suffice. For production deployments, the recommended solution is to instead set up a separate HA management cluster in an isolated OpenStack project by leveraging the CAPI management cluster configuration used in `Azimuth `_. This approach will provide a resilient HA management cluster with a standard set of component versions that are regularly tested in Azimuth CI. +The general process for setting up this CAPI management cluster using Azimuth tooling is described here, but the `Azimuth operator documentation `_ should be consulted for additional information if required. + +The diagram below shows the general architecture of the CAPI management cluster provisioned using Azimuth tooling. It consists of a Seed VM (a terraform-provisioned OpenStack VM) running a small k3s cluster (which itself is actually a CAPI management cluster but only for the purpose of managing the HA cluster) as well as a HA management cluster made up of (by default) 3 control plane VMs and 3 worker VMs. This HA cluster runs the various Kubernetes components responsible for managing Magnum tenant clusters. + +.. image:: /_static/images/capi-architecture-diagram.png + :width: 100% + +The setup and configuration of a CAPI management cluster using Azimuth tooling follow a pattern that should be familiar to Kayobe operators. There is an 'upstream' `azimuth-config `_ repository which contains recommended defaults for various configuration options (equivalent to stackhpc-kayobe-config), and then each client site will maintain an independent copy of this repository which will contain site-specific configuration. Together, these upstream and site-specific configuration repositories can set or override Ansible variables for the `azimuth-ops `_ Ansible collection, which contains the playbooks required to deploy or update a CAPI management cluster (or a full Azimuth deployment). + +In order to deploy a CAPI management cluster for use with Magnum, first create a copy of the upstream Azimuth config repository in the client's GitHub/GitLab. To do so, follow the instructions found in the `initial repository setup `_ section of the Azimuth operator docs. The site-specific repository should then be encrypted following `these instructions `_ to avoid leaking any secrets (such as cloud credentials) that will be added to the configuration later on. + +Next, rather than copying the ``example`` environment as recommended in the Azimuth docs, instead copy the ``capi-mgmt-example`` environment and give it a suitable site-specific name: + +.. code-block:: bash + + cp -r ./environments/capi-mgmt-example ./environments/ + +By default, both the seed VM name and the CAPI cluster VM names will be derived by prefixing the environment name with `capi-mgmt-` so naming the environment after the cloud (e.g. `sms-lab-prod`) is recommended. + +Having created this concrete environment to hold site-specific configuration, next open ``environments//inventory/group-vars/all/variables.yml`` and, at a minimum, set the following options to the desired values for the target cloud: + +.. code-block:: yaml + + infra_external_network_id: + infra_flavor_id: + capi_cluster_control_plane_flavor: + capi_cluster_worker_flavor: + +The comments surrounding each option in the ``variables.yml`` provide some tips on choosing sensible values (e.g. resource requirements for each flavor). In most cases, other configuration options can be left blank since they will fall back to the upstream defaults; however, if the default configuration is not suitable, the roles in `ansible-collection-azimuth-ops `_ contain a range of config variables which can be overridden in ``variables.yml`` as required. In particular, the `infra role variables `_ are mostly relevant to the seed VM configuration, and the `capi_cluster role variables `_ are relevant for HA cluster config. + +.. note:: + + One important distinction between azimuth-config and stackhpc-kayobe-config is that the environments in azimuth-config are `layered`. This can be seen in the ``ansible.cfg`` file for each environment, which will contain a line of the form ``inventory = `` showing the inheritance chain for variables defined in each environment. See `these docs `_ for more details. + +In addition to setting the required infrastructure variables, Terraform must also be configured to use a remote state store (either GitLab or S3) for the seed VM state. To do so, follow the instructions found `here `_. + +The HA cluster also contains a deployment of `kube-prometheus-stack `_ for monitoring and alerting. To send the cluster alerts to Slack, the ``alertmanager_config_slack_webhook_url`` variable should be set in ``environments//inventory/group-vars/all/secrets.yml``. If the repository was encrypted correctly above, this file will automatically be encrypted before a git push. Run ``git-crypt status -e`` to verify that this file is included in the encrypted list before git-committing the webhook URL. + +The final step before beginning deployment of the CAPI management cluster is to provide some cloud credentials. It is recommended that the CAPI management cluster is deployed in an isolated OpenStack project. After creating the target project (preferably using `openstack-config `_), generate an application credential for the project using the Identity tab in Horizon and then download the corresponding ``clouds.yaml`` file and place it in ``environments//clouds.yaml``. + +To deploy the CAPI management cluster using this site-specific environment, run + +.. code-block:: bash + + # Install or update the local Ansible Python venv + ./bin/ensure-venv + + # Activate the environment + source bin/activate + + # Install or update Ansible dependencies + ansible-galaxy install -f -r ./requirements.yml + + # Run the provision playbook from the azimuth-ops collection + # NOTE: THIS COMMAND RUNS A DIFFERENT PLAYBOOK FROM + # THE STANDARD AZIMUTH DEPLOYMENT INSTRUCTIONS + ansible-playbook azimuth_cloud.azimuth_ops.provision_capi_mgmt + +The general running order of the provisioning playbook is the following: + +- Ensure Terraform is installed locally + +- Use Terraform to provision the seed VM (and create any required internal networks, volumes etc.) + +- Install k3s on the seed (with all k3s data stored on the attached Cinder volume) + +- Install the required components on the k3s cluster to provision the HA cluster + +- Provision the HA cluster + +- Install the required components on the HA cluster to manage Magnum user clusters + +Once the seed VM has been provisioned, it can be accessed via SSH by running ``./bin/seed-ssh`` from the root of the azimuth-config repository. Within the seed VM, the k3s cluster and the HA cluster can both be accessed using the pre-installed ``kubectl`` and ``helm`` command line tools. Both of these tools will target the k3s cluster by default; however, the ``kubeconfig`` file for the HA cluster can be found in the seed's home directory (named e.g. ``kubeconfig-capi-mgmt-.yaml``). + +.. note:: + + The provision playbook is responsible for copying the HA ``kubeconfig`` to this location *after* the HA cluster is up and running. If you need to access the HA cluster while it is still deploying, the ``kubeconfig`` file can be found stored as a Kubernetes secret on the k3s cluster. + +It is possible to reconfigure or upgrade the management cluster after initial deployment by simply re-running the ``provision_capi_mgmt`` playbook. However, it's preferable that most Day 2 ops (i.e. reconfigures and upgrades) be done via a CD Pipeline. See `these Azimuth docs `_ for more information. + +Kayobe Config +============== + +To configure the Magnum service with the Cluster API driver enabled, first ensure that your kayobe-config branch is up to date with |current_release_git_branch_name|. + +Next, copy the CAPI management cluster's kubeconfig file into your stackhpc-kayobe-config environment (e.g. ``/kolla/config/magnum/kubeconfig``). This file must be Ansible vault encrypted. + +The presence of a kubeconfig file in the Magnum config directory is used by Kolla to determine whether the CAPI Helm driver should be enabled. + +To apply the configuration, run ``kayobe overcloud service reconfigure -kt magnum``. + +Magnum Cluster Templates +======================== + +The clusters deployed by the Cluster API driver make use of the Ubuntu Kubernetes images built in the `azimuth-images `_ repository and then use `capi-helm-charts `_ to provide the Helm charts which define the clusters based on these images. Between them, these two repositories have CI jobs that regularly build and test images and Helm charts for the latest Kubernetes versions. It is therefore important to update the cluster templates on each cloud regularly to make use of these new releases. + +Note that these templates are a tested set against the specific CAPI management cluster release. As such, you should make sure to update your CAPI management cluster to the latest release before updating to the latest templates. + +Magnum templates should be defined within an existing client-specific `openstack-config `_ repository. See the openstack-config `README `_ for more details. diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst new file mode 100644 index 000000000..931324e31 --- /dev/null +++ b/doc/source/configuration/monitoring.rst @@ -0,0 +1,302 @@ +========== +Monitoring +========== + +.. _monitoring-service-configuration: + +Monitoring Configuration +======================== + +StackHPC kayobe config includes a reference monitoring and alerting stack based +on Prometheus, Alertmanager, Grafana, Fluentd, OpenSearch & OpenSearch +Dashboards. These services by default come enabled and configured. + +Monitoring hosts, usually the controllers, should be added to the monitoring +group. The group definition can be applied in various different places. For +example, this configuration could be added to etc/kayobe/inventory/groups: + +.. code-block:: yaml + + [monitoring:children] + controllers + +Central OpenSearch cluster collects OpenStack logs, with an option to receive +operating system logs too. In order to enable this, execute custom playbook +after deployment: + +.. code-block:: console + + cd $KAYOBE_CONFIG_PATH + kayobe playbook run ansible/rsyslog.yml + +`Prometheus `__ comes with a comprehensive set of +metrics gathered from enabled exporters; every exporter's data is visualised +by at least one `Grafana `__ dashboard. Standard set of +alerting rules is present as well. + +While the default configuration often works out of the box, there +are some tunables which can be customised to adapt the configuration to a +particular deployment's needs. + +The configuration options can be found in +``etc/kayobe/stackhpc-monitoring.yml``: + +.. literalinclude:: ../../../etc/kayobe/stackhpc-monitoring.yml + :language: yaml + +SMART Drive Monitoring +====================== + +StackHPC kayobe config also includes drive monitoring for spinning disks and +NVME's. + +By default, node exporter doesn't provide SMART metrics, hence we make use +of 2 scripts (one for NVME’s and one for spinning drives), which are run by +a cronjob, to output the metrics and we use node exporter's Textfile collector +to report the metrics output by the scripts to Prometheus. These metrics can +then be visualised in Grafana with the bundled dashboard. + +After pulling in the latest changes into your local kayobe config, reconfigure +Prometheus and Grafana + +.. code-block:: console + + kayobe overcloud service reconfigure -kt grafana,prometheus + +(Note: If you run into an error when reconfiguring Grafana, it could be due to +`this `__ bug and at +present, the workaround is to go into each node running Grafana and manually +restart the process with ``systemctl restart kolla-grafana-container.service`` +and then try the reconfigure command again.) + +Once the reconfigure has completed you can now run the custom playbook which +copies over the scripts and sets up the cron jobs to start SMART monitoring +on the overcloud hosts: + +.. code-block:: console + + (kayobe) [stack@node ~]$ cd etc/kayobe + (kayobe) [stack@node kayobe]$ kayobe playbook run ansible/smartmon-tools.yml + +SMART reporting should now be enabled along with a Prometheus alert for +unhealthy disks and a Grafana dashboard called ``Hardware Overview``. + +Alertmanager, Slack and Microsoft Teams +======================================= + +StackHPC Kayobe configuration comes bundled with an array of alerts but does not +enable any receivers for notifications by default. Various receivers can be +configured for Alertmanager. Slack is currently the most common. + +To set up a receiver, create a ``prometheus-alertmanager.yml`` file under +``etc/kayobe/kolla/config/prometheus/``. An example config is stored in this +directory. The example configuration uses two Slack channels. One channel +receives all alerts while the other only receives alerts tagged as critical. It +also adds a silence button to temporarily mute alerts. To use the example in a +deployment, you will need to generate two webhook URLs, one for each channel. + +To generate a slack webhook, `create a new app +`__ in the workspace you want to add alerts to. +From the Features page, toggle Activate incoming webhooks on. Click Add new +webhook to workspace. Pick a channel that the app will post to, then click +Authorise. You only need one app to generate both webhooks. + +Both URLs should be encrypted using ansible vault, as they give anyone access to +your slack channels. The standard practice is to store them in +``kayobe/secrets.yml`` as: + +.. code-block:: yaml + + secrets_slack_notification_channel_url: + secrets_slack_critical_notification_channel_url: + +These should then be set as the ``slack_api_url`` and ``api_url`` for the +regular and critical alerts channels respectively. Both slack channel names will +need to be set, and the proxy URL sould be set or removed. + +If you want to add an alerting rule, there are many good examples of alerts are +available `here `__. They simply +need to be added to one of the ``*.rules`` files in the prometheus configuration +directory. + +If however you are using Microsoft Teams instead of Slack, you can use Prometheus +Alertmanager's built-in support for the new message format based on Power Automate flows. +You will need an incoming webhook URL for your Teams channel. +This can be done by following `these instructions `__. +To set up a receiver, create a ``prometheus-alertmanager.yml`` file under +``etc/kayobe/kolla/config/prometheus/``. +An example config is stored in this directory known as ``prometheus-alertmanager.msteamvs2.yml.example``. +The example configuration uses two Slack channels. +One channel receives all alerts while the other only receives alerts tagged as critical. +Feel free to modify the example configuration to suit your needs. + +Ceph Monitoring +=============== + +There is code in the globals.yml file to extract the ceph mgr nodes from the +mgrs group and list them as the endpoints for prometheus. Additionally, +depending on your configuration, you may need set the +``kolla_enable_prometheus_ceph_mgr_exporter`` variable to ``true`` in order to +enable the ceph mgr exporter. + +.. _os-capacity: + +OpenStack Capacity +================== + +OpenStack Capacity allows you to see how much space you have available +in your cloud. StackHPC Kayobe Config will deploy OpenStack Capacity +by default on a service deploy, this can be disabled by setting +``stackhpc_enable_os_capacity`` to false. + +OpenStack Capacity is deployed automatically using a service deploy hook +with the generated kolla-ansible admin credentials, you can override these +by setting the authentication url, username, password, project name and +project domain name in ``stackhpc-monitoring.yml``: + +.. code-block:: yaml + + stackhpc_os_capacity_auth_url: + stackhpc_os_capacity_username: + stackhpc_os_capacity_password: + stackhpc_os_capacity_project_name: + stackhpc_os_capacity_domain_name: + stackhpc_os_capacity_openstack_region_name: + +Additionally, you should ensure these credentials have the correct permissions +for the exporter. + +If you are deploying in a cloud with internal TLS, you may be required +to provide a CA certificate for the OpenStack Capacity exporter if your +certificate is not signed by a trusted CA. For example, to use a CA certificate +named ``vault.crt`` or ``openbao.crt`` that is also added to the Kolla containers: + +.. code-block:: yaml + + stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" + +or + +.. code-block:: yaml + + stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt" + +Alternatively, to disable certificate verification for the OpenStack Capacity +exporter: + +.. code-block:: yaml + + stackhpc_os_capacity_openstack_verify: false + +If you've modified your credentials, you will need to re-deploy OpenStack Capacity +using the ``deploy-os-capacity-exporter.yml`` Ansible playbook +via Kayobe. + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-os-capacity-exporter.yml + +If you notice ``HaproxyServerDown`` or ``HaproxyBackendDown`` prometheus +alerts after deployment it's likely the os_exporter secrets have not been +set correctly, double check you have entered the correct authentication +information appropiate to your cloud and re-deploy. + +Friendly Network Names +======================= +For operators that prefer to see descriptive or friendly interface names the +following play can be run. This takes network names as defined in kayobe and +relabels the devices/interfaces in Prometheus to make use of these names. + +**Check considerations and known limitations to see if this is suitable in any +given environment before applying.** + +This reuses existing fields to provide good compatibility with existing +dashboards and alerts. + +To enable the change: + +.. code-block:: console + + kayobe playbook run etc/kayobe/ansible/prometheus-network-names.yml + kayobe overcloud service reconfigure --kt prometheus + +This first generates a template based on the prometheus.yml.j2 +``etc/kayobe/ansible/`` and which is further templated for use with +kolla-ansible. +This is then rolled out via service reconfigure. + + +This helps Prometheus provide insights that can be more easily understood by +those without an intimate understanding of a given site. Prometheus Node +Exporter and cAdvisor both provide network statistics using the +interface/device names. This play causes Prometheus to relabel these fields to +human readable names based on the networks as defined in kayobe +e.g. bond1.1838 may become storage_network. + +The default labels are preserved with the prefix ``original_``. + +* For node_exporter, ``device`` is then used for network names, while + ``original_device`` is used for the interface itself. +* For cAdvisor, ``interface`` is used for network names, and + ``original_interface`` is used to preserve the interface name. + +:Known-Limitations/Considerations/Requirements: + +Before enabling this feature, the implications must be discussed with the +customer. The following are key considerations for that conversation: + +* Only network names defined within kayobe are within scope. +* Tenant network interfaces, including SR-IOV are not considered or modified. +* Only the interface directly attributed to a network will be relabelled. + This may be a bond, a vlan tagged sub-interface, or both. + The parent bond, or bond members are not relabelled unless they are + captured within a distinct defined network. +* Modified entries will be within existing labels. This may be breaking for + anything that expects the original structure, including custom dashboards, + alerting, billing, etc. +* After applying, there will be inconsistency in the time-series db for the + duration of the retention period i.e until previously ingested entries + expire. + The metrics gathered prior to applying these modifications will be unaltered, + with all new metrics using the new structure. +* The interface names and their purpose must be consistent and unique within + the environment. i.e if eth0 is defined as admin_interface on one node, no + other node can include a different network definition using eth0. + This does not apply in the case when both devices are bond members. + e.g. bond0 on a controller has eth0 and eth1 as members. bond1 on a compute + uses eth0 and eth1 as members. This is not problematic as it is only + the bond itself that is relabelled. + +Redfish exporter +================ + +Redfish exporter will query the overcloud BMCs via their redfish interfaces +to produce various metrics relating to the hardware, and system health. + +To configure the exporter, adjust the variables in +``$KAYOBE_CONFIG_PATH/stackhpc-monitoring.yml`` to use appropriate values: + +.. code-block:: yaml + + # Whether the redfish exporter is enabled. + stackhpc_enable_redfish_exporter: true + + # Redfish exporter credentials + redfish_exporter_default_username: "{{ ipmi_username }}" + redfish_exporter_default_password: "{{ ipmi_password }}" + + # The address of the BMC that is queried by redfish exporter for metrics. + redfish_exporter_target_address: "{{ ipmi_address }}" + +Deploy the exporter on the seed: + +.. code-block:: console + + kayobe seed service deploy -t seed-deploy-containers -kt none + +It is required that you re-configure the Prometheus, Grafana +services following deployment, to do this run the following Kayobe command. + +.. code-block:: console + + kayobe overcloud service reconfigure -kt grafana,prometheus diff --git a/doc/source/configuration/openbao.rst b/doc/source/configuration/openbao.rst new file mode 100644 index 000000000..b777c1e87 --- /dev/null +++ b/doc/source/configuration/openbao.rst @@ -0,0 +1,440 @@ +.. _openbao: + +======================== +OpenBao for internal PKI +======================== + +This document describes how to deploy OpenBao for +internal PKI purposes using the +`StackHPC Hashicorp collection `_ + +OpenBao may be used as a Certificate Authority to generate certificates for: + +* OpenStack internal API +* OpenStack backend APIs +* RabbitMQ + +TLS support is described in the :kolla-ansible-doc:`Kolla Ansible documentation +` and the :kayobe-doc:`Kayobe documentation +`. + +OpenBao may also be used as the secret store for Barbican. + +Background +========== + +Our OpenStack environment employs two separate OpenBao instances. +These instances manage the Public Key Infrastructure (PKI) by handling the +creation and issuance of certificates. + +- The first OpenBao instance is located on the seed host. + It handles infrastructure-level certificates, generating the root + Certificate Authority (CA) and intermediate CA for the second OpenBao. + The ``openbao-deploy-seed.yml`` playbook sets up this instance. + +- The second OpenBao instance is within the OpenStack + overcloud, located on the controller nodes. This instance uses the + intermediate CA from the seed OpenBao to issue application-specific + certificates. The ``vault-openbao-overcloud.yml`` playbook is used + for its setup. It ensures that all controller nodes trust the + intermediate CA from the root OpenBao. + +The dual OpenBao setup enhances security by protecting the root CA's key. The more +exposed overcloud OpenBao only possesses the intermediate key, ensuring that +the root key remains secure even if the overcloud OpenBao instance is compromised. + +Prerequisites +============= + +Before beginning the deployment of OpenBao for openstack internal TLS and backend TLS you should ensure that you have the following. + + * Seed Node or a host to run the vault container on + * Overcloud controller hosts to install second vault on + * Ansible Galaxy dependencies installed: ``kayobe control host bootstrap`` + * Python dependencies installed: ``pip install -r kayobe-config/requirements.txt`` + +By default OpenBao image is not synced from Docker Hub to the local +Pulp. To sync this image, set ``stackhpc_sync_openbao_images`` to ``true``. +The OpenBao deployment configuration will be automatically updated to pull images +from Pulp. + +Deployment +========== + +Setup OpenBao on the seed node +------------------------------ + +1. Run openbao-deploy-seed.yml custom playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-deploy-seed.yml + +2. Encrypt generated certs/keys with ansible-vault (use proper location of vault password file) + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/openbao/OS-TLS-INT.pem + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/openbao/seed-openbao-keys.json + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/openbao/overcloud.key + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/openbao/OS-TLS-INT.pem + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/openbao/seed-openbao-keys.json + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/openbao/overcloud.key + +Setup OpenBao HA on the overcloud hosts +--------------------------------------- + +1. Run openbao-deploy-overcloud.yml custom playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-deploy-overcloud.yml + +2. Encrypt overcloud openbao keys (use proper location of vault password file) + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/openbao/overcloud-openbao-keys.json + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/openbao/overcloud-openbao-keys.json + +Rotating OpenBao certificate on the overcloud hosts +--------------------------------------------------- + +The certificate for the overcloud OpenBao has an expiration of one or two years after the certificate was generated. +The expiration date of a certificate can be determined with ``openssl x509 -enddate -noout -in overcloud.crt`` +This will be problematic if anything needs to interact with the OpenBao API such as issuing new certificates or Barbican integration. + +1. Delete the old certificate: + + .. code-block:: + + rm $KAYOBE_CONFIG_PATH/openbao/overcloud.crt + + Or if environments are being used + + .. code-block:: + + rm $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/openbao/overcloud.crt + +2. Generate a new certificate (and key): + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-deploy-seed.yml + +3. Encrypt generated key with ansible-vault (use proper location of vault password file) + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/openbao/overcloud.key + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/openbao/overcloud.key + +4. Copy the new certificate to the overcloud hosts. Note, if the old + certificate has expired this will fail on the unseal step. + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-deploy-overcloud.yml + +5. Restart the containers to use the new certificate: + + .. code-block:: + + kayobe overcloud host command run --command "docker restart openbao" -l controllers + +6. If sealed, unseal OpenBao: + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-unseal-overcloud.yml + +Certificates generation +======================= + +.. note:: + + Generating certificates will fail if the OpenBao on the overcloud is sealed. This will happen whenever the openbao containers are restarted. To unseal the + overcloud OpenBao, run: + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-unseal-overcloud.yml + +Create the external TLS certificates (testing only) +--------------------------------------------------- + +This method should only be used for testing. For external TLS on production systems, +See `Installing External TLS Certificates `__. + +Typically external API TLS certificates should be generated by a organisation's trusted internal or third-party CA. +For test and development purposes it is possible to use OpenBao as a CA for the external API. + +1. Run the playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-generate-test-external-tls.yml + +2. Use ansible-vault to encrypt the PEM bundle in $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem. Commit the PEM bundle to the kayobe configuration. + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem + +Create the internal TLS certificates +------------------------------------ + +1. Run the playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-generate-internal-tls.yml + +2. Use ansible-vault to encrypt the PEM bundle in $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy-internal.pem. Commit the PEM bundle and root CA to the kayobe configuration. + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy-internal.pem + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy-internal.pem + +Create the backend TLS and RabbitMQ TLS certificates +---------------------------------------------------- + +1. Run the playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-generate-backend-tls.yml + +2. Use ansible-vault to encrypt the keys in $KAYOBE_CONFIG_PATH/kolla/certificates/-key.pem. Commit the certificates and keys to the kayobe configuration. + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/-key.pem + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/-key.pem + +.. _openbao-haproxy: + +HAProxy integration +=================== + +It is possible to expose the overcloud OpenBao service via the Kolla Ansible HAProxy load balancer. +This provides a single highly available API endpoint, as well as monitoring of the OpenBao backends when combined with Prometheus. +HAProxy integration is no longer required for generating OpenStack control plane certificates, making it possible to deploy OpenBao and generate certificates before any containers have been deployed by Kolla Ansible. + +1. Create the HAProxy config to reverse proxy the OpenBao HA container + + Set the openbao_front to the external VIP address or internal VIP address depending on the installation. Set the openbao_back to the IPs of the control nodes. + + Set the following in etc/kayobe/kolla/config/haproxy/services.d/openbao.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/openbao.cfg + + .. code-block:: + + # Delete "verify none" if not using self-signed/unknown issuer + {% raw %} + frontend openbao_front + mode tcp + option tcplog + bind {{ kolla_internal_vip_address }}:8200 + default_backend openbao_back + + backend openbao_back + mode tcp + option httpchk GET /v1/sys/health + # https://openbao.org/api-docs/system/health/ + # 200: initialized, unsealed, and active + # 429: standby + http-check expect rstatus (200|429) + + {% for host in groups['control'] %} + {% set host_name = hostvars[host].ansible_facts.hostname %} + {% set host_ip = 'api' | kolla_address(host) %} + server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5 + {% endfor %} + {% endraw %} + +2. If HAProxy has not yet been deployed, continue to :ref:`certificates deployment `. + If HAProxy has been deployed, it may be redeployed with the new OpenBao service configuration: + + .. code-block:: + + kayobe overcloud service deploy -kt haproxy + +.. _openbao-certificates: + +Certificates deployment +======================= + +.. warning:: + + The switch from HTTP to HTTPS during the deployment of internal/backend TLS certificates can temporarily disrupt service availability and necessitates a restart of all services. During this transition, endpoints may become unreachable following the HAProxy restart, persisting until the endpoint catalogue and client have been reconfigured to use HTTPS. + +Enable the required TLS variables in kayobe and kolla +----------------------------------------------------- + +1. If using OpenBao as a CA for the external API, set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml + + .. code-block:: + + # Whether TLS is enabled for the external API endpoints. Default is 'no'. + kolla_enable_tls_external: yes + kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}" + + See :ref:`tempest-cacert` for information on adding CA certificates to the trust store when running Tempest. + +2. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml + + .. code-block:: + + # Whether TLS is enabled for the internal API endpoints. Default is 'no'. + kolla_enable_tls_internal: yes + kolla_admin_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}" + + See :ref:`os-capacity` for information on adding CA certificates to the trust store when deploying the OpenStack Capacity exporter. + +3. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml + + .. code-block:: + + # Internal TLS configuration + # Copy the self-signed CA into the kolla containers + kolla_copy_ca_into_containers: "yes" + # Use the following trust store within the container + openstack_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution == 'rocky' else '/etc/ssl/certs/ca-certificates.crt' }}" + + # Backend TLS config + # Enable backend TLS + kolla_enable_tls_backend: "yes" + + # If using RabbitMQ TLS: + rabbitmq_enable_tls: "yes" + +4. Deploy OpenStack + + .. warning:: + + It is important that you are only using admin endpoints for keystone. If + any admin endpoints exist for other services, they must be deleted e.g. + + .. code-block:: + + openstack endpoint list --interface admin -f value | \ + awk '!/keystone/ {print $1}' | xargs openstack endpoint delete + + .. code-block:: + + kayobe overcloud service deploy + + If VM provisioning fails with an error with this format: + + .. code-block:: + + Unable to establish connection to http://:9696/v2.0/ports/some-sort-of-uuid: Connection aborted + + Restart the nova-compute container on all hypervisors: + + .. code-block:: + + kayobe overcloud host command run --command "systemctl restart kolla-nova_compute-container.service" --become --show-output -l compute + +Barbican integration +==================== + +Barbican integration depends on :ref:`HAProxy integration `. + +Enable Barbican in kayobe +------------------------- + +1. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml + + .. code-block:: + + kolla_enable_barbican: yes + +Generate secrets_barbican_approle_secret_id +------------------------------------------- + +1. Run ``uuidgen`` to generate secret id +2. Insert into secrets.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/secrets.yml + + .. code-block:: + + secrets_barbican_approle_secret_id: "YOUR-SECRET-GOES-HERE" + +Create required configuration in OpenBao +---------------------------------------- + +1. Run openbao-deploy-barbican.yml custom playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/openbao-deploy-barbican.yml + +Add secrets_barbican_approle_id to secrets +------------------------------------------ + +1. Note the role id from playbook output and insert into secrets.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/secrets.yml + + .. code-block:: + + secrets_barbican_approle_role_id: "YOUR-APPROLE-ID-GOES-HERE" + +Configure Barbican +------------------ + +1. Put required configuration in kayobe-config/etc/kayobe/kolla/config/barbican.conf or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/barbican.conf + + .. code-block:: + + [secretstore] + namespace=barbican.secretstore.plugin + enable_multiple_secret_stores=false + enabled_secretstore_plugins=vault_plugin + + [vault_plugin] + vault_url = https://{{ kolla_internal_fqdn }}:8200 + use_ssl = True + {% raw %} + ssl_ca_crt_file = {{ openstack_cacert }} + {% endraw %} + approle_role_id = {{ secrets_barbican_approle_role_id }} + approle_secret_id = {{ secrets_barbican_approle_secret_id }} + kv_mountpoint = barbican + +Deploy Barbican +--------------- + + .. code-block:: + + kayobe overcloud service deploy -kt barbican diff --git a/doc/source/configuration/release-train.rst b/doc/source/configuration/release-train.rst new file mode 100644 index 000000000..ad489f502 --- /dev/null +++ b/doc/source/configuration/release-train.rst @@ -0,0 +1,354 @@ +.. _stackhpc-release-train: + +====================== +StackHPC Release Train +====================== + +StackHPC provides packages, container images, and host images for OpenStack via +`Ark `__. These artifacts are built and released using +a process known as the `Release Train +`__. + +Deployments should use a local `Pulp `__ repository +server to synchronise content from Ark and serve it locally. This reduces +Internet bandwidth requirements for package and container downloads. Content is +synced on demand from Ark to the local Pulp, meaning that the local Pulp acts +like a pull-through cache. + +Access to the repositories on Ark is controlled via user accounts issued by +StackHPC. + +.. image:: /_static/images/release-train.svg + :width: 75% + +All content on Ark is versioned, meaning that a deployment may continue to use +older package repository snapshots and container images when newer content is +released. This allows for improved reliability & repeatability of deployments. + +This configuration defines two `Pulp distributions +`__ for +packages, ``development`` and ``production``. This allows packages to be +updated and tested in a development or staging environment before rolling them +out to production. Typically a given environment will always use the same +distribution, meaning that package repository configuration files do not need +to be updated on control plane hosts in order to consume a package update. + +Configuration +============= + +This configuration provides the following: + +* Configuration to deploy a local Pulp service as a container on the seed +* Pulp repository definitions for Rocky Linux 9 and Ubuntu Jammy 22.04 +* Playbooks to synchronise a local Pulp service with Ark +* Configuration to use the local Pulp repository mirrors on control plane hosts +* Configuration to use the local Pulp container registry on control plane hosts +* Configuration to deploy pre-built OS images to overcloud hosts using Bifrost + +Local Pulp server +----------------- + +The Pulp container is deployed on the seed by default, but may be disabled by +setting ``seed_pulp_container_enabled`` to ``false`` in +``etc/kayobe/seed.yml``. + +The URL for the local Pulp server is configured by ``pulp_url`` within +``etc/kayobe/pulp.yml``. + +The Pulp service can be configured with two sets of credentials; one for +administrator operations and another read-only for overcloud hosts +to use. +The administrator credentials can be configured ``pulp_username``, +``pulp_password`` +The basic user account credentials can be configured with ``pulp_stack_username`` +and ``pulp_stack_password``. +Both sets of credentials can be found within ``etc/kayobe/pulp.yml``. + +Both the ``pulp_password`` and ``pulp_stack_password`` are intended to be +configured via their ``secrets_*`` counterparts, i.e. +``secrets_pulp_password`` and ``secrets_pulp_stack_password``. These variables +are expected to be set in an Ansible Vault encrypted +``etc/kayobe/secrets.yml`` file. + +Passwords can be generated using ``OpenSSL`` + +.. code-block:: console + + openssl rand -base64 32 + +Host images are not synchronised to the local Pulp server, since they should +only be pulled to the seed node once. More information on host images can be +found :ref:`here `. + +StackHPC Ark +------------ + +The Ark pulp credentials issued by StackHPC should be configured in +``etc/kayobe/pulp.yml``, using Ansible Vault to encrypt the password: + +.. code-block:: yaml + + stackhpc_release_pulp_username: + stackhpc_release_pulp_password: + +Package repositories +-------------------- + +Rocky Linux 9 and Ubuntu Jammy package repositories are synced based on the +value of ``os_distribution`` and ``os_release``. + +On Ark, each package repository provides versioned snapshots using a datetime +stamp (e.g. ``20220817T082321``). The current set of tested versions is defined +in ``etc/kayobe/pulp-repo-versions.yml``. This file is managed by the StackHPC +Release Train and should generally not be modified by consumers of this +repository. + +Package managers +---------------- + +For Ubuntu Jammy systems, the package manager configuration is provided by +``stackhpc_apt_repositories`` in ``etc/kayobe/apt.yml``. + +The configuration is applied by default to all Ubuntu hosts. The configuration +can be overridden by changing the repository definitions in +``apt_repositories`` or toggling ``apt_disable_sources_list`` to use the +default apt repositories. This can be done on a host-by host basis by defining +the variables as host or group vars under ``etc/kayobe/inventory/host_vars`` or +``etc/kayobe/inventory/group_vars``. + +For Ubuntu-based deployments, Pulp currently `lacks support +`_ for certain types of content, +including i18n files and command-not-found indices. This breaks APT when the +``command-not-found`` package is installed: + +.. code:: console + + E: Failed to fetch https://pulp.example.com/pulp/content/ubuntu/jammy-security/development/dists/jammy-security/main/cnf/Commands-amd64 404 Not Found + +The ``purge-command-not-found.yml`` custom playbook can be used to uninstall +the package, prior to running any other APT commands. It may be installed as a +:kayobe-doc:`pre-hook ` to the ``host +configure`` commands. Note that if used as a hook, this playbook matches all +hosts, so will run against the seed, even when running ``overcloud host +configure``. Depending on the stage of deployment, some hosts may be +unreachable. + +For Rocky Linux based systems, package manager configuration is provided by +``stackhpc_dnf_repos`` in ``etc/kayobe/dnf.yml``, which points to package +repositories on the local Pulp server. To use this configuration, the +``dnf_custom_repos`` variable must be set, and this is done for hosts in the +``overcloud`` group via the group_vars file +``etc/kayobe/inventory/group_vars/overcloud/stackhpc-dnf-repos``. Similar +configuration may be added for other groups, however there may be ordering +issues during initial deployment when Pulp has not yet been deployed. + +In both instances, the configuration points to package repositories on the +local Pulp server. + +The distribution name for the environment should be configured as either +``development`` or ``production`` via ``stackhpc_repo_distribution`` in +``etc/kayobe/stackhpc.yml``. + +Ceph container images +--------------------- + +By default, Ceph images are not synced from quay.io to the local Pulp. To sync +these images, set ``stackhpc_sync_ceph_images`` to ``true``. + +HashiCorp container images +-------------------------- + +By default, HashiCorp images (Consul and Vault) are not synced from Docker Hub +to the local Pulp. To sync these images, set ``stackhpc_sync_hashicorp_images`` +to ``true``. + +Custom container images +----------------------- + +A custom list of container images can be synced to the local Pulp using the +``stackhpc_pulp_repository_container_repos_extra`` and +``stackhpc_pulp_distribution_container_extra`` variables. + +.. code-block:: yaml + + # List of extra container image repositories. + stackhpc_pulp_repository_container_repos_extra: + - name: "certbot/certbot" + url: "https://registry-1.docker.io" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + state: present + include_tags: "nightly" + required: True + + # List of extra container image distributions. + stackhpc_pulp_distribution_container_extra: + - name: certbot + repository: certbot/certbot + base_path: certbot/certbot + state: present + required: True + +Usage +===== + +The local Pulp service will be deployed as a :kayobe-doc:`Seed custom container +` +on next ``kayobe seed service deploy`` or ``kayobe seed service upgrade``. + +The following custom playbooks are provided in ``etc/kayobe/ansible/``: + +See the Kayobe :kayobe-doc:`custom playbook documentation +` for information on how to run them. + +* ``pulp-repo-sync.yml``: Pull packages from Ark to the local Pulp. This will + create a new repository version (snapshot) for each repository in the local + Pulp server when new packages are available. The new packages will not be + available to cloud nodes until they have been published. +* ``pulp-repo-publish.yml``: Publish synchronised packages to the + ``development`` distribution in the local Pulp. This will make synchronised + packages available to cloud nodes using the ``development`` distribution + (typically a development or staging environment). The new packages will not + be available to cloud nodes using the ``production`` distribution until they + have been promoted. +* ``pulp-repo-promote-production.yml``: Promote packages in the ``development`` + distribution to the ``production`` distribution in the local Pulp. This will + make all packages currently available to cloud nodes using the + ``development`` distribution also available to cloud nodes using the + ``production`` distribution. Typically this would be done only once the new + packages have been validated in a development or staging environment. +* ``pulp-container-sync.yml``: Pull container images from Ark to the local + Pulp. This will create a new repository version (snapshot) for each + repository in the local Pulp server when new image tags are available. If + these are new container image repositories, then the new image tags will not + be available to cloud nodes until they have been published. +* ``pulp-container-publish.yml``: Publish synchronised container images in the + local Pulp. This will make synchronised container images available to cloud + nodes. + +Syncing content +--------------- + +A typical workflow to sync all packages and containers is as follows: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + +Once the content has been tested in a test/staging environment, it may be +promoted to production: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml + +Synchronising all Kolla container images can take a long time. A limited list +of images can be synchronised using the ``stackhpc_pulp_images_kolla_filter`` +variable, which accepts a whitespace-separated list of regular expressions +matching Kolla image names. Usage is similar to ``kolla-build`` CLI arguments. +For example: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml -e stackhpc_pulp_images_kolla_filter='"^glance nova-compute$"' + +Initial seed deployment +----------------------- + +During the initial seed deployment, there is an ordering issue where the +Bifrost container will not yet have been synced, but the local Pulp container +has not yet been deployed. This can be avoided with the following workflow: + +.. code-block:: console + + kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + kayobe seed service deploy + +Working with pulp +================= + +The `pulp CLI +`__ tool can be used to administer your local +pulp installation. Please follow the upstream documentation for installation +instructions. + +pulp CLI tricks +--------------- + +Saving credentials +~~~~~~~~~~~~~~~~~~ + +This is useful to avoid the need to always supply your credentials when running commands +from the command line: + +.. code-block:: console + + (venv-pulp) [stack@seed ~]$ pulp config create --username admin --base-url http://:8080 --password + + +Troubleshooting +--------------- + +HTTP Error 400: Bad Request {"name":["This field must be unique."]} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you have previously tried to push an image to pulp e.g for local testing, you may +see this message when you later try to run ``pulp-container-sync.yml``: + +.. code-block:: console + + TASK [stackhpc.pulp.pulp_repository : Setup container repositories] ***************************** + failed: [localhost] (item=stackhpc/rocky-source-prometheus-jiralert) => changed=false + ansible_loop_var: item + item: + name: stackhpc/rocky-source-prometheus-jiralert + policy: on_demand + remote_password: password + remote_username: username + state: present + url: https://ark.stackhpc.com + msg: 'HTTP Error 400: Bad Request b''{"name":["This field must be unique."]}''' + +The issue is that pushing an image automatically creates a `container-push repository +`__ +which conflicts with the creation of a regular container repository of the same +name. You can resolve this conflict by deleting the distribution associated +with the push repository using the pulp CLI: + +.. code-block:: console + + (venv-pulp) [stack@seed ~]$ pulp --base-url http://:8080--username admin --password container distribution destroy --name stackhpc/rocky-source-prometheus-jiralert + Started background task /pulp/api/v3/tasks/1f0a474a-b7c0-44b4-9ef4-ed633077f4d8/ + .Done. + +HTTP Error 404: Not Found +~~~~~~~~~~~~~~~~~~~~~~~~~ + +If your login credentials are incorrect, or lack the required permissions, +you will see a 404 error during ``pulp-repo-sync.yml``: + +.. code-block:: console + + TASK [stackhpc.pulp.pulp_repository : Sync RPM remotes into repositories] **************************************************************************************************************************************** + An exception occurred during task execution. To see the full traceback, use -vvv. The error was: Exception: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/rocky/9/BaseOS/x86_64/os/20211122T102435')) + failed: [localhost] (item=rocky-9-baseos-development) => changed=false + ansible_loop_var: item + item: + name: rocky-9-baseos-development + policy: on_demand + proxy_url: __omit_place_holder__d35452c39719f081229941a64fd2cdce1188a287 + remote_password: + remote_username: + required: true + state: present + sync_policy: mirror_complete + url: https://ark.stackhpc.com/pulp/content/rocky/9/BaseOS/x86_64/os/20211122T102435 + msg: Task failed to complete. (failed; 404, message='Not Found', url=URL('https://ark.stackhpc.com/pulp/content/rocky/9/BaseOS/x86_64/os/20211122T102435')) ''' + +The issue can be rectified by updating the ``stackhpc_release_pulp_username`` +and ``stackhpc_release_pulp_password`` variables. diff --git a/doc/source/configuration/security-hardening.rst b/doc/source/configuration/security-hardening.rst new file mode 100644 index 000000000..1bb8216b5 --- /dev/null +++ b/doc/source/configuration/security-hardening.rst @@ -0,0 +1,80 @@ +================== +Security Hardening +================== + +CIS Benchmark Hardening +----------------------- + +The roles from the `Ansible-Lockdown `_ +project are used to harden hosts in accordance with the CIS benchmark criteria. +It won't get your benchmark score to 100%, but should provide a significant +improvement over an unhardened system. A typical score would be 70%. + +The following operating systems are supported: + +- Ubuntu 22.04 +- Rocky 9 + +Configuration +-------------- + +Some overrides to the role defaults are provided in +``$KAYOBE_CONFIG_PATH/inventory/group_vars/cis-hardening/cis``. These may not be +suitable for all deployments and so some fine tuning may be required. For +instance, you may want different rules on a network node compared to a +controller. It is best to consult the upstream role documentation for details +about what each variable does. The documentation can be found here: + +- `Ubuntu 22.04 `__ +- `Rocky 9 `__ + +Running the playbooks +--------------------- + +.. note:: + + The hosts may need rebooting to fully pick up all of the changes. The CIS + roles will warn you when this needs to be done, but the actual reboot is left + as a manual operation to allow you to select a convenient time. Generally, if + you are applying the hardening for the first time, then you will need to + reboot. + +As there is potential for unintended side effects when applying the hardening +playbooks, the playbooks are not currently enabled by default. It is recommended +that they are first applied to a representative staging environment to determine +whether or not workloads or API requests are affected by any configuration changes. + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cis.yml + +Targetting additional hosts +--------------------------- + +The ``cis.yml`` playbook targets hosts in the ``cis-hardening`` group. By +default this includes the ``overcloud`` group. You can adjust this group +to suit your needs, e.g to add the seed VM: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/inventory/groups + + [cis-hardening:children] + overcloud + seed + +Enabling the host configure hook +-------------------------------- + +A hook is pre-installed but its execution is guarded by the +``stackhpc_enable_cis_benchmark_hardening_hook`` configuration option. +If you want the hardening playbooks to run automatically, as part of +host configure, simply set this flag to ``true``: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/stackhpc.yml + + stackhpc_enable_cis_benchmark_hardening_hook: true + +Alternatively, this can be toggled on a per-environment basis by +setting it in an environment specific config file, or even on +targeted hosts by using group or host vars. diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst new file mode 100644 index 000000000..3d1b4d256 --- /dev/null +++ b/doc/source/configuration/vault.rst @@ -0,0 +1,447 @@ +.. _hashicorp-vault: + +================================ +Hashicorp Vault for internal PKI +================================ + +.. warning:: + + Deployment of Hashicorp Vault is deprecated and will be removed in a future release. + New deployments should use OpenBao and existing deployments will be migrated once an + method for migration is available. + See :ref:`here ` for more information. + +This document describes how to deploy Hashicorp Vault for +internal PKI purposes using the +`StackHPC Hashicorp collection `_ + +Vault may be used as a Certificate Authority to generate certificates for: + +* OpenStack internal API +* OpenStack backend APIs +* RabbitMQ + +TLS support is described in the :kolla-ansible-doc:`Kolla Ansible documentation +` and the :kayobe-doc:`Kayobe documentation +`. + +Vault may also be used as the secret store for Barbican. + +Background +========== + +Our OpenStack environment employs two separate HashiCorp Vault instances. +These instances manage the Public Key Infrastructure (PKI) by handling the +creation and issuance of certificates. + +- The first HashiCorp Vault instance is located on the seed host. + It handles infrastructure-level certificates, generating the root + Certificate Authority (CA) and intermediate CA for the second Vault. + The ``vault-deploy-seed.yml`` playbook sets up this instance. + +- The second HashiCorp Vault instance is within the OpenStack + overcloud, located on the controller nodes. This instance uses the + intermediate CA from the seed Vault to issue application-specific + certificates. The ``vault-deploy-overcloud.yml`` playbook is used + for its setup. It ensures that all controller nodes trust the + intermediate CA from the root Vault. + +The dual Vault setup enhances security by protecting the root CA's key. The more +exposed overcloud vault only possesses the intermediate key, ensuring that +the root key remains secure even if the overcloud Vault instance is compromised. + +Prerequisites +============= + +Before beginning the deployment of vault for openstack internal TLS and backend TLS you should ensure that you have the following. + + * Seed Node or a host to run the vault container on + * Overcloud controller hosts to install second vault on + * Ansible Galaxy dependencies installed: ``kayobe control host bootstrap`` + * Python dependencies installed: ``pip install -r kayobe-config/requirements.txt`` + +By default, Consul and Vault images are not synced from Docker Hub to the local +Pulp. To sync these images, set ``stackhpc_sync_hashicorp_images`` to ``true``. +The Vault deployment configuration will be automatically updated to pull images +from Pulp. + +Deployment +========== + +Setup Vault on the seed node +---------------------------- + +1. Run vault-deploy-seed.yml custom playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml + +2. Encrypt generated certs/keys with ansible-vault (use proper location of vault password file) + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/vault/OS-TLS-INT.pem + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/vault/seed-vault-keys.json + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/vault/overcloud.key + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/OS-TLS-INT.pem + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/seed-vault-keys.json + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud.key + +Setup Vault HA on the overcloud hosts +------------------------------------- + +1. Run vault-deploy-overcloud.yml custom playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml + +2. Encrypt overcloud vault keys (use proper location of vault password file) + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/vault/overcloud-vault-keys.json + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud-vault-keys.json + +Rotating Vault certificate on the overcloud hosts +------------------------------------------------- + +The certificate for the overcloud Vault has an expiration of one or two years after the certificate was generated. +The expiration date of a certificate can be determined with ``openssl x509 -enddate -noout -in overcloud.crt`` +This will be problematic if anything needs to interact with the Vault API such as issuing new certificates or Barbican integration. + +1. Delete the old certificate: + + .. code-block:: + + rm $KAYOBE_CONFIG_PATH/vault/overcloud.crt + + Or if environments are being used + + .. code-block:: + + rm $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud.crt + +2. Generate a new certificate (and key): + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml + +3. Encrypt generated key with ansible-vault (use proper location of vault password file) + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/vault/overcloud.key + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud.key + +4. Copy the new certificate to the overcloud hosts. Note, if the old + certificate has expired this will fail on the unseal step. + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml + +5. Restart the containers to use the new certificate: + + .. code-block:: + + kayobe overcloud host command run --command "docker restart vault" -l controllers + +6. If sealed, unseal the vault: + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-unseal-overcloud.yml + +Certificates generation +======================= + +.. note:: + + Generating certificates will fail if the Vault on the overcloud is sealed. This will happen whenever the vault containers are restarted. To unseal the + overcloud Vault, run: + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-unseal-overcloud.yml + +Create the external TLS certificates (testing only) +--------------------------------------------------- + +This method should only be used for testing. For external TLS on production systems, +See `Installing External TLS Certificates `__. + +Typically external API TLS certificates should be generated by a organisation's trusted internal or third-party CA. +For test and development purposes it is possible to use Vault as a CA for the external API. + +1. Run the playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml + +2. Use ansible-vault to encrypt the PEM bundle in $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem. Commit the PEM bundle to the kayobe configuration. + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem + +Create the internal TLS certificates +------------------------------------ + +1. Run the playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-internal-tls.yml + +2. Use ansible-vault to encrypt the PEM bundle in $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy-internal.pem. Commit the PEM bundle and root CA to the kayobe configuration. + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy-internal.pem + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy-internal.pem + +Create the backend TLS and RabbitMQ TLS certificates +---------------------------------------------------- + +1. Run the playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-backend-tls.yml + +2. Use ansible-vault to encrypt the keys in $KAYOBE_CONFIG_PATH/kolla/certificates/-key.pem. Commit the certificates and keys to the kayobe configuration. + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/-key.pem + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/-key.pem + +.. _vault-haproxy: + +HAProxy integration +=================== + +It is possible to expose the overcloud Vault service via the Kolla Ansible HAProxy load balancer. +This provides a single highly available API endpoint, as well as monitoring of the Vault backends when combined with Prometheus. +HAProxy integration is no longer required for generating OpenStack control plane certificates, making it possible to deploy Vault and generate certificates before any containers have been deployed by Kolla Ansible. + +1. Create the HAProxy config to reverse proxy the Vault HA container + + Set the vault_front to the external VIP address or internal VIP address depending on the installation. Set the vault_back to the IPs of the control nodes. + + Set the following in etc/kayobe/kolla/config/haproxy/services.d/vault.cfg or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/haproxy/services.d/vault.cfg + + .. code-block:: + + # Delete "verify none" if not using self-signed/unknown issuer + {% raw %} + frontend vault_front + mode tcp + option tcplog + bind {{ kolla_internal_vip_address }}:8200 + default_backend vault_back + + backend vault_back + mode tcp + option httpchk GET /v1/sys/health + # https://www.vaultproject.io/api-docs/system/health + # 200: initialized, unsealed, and active + # 429: standby + http-check expect rstatus (200|429) + + {% for host in groups['control'] %} + {% set host_name = hostvars[host].ansible_facts.hostname %} + {% set host_ip = 'api' | kolla_address(host) %} + server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5 + {% endfor %} + {% endraw %} + +2. If HAProxy has not yet been deployed, continue to :ref:`certificates deployment `. + If HAProxy has been deployed, it may be redeployed with the new Vault service configuration: + + .. code-block:: + + kayobe overcloud service deploy -kt haproxy + +.. _vault-certificates: + +Certificates deployment +======================= + +.. warning:: + + The switch from HTTP to HTTPS during the deployment of internal/backend TLS certificates can temporarily disrupt service availability and necessitates a restart of all services. During this transition, endpoints may become unreachable following the HAProxy restart, persisting until the endpoint catalogue and client have been reconfigured to use HTTPS. + +Enable the required TLS variables in kayobe and kolla +----------------------------------------------------- + +1. If using Vault as a CA for the external API, set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml + + .. code-block:: + + # Whether TLS is enabled for the external API endpoints. Default is 'no'. + kolla_enable_tls_external: yes + kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}" + + See :ref:`tempest-cacert` for information on adding CA certificates to the trust store when running Tempest. + +2. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml + + .. code-block:: + + # Whether TLS is enabled for the internal API endpoints. Default is 'no'. + kolla_enable_tls_internal: yes + kolla_admin_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}" + + See :ref:`os-capacity` for information on adding CA certificates to the trust store when deploying the OpenStack Capacity exporter. + +3. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml + + .. code-block:: + + # Internal TLS configuration + # Copy the self-signed CA into the kolla containers + kolla_copy_ca_into_containers: "yes" + # Use the following trust store within the container + openstack_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution == 'rocky' else '/etc/ssl/certs/ca-certificates.crt' }}" + + # Backend TLS config + # Enable backend TLS + kolla_enable_tls_backend: "yes" + + # If using RabbitMQ TLS: + rabbitmq_enable_tls: "yes" + +4. Deploy OpenStack + + .. warning:: + + It is important that you are only using admin endpoints for keystone. If + any admin endpoints exist for other services, they must be deleted e.g. + + .. code-block:: + + openstack endpoint list --interface admin -f value | \ + awk '!/keystone/ {print $1}' | xargs openstack endpoint delete + + .. code-block:: + + kayobe overcloud service deploy + + If VM provisioning fails with an error with this format: + + .. code-block:: + + Unable to establish connection to http://:9696/v2.0/ports/some-sort-of-uuid: Connection aborted + + Restart the nova-compute container on all hypervisors: + + .. code-block:: + + kayobe overcloud host command run --command "systemctl restart kolla-nova_compute-container.service" --become --show-output -l compute + +Barbican integration +==================== + +Barbican integration depends on :ref:`HAProxy integration `. + +Enable Barbican in kayobe +------------------------- + +1. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml + + .. code-block:: + + kolla_enable_barbican: yes + +Generate secrets_barbican_approle_secret_id +------------------------------------------- + +1. Run ``uuidgen`` to generate secret id +2. Insert into secrets.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/secrets.yml + + .. code-block:: + + secrets_barbican_approle_secret_id: "YOUR-SECRET-GOES-HERE" + +Create required configuration in Vault +-------------------------------------- + +1. Run vault-deploy-barbican.yml custom playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-barbican.yml + +Add secrets_barbican_approle_id to secrets +------------------------------------------ + +1. Note the role id from playbook output and insert into secrets.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/secrets.yml + + .. code-block:: + + secrets_barbican_approle_role_id: "YOUR-APPROLE-ID-GOES-HERE" + +Configure Barbican +------------------ + +1. Put required configuration in kayobe-config/etc/kayobe/kolla/config/barbican.conf or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/config/barbican.conf + + .. code-block:: + + [secretstore] + namespace=barbican.secretstore.plugin + enable_multiple_secret_stores=false + enabled_secretstore_plugins=vault_plugin + + [vault_plugin] + vault_url = https://{{ kolla_internal_fqdn }}:8200 + use_ssl = True + {% raw %} + ssl_ca_crt_file = {{ openstack_cacert }} + {% endraw %} + approle_role_id = {{ secrets_barbican_approle_role_id }} + approle_secret_id = {{ secrets_barbican_approle_secret_id }} + kv_mountpoint = barbican + +Deploy Barbican +--------------- + + .. code-block:: + + kayobe overcloud service deploy -kt barbican diff --git a/doc/source/configuration/walled-garden.rst b/doc/source/configuration/walled-garden.rst new file mode 100644 index 000000000..937619a5a --- /dev/null +++ b/doc/source/configuration/walled-garden.rst @@ -0,0 +1,120 @@ +============================== +Walled Garden deployment guide +============================== + +This document describes how to configure Kayobe for a “walled garden” +deployment, where most hosts do not have external network access. + +Routing +======= + +With the exception of hosts running HAProxy (usually controllers), +overcloud hosts typically do not have external network access. + +The hosts running HAProxy typically have a gateway on the public API +network for external network access. + +NTP +=== + +Overcloud hosts initially use the seed’s NTP server. + +In ``etc/kayobe/inventory/group_vars/overcloud/time``: + +.. code:: yaml + + --- + # NTP services for overcloud hosts + # During early initialisation we use the seed Node + # Following deployment we include the OpenStack VIP + + chrony_ntp_servers: + - server: "{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}" + +Proxy +===== + +We use a Squid caching proxy to access external web resources. + +In ``etc/kayobe/seed.yml``, enable the Squid proxy container on the +seed: + +.. code:: yaml + + # Seed container running a Squid caching proxy. This can be used to proxy + # HTTP(S) requests from control plane hosts. + seed_squid_container_enabled: true + +In some environments we have found that squid’s preference for IPv6 can +cause problems. It can be forced to prefer IPv4, by adding the following +in ``etc/kayobe/containers/squid_proxy/squid.conf``: + +.. code:: + + dns_v4_first on + +In ``etc/kayobe/inventory/group_vars/overcloud/proxy`` (and any other +groups that need to use the proxy), configure overcloud hosts to use the +proxy: + +.. code:: yaml + + --- + # HTTP proxy URL (format: http(s)://[user:password@]proxy_name:port). By + # default no proxy is used. + http_proxy: "http://{{ admin_oc_net_name | net_ip(inventory_hostname=groups['seed'][0]) }}:3128" + + # HTTPS proxy URL (format: http(s)://[user:password@]proxy_name:port). By + # default no proxy is used. + https_proxy: "{{ http_proxy }}" + + # List of domains, hostnames, IP addresses and networks for which no proxy is + # used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~ + # docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or + # ["127.0.0.1", "localhost"] otherwise. This is configured only if either + # http_proxy or https_proxy is set. + no_proxy: + - "127.0.0.1" + - "localhost" + - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" + - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[groups.seed.0] }}" + - "{{ lookup('vars', admin_oc_net_name ~ '_ips')[inventory_hostname] }}" + - "{{ kolla_external_fqdn }}" + - "{{ kolla_internal_fqdn }}" + + # PyPI proxy URL (format: http(s)://[user:password@]proxy_name:port) + pip_proxy: "{{ https_proxy }}" + +We typically don’t use the proxy for DNF package updates, or for +container image downloads, since the Pulp server is hosted on the seed. +The ``no_proxy`` setting should handle this. + +For Ubuntu hosts, where package repos are not hosted in a local Pulp +server, you will also want to proxy APT requests. This can be done by +adding the following in +``etc/kayobe/inventory/group_vars/overcloud/proxy``: + +.. code:: yaml + + # Apt proxy URL for HTTP. Default is empty (no proxy). + apt_proxy_http: "{{ http_proxy }}" + + # Apt proxy URL for HTTPS. Default is {{ apt_proxy_http }}. + apt_proxy_https: "{{ https_proxy }}" + +Typically, container images are pulled from the local Pulp server. If +you need to be able to pull container images from external sources, it +may be necessary to add proxy configuration for Docker. This is Kolla +Ansible configuration, rather than Kayobe, in +``etc/kayobe/kolla/inventory/group_vars/overcloud``: + +.. code:: yaml + + --- + # Use a proxy for external Docker image pulls + docker_http_proxy: "http://:3128" + docker_https_proxy: "http://:3128" + docker_no_proxy: + - "127.0.0.1" + - "localhost" + - "{{ ('http://' ~ docker_registry) | urlsplit('hostname') if docker_registry else '' }}" diff --git a/doc/source/configuration/wazuh.rst b/doc/source/configuration/wazuh.rst new file mode 100644 index 000000000..a4ecedf93 --- /dev/null +++ b/doc/source/configuration/wazuh.rst @@ -0,0 +1,431 @@ +===== +Wazuh +===== + +`Wazuh `_ is a security monitoring platform. +It monitors for: + +* Security-related system events. +* Known vulnerabilities (CVEs) in versions of installed software. +* Misconfigurations in system security. + +The short version +================= + +#. Create an infrastructure VM for the Wazuh manager, and add it to the wazuh-manager group +#. Configure the infrastructure VM with kayobe: ``kayobe infra vm host configure`` +#. Edit your config under + ``$KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/wazuh-manager``, in + particular the defaults assume that the ``provision_oc_net`` network will be + used. +#. Generate secrets: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml`` +#. Deploy the Wazuh manager: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml`` +#. Deploy the Wazuh agents: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml`` + + +Wazuh Manager Host +================== + +Provision using infra-vms +------------------------- + +Provisioning an infra VM for Wazuh Manager. + +Kayobe supports :kayobe-doc:`provisioning infra VMs `. +The following configuration may be used as a guide. Config for infra VMs is documented :kayobe-doc:`here `. + +Add a Wazuh Manager host to the ``wazuh-manager`` group in ``$KAYOBE_CONFIG_PATH/inventory/hosts``. + +.. code-block:: ini + + [wazuh-manager] + os-wazuh + +Add the ``wazuh-manager`` group to the ``infra-vms`` group in ``$KAYOBE_CONFIG_PATH/inventory/groups``. + +.. code-block:: ini + + [wazuh-manager] + + [infra-vms:children] + wazuh-manager + +Define VM sizing in ``$KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/infra-vms``: + +.. code-block:: yaml + + --- + # Memory in MB. + infra_vm_memory_mb: 16384 + + # Number of vCPUs. + infra_vm_vcpus: 8 + + # Capacity of the infra VM data volume. + infra_vm_data_capacity: "200G" + +Optional: define LVM volumes in ``$KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/lvm``. +``/var/ossec`` often requires greater storage space, and ``/var/lib/wazuh-indexer`` +may be beneficial too. + +.. code-block:: yaml + + # List of infra VM volume groups. See mrlesmithjr.manage-lvm role for + # format. + infra_vm_lvm_groups: + - vgname: "data" + disks: + - "/dev/vdb" + create: true + lvnames: + - lvname: "data" + size: "100%VG" + filesystem: "ext4" + mount: true + mntp: "/var/ossec" + create: true + + +Define network interfaces ``$KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/network-interfaces``: + +(The following is an example - the names will depend on your particular network configuration.) + +.. code-block:: yaml + + --- + # Overcloud provisioning network IP information. + provision_oc_net_interface: "ens3" + + +The Wazuh manager may need to be exposed externally, in which case it may require another interface. +This can be done as follows in ``$KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/network-interfaces``, +with the network defined in ``networks.yml`` as usual. + +.. code-block:: yaml + + infra_vm_extra_network_interfaces: + - "extra_net" + + # External network connectivity on ens2 + extra_net_interface: "ens2" + + +Follow the Kayobe instructions to :kayobe-doc:`provision the VM ` and configure the host. + + +Network Setup +------------- + +Your wazuh-manager VM needs to have network connection with servers which will have the wazuh-agent installed, preferably it should be in the `provision_oc_net`. + + +Required ports +-------------- + +Several services are used for the communication of Wazuh components. Below is the list of default ports used by these services. + ++-----------------+-----------+----------------+------------------------------------------------+ +| Component | Port | Protocol | Purpose | ++=================+===========+================+================================================+ +| | 1514 | TCP (default) | Agent connection service | ++ +-----------+----------------+------------------------------------------------+ +| | 1514 | UDP (optional) | Agent connection service (disabled by default) | ++ +-----------+----------------+------------------------------------------------+ +| Wazuh server | 1515 | TCP | Agent enrollment service | ++ +-----------+----------------+------------------------------------------------+ +| | 1516 | TCP | Wazuh cluster daemon | ++ +-----------+----------------+------------------------------------------------+ +| | 514 | UDP (default) | Wazuh Syslog collector (disabled by default) | ++ +-----------+----------------+------------------------------------------------+ +| | 514 | TCP (optional) | Wazuh Syslog collector (disabled by default) | ++ +-----------+----------------+------------------------------------------------+ +| | 55000 | TCP | Wazuh server RESTful API | ++-----------------+-----------+----------------+------------------------------------------------+ +| | 9200 | TCP | Wazuh indexer RESTful API | ++ Wazuh indexer +-----------+----------------+------------------------------------------------+ +| | 9300-9400 | TCP | Wazuh indexer cluster communication | ++-----------------+-----------+----------------+------------------------------------------------+ +| Wazuh dashboard | 443 | TCP | Wazuh web user interface | ++-----------------+-----------+----------------+------------------------------------------------+ + + +Manually provisioned VM +----------------------- + +In cases where you can’t use infra-vms to deploy your wazuh-manager VM but you want to configure +the host using kayobe, here are some tips (note that depending on your setup this doesn't have to always apply): + +* Depending on preferences, hosts have to be part of some group in inventory. ``infra-vms`` group still seems like the best choice. + You can use ``kayobe infra vm host configure`` to configure host in this case. + The tips below are based on the assumption that infra-vm will be used. +* user ``stack`` with passwordless sudo and access with ssh keys needs to be present on the host. + It can be achieved in many different ways, depending on your setup. +* lvm configuration should be placed in ``host_vars/`` +* wazuh-manager hosts have to be part of ``infra-vms`` group (directly or as child) +* The network used on the host needs to be defined in ``networks.yml`` and + if you have pre-alocated an IP, it should be added to ``network-allocation.yml``. + For example, if using host with IP 10.10.224.5 in network 10.10.224.0/24 one have to add: + + +``networks.yml``: + +.. code-block:: yaml + + provision_oc_net_cidr: 10.10.224.0/24 + provision_oc_net_allocation_pool_start: 10.10.224.3 + provision_oc_net_allocation_pool_end: 10.10.224.200 + provision_oc_net_gateway: 10.10.224.254 + + +``network-allocation.yml``: + +.. code-block:: yaml + + provision_oc_net_ips: + nesmetprd01: 10.10.224.5 + +Deploying Wazuh Manager services +================================ + +Setup +----- + +To install a specific version modify the wazuh-ansible entry in ``$KAYOBE_CONFIG_PATH/ansible/requirements.yml``: + +.. code-block:: yaml + + roles: + - name: wazuh-ansible + src: https://github.com/stackhpc/wazuh-ansible + version: custom-branch + +The default version has been tested and verified, but there is no reason not to use a different one. + +Reinstall the role if required: + +``kayobe control host bootstrap`` + + +Edit the playbook and variables to your needs: + +Wazuh manager configuration +--------------------------- + +Wazuh manager playbook is located in ``$KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml``. +Running this playbook will: + +* generate certificates for wazuh-manager +* setup and deploy filebeat on wazuh-manager vm +* setup and deploy wazuh-indexer on wazuh-manager vm +* setup and deploy wazuh-manager on wazuh-manager vm +* setup and deploy wazuh-dashboard on wazuh-manager vm +* copy certificates over to wazuh-manager vm + +Wazuh manager variables file is located in ``$KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/wazuh-manager``. + +You may need to modify some of the variables, including: + +* wazuh_manager_ip + + +.. note:: + + If you are using multiple environments, and you need to customise Wazuh in + each environment, create override files in an appropriate directory, + for example ``$KAYOBE_CONFIG_PATH/environments/production/inventory/group_vars/``. + + Files which values can be overridden (in the context of Wazuh): + + - $KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh/wazuh-manager/wazuh-manager + - $KAYOBE_CONFIG_PATH/wazuh-manager.yml + - $KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh/wazuh-agent/wazuh-agent + +You'll need to run ``wazuh-manager.yml`` playbook again to apply customisation. + +Secrets +------- + +Wazuh requires that secrets or passwords are set for itself and the services with which it communiticates. +Wazuh secrets playbook is located in ``$KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml``. +Running this playbook will generate and put pertinent security items into secrets +vault file which will be placed in ``$KAYOBE_CONFIG_PATH/wazuh-secrets.yml``. +If using environments it ends up in ``$KAYOBE_CONFIG_PATH/environments//wazuh-secrets.yml`` +Remember to encrypt! + +Wazuh secrets template is located in ``$KAYOBE_CONFIG_PATH/ansible/templates/wazuh-secrets.yml.j2``. +It will be used by wazuh secrets playbook to generate wazuh secrets vault file. + + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml + +.. note:: Use ``ansible-vault`` to view the secrets: + + ``ansible-vault view --vault-password-file ~/vault.password $KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/wazuh-secrets.yml`` + +Configure Wazuh Dashboard's Server Host +--------------------------------------- + +It is common to want to configure the Wazuh dashboard to serve on a different network than the overcloud provisioning network used for Wazuh's internal communication. + +In order to do so, either create or edit the ``$KAYOBE_CONFIG_PATH/environments//inventory/group_vars/wazuh-manager/wazuh-manager.yml`` configuration file to include the dashboard variable: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/environments//inventory/group_vars/wazuh-manager/wazuh-manager.yml + + dashboard_server_host: "{{ _net_name | net_ip }}" + +For example: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/environments//inventory/group_vars/wazuh-manager/wazuh-manager.yml + + dashboard_server_host: "{{ public_net_name | net_ip }}" + +If this is being added post deployment the user will be required to re-run the ``wazuh-manager.yml`` ansible playbook via the following command: + +.. code-block:: bash + :caption: Deploy or re-run the ``wazuh-manager.yml`` ansible playbook to apply changes made to the configuration. + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml + +More on the deployment of Wazuh dashboard can be found below in the :ref:`subsequent section `. + +TLS (optional) +-------------- + +You can generate your own TLS certificates, otherwise skip this section. +By default, Wazuh Ansible uses `wazuh-cert-tool.sh `__ +to automatically +generate certificates for wazuh-indexer (previously Elasticsearch and opendistro) +and wazuh-dashbooard (previously Kibana) using a local CA. +If the certificates directory ``{{ kayobe_env_config_path }}/wazuh`` +does not exist, it will generate the following certificates in ``{{ kayobe_env_config_path }}/wazuh/wazuh-certificates/`` +(here os-wazuh is set as ``elasticsearch_node_name`` and ``kibana_node_name``: + + +* Admin certificate for opendistro security + * admin-key.pem, admin.pem +* Node certificate + * os-wazuh-key.pem, os-wazuh.pem +* HTTP certificate for wazuh-dashboard (port 5601) & wazuh-indexer (port 9200) + * os-wazuh_http.key, os-wazuh_http.pem +* Root CA certificate + * root-ca.key root-ca.pem + + +It is also possible to use externally generated certificates for +wazuh-dashboard. Customise the ``dashboard_node_name`` variable so that you can +use a separate certificate and key for this service e.g: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-manager/wazuh-manager + + dashboard_node_name: "wazuh-dashboard" + +You will need to create two files matching the following pattern: + +- ``{{ dashboard_node_name }}-key.pem`` for the private key +- ``{{ dashboard_node_name }}.pem`` for the certificate + +Drop these files into ``$KAYOBE_CONFIG_PATH/environments//wazuh/wazuh-certificates/`` if +using the kayobe environments feature, or ``$KAYOBE_CONFIG_PATH/wazuh/wazuh-certificates/`` if not. +The key for the external certificate should be in PKCS#8 format (in its header it may have BEGIN +PRIVATE KEY instead of BEGIN RSA PRIVATE KEY or BEGIN OPENSSH PRIVATE KEY). + +Example OpenSSL rune to convert to PKCS#8: + +``openssl pkcs8 -topk8 -nocrypt -in wazuh.key -out wazuh.key.pkcs8`` + +TODO: document how to use a local certificate. Do we need to override all certificates? + +Custom SCA Policies (optional) +------------------------------ + +Wazuh ships with a large selection of Security Configuration Assessment +rulesets. However, you may find you want to add more. This can be achieved via +`custom policies `_. + +SKC supports this automatically, just add the policy file from this PR to +``{{ kayobe_env_config_path }}/wazuh/custom_sca_policies``. + +.. _Deploy: + +Deploy +------ + +Deploy Wazuh manager: + +``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml`` + +If you are using the wazuh generated certificates, +this will result in the creation of some certificates and keys (in case of custom certs adjust path to it). +Encrypt the keys (and remember to commit to git): + +``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments//wazuh/wazuh-certificates/*.key $KAYOBE_CONFIG_PATH/environments//wazuh/wazuh-certificates/*-key.pem`` + +If using the kayobe environments feature, otherwise: + +``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*.key $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*-key.pem`` + +.. _wazuh-verification: + +Verification +------------ + +The Wazuh portal should be accessible on port 443 of the Wazuh +manager’s IPs (using HTTPS, with the root CA cert in ``$KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/wazuh-certificates/root-ca.pem``). +The first login should be as the admin user, +with the opendistro_admin_password password in ``$KAYOBE_CONFIG_PATH/wazuh-secrets.yml``. +This will create the necessary indices. + +Troubleshooting + +Logs are in ``/var/log/wazuh-indexer/wazuh.log``. There are also logs in the journal. + +Wazuh agents +============ + +Wazuh agent playbook is located in ``$KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml``. + +Wazuh agent variables file is located in ``$KAYOBE_CONFIG_PATH/inventory/group_vars/wazuh-agent/wazuh-agent``. + +You may need to modify some variables, including: + +* wazuh_manager_address + +Deploy the Wazuh agents: + +``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml`` + +The Wazuh Agent is deployed to all hosts in the ``wazuh-agent`` +inventory group, comprising the ``seed`` group +plus the ``overcloud`` group (containing all hosts in the +OpenStack control plane). + +.. code-block:: ini + + [wazuh-agent:children] + seed + overcloud + +The hosts running Wazuh Agent should automatically be registered +and visible within the Wazuh Manager dashboard. + +.. note:: It is good practice to use a `Kayobe deploy hook + `_ + to automate deployment and configuration of the Wazuh Agent + following a run of ``kayobe overcloud host configure``. + +Verification +------------ + +The Wazuh agents should register with the Wazuh manager. This can be verified via the agents page in Wazuh Portal. +Check CIS benchmark output in agent section. + +Additional resources +-------------------- + +For times when you need to upgrade wazuh with elasticsearch to version with opensearch or you just need to deinstall all wazuh components: +Wazuh purge script: https://github.com/stackhpc/wazuh-server-purge diff --git a/doc/source/contributor/documentation.rst b/doc/source/contributor/documentation.rst new file mode 100644 index 000000000..b8a263f6c --- /dev/null +++ b/doc/source/contributor/documentation.rst @@ -0,0 +1,20 @@ +.. _documentation: + +============= +Documentation +============= + +This documentation is provided in the ``doc`` directory. To build it, run: + +.. code-block:: console + + tox -e docs + +The HTML will be written to ``doc/build``, and may be viewed in a web browser. + +Documentation is hosted on readthedocs.io. Multiple maintainers can be added +for the project. GitHub integration ensures that each version is built when the +corresponding branch is pushed to. + +Each release is hosted as a separate version, and new versions will need to be +added manually. diff --git a/doc/source/contributor/environments/aufn-ceph.rst b/doc/source/contributor/environments/aufn-ceph.rst new file mode 100644 index 000000000..26e7fa7c9 --- /dev/null +++ b/doc/source/contributor/environments/aufn-ceph.rst @@ -0,0 +1,190 @@ +================================= +A universe from nothing with Ceph +================================= + +This environment creates a Universe-from-nothing_-style deployment of Kayobe consisting of multiple 'virtual baremetal nodes' running as VMs on a single physical hypervisor. + +.. _Universe-from-nothing: https://github.com/stackhpc/a-universe-from-nothing + +.. warning:: + + This guide was written for the Yoga release and has not been validated for + Caracal. Proceed with caution. + +Prerequisites +============= + +* a baremetal node with at least 64GB of RAM running Rocky Linux 9 or Ubuntu Jammy. + +* access to the test pulp server on SMS lab + +Setup +===== + +--- + +**Note**: The steps detailed below are combined into a convenient script which is packaged with this repo at ``etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh``. For an automated deployment, this script can simply be copied to the baremetal host and then executed as ``bash ~/a-universe-from-nothing.sh``. + +--- + +To begin the manual setup, access the baremetal node via SSH and install some basic dependencies. + +Rocky: + +.. parsed-literal:: + + sudo dnf install -y gcc python3-devel + +Ubuntu: + +.. parsed-literal:: + + sudo apt update + sudo apt -y install gcc libffi-dev python3-dev python-is-python3 + + +Configure the system firewall and security settings: + +.. parsed-literal:: + + # Disable the firewall. + sudo systemctl is-enabled firewalld && sudo systemctl stop firewalld && sudo systemctl disable firewalld + + # Disable SELinux both immediately and permanently. + if $(which setenforce 2>/dev/null >/dev/null); then + sudo setenforce 0 + sudo sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config + fi + + # Prevent sudo from performing DNS queries. + echo 'Defaults !fqdn' | sudo tee /etc/sudoers.d/no-fqdn + +Clone the Kayobe, Kayobe configuration (this one) and Tenks repositories: + +.. parsed-literal:: + + cd + mkdir -p src + pushd src + git clone https://github.com/stackhpc/kayobe.git -b |current_release_git_branch_name| + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b |current_release_git_branch_name| kayobe-config + pushd kayobe + git clone https://opendev.org/openstack/tenks.git + popd + popd + +Create a virtual environment and install Kayobe: + +.. parsed-literal:: + + cd + mkdir -p venvs + pushd venvs + python3 -m venv kayobe + source kayobe/bin/activate + pip install -U pip + pip install ../src/kayobe + popd + + +Installation +============ + +The following commands activate the correct kayobe environment and prepare the Ansible control host: + +.. parsed-literal:: + + pushd ~/src/kayobe-config + source kayobe-env --environment aufn-ceph + $KAYOBE_CONFIG_PATH/environments/aufn-ceph/configure-local-networking.sh + kayobe control host bootstrap + +Deployment +========== + +Next, configure the seed VM: + +.. parsed-literal:: + + kayobe seed hypervisor host configure + kayobe seed vm provision + kayobe seed host configure + +Once the seed vm is provisioned, deploy a local pulp server on the seed and then re-add the relevant local networking configuration since ``service deploy`` restarts the networking interface: + +.. parsed-literal:: + + kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none + $KAYOBE_CONFIG_PATH/environments/aufn-ceph/configure-local-networking.sh + +We can now sync the contents of the local pulp server with that of SMS test pulp and then complete the seed VM setup: + +.. parsed-literal:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + kayobe seed service deploy + +With the seed VM configured, we use Tenks_ to deploy an additional set of VMs on the same baremetal node and configure them as 'virual baremetal' hosts in order to replicate a true multi-node kayobe deployment within a single node. + +.. _Tenks: https://github.com/stackhpc/tenks + +.. parsed-literal:: + + export TENKS_CONFIG_PATH=$KAYOBE_CONFIG_PATH/environments/aufn-ceph/tenks.yml + export KAYOBE_CONFIG_SOURCE_PATH=~/src/kayobe-config + export KAYOBE_VENV_PATH=~/venvs/kayobe + pushd ~/src/kayobe + ./dev/tenks-deploy-overcloud.sh ./tenks + popd + +These nodes can then be provisioned as overcloud control, compute and storage hosts with + +.. parsed-literal:: + + kayobe overcloud inventory discover + kayobe overcloud hardware inspect + kayobe overcloud provision + kayobe overcloud host configure + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml + kayobe overcloud container image pull + kayobe overcloud service deploy + source $KOLLA_CONFIG_PATH/public-openrc.sh + kayobe overcloud post configure + source $KOLLA_CONFIG_PATH/public-openrc.sh + +Finally, we create the bare minimum cloud infrastructure (networks, images, flavors etc.) by running the following shell script. + +.. parsed-literal:: + + $KAYOBE_CONFIG_PATH/environments/aufn-ceph/configure-openstack.sh ~ + +This completes the deployment process. + + +Testing +======= + +We can deploy a test VM to ensure that our 'universe' is up and running by first creating a python virtual environment with the OpenStack CLI installed. + +.. parsed-literal:: + + python3 -m venv ~/openstack-env + source ~/openstack-env/bin/activate + pip install -U pip + pip install python-openstackclient + +We then use the CLI to create a keypair, floating IP and test VM: + +.. parsed-literal:: + + openstack keypair create --public-key ~/.ssh/id_rsa.pub mykey + openstack server create --key-name mykey --flavor m1.tiny --image cirros --network admin-tenant test-vm-1 + openstack floating ip create external + openstack server add floating ip test-vm-1 `openstack floating ip list -c ID -f value` + openstack server list + +which will create a VM named ``test-vm-1`` with a Cirros OS image and a default login password of 'gocubsgo'. diff --git a/doc/source/contributor/environments/ci-aio.rst b/doc/source/contributor/environments/ci-aio.rst new file mode 100644 index 000000000..953782568 --- /dev/null +++ b/doc/source/contributor/environments/ci-aio.rst @@ -0,0 +1,266 @@ +====== +ci-aio +====== + +This environment deploys an all-in-one converged control/compute cloud for +testing. + +There are two ways to set up the environment. The automated setup script +automates the manual setup steps below, and is recommended for most users. +The manual setup steps are provided for reference, and for users who wish to +make changes to the setup process. + +.. seealso:: + + All-in-one GitHub CI testing using this environment is described :ref:`here + `. + +Prerequisites +============= + +* a Rocky Linux 9 or Ubuntu Jammy 22.04 host + +Automated Setup +=============== + +Access the host via SSH. You may wish to start a ``tmux`` session. + +Download the setup script: + +.. parsed-literal:: + + wget https://raw.githubusercontent.com/stackhpc/stackhpc-kayobe-config/stackhpc/2024.1/etc/kayobe/environments/ci-aio/automated-setup.sh + +Change the permissions on the script: + +.. parsed-literal:: + + sudo chmod 700 automated-setup.sh + +Acquire the Ansible Vault password for this repository, and store a +copy at ``~/vault-pw``. + +Run the setup script: + +.. parsed-literal:: + + ./automated-setup.sh + +The script will pull the current version of Kayobe and this repository, and +then run the manual setup steps below. The script can be easily edited with the +following options: + +* ``BASE_PATH`` (default: ``~``) - Directory to deploy from. The directory must + exist before running the script. +* ``KAYOBE_BRANCH`` (default: ``stackhpc/2024.1``) - The branch of Kayobe + source code to use. +* ``KAYOBE_CONFIG_BRANCH`` (default: ``stackhpc/2024.1``) - The branch of + ``stackhpc-kayobe-config`` to use. +* ``KAYOBE_AIO_LVM`` (default: ``true``) - Whether the image uses LVM. +* ``KAYOBE_CONFIG_EDIT_PAUSE`` (default: ``false``) - Option to pause + deployment after cloning the kayobe-config branch, so the environment can be + customised before continuing. +* ``AIO_RUN_TEMPEST`` (default: ``false``) - Whether to run Tempest Refstack + after deployment instead of the default VM smoke test. + +Manual Setup +============ + +Host Configuration +------------------ + +Access the host via SSH. + +If using an LVM-based image, extend the ``lv_home`` and ``lv_tmp`` logical +volumes. + +.. parsed-literal:: + + sudo pvresize $(sudo pvs --noheadings | head -n 1 | awk '{print $1}') + sudo lvextend -L 4G /dev/rootvg/lv_home -r + sudo lvextend -L 4G /dev/rootvg/lv_tmp -r + +Install package dependencies. + +On Rocky Linux: + +.. parsed-literal:: + + sudo dnf install -y git + +On Ubuntu: + +.. parsed-literal:: + + sudo apt update + sudo apt install -y gcc git libffi-dev python3-dev python-is-python3 python3-venv + +Clone the Kayobe and Kayobe configuration repositories (this one): + +.. parsed-literal:: + + cd + mkdir -p src + pushd src + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b |current_release_git_branch_name| kayobe-config + popd + +Create a virtual environment and install Kayobe: + +.. parsed-literal:: + + cd + mkdir -p venvs + pushd venvs + python3 -m venv kayobe + source kayobe/bin/activate + pip install -U pip + pip install -r ../src/kayobe-config/requirements.txt + popd + +Add initial network configuration: + +.. parsed-literal:: + + sudo ip l add breth1 type bridge + sudo ip l set breth1 up + sudo ip a add 192.168.33.3/24 dev breth1 + sudo ip l add dummy1 type dummy + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + +On Ubuntu systems, persist the running network configuration. + +.. parsed-literal:: + + sudo cp /run/systemd/network/* /etc/systemd/network + +Configuration +============= + +If using Ironic: + +.. parsed-literal:: + + cd src/kayobe-config + cat << EOF > etc/kayobe/aio.yml + kolla_enable_ironic: true + EOF + +Installation +------------ + +Acquire the Ansible Vault password for this repository, and store a copy at +``~/vault-pw``. + +The following commands install Kayobe and its dependencies, and prepare the +Ansible control host. + +.. parsed-literal:: + + export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) + pushd ~/venvs/kayobe + source bin/activate + popd + pushd ~/src/kayobe-config + source kayobe-env --environment ci-aio + kayobe control host bootstrap + +Deployment +---------- + +If using an LVM-based image, grow the root volume group. + +.. parsed-literal:: + + kayobe playbook run etc/kayobe/ansible/growroot.yml + +On Ubuntu systems, purge the command-not-found package. + +.. parsed-literal:: + + kayobe playbook run etc/kayobe/ansible/purge-command-not-found.yml + +Next, configure the host OS & services. + +.. parsed-literal:: + + kayobe overcloud host configure + +Finally, deploy the overcloud services. + +.. parsed-literal:: + + kayobe overcloud service deploy + +The control plane should now be running. + +If using Ironic, run overcloud post configuration: + +.. parsed-literal:: + + source ~/src/kayobe-config/etc/kolla/public-openrc.sh + kayobe overcloud post configure + +Testing +------- + +Run a smoke test: + +.. parsed-literal:: + + cd ~/src/kayobe + ./dev/overcloud-test-vm.sh + +Ironic +------ + +For a control plane with Ironic enabled, a "bare metal" instance can be +deployed. We can use the Tenks project to create fake bare metal nodes. + +Clone the tenks repository: + +.. parsed-literal:: + + cd ~/src/kayobe + git clone https://opendev.org/openstack/tenks.git + +Optionally, edit the Tenks configuration file, +``~/src/kayobe/dev/tenks-deploy-config-compute.yml``. + +Run the ``dev/tenks-deploy-compute.sh`` script to deploy Tenks: + +.. parsed-literal:: + + cd ~/src/kayobe + export KAYOBE_CONFIG_SOURCE_PATH=~/src/kayobe-config + export KAYOBE_VENV_PATH=~/venvs/kayobe + ./dev/tenks-deploy-compute.sh ./tenks/ + +Check that Tenks has created VMs called tk0 and tk1: + +.. parsed-literal:: + + sudo virsh list --all + +Verify that VirtualBMC is running: + +.. parsed-literal:: + + ~/tenks-venv/bin/vbmc list + +We are now ready to run the ``dev/overcloud-test-baremetal.sh`` script. This +will run the ``init-runonce`` setup script provided by Kolla Ansible that +registers images, networks, flavors etc. It will then deploy a bare metal +server instance, and delete it once it becomes active: + +.. parsed-literal:: + + ./dev/overcloud-test-baremetal.sh + +The machines and networking created by Tenks can be cleaned up via +``dev/tenks-teardown-compute.sh``: + +.. parsed-literal:: + + ./dev/tenks-teardown-compute.sh ./tenks diff --git a/doc/source/contributor/environments/ci-builder.rst b/doc/source/contributor/environments/ci-builder.rst new file mode 100644 index 000000000..ec5bbb779 --- /dev/null +++ b/doc/source/contributor/environments/ci-builder.rst @@ -0,0 +1,207 @@ +========== +ci-builder +========== + +The ``ci-builder`` Kayobe environment is used to build Kolla container images. +Images are built using package repositories in the StackHPC Ark Pulp service, +and pushed there once built. + +.. warning:: + + This guide was written for the Yoga release and has not been validated for + Caracal. Proceed with caution. + +In general it is preferable to use the `container image build CI workflow +`_ +to build container images, but this manual approach may be useful in some +cases. + +.. seealso:: + + A container image build GitHub Actions workflow using this environment is + described :ref:`here `. + +.. seealso:: + + An overcloud host image build GitHub Actions workflow using this environment + is described :ref:`here `. + +Prerequisites +============= + +* a Rocky Linux 9 or Ubuntu Jammy 22.04 host + +Setup +===== + +Access the host via SSH. You may wish to start a ``tmux`` session. + +If using an LVM-based image, extend the ``lv_home`` and ``lv_tmp`` logical +volumes. + +.. parsed-literal:: + + sudo pvresize $(sudo pvs --noheadings | head -n 1 | awk '{print $1}') + sudo lvextend -L 4G /dev/rootvg/lv_home -r + sudo lvextend -L 4G /dev/rootvg/lv_tmp -r + +Install package dependencies. + +On Rocky Linux: + +.. parsed-literal:: + + sudo dnf install -y git + +On Ubuntu: + +.. parsed-literal:: + + sudo apt update + sudo apt install -y gcc git libffi-dev python3-dev python-is-python3 python3-venv + +Clone the Kayobe and Kayobe configuration repositories (this one): + +.. parsed-literal:: + + cd + mkdir -p src + pushd src + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b |current_release_git_branch_name| kayobe-config + popd + +Create a virtual environment and install Kayobe: + +.. parsed-literal:: + + cd + mkdir -p venvs + pushd venvs + python3 -m venv kayobe + source kayobe/bin/activate + pip install -U pip + pip install -r ../src/kayobe-config/requirements.txt + popd + +Add initial network configuration: + +.. parsed-literal:: + + sudo ip l add breth1 type bridge + sudo ip l set breth1 up + sudo ip a add 192.168.33.3/24 dev breth1 + sudo ip l add dummy1 type dummy + sudo ip l set dummy1 up + sudo ip l set dummy1 master breth1 + +On Ubuntu systems, persist the running network configuration. + +.. parsed-literal:: + + sudo cp /run/systemd/network/* /etc/systemd/network + +Installation +============ + +Acquire the Ansible Vault password for this repository, and store a copy at +``~/vault-pw``. + +The following commands install Kayobe and its dependencies, and prepare the +Ansible control host. + +.. parsed-literal:: + + export KAYOBE_VAULT_PASSWORD=$(cat ~/vault-pw) + pushd ~/venvs/kayobe + source bin/activate + popd + pushd ~/src/kayobe-config + source kayobe-env --environment ci-builder + kayobe control host bootstrap + +Deployment +========== + +If using an LVM-based image, uncomment the ``seed_lvm_groups`` variable in +``etc/kayobe/environments/ci-builder/seed.yml``. + +If using an LVM-based image, grow the root volume group. + +.. parsed-literal:: + + kayobe playbook run etc/kayobe/ansible/growroot.yml -e growroot_group=seed + +On Ubuntu systems, purge the command-not-found package. + +.. parsed-literal:: + + kayobe playbook run etc/kayobe/ansible/purge-command-not-found.yml + +Next, configure the host OS & services. + +.. parsed-literal:: + + kayobe seed host configure + +.. _authenticating-pulp-proxy: + +Authenticating Pulp proxy +------------------------- + +If you are building against authenticated package repositories such as those in +`Ark `_, you will need to provide secure access to +the repositories without leaking credentials into the built images or their +metadata. This is typically not the case for a client-local Pulp, which +provides unauthenticated read-only access to the repositories on a trusted +network. + +Docker provides `build +secrets `_, but these must be +explicitly requested for each RUN statement, making them challenging to use in +Kolla. + +StackHPC Kayobe Configuration provides support for deploying an authenticating +Pulp proxy that injects an HTTP basic auth header into requests that it +proxies. Because this proxy bypasses Pulp's authentication, it must not be +exposed to any untrusted environment. + +Ensure that ``localhost`` is resolvable if Docker bridge networking is +disabled. This may be achieved by adding the following to ``/etc/hosts``: + +.. parsed-literal:: + + 127.0.0.1 localhost + +To deploy the proxy: + +.. parsed-literal:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml + +Building images +=============== + +At this point you are ready to build and push some container images. + +.. parsed-literal:: + + kayobe seed container image build --push + kayobe overcloud container image build --push + +If using an :ref:`authenticating Pulp proxy `, +append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands. + +The container images are tagged as |current_release|--. +Check ``tag`` in ``/opt/kayobe/etc/kolla/kolla-build.conf`` or run ``docker +image ls`` to see the tag of the new images. + +To build images for a different base distribution, set ``-e +kolla_base_distro=``. + +To build images using a specific tag, set ``-e kolla_tag=``. + +Using the new images +==================== + +To use the new images, edit +``~/src/kayobe-config/etc/kayobe/kolla-image-tags.yml`` to reference the tag. diff --git a/doc/source/contributor/environments/ci-multinode.rst b/doc/source/contributor/environments/ci-multinode.rst new file mode 100644 index 000000000..165ef5120 --- /dev/null +++ b/doc/source/contributor/environments/ci-multinode.rst @@ -0,0 +1,376 @@ +========================== +Multinode Test Environment +========================== + +.. warning:: + + This guide was written for the Yoga release and has not been validated for + Caracal. Proceed with caution. + +The ``ci-multinode`` environment provides a Kayobe configuration for multi-node +clouds to be used for test and development purposes. It is designed to be used +in combination with the `terraform-kayobe-multinode +`__ repository. Follow +the instructions in terraform-kayobe-multinode to deploy a cluster using this +configuration. This documentation covers configuration of additional services +beyond the defaults. This includes: + +* Manila +* Magnum +* Wazuh + +.. seealso:: + + On-demand and nightly GitHub Actions workflows workflow using this + environment are described :ref:`here `. + +Manila +====== +The Multinode environment supports Manila with the CephFS native backend, but it +is not enabled by default. To enable it, set the following in +``etc/kayobe/environments/ci-multinode/kolla.yml``: + +.. code-block:: yaml + + kolla_enable_manila: true + kolla_enable_manila_backend_cephfs_native: true + +If you are working on an existing deployment, you need to do the following first. + +1. Create CephFS pools: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-pools.yml`` +2. Create cephx key for Manila: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-keys.yml`` +3. Run Manila related Ceph commands: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-commands-post.yml`` +4. Gather Ceph configuration and keyring for Manila: ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml`` +5. Configure Storage network on Seed node: ``kayobe seed host configure -t network,ip-allocation,snat`` + +Then, run ``kayobe overcloud service deploy`` to deploy Manila. + +To test it, you will need two virtual machines. Cirros does not support the Ceph +kernel client, so you will need to use a different image. Any regular Linux +distribution should work. As an example, this guide will use Ubuntu 22.04. + +Download the image locally: + +.. code-block:: bash + + wget https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img + +Upload the image to Glance: + +.. code-block:: bash + + openstack image create --container-format bare --disk-format qcow2 --file jammy-server-cloudimg-amd64.img Ubuntu-22.04 --progress + +Create a keypair: + +.. code-block:: bash + + openstack keypair create --private-key ~/.ssh/id_rsa id_rsa + +Create two virtual machines from the image: + +.. code-block:: bash + + openstack server create --flavor m1.small --image Ubuntu-22.04 --key-name id_rsa --network admin-tenant ubuntu-client-1 + openstack server create --flavor m1.small --image Ubuntu-22.04 --key-name id_rsa --network admin-tenant ubuntu-client-2 + +Wait until the instances are active. It is worth noting that this process can +take a while, especially if the overcloud is deployed to virtual machines. You +can monitor the progress with the following command: + +.. code-block:: bash + + watch openstack server list + +Once they are active, create two floating IPs: + +.. code-block:: bash + + openstack floating ip create external + openstack floating ip create external + +Associate the floating IPs to the instances: + +.. code-block:: bash + + openstack server add floating ip ubuntu-client-1 + openstack server add floating ip ubuntu-client-2 + + +Then SSH into each instance and install the Ceph client: + +.. code-block:: bash + + sudo apt update + sudo apt install -y ceph-common + + +Back on the host, install the Manila client: + +.. code-block:: bash + + pip install python-manilaclient + +Then create a share type and share: + +.. code-block:: bash + + openstack share type create cephfs-type false --public true + openstack share type set cephfs-type --extra-specs vendor_name=Ceph, storage_protocol=CEPHFS + openstack share create --name test-share --share-type cephfs-type --public true CephFS 2 + +Wait until the share is available: + +.. code-block:: bash + + openstack share list + +Then allow access to the shares to two users: + +.. code-block:: bash + + openstack share access create test-share cephx alice + openstack share access create test-share cephx bob + +Show the access list to make sure the state of both entries is ``active`` and +take note of the access keys: + +.. code-block:: bash + + openstack share access list test-share + +And take note of the path to the share: + +.. code-block:: bash + + openstack share export location list test-share + +SSH into the first instance, create a directory for the share, and mount it: + +.. code-block:: bash + + mkdir testdir + sudo mount -t ceph {path} -o name=alice,secret='{access_key}' testdir + +Where the path is the path to the share from the previous step, and the secret +is the access key for the user alice. + +Then create a file in the share: + +.. code-block:: bash + + sudo touch testdir/testfile + +SSH into the second instance, create a directory for the share, and mount it: + +.. code-block:: bash + + mkdir testdir + sudo mount -t ceph {path} -o name=bob,secret='{access_key}' testdir + +Where the path is the same as before, and the secret is the access key for the +user bob. + +Then check that the file created in the first instance is visible in the second +instance: + +.. code-block:: bash + + ls testdir + +If it shows the test file then the share is working correctly. + +Magnum +====== + +The Multinode environment has Magnum enabled by default. To test it, you will +need to create a Kubernetes cluster. It is recommended that you use the +specified Fedora 35 image, as others may not work. Download the image locally, +then extract it and upload it to glance: + +.. code-block:: bash + + wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/35.20220410.3.1/x86_64/fedora-coreos-35.20220410.3.1-openstack.x86_64.qcow2.xz + unxz fedora-coreos-35.20220410.3.1-openstack.x86_64.qcow2.xz + openstack image create --container-format bare --disk-format qcow2 --property os_distro='fedora-coreos' --property os_version='35' --file fedora-coreos-35.20220410.3.1-openstack.x86_64.qcow2 fedora-coreos-35 --progress + +Create a keypair: + +.. code-block:: bash + + openstack keypair create --private-key ~/.ssh/id_rsa id_rsa + +Install the Magnum, Heat, and Octavia clients: + +.. code-block:: bash + + pip install python-magnumclient + pip install python-heatclient + pip install python-octaviaclient + +Create a cluster template: + +.. code-block:: bash + + openstack coe cluster template create test-template --image fedora-coreos-35 --external-network external --labels etcd_volume_size=8,boot_volume_size=50,cloud_provider_enabled=true,heat_container_agent_tag=wallaby-stable-1,kube_tag=v1.23.6,cloud_provider_tag=v1.23.1,monitoring_enabled=true,auto_scaling_enabled=true,auto_healing_enabled=true,auto_healing_controller=magnum-auto-healer,magnum_auto_healer_tag=v1.23.0.1-shpc,etcd_tag=v3.5.4,master_lb_floating_ip_enabled=true,cinder_csi_enabled=true,container_infra_prefix=ghcr.io/stackhpc/,min_node_count=1,max_node_count=50,octavia_lb_algorithm=SOURCE_IP_PORT,octavia_provider=ovn --dns-nameserver 8.8.8.8 --flavor m1.medium --master-flavor m1.medium --network-driver calico --volume-driver cinder --docker-storage-driver overlay2 --floating-ip-enabled --master-lb-enabled --coe kubernetes + +Create a cluster: + +.. code-block:: bash + + openstack coe cluster create --cluster-template test-template --keypair id_rsa --master-count 1 --node-count 1 --floating-ip-enabled test-cluster + +This command will take a while to complete. You can monitor the progress with +the following command: + +.. code-block:: bash + + watch "openstack --insecure coe cluster list ; openstack --insecure stack list ; openstack --insecure server list" + +Once the cluster is created, you can SSH into the master node and check that +there are no failed containers: + +.. code-block:: bash + + ssh core@{master-ip} + +List the podman and docker containers: + +.. code-block:: bash + + sudo docker ps + sudo podman ps + +If there are any failed containers, you can check the logs with the following +commands: + +.. code-block:: bash + + sudo docker logs {container-id} + sudo podman logs {container-id} + +Or look at the logs under ``/var/log``. In particular, pay close attention to +``/var/log/heat-config`` on the master and +``/var/log/kolla/{magnum,heat,neutron}/*`` on the controllers. + +Otherwise, the ``state`` of the cluster should eventually become +``CREATE_COMPLETE`` and the ``health_status`` should be ``HEALTHY``. + +You can interact with the cluster using ``kubectl``. The instructions for +installing ``kubectl`` are available `here +`_. You can then +configure ``kubectl`` to use the cluster, and check that the pods are all +running: + +.. code-block:: bash + + openstack coe cluster config test-cluster --dir $PWD + export KUBECONFIG=$PWD/config + kubectl get pods -A + +Finally, you can optionally use sonobuoy to run a complete set of Kubernetes +conformance tests. + +Find the latest release of sonobuoy on their `github releases page +`_. Then download it with wget, e.g.: + +.. code-block:: bash + + wget https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.56.16/sonobuoy_0.56.16_linux_amd64.tar.gz + +Extract it with tar: + +.. code-block:: bash + + tar -xvf sonobuoy_0.56.16_linux_amd64.tar.gz + +And run it: + +.. code-block:: bash + + ./sonobuoy run --wait + +This will take a while to complete. Once it is done you can check the results +with: + +.. code-block:: bash + + results=$(./sonobuoy retrieve) + ./sonobuoy results $results + +There are various other options for sonobuoy, see the `documentation +`_ for more details. + +Wazuh +====== + +Adding Wazuh to a new deployment +-------------------------------- + +Wazuh is supported but not deployed by default. If you are using the standard +[StackHPC multinode +terraform](https://github.com/stackhpc/terraform-kayobe-multinode), there is a +``deploy_wazuh`` terraform variable that will add it to the automated setup. + +Adding Wazuh to an existing deployment +-------------------------------------- + +Create an additional VM with the same basic configuration (key, image, flavour +etc.) as the existing deployment. + +Add the IP and hostname to ``/etc/hosts`` on the ansible control host. + +Add the hostname to the ``[wazuh-manager]`` group in +``$KAYOBE_CONFIG_PATH/environments/ci-multinode/inventory/hosts``. + +Add the host to the ``[infra-vms]`` group, either directly or by making the +``wazuh-manager`` group a child group of ``infra-vms``. + +Create the following directory structure: +``$KAYOBE_CONFIG_PATH/hooks/infra-vm-host-configure/pre.d/`` + +Either copy or symlink in the growroot, networking, and vxlan playbooks as +shown in ``$KAYOBE_CONFIG_PATH/hooks/seed-host-configure/pre.d/``. + +Configure the Wazuh manager VM: + +.. code-block:: bash + + kayobe infra vm host configure + +Create and encrypt the Wazuh secrets + +.. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml + ansible-vault encrypt --vault-password-file ~/vault.password $KAYOBE_CONFIG_PATH/environments/ci-multinode/wazuh-secrets.yml + +Run the Wazuh manager and agent deployment playbooks: + +.. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml + +Wazuh should now be fully deployed. To test the service, you can use sshuttle +or some other forwarding protocol to access the Wazuh dashboard. + +.. code-block:: bash + + sshuttle -r + +The above example assumes an SSH configuration that allows access with +``ssh ``. + +Open ``https:///`` in a web browser, and you should see a +login screen. + +The default username is ``admin`` and the password is the +``opendistro_admin_password`` which can be found in ``wazuh-secrets.yml`` e.g. + +.. code-block:: bash + + ansible-vault view $KAYOBE_CONFIG_PATH/environments/ci-multinode/wazuh-secrets.yml --vault-password-file ~/vault.password | grep opendistro_admin_password + +If the deployment has been successful, you should be able to see a Wazuh agent +for each host in your deployment (aside from the Wazuh manager itself). diff --git a/doc/source/contributor/environments/index.rst b/doc/source/contributor/environments/index.rst new file mode 100644 index 000000000..888608973 --- /dev/null +++ b/doc/source/contributor/environments/index.rst @@ -0,0 +1,12 @@ +Environments +============ + +The following Kayobe environments are provided with this configuration: + +.. toctree:: + :maxdepth: 1 + + ci-aio + ci-builder + ci-multinode + aufn-ceph diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 000000000..f16a6cbee --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1,16 @@ +================= +Contributor Guide +================= + +This guide is for contributors of the StackHPC Kayobe configuration project. + +.. toctree:: + :maxdepth: 1 + + documentation + release-notes + environments/index + package-updates + pre-commit + ofed + testing-ci-automation diff --git a/doc/source/contributor/ofed.rst b/doc/source/contributor/ofed.rst new file mode 100644 index 000000000..cbd57b421 --- /dev/null +++ b/doc/source/contributor/ofed.rst @@ -0,0 +1,104 @@ +==== +OFED +==== + +Warning: Experimental workflow subject to change + +The Nvidia DOCA framework is distributed as part of StackHPC Release Train for OFED driver support, +this repository is synced into Ark as part of the Release Train workflows, however to ensure +compatibility with Release Train packages, we are required to build OFED modules with support for +the latest Release Train kernel. + +Workflow +======== + +The workflow uses workflow_dispatch to manually request an OFED build, which will deploy a builder +VM, apply kayobe config to the builder, upgrade the kernel, reboot, then run two Ansible playbooks +for building and uploading OFED modules to Ark. + +Pre-requisites +-------------- + +Before building OFED packages, the workflow will ensure that: + +* A full distro-sync has taken place, ensuring the kernel is upgraded. + +* The bootloader has been configured to use the latest kernel (reset-bls-entries.yml) + +* noexec is disabled in the temporary logical volume. + +build-ofed +---------- + +Currently we only support building Rocky Linux 9 OFED kernel module packages. + +The Build OFED module workflow will check that the filesystem is configured (noexec disabled) +to allow the DOCA build script to run. The workflow will also install any necessary dependencies +for the module build. + +The build script will output a ``doca-kernel-repo`` RPM which contains all kernel modules built +as part of the workflow. When this RPM is installed, the repofile is created pointing to the +modules in ``/usr/share/doca-host-/Modules//`` on the host. + +push-ofed +--------- + +As mentioned above, the DOCA repository is synced into the ``doca`` repository in Ark. This workflow +will upload the ``doca-kernel-repo`` RPM to a separate repository named ``doca-modules``. The version +for this repository is set in ``pulp-repo-versions.yml`` and is disabled for local pulp syncs by +default. + +Install process +=============== + +Release Train configuration +--------------------------- + +DOCA repositories will need to be synced to the local Pulp service, Ensure the DOCA +hosts added to the ``mlnx`` group before running a package sync, if the group is not +empty DOCA will be synced into the local Pulp. The local Pulp can be synced with Ark +by running: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml + +DOCA repositories can be templated to hosts by running Kayobe host configure. + +.. code-block:: console + + kayobe overcloud host configure -t dnf + +StackHPC DOCA kernel modules will require the latest kernel version available in Ark for +the current Rocky minor version. You should ensure that packages are up to date by running +a package update, which can also be limited to hosts in the ``mlnx`` group. + +.. code-block:: console + + kayobe overcloud host package update --packages "*" --limit mlnx + +To ensure the latest kernel is the default on boot, the bootloader entries will need +to be reset before rebooting. + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reset-bls-entries.yml -e reset_bls_host=mlnx + +The hosts can now be rebooted to use the latest kernel, a rolling reboot may be applicable +here to reduce distruptions. See the `package updates documentation `. + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit mlnx + +install-doca +------------ + +A playbook is provided to install DOCA on hosts in the ``mlnx`` group. Ensure this group +is configured to include the hosts you wish to install DOCA on. To run the install +playbook: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/install-doca.yml diff --git a/doc/source/contributor/package-updates.rst b/doc/source/contributor/package-updates.rst new file mode 100644 index 000000000..5469745c7 --- /dev/null +++ b/doc/source/contributor/package-updates.rst @@ -0,0 +1,267 @@ +============================= +Package and Container Updates +============================= + +This section describes the Release Train process of creating new package repository snapshots and updating the container images to use these new packages. It includes instructions on the recommended way to test these updates, using the multinode test environment. + +Preparations +============ + +1. Before building images, you should check for any outstanding PRs into the earliest supported release. Below are the links for the 2024.1 (Caracal) branches. + + kayobe-config: https://github.com/stackhpc/stackhpc-kayobe-config/pulls?q=is%3Apr+is%3Aopen+base%3Astackhpc%2F2024.1 + + kolla: https://github.com/stackhpc/kolla/pulls?q=is%3Apr+is%3Aopen+base%3Astackhpc%2F2024.1 + + kolla-ansible: https://github.com/stackhpc/kolla-ansible/pulls?q=is%3Apr+is%3Aopen+base%3Astackhpc%2F2024.1 + + You should also check any referenced source trees in etc/kayobe/kolla.yml. + + e.g: https://github.com/stackhpc/stackhpc-kayobe-config/blob/68026c4b2e605fcb0e60786ad438b32a74a50a13/etc/kayobe/kolla.yml#L113-L143 + +2. Follow the workflows documented `here `_. Sync the package repositories. Then, for each release: + +* Make sure the nightly sync of package repositories has succeeded. + +* Update the Kayobe package repository versions. + +* Build and push Kolla container images. + +* Open a draft PR into ``stackhpc-kayobe-config`` with the updated container image tags. + +* Test. + +* Review the PR. Once approved, promote the container images before then merging the PR. + +The rest of this document describes the final "test" stage. + +Testing +======= + +The following steps describe the process to test the new package and container repositories. See the subsections beneath for further explanations. + +1. Build two multinode environments for OVS and OVN, both on the earliest supported release. + +2. Run tests on current package versions as a baseline. + +3. Upgrade host packages. + +4. Upgrade containers. + +5. Run tests again with the new packages. + +6. Request reviews for your proposed PR to bring in the new packages. + +7. Promote these packages before merging the PR, but after the CI checks have passed. + +8. Upgrade OpenStack to the next release. + +9. Repeat steps 2 and 4-7. (Step 3 is skipped as the host packages will be shared across these releases.) + +10. Repeat 8 and 9 for any further releases. + +Creating the multinode environments +----------------------------------- + +The `Multinode deployment workflow `_ can be used to automatically test changes. + +To manually test the changes, there is a comprehensive guide to set up a Multinode environment with Terraform, found here: https://github.com/stackhpc/terraform-kayobe-multinode. There are some things to note: + +* OVN is enabled by default, you should override it under ``etc/kayobe/environments/ci-multinode/kolla.yml kolla_enable_ovn: false`` for the OVS multinode environment. + +* Remember to set a different ``vxlan_vni`` for each. + +* Before starting any tests, run ``dnf distro-sync -y`` on each host to ensure you are using the same snapshots as in the release train. Option ``-y`` is used to prevent hosts hang waiting for the confirmation input. You can do this using the following commands: + + .. code-block:: console + + kayobe seed host command run -b --command "dnf distro-sync -y" + kayobe overcloud host command run -b --command "dnf distro-sync -y" + +* This may have installed a new kernel version. If so, you will need to reboot the overcloud hosts. You can check the installed kernels and the currently running kernel with the following commands. If the latest listed version is not running, you will need to reboot. + +.. code-block:: console + + kayobe seed host command run -b --show-output --command "dnf list installed kernel" + kayobe seed host command run -b --show-output --command "uname -a" + + kayobe playbook run --limit seed,overcloud $KAYOBE_CONFIG_PATH/ansible/reboot.yml + +* The tempest tests run automatically at the end of the multinode deployment script. If you have the time, it is worth fixing any failing tests you can so that there is greater coverage for the package updates. (Also remember to propose these fixes in the relevant repos where applicable.) + +Upgrading host packages +----------------------- + +Checkout the new kayobe-config branch (from the draft PR): + +.. code-block:: console + + cd $KAYOBE_CONFIG_ROOT + git fetch + git checkout + +For Rocky Linux 9, bump the snapshot versions in /etc/yum/repos.d with: + +.. code-block:: console + + kayobe seed host configure -t dnf + kayobe overcloud host configure -t dnf + +Install new packages: + +.. code-block:: console + + kayobe overcloud host package update --packages "*" + +Perform a rolling reboot of hosts: + +.. note:: + In the Multinode environment, the seed-hypervisor cannot access control + plane instances with the Openstack client. To use Openstack client, connect + to the Seed instance via SSH first. For authentication, use scp to copy + ``public-openrc.sh`` to the Seed + +.. code-block:: console + + # Check your hypervisor hostname + (seed) openstack hypervisor list + + # Reboot controller instances and zeroth compute instance + (seed-hypervisor) export ANSIBLE_SERIAL=1 + (seed-hypervisor) kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit controllers + (seed-hypervisor) kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit compute[0] + + # Test live migration + (seed) openstack server create --image cirros --flavor m1.tiny --network external --hypervisor-hostname --os-compute-api-version 2.74 server1 + (seed) openstack server migrate --live-migration server1 + (seed) watch openstack server show server1 + + (seed-hypervisor) kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit compute[1] + + # Try and migrate back + (seed) openstack server migrate --live-migration server1 + (seed) watch openstack server show server1 + +Upgrading containers within a release +------------------------------------- + +Checkout the new kayobe-config branch (from the draft PR): + +.. code-block:: console + + cd $KAYOBE_CONFIG_ROOT + git fetch + git checkout + +Deploy the services, once the new tags are set in the kayobe_config: + +.. code-block:: console + + kayobe overcloud service deploy + +Upgrading OpenStack to the next release in a multinode environment +------------------------------------------------------------------ + +.. warning:: + + This guide was written for the Wallaby release and has not been validated + for Caracal. Proceed with caution. + +As this is not a full production system, only a reduced number of steps need to be followed to upgrade to a new release. Below describes these steps, with ``stackhpc/wallaby`` as the starting branch: + +.. code-block:: console + + cd /src/kayobe-config/ + git fetch + git checkout -b xena_uber_merge + git merge origin/stackhpc/xena + + source /venvs/kayobe/bin/activate + cd /src/kayobe + git checkout stackhpc/xena + git fetch + pip install -U ~/src/kayobe + + kayobe control host upgrade + kayobe overcloud host upgrade + + kayobe overcloud container image pull + + ---Optional + kayobe overcloud service configuration save --output-dir config/wallaby + kayobe overcloud service configuration generate --node-config-dir /tmp/kolla-xena-config + kayobe overcloud service configuration save --output-dir config/xena --node-config-dir /tmp/kolla-xena-config + kayobe overcloud host command run --command 'rm -rf /tmp/kolla-xena-config' --become + # Check the diff between the old and new configs + --- + + kayobe overcloud service upgrade + +Tests +----- + +Tempest +####### + +Run tempest, you can then perform the other tests while it runs. Once complete, check if any tests are failing. + +As of February 2023, only one test was expected to fail. This may no longer be the case, so any additional failures are worth exploring. + +.. code-block:: console + + tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_port_security_macspoofing_port + +Poke around horizon +################### + +Perform some basic operations such as spawning VMs or attaching/detaching volumes and check that each page works correctly. + +Monitoring +########## + +Check for any ERROR log messages in Kibana. + +Check that the Grafana dashboards are all populated with data. + +Check that there are no active alerts. + +Check that there are no flapping alerts. + +Octavia (OVN only) +################## + +You will need to add an Ubuntu image and create a keypair. + +.. code-block:: console + + wget http://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img + + openstack image create \ + --progress \ + --container-format bare \ + --disk-format qcow2 \ + --file focal-server-cloudimg-amd64.img \ + Ubuntu-20.04 + + openstack keypair create --private-key ~/.ssh/os-admin os-admin + +Then run Octavia test script: + +https://gist.github.com/MoteHue/ee5990bddea0677f54d8bb93d307aa71#file-octavia_test-sh + + +Attempt to build OFED against the latest kernel in ARK +###################################################### + +Note that this only needs to be performed once. + +.. code-block:: console + + kayobe overcloud host configure -t dnf + kayobe overcloud host package update --packages "*" + +Then run the OFED test script on one of the upgraded overcloud hosts: + +https://gist.github.com/cityofships/b4883ee19f75d14534f04115892b8465 + + diff --git a/doc/source/contributor/pre-commit.rst b/doc/source/contributor/pre-commit.rst new file mode 100644 index 000000000..dc9f691bf --- /dev/null +++ b/doc/source/contributor/pre-commit.rst @@ -0,0 +1,47 @@ +================ +Pre-commit Hooks +================ + +StackHPC Kayobe configuration carries support for +`pre-commit hooks `_ which simplify the use of git +hooks enabling the identification and repairing of broken or poor code +before committing. +These hooks are designed to make working within SKC easier and less error prone. + +Currently the following hooks are provided: + +- ``check-yaml``: perform basic yaml syntax linting +- ``end-of-file-fixer``: identify and automatically fix missing newline +- ``trailing-whitespace``: identify and automatically fix excessive white space +- ``ripsecrets``: identify and prevent secrets from being committed to the branch + +.. warning:: + The hook ``ripsecrets`` is capable of preventing the accidental leaking of secrets + such as those found within `secrets.yml` or `passwords.yml`. + However if the secret is contained within a file on it's own and lacks a certain level + of entropy then the secret will not be identified as such as and maybe leaked as a result. + +Installation of `pre-commit` hooks is handled via the `install-pre-commit-hooks` playbook +found within the Ansible directory. +Either run the playbook manually or add the playbook as a hook within Kayobe config such as +within `control-host-bootstrap/post.d`. +Once done you should find `pre-commit` is available within the `kayobe` virtualenv. + +To run the playbook using the following command + +- ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/install-pre-commit-hooks.yml`` + +Whereas to run the playbook when control host bootstrap runs ensure it registered as symlink using the following command + +- ``mkdir -p $KAYOBE_CONFIG_PATH/hooks/control-host-bootstrap/post.d`` +- ``ln -s $KAYOBE_CONFIG_PATH/ansible/install-pre-commit-hooks.yml $KAYOBE_CONFIG_PATH/hooks/control-host-bootstrap/post.d/install-pre-commit-hooks.yml`` + +All that remains is the installation of the hooks themselves which can be accomplished either by +running `pre-commit run` or using `git commit` when you have changes that need to be committed. +This will trigger a brief installation process of the hooks which may take a few minutes. +This a one time process and will not be required again unless new hooks are added or existing ones are updated. + +.. note:: + Currently if you run ``pre-commit run --all-files`` it will make a series of changes to + release notes that lack new lines as well configuration files that ``check-yaml`` does not + approve of. diff --git a/doc/source/contributor/release-notes.rst b/doc/source/contributor/release-notes.rst new file mode 100644 index 000000000..8deb85730 --- /dev/null +++ b/doc/source/contributor/release-notes.rst @@ -0,0 +1,38 @@ +============= +Release notes +============= + +StackHPC Kayobe configuration uses the following release notes sections: + +- ``features`` --- for new features or functionality; these should ideally + refer to the blueprint being implemented; +- ``fixes`` --- for fixes closing bugs; these must refer to the bug being + closed; +- ``upgrade`` --- for notes relevant when upgrading from previous version; + these should ideally be added only between major versions; required when + the proposed change affects behaviour in a non-backwards compatible way or + generally changes something impactful; +- ``deprecations`` --- to track deprecated features; relevant changes may + consist of only the commit message and the release note; +- ``prelude`` --- filled in by the PTL before each release or RC. + +Other release note types may be applied per common sense. +Each change should include a release note unless being a ``TrivialFix`` +change or affecting only docs or CI. Such changes should `not` include +a release note to avoid confusion. +Remember release notes are mostly for end users which, in case of Kolla, +are OpenStack administrators/operators. + +To add a release note, install the ``reno`` package in a Python virtual +environment, then run the following command: + +.. code-block:: console + + reno new + +Release notes for the current release are included in the :ref:`documentation`. +Note that a note won't be included in the generated documentation until it is +tracked by ``git``. + +All release notes can be inspected by browsing ``releasenotes/notes`` +directory. diff --git a/doc/source/contributor/testing-ci-automation.rst b/doc/source/contributor/testing-ci-automation.rst new file mode 100644 index 000000000..427a61b78 --- /dev/null +++ b/doc/source/contributor/testing-ci-automation.rst @@ -0,0 +1,422 @@ +======================== +Testing, CI & Automation +======================== + +CI hosting clouds +================= + +Several GitHub Actions workflows run on public runners, but some require +private runners. For these, we use a CI hosting cloud. + +Leafcloud +--------- + +`Leafcloud `_ is currently the only supported cloud +for hosting CI workloads. + +Workloads run in the ``stackhpc-ci`` project, and CI workflows authenticate +using a ci+skc@stackhpc.com user account. This is an alias for the +`ci@stackhpc.com Google group +`_, which may be subscribed to +by multiple StackHPC Engineers. Credentials for this account should be shared +amongst a few StackHPC Engineers. + +An autoscaling `Actions Runner Controller (ARC) +`_ +cluster also lives in the ``stackhpc-ci`` project, and runs several jobs that +require access to the cloud or benefit from data-locality with Ark. + +SMS Lab +------- + +SMS lab will soon be added as a supported CI hosting cloud. Several +considerations must be made when porting CI to Leafcloud. + +Many of our CI and other workflows require access to large volumes of data +stored in Ark. This includes package repositories, container images, disk +images, etc. Naively accessing this data from SMS lab will result in high +Internet usage and slow jobs. The previous incarnation of SMS lab hosted a +"Test Pulp" instance that acted as a local mirror of package repositories and +container images. This worked, but required explicit syncing with Ark when +content is updated, and was a bit brittle. + +For SMS lab 2.0, we propose a different approach. Package repository data is +smaller than container images, but we might still benefit from the use of a +Squid caching proxy. For container images we will use a `Docker registry mirror +`_ as a pull-through cache. + +Container and host image build jobs require significant data uploads, and may +still need to run on Leafcloud to avoid long delays while transferring data to +Ark. + +CI for pull requests (PRs) +========================== + +Continuous Integration (CI) is used within StackHPC Kayobe Configuration to +perform various tests against pull requests. The top-level workflow is in +``.github/workflows/stackhpc-pull-request.yml``. It includes the following +jobs: + +``check-changes`` + Determines which other jobs need to run, based on files that are changed in + the PR. The ``.github/path-filters.yml`` file contains the paths. +``Tox pep8 with Python 3.10`` + Runs the Tox ``pep8`` environment. +``Tox releasenotes with Python 3.10`` + Builds the release notes using the Tox ``releasenotes`` environment. The + separate release notes are not really used - rather they are integrated into + the main documentation. +``Tox docs with Python 3.10`` + Builds the documentation using the Tox ``docs`` environment. +``Build Kayobe Image`` + Builds a Kayobe container image for the pull request and pushes it to GHCR. + Uses the ``.github/workflows/stackhpc-build-kayobe-image.yml`` reusable + workflow. +``Check container image tags`` + Checks that: + + - the image to container mapping in ``tools/kolla-images.py`` matches Kolla + Ansible. + - the container tag hierarchy in ``tools/kolla-images.py`` matches Kolla + Ansible. + - the container image tags defined in ``etc/kayobe/kolla-image-tags.yml`` are + present in the ``stackhpc-dev`` namespace in Ark. + + Uses the ``.github/workflows/stackhpc-check-tags.yml`` reusable workflow, + which runs the ``etc/kayobe/ansible/check-tags.yml`` and + ``etc/kayobe/ansible/check-kolla-images-py.yml`` playbooks. +``aio [upgrade] ( )`` + Runs an all-in-one OpenStack deployment test. + Various jobs are run using different parameters. + Uses the ``.github/workflows/stackhpc-all-in-one.yml`` reusable workflow. + See :ref:`below ` for further details. + +.. _testing-ci-aio: + +All in one testing +------------------ + +The ``.github/workflows/stackhpc-all-in-one.yml`` reusable workflow accepts +various parameters, and the following are used to create a test matrix for PRs: + + - Operating System (Rocky 9, Ubuntu Jammy) + - Neutron plugin (OVS, OVN) + - Upgrade or no upgrade + +The workflow runs on an autoscaling `Actions Runner Controller (ARC) +`_ +cluster, and the GitHub runner acts as both a Terraform client and an Ansible +control host. Kayobe is executed using kayobe-automation within another +container, using the Kayobe container image built in the ``Build Kayobe Image`` +job. + +The workflow performs the following high-level steps: + +#. Deploy a VM on an OpenStack cloud using the `aio + `_ + Terraform configuration. +#. Deploy OpenStack in the VM using Kayobe and the :doc:`ci-aio + ` environment. If this is an upgrade job, the previous + OpenStack release is deployed. +#. Register test resources in the cloud under test (images, flavors, networks, + subnets, routers, etc.). +#. If this is an upgrade job, upgrade the cloud under test to the target + release. +#. Run Tempest and `StackHPC Cloud Tests + `_ to test the cloud. +#. Collect diagnostic information. +#. Upload results as an artifact. +#. Destroy the VM using Terraform. + +In order to create VMs on the cloud hosting the CI, we need a few things: + +- an OpenStack project with sufficient quota to run CI jobs for several PRs + concurrently +- an OpenStack user account +- a ``clouds.yaml`` file +- an application credential to authenticate with the cloud +- a flavor for the VM (minimum 8GiB RAM) +- a set of images for the VM +- a network and subnet for the VM +- SSH connectivity from the GitHub runner to the VM +- access from the VM to the Internet + +This information is provided to GitHub Actions using `secrets +`_ +and `variables +`_. +`GitHub environments +`_ +are used to allow running jobs on different clouds. + +``KAYOBE_VAULT_PASSWORD`` is a repository-scoped GitHub secret containing the +Ansible Vault password for the ``ci-aio`` Kayobe environment. + +The following GitHub secrets are defined in each GitHub environment: + +- ``CLOUDS_YAML`` +- ``OS_APPLICATION_CREDENTIAL_ID`` +- ``OS_APPLICATION_CREDENTIAL_SECRET`` + +The following GitHub variables are defined in each GitHub environment: + +- ``AIO_FLAVOR`` +- ``AIO_NETWORK`` +- ``AIO_SUBNET`` +- ``OS_CLOUD`` + +Glance images for all-in-one VMs are not configured using GitHub variables. +Instead we use the overcloud host images that are built and uploaded to Ark. +These are also uploaded to clouds running CI, with well-known names using the +versions defined in ``etc/kayobe/pulp-host-image-versions.yml``. + +.. _ci-promotion: + +Promotion +========= + +The ``.github/workflows/stackhpc-promote.yml`` workflow runs on a push to any +release branch of StackHPC Kayobe Configuration. It triggers other workflows in +StackHPC Release Train to promote the `package repositories +`_ +and `container images +`_ +referenced in the configuration. + +The standard GitHub API token available in the workflow (``GITHUB_TOKEN``) is +not allowed to trigger a workflow in another repository. To do this, we use a +`fine-grained PAT token +`_ +owned by the ``stackhpc-ci`` GitHub user. This token has read/write permissions +on actions for the ``stackhpc/stackhpc-release-train`` repository. The token is +stored as the ``STACKHPC_RELEASE_TRAIN_TOKEN`` GitHub secret in the StackHPC +Kayobe Configuration repository. The token expires periodically and must be +regenerated, after which the secret must be updated. + +Tag and release +=============== + +The ``.github/workflows/tag-and-release.yml`` workflow runs on a push to any +release branch of StackHPC Kayobe Configuration. It generates a Git tag and +an accompanying GitHub release. See also the `Release Train documentation +`__. + +CI cleanup +========== + +The ``.github/workflows/stackhpc-ci-cleanup.yml`` workflow runs periodically +(currently every 2 hours). It checks for all-in-one CI VMs older than 3 hours +and deletes them, to avoid excess cloud resource consumption. + +.. _testing-container-images: + +Container images +================ + +The ``.github/workflows/stackhpc-container-image-build.yml`` workflow runs on +demand, and is used to build Kolla container images. The process for building +images and updating the configuration to use them is described in the `Release +Train documentation +`__. + +The workflow runs as a matrix, with a job for each supported container OS +distribution. The workflow runs on an autoscaling `Actions Runner Controller +(ARC) +`_ +cluster, and the GitHub runner acts as both the Ansible control host and +container image build host. + +A Pulp authentication proxy container is deployed on the runner that provides +unauthenticated access to the package repositories in Ark. This avoids leaking +Ark credentials into the built container images. + +Once built, images are scanned for vulnerabilities using `Trivy +`_. Any critical vulnerabilities will break the build, +unless the ``push-dirty`` input is true. + +If the ``push`` input is true, images are pushed to Ark, and a `container sync +`_ +workflow is triggered in the StackHPC Release Train repository. See +:ref:`here ` for information on triggering workflows in another repository. + +An artifact containing image build logs is uploaded on completion. + +.. _testing-host-images: + +Overcloud host images +===================== + +The ``.github/workflows/overcloud-host-image-build.yml`` workflow runs on +demand, and is used to build overcloud host images. + +The workflow runs as a single job, building each supported container OS +distribution sequentially. The workflow runs on an autoscaling `Actions Runner +Controller (ARC) +`_ +cluster, and the GitHub runner acts as both a Terraform client and an Ansible +control host. Similarly to the all-in-one CI testing, Terraform is used to +create a VM on a cloud that is then used for building images. + +The following steps are taken for each supported image: + +#. Build an image using Kayobe +#. Upload the image to Ark +#. Upload the image to clouds hosting CI + +At the end of the job, build logs are uploaded as an artifact and the VM is +destroyed. + +In order to create a VM on the cloud hosting the CI, we need a few things: + +- an OpenStack project with sufficient quota to run at least one build VM +- an OpenStack user account +- a ``clouds.yaml`` file +- an application credential to authenticate with the cloud +- a flavor for the VM (minimum 8GiB RAM) +- a Rocky Linux 9 image for the VM +- a network and subnet for the VM +- SSH connectivity from the GitHub runner to the VM +- access from the VM to the Internet + +This information is provided to GitHub Actions using `secrets +`_ +and `variables +`_. +`GitHub environments +`_ +are used to allow running jobs on different clouds. + +``KAYOBE_VAULT_PASSWORD_CI_BUILDER`` is a repository-scoped GitHub secret +containing the Ansible Vault password for the ``ci-builder`` Kayobe +environment. + +The following GitHub secrets are defined in each GitHub environment: + +- ``CLOUDS_YAML`` +- ``OS_APPLICATION_CREDENTIAL_ID`` +- ``OS_APPLICATION_CREDENTIAL_SECRET`` + +The following GitHub variables are defined in each GitHub environment: + +- ``HOST_IMAGE_BUILD_FLAVOR`` +- ``HOST_IMAGE_BUILD_IMAGE`` +- ``HOST_IMAGE_BUILD_NETWORK`` +- ``HOST_IMAGE_BUILD_SUBNET`` +- ``OS_CLOUD`` + +The ``.github/workflows/overcloud-host-image-promote.yml`` workflow runs on +demand and is used to promote overcloud host images. Unlike package +repositories and container images, host image promotion is still an manual +step. + +The ``.github/workflows/overcloud-host-image-upload.yml`` workflow runs on +demand and is used to upload images to clouds hosting CI. It is mainly used +when this step failed in a previous host image build job. + +.. _testing-multinode: + +Multinode test clusters +======================= + +The ``.github/workflows/stackhpc-multinode.yml`` workflow runs on demand and is +used to create a multinode test cluster. The +``.github/workflows/stackhpc-multinode-periodic.yml`` workflow runs +periodically (currently nightly) and runs a random test configuration +(generated by ``.github/workflows/multinode-inputs.py``). + +Both workflows use a `reusable workflow +`_ +in the StackHPC OpenStack GitHub Workflows repository. Note that since this +workflow is in a different repository and we reference it with a tag, changes +to the reusable workflow are not picked up until the tag is bumped. + +The workflow runs on an autoscaling `Actions Runner Controller (ARC) +`_ +cluster, and the GitHub runner acts as a Terraform client. Kayobe is executed +on another VM that acts as the Ansible control host. + +The workflow performs the following high-level steps: + +#. Deploy a set of VMs on an OpenStack cloud using the `Terraform Kayobe + Multinode `_ + Terraform configuration. +#. Configure one of the VMs as an Ansible control host for Kayobe. +#. Deploy OpenStack in the other VMs using Kayobe and the :doc:`ci-multinode + ` environment. If this is an upgrade job, the + previous OpenStack release is deployed. +#. Register test resources in the cloud under test (images, flavors, networks, + subnets, routers, etc.). +#. Run Tempest and `StackHPC Cloud Tests + `__ to test the cloud. +#. If this is an upgrade job, upgrade the cloud under test to the target + release. +#. Run Tempest and `StackHPC Cloud Tests + `__ to test the cloud. +#. Collect diagnostic information. +#. Upload results as an artifact. +#. Destroy the VMs using Terraform. +#. For nightly jobs, send a Slack alert to ``#release-train-alerts`` on + failure. + +In order to create VMs on the cloud hosting the CI, we need a few things: + +- an OpenStack project with sufficient quota to create several clusters + concurrently +- an OpenStack user account +- a ``clouds.yaml`` file +- an application credential to authenticate with the cloud +- flavors for each type of VM +- a set of images for the VMs +- a network and subnet for the VMs +- a floating IP pool or external network for the Ansible control host (optional) +- SSH connectivity from the GitHub runner to the Ansible control host VM +- access from the VMs to the Internet + +This information is provided to GitHub Actions using `secrets +`_ +and `variables +`_. +`GitHub environments +`_ +are used to allow running jobs on different clouds. + +``KAYOBE_VAULT_PASSWORD_CI_MULTINODE`` is a repository-scoped GitHub secret +containing the Ansible Vault password for the ``ci-multinode`` Kayobe +environment. + +The following GitHub secrets are defined in each GitHub environment: + +- ``CLOUDS_YAML`` +- ``OS_APPLICATION_CREDENTIAL_ID`` +- ``OS_APPLICATION_CREDENTIAL_SECRET`` + +The following GitHub variables are defined in each GitHub environment: + +- ``MULTINODE_ANSIBLE_CONTROL_VM_FLAVOR`` +- ``MULTINODE_FIP_POOL`` +- ``MULTINODE_FLAVOR`` +- ``MULTINODE_INFRA_VM_FLAVOR`` +- ``MULTINODE_NETWORK`` +- ``MULTINODE_SEED_VM_FLAVOR`` +- ``MULTINODE_STORAGE_FLAVOR`` +- ``MULTINODE_SUBNET`` +- ``OS_CLOUD`` + +Glance images for multinode VMs are not configured using GitHub variables. +Instead we use the overcloud host images that are built and uploaded to Ark. +These are also uploaded to clouds running CI, with well-known names using the +versions defined in ``etc/kayobe/pulp-host-image-versions.yml``. + +For multinode clusters created on demand, it is possible to pause the workflow +execution on certain conditions and gain access to the cluster for a limited +period of time. This can be used to interact with the system to investigate +faults, debug, etc. To do this, use the ``break_on`` and ``break_duration`` +workflow inputs. + +Slack alerts +============ + +Slack alerts are sent when certain automatically-triggered workflows fail. See +the `Release Train documentation +`__ for +more details. diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 000000000..85d3c4697 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,35 @@ +.. kayobe documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +================================================== +Welcome to StackHPC's Kayobe Config documentation! +================================================== + + +Overview +======== + +This documentation covers the StackHPC Kayobe configuration. It is intended to +complement, rather than replace, the upstream :kayobe-doc:`Kayobe <>`, +:kolla-doc:`Kolla <>` and :kolla-ansible-doc:`Kolla Ansible<>` +documentation. + +The configuration includes various things, such as: + +* Opinionated configuration +* Custom playbooks +* Continuous Integration (CI) workflows + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + usage + configuration/index + operations/index + release-notes + contributor/index diff --git a/doc/source/operations/bifrost-hardware-inventory-management.rst b/doc/source/operations/bifrost-hardware-inventory-management.rst new file mode 100644 index 000000000..22970463d --- /dev/null +++ b/doc/source/operations/bifrost-hardware-inventory-management.rst @@ -0,0 +1,283 @@ +===================================== +Bifrost Hardware Inventory Management +===================================== + +In most deployments, hardware inventory is managed by the Bifrost service. + +Reconfiguring Control Plane Hardware +==================================== + +If a server's hardware or firmware configuration is changed, it should be +re-inspected in Bifrost before it is redeployed into service. A single server +can be reinspected like this: + +.. code-block:: console + + kayobe overcloud hardware inspect --limit + +.. _enrolling-new-hypervisors: + +Enrolling New Hypervisors +------------------------- + +New hypervisors can be added to the Bifrost inventory by using its discovery +capabilities. Assuming that new hypervisors have IPMI enabled and are +configured to network boot on the provisioning network, the following commands +will instruct them to PXE boot. The nodes will boot on the Ironic Python Agent +kernel and ramdisk, which is configured to extract hardware information and +send it to Bifrost. Note that IPMI credentials can be found in the encrypted +file located at ``$KAYOBE_CONFIG_PATH/secrets.yml``. + +.. code-block:: console + + ipmitool -I lanplus -U -H -ipmi chassis bootdev pxe + +If node is are off, power them on: + +.. code-block:: console + + ipmitool -I lanplus -U -H -ipmi power on + +If nodes is on, reset them: + +.. code-block:: console + + ipmitool -I lanplus -U -H -ipmi power reset + +Once node have booted and have completed introspection, they should be visible +in Bifrost: + +.. code-block:: console + + baremetal node list --provision-state enroll + +--------------------------------------+-----------------------+---------------+-------------+--------------------+-------------+ + | UUID | Name | Instance UUID | Power State | Provisioning State | Maintenance | + +--------------------------------------+-----------------------+---------------+-------------+--------------------+-------------+ + | da0c61af-b411-41b9-8909-df2509f2059b | example-hypervisor-01 | None | power off | enroll | False | + +--------------------------------------+-----------------------+---------------+-------------+--------------------+-------------+ + +After editing ``$KAYOBE_CONFIG_PATH/overcloud.yml`` (or +``$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/overcloud.yml`` +if Kayobe environment is used) to add these new hosts to +the correct groups, import them in Kayobe's inventory with: + +.. code-block:: console + + kayobe overcloud inventory discover + +We can then provision and configure them: + +.. code-block:: console + + kayobe overcloud provision --limit + kayobe overcloud host configure --limit + kayobe overcloud service deploy --limit --kolla-limit + +.. note:: + + Reconfiguring monitoring services on controllers is required after provisioning them. + Otherwise, they will not show up. + +Replacing a Failing Hypervisor +------------------------------ + +To replace a failing hypervisor, proceed as follows: + +* :ref:`Disable the hypervisor to avoid scheduling any new instance on it ` +* :ref:`Evacuate all instances ` +* :ref:`Set the node to maintenance mode in Bifrost ` +* Physically fix or replace the node +* It may be necessary to reinspect the node if hardware was changed (this will require deprovisioning and reprovisioning) +* If the node was replaced or reprovisioned, follow :ref:`enrolling-new-hypervisors` + +To deprovision an existing hypervisor, run: + +.. code-block:: console + + kayobe overcloud deprovision --limit + +.. warning:: + + Always use ``--limit`` with ``kayobe overcloud deprovision`` on a production + system. Running this command without a limit will deprovision all overcloud + hosts. + +.. _evacuating-all-instances: + +Evacuating all instances +------------------------ + +.. code-block:: console + + openstack server evacuate $(openstack server list --host --format value --column ID) + +You should now check the status of all the instances that were running on that +hypervisor. They should all show the status ACTIVE. This can be verified with: + +.. code-block:: console + + openstack server show + +Troubleshooting +=============== + +Servers that have been shut down +-------------------------------- + +If there are any instances that are SHUTOFF they won’t be migrated, but you can +use ``openstack server migrate`` for them once the live migration is finished. + +Also if a VM does heavy memory access, it may take ages to migrate (Nova tries +to incrementally increase the expected downtime, but is quite conservative). +You can use ``openstack server migration force complete --os-compute-api-version 2.22 +`` to trigger the final move. + +You get the migration ID via ``openstack server migration list --server ``. + +For more details see: +http://www.danplanet.com/blog/2016/03/03/evacuate-in-nova-one-command-to-confuse-us-all/ + +Flavors have changed +-------------------- + +If the size of the flavors has changed, some instances will also fail to +migrate as the process needs manual confirmation. You can do this with: + +.. code-block:: console + + openstack server resize confirm + +The symptom to look out for is that the server is showing a status of ``VERIFY +RESIZE`` as shown in this snippet of ``openstack server show ``: + +.. code-block:: console + + | status | VERIFY_RESIZE | + +.. _set-bifrost-maintenance-mode: + +Set maintenance mode on a node in Bifrost +----------------------------------------- + +.. code-block:: console + + docker exec -it bifrost_deploy /bin/bash + (bifrost-deploy)[root@seed bifrost-base]# export OS_CLOUD=bifrost + (bifrost-deploy)[root@seed bifrost-base]# baremetal node maintenance set + +.. _unset-bifrost-maintenance-mode: + +Unset maintenance mode on a node in Bifrost +------------------------------------------- + +.. code-block:: console + + docker exec -it bifrost_deploy /bin/bash + (bifrost-deploy)[root@seed bifrost-base]# export OS_CLOUD=bifrost + (bifrost-deploy)[root@seed bifrost-base]# baremetal node maintenance unset + +Detect hardware differences with ADVise +======================================= + +Hardware information captured during the Ironic introspection process can be +analysed to detect hardware differences, such as mismatches in firmware +versions or missing storage devices. The `ADVise `__ +tool can be used for this purpose. + +Extract Bifrost introspection data +---------------------------------- + +The ADVise tool assumes that hardware introspection data has already been gathered in JSON format. +The ``extra-hardware`` disk builder element enabled when building the IPA image for the required data to be available. + +To build ipa image with extra-hardware you need to edit ``ipa.yml`` and add this: + +.. code-block:: console + + # Whether to build IPA images from source. + ipa_build_images: true + + # List of additional Diskimage Builder (DIB) elements to use when building IPA + images. Default is none. + ipa_build_dib_elements_extra: + - "extra-hardware" + + # List of additional inspection collectors to run. + ipa_collectors_extra: + - "extra-hardware" + +Extract introspection data from Bifrost with Kayobe. JSON files will be created +into ``$KAYOBE_CONFIG_PATH/overcloud-introspection-data``: + +.. code-block:: console + + kayobe overcloud introspection data save + +Using ADVise +------------ + +The Ansible playbook ``advise-run.yml`` can be found at ``$KAYOBE_CONFIG_PATH/ansible/advise-run.yml``. + +The playbook will: + +1. Install ADVise and dependencies +2. Run the mungetout utility for extracting the required information from the introspection data ready for use with ADVise. +3. Run ADVise on the data. + +.. code-block:: console + + cd $KAYOBE_CONFIG_PATH + ansible-playbook $KAYOBE_CONFIG_PATH/ansible/advise-run.yml + +The playbook has the following optional parameters: + +- venv : path to the virtual environment to use. Default: ``"~/venvs/advise-review"`` +- input_dir: path to the hardware introspection data. Default: ``"{{ lookup('env', 'PWD') }}/overcloud-introspection-data"`` +- output_dir: path to where results should be saved. Default: ``"{{ lookup('env', 'PWD') }}/review"`` +- advise-pattern: regular expression to specify what introspection data should be analysed. Default: ``".*.eval"`` + +You can override them by provide new values with ``-e =`` + +Example command to run the tool on data about the compute nodes in a system, where compute nodes are named cpt01, cpt02, cpt03…: + +.. code-block:: console + + ansible-playbook advise-run.yml -e advise_pattern=’(cpt)(.*)(.eval)’ + + +.. note:: + The mungetout utility will always use the file extension .eval + +Using the results +----------------- + +The ADVise tool will output a selection of results found under output_dir/results these include: + +- ``.html`` files to display network visualisations of any hardware differences. +- The folder ``Paired_Comparisons`` which contains information on the shared and differing fields found between the systems. + This is a reflection of the network visualisation webpage, with more detail as to what the differences are. +- ``_summary``, a listing of how the systems can be grouped into sets of identical hardware. +- ``_performance``, the results of analysing the benchmarking data gathered. +- ``_perf_summary``, a subset of the performance metrics, just showing any potentially anomalous data such as where variance + is too high, or individual nodes have been found to over/underperform. + +The ADVise tool will also launch an interactive Dash webpage, which displays the network visualisations, +tables with information on the differing hardware attributes, the performance metrics as a range of box-plots, +and specifies which individual nodes may be anomalous via box-plot outliers. This can be accessed at ``localhost:8050``. +To close this service, simply ``Ctrl+C`` in the terminal where you ran the playbook. + +To get visuallised result, It is recommanded to copy instrospection data to your local machine then run ADVise playbook locally. + +Recommanded Workflow +-------------------- + +1. Run the playbook as outlined above. +2. Open the Dash webpage at ``localhost:8050``. +3. Review the hardware differences. Note that hovering over a group will display the nodes it contains. +4. Identify any unexpected differences in the systems. If multiple differing fields exist they will be graphed separately. + As an example, here we expected all compute nodes to be identical. +5. Use the dropdown menu beneath each graph to show a table of the differences found between two sets of groups. + If required, information on shared fields can be found under ``output_dir/results/Paired_Comparisons``. +6. Scroll down the webpage to the performance review. Identify if any of the discovered performance results could be + indicative of a larger issue. +7. Examine the ``_performance`` and ``_perf_summary`` files if you require any more information. diff --git a/doc/source/operations/ceph-management.rst b/doc/source/operations/ceph-management.rst new file mode 100644 index 000000000..98988959b --- /dev/null +++ b/doc/source/operations/ceph-management.rst @@ -0,0 +1,294 @@ +=========================== +Managing and Operating Ceph +=========================== + +Working with Cephadm +==================== + +This documentation provides guide for Ceph operations. For deploying Ceph, +please refer to :ref:`cephadm-kayobe` documentation. + +Cephadm configuration location +------------------------------ + +In kayobe-config repository, under ``$KAYOBE_CONFIG_PATH/cephadm.yml`` (or in a specific +Kayobe environment when using multiple environment, e.g. +``$KAYOBE_CONFIG_PATH/environments//cephadm.yml``) + +StackHPC's Cephadm Ansible collection relies on multiple inventory groups: + +- ``mons`` +- ``mgrs`` +- ``osds`` +- ``rgws`` (optional) + +Those groups are usually defined in ``$KAYOBE_CONFIG_PATH/inventory/groups``. + +Running Cephadm playbooks +------------------------- + +In kayobe-config repository, under ``$KAYOBE_CONFIG_PATH/ansible`` there is a set of +Cephadm based playbooks utilising stackhpc.cephadm Ansible Galaxy collection. + +``cephadm.yml`` runs the end to end process of Cephadm deployment and +configuration. It is composed with following list of other Cephadm playbooks +and they can be run separately. + +- ``cephadm-deploy.yml`` - Runs the bootstrap/deploy playbook without the + additional playbooks +- ``cephadm-commands-pre.yml`` - Runs Ceph commands before post-deployment + configuration (You can set a list of commands at ``cephadm_commands_pre_extra`` + variable in ``$KAYOBE_CONFIG_PATH/cephadm.yml``) +- ``cephadm-ec-profiles.yml`` - Defines Ceph EC profiles +- ``cephadm-crush-rules.yml`` - Defines Ceph crush rules according +- ``cephadm-pools.yml`` - Defines Ceph pools +- ``cephadm-keys.yml`` - Defines Ceph users/keys +- ``cephadm-commands-post.yml`` - Runs Ceph commands after post-deployment + configuration (You can set a list of commands at ``cephadm_commands_post_extra`` + variable in ``$KAYOBE_CONFIG_PATH/cephadm.yml``) + +There are also other Ceph playbooks that are not part of ``cephadm.yml`` + +- ``cephadm-gather-keys.yml`` - Populate ``ceph.conf`` and keyrings in kayobe-config by + gathering Ceph configuration and keys +- ``ceph-enter-maintenance.yml`` - Set Ceph to maintenance mode for storage + hosts (Can limit the hosts with ``-l ``) +- ``ceph-exit-maintenance.yml`` - Unset Ceph to maintenance mode for storage + hosts (Can limit the hosts with ``-l ``) + +Running Ceph commands +--------------------- + +Ceph commands are usually run inside a ``cephadm shell`` utility container: + +.. code-block:: console + + # From storage host + sudo cephadm shell + +Operating a cluster requires a keyring with an admin access to be available for Ceph +commands. Cephadm will copy such keyring to the nodes carrying +`_admin `__ +label - present on MON servers by default when using +`StackHPC Cephadm collection `__. + +Adding a new storage node +------------------------- + +Add a node to a respective group (e.g. osds) and run ``cephadm-deploy.yml`` +playbook. + +.. note:: + To add other node types than osds (mons, mgrs, etc) you need to specify + ``-e cephadm_bootstrap=True`` on playbook run. + +Removing a storage node +----------------------- + +First drain the node + +.. code-block:: console + + # From storage host + sudo cephadm shell + ceph orch host drain + +Once all daemons are removed - you can remove the host: + +.. code-block:: console + + # From storage host + sudo cephadm shell + ceph orch host rm + +And then remove the host from inventory (usually in +``$KAYOBE_CONFIG_PATH/inventory/overcloud``) + +Additional options/commands may be found in +`Host management `_ + +Replacing failing drive +----------------------- + +A failing drive in a Ceph cluster will cause OSD daemon to crash. +In this case Ceph will go into `HEALTH_WARN` state. +Ceph can report details about failed OSDs by running: + +.. code-block:: console + + # From storage host + sudo cephadm shell + ceph health detail + +.. note:: + + Remember to run ceph/rbd commands from within ``cephadm shell`` + (preferred method) or after installing Ceph client. Details in the + official `documentation `__. + It is also required that the host where commands are executed has admin + Ceph keyring present - easiest to achieve by applying + `_admin `__ + label (Ceph MON servers have it by default when using + `StackHPC Cephadm collection `__). + +A failed OSD will also be reported as down by running: + +.. code-block:: console + + ceph osd tree + +Note the ID of the failed OSD. + +The failed disk is usually logged by the Linux kernel too: + +.. code-block:: console + + # From storage host + dmesg -T + +Cross-reference the hardware device and OSD ID to ensure they match. +(Using `pvs` and `lvs` may help make this connection). + +See upstream documentation: +https://docs.ceph.com/en/latest/cephadm/services/osd/#replacing-an-osd + +In case where disk holding DB and/or WAL fails, it is necessary to recreate +all OSDs that are associated with this disk - usually NVMe drive. The +following single command is sufficient to identify which OSDs are tied to +which physical disks: + +.. code-block:: console + + ceph device ls + +Once OSDs on failed disks are identified, follow procedure below. + +If rebooting a Ceph node, first set ``noout`` to prevent excess data +movement: + +.. code-block:: console + + # From storage host + sudo cephadm shell + ceph osd set noout + +Reboot the node and replace the drive + +Unset noout after the node is back online + +.. code-block:: console + + # From storage host + sudo cephadm shell + ceph osd unset noout + +Remove the OSD using Ceph orchestrator command: + +.. code-block:: console + + # From storage host + sudo cephadm shell + ceph orch osd rm --replace + +After removing OSDs, if the drives the OSDs were deployed on once again become +available, Cephadm may automatically try to deploy more OSDs on these drives if +they match an existing drivegroup spec. +If this is not your desired action plan - it's best to modify the drivegroup +spec before (``cephadm_osd_spec`` variable in ``$KAYOBE_CONFIG_PATH/cephadm.yml``). +Either set ``unmanaged: true`` to stop Cephadm from picking up new disks or +modify it in some way that it no longer matches the drives you want to remove. + +Host maintenance +---------------- + +https://docs.ceph.com/en/latest/cephadm/host-management/#maintenance-mode + +Upgrading +--------- + +https://docs.ceph.com/en/latest/cephadm/upgrade/ + + +Troubleshooting +=============== + +Inspecting a Ceph Block Device for a VM +--------------------------------------- + +To find out what block devices are attached to a VM, go to the hypervisor that +it is running on (an admin-level user can see this from ``openstack server +show``). + +On this hypervisor, enter the libvirt container: + +.. code-block:: console + + # From hypervisor host + docker exec -it nova_libvirt /bin/bash + +Find the VM name using libvirt: + +.. code-block:: console + + (nova-libvirt)[root@compute-01 /]# virsh list + Id Name State + ------------------------------------ + 1 instance-00000001 running + +Now inspect the properties of the VM using ``virsh dumpxml``: + +.. code-block:: console + + (nova-libvirt)[root@compute-01 /]# virsh dumpxml instance-00000001 | grep rbd + + +On a Ceph node, the RBD pool can be inspected and the volume extracted as a RAW +block image: + +.. code-block:: console + + # From storage host + sudo cephadm shell + rbd ls + rbd export /51206278-e797-4153-b720-8255381228da_disk blob.raw + +The raw block device (blob.raw above) can be mounted using the loopback device. + +Inspecting a QCOW Image using LibGuestFS +---------------------------------------- + +The virtual machine's root image can be inspected by installing +libguestfs-tools and using the guestfish command: + +.. code-block:: console + + # From storage host + export LIBGUESTFS_BACKEND=direct + guestfish -a blob.qcow + > run + 100% [XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] 00:00 + > list-filesystems + /dev/sda1: ext4 + > mount /dev/sda1 / + > ls / + bin + boot + dev + etc + home + lib + lib64 + lost+found + media + mnt + opt + proc + root + run + sbin + srv + sys + tmp + usr + var + > quit diff --git a/doc/source/operations/control-plane-operation.rst b/doc/source/operations/control-plane-operation.rst new file mode 100644 index 000000000..3dfd1ec44 --- /dev/null +++ b/doc/source/operations/control-plane-operation.rst @@ -0,0 +1,386 @@ +======================= +Operating Control Plane +======================= + +Backup of the OpenStack Control Plane +===================================== + +As the backup procedure is constantly changing, it is normally best to check +the upstream documentation for an up to date procedure. Here is a high level +overview of the key things you need to backup: + +Controllers +----------- + +* `Back up SQL databases `__ +* `Back up configuration in /etc/kolla `__ + +Compute +------- + +The compute nodes can largely be thought of as ephemeral, but you do need to +make sure you have migrated any instances and disabled the hypervisor before +rebooting, decommissioning or making any disruptive configuration change. + +Monitoring +---------- + +* `Back up InfluxDB `__ +* `Back up OpenSearch `__ +* `Back up Prometheus `__ + +Seed +---- + +* `Back up bifrost `__ + +Ansible control host +-------------------- + +* Back up service VMs such as the seed VM + +Control Plane Monitoring +======================== + +This section shows user guide of monitoring control plane. To see how to +configure monitoring services, read :ref:`monitoring-service-configuration`. + +The control plane has been configured to collect logs centrally using Fluentd, +OpenSearch and OpenSearch Dashboards. + +Telemetry monitoring of the control plane is performed by Prometheus. Metrics +are collected by Prometheus exporters, which are either running on all hosts +(e.g. node exporter), on specific hosts (e.g. controllers for the memcached +exporter or monitoring hosts for the OpenStack exporter). These exporters are +scraped by the Prometheus server. + +Configuring Prometheus Alerts +----------------------------- + +Alerts are defined in code and stored in Kayobe configuration. See ``*.rules`` +files in ``$KAYOBE_CONFIG_PATH/kolla/config/prometheus`` as a model to add +custom rules. + +Silencing Prometheus Alerts +--------------------------- + +Sometimes alerts must be silenced because the root cause cannot be resolved +right away, such as when hardware is faulty. For example, an unreachable +hypervisor will produce several alerts: + +* ``InstanceDown`` from Node Exporter +* ``OpenStackServiceDown`` from the OpenStack exporter, which reports status of + the ``nova-compute`` agent on the host +* ``PrometheusTargetMissing`` from several Prometheus exporters + +Rather than silencing each alert one by one for a specific host, a silence can +apply to multiple alerts using a reduced list of labels. Log into Alertmanager, +click on the ``Silence`` button next to an alert and adjust the matcher list +to keep only ``instance=`` label. +Then, create another silence to match ``hostname=`` (this is +required because, for the OpenStack exporter, the instance is the host running +the monitoring service rather than the host being monitored). + +Control Plane Shutdown Procedure +================================ + +Overview +-------- + +* Verify integrity of clustered components (RabbitMQ, Galera, Keepalived). They + should all report a healthy status. +* Put node into maintenance mode in bifrost to prevent it from automatically + powering back on +* Shutdown down nodes one at a time gracefully using systemctl poweroff + +Controllers +----------- + +If you are restarting the controllers, it is best to do this one controller at +a time to avoid the clustered components losing quorum. + +Checking Galera state ++++++++++++++++++++++ + +On each controller perform the following: + +.. code-block:: console + + [stack@controller0 ~]$ docker exec -i mariadb mysql -u root -p -e "SHOW STATUS LIKE 'wsrep_local_state_comment'" + Variable_name Value + wsrep_local_state_comment Synced + +The password can be found using: + +.. code-block:: console + + ansible-vault view $KAYOBE_CONFIG_PATH/kolla/passwords.yml \ + --vault-password-file | grep ^database + +Checking RabbitMQ ++++++++++++++++++ + +RabbitMQ health is determined using the command ``rabbitmqctl cluster_status``: + +.. code-block:: console + + [stack@controller0 ~]$ docker exec rabbitmq rabbitmqctl cluster_status + + Cluster status of node rabbit@controller0 ... + [{nodes,[{disc,['rabbit@controller0','rabbit@controller1', + 'rabbit@controller2']}]}, + {running_nodes,['rabbit@controller1','rabbit@controller2', + 'rabbit@controller0']}, + {cluster_name,<<"rabbit@controller2">>}, + {partitions,[]}, + {alarms,[{'rabbit@controller1',[]}, + {'rabbit@controller2',[]}, + {'rabbit@controller0',[]}]}] + +Checking Keepalived ++++++++++++++++++++ + +On (for example) three controllers: + +.. code-block:: console + + [stack@controller0 ~]$ docker logs keepalived + +Two instances should show: + +.. code-block:: console + + VRRP_Instance(kolla_internal_vip_51) Entering BACKUP STATE + +and the other: + +.. code-block:: console + + VRRP_Instance(kolla_internal_vip_51) Entering MASTER STATE + +Ansible Control Host +-------------------- + +The Ansible control host is not enrolled in bifrost. This node may run services +such as the seed virtual machine which will need to be gracefully powered down. + +Compute +------- + +If you are shutting down a single hypervisor, to avoid down time to tenants it +is advisable to migrate all of the instances to another machine. See +:ref:`evacuating-all-instances`. + +Ceph +---- + +The following guide provides a good overview: +https://access.redhat.com/documentation/en-us/red_hat_openstack_platform/8/html/director_installation_and_usage/sect-rebooting-ceph + +Shutting down the seed VM +------------------------- + +.. code-block:: console + + virsh shutdown + +.. _full-shutdown: + +Full shutdown +------------- + +In case a full shutdown of the system is required, we advise to use the +following order: + +* Perform a graceful shutdown of all virtual machine instances +* Shut down compute nodes +* Shut down monitoring node (if separate from controllers) +* Shut down network nodes (if separate from controllers) +* Shut down controllers +* Shut down Ceph nodes (if applicable) +* Shut down seed VM +* Shut down Ansible control host + +Rebooting a node +---------------- + +Use ``reboot.yml`` playbook to reboot nodes +Example: Reboot all compute hosts apart from compute0: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml --limit 'compute:!compute0' + +References +---------- + +* https://galeracluster.com/library/training/tutorials/restarting-cluster.html + +Control Plane Power on Procedure +================================ + +Overview +-------- + +* Remove the node from maintenance mode in bifrost +* Bifrost should automatically power on the node via IPMI +* Check that all docker containers are running +* Check OpenSearch Dashboards for any messages with log level ERROR or equivalent + +Controllers +----------- + +If all of the servers were shut down at the same time, it is necessary to run a +script to recover the database once they have all started up. This can be done +with the following command: + +.. code-block:: console + + kayobe overcloud database recover + +Ansible Control Host +-------------------- + +The Ansible control host is not enrolled in Bifrost and will have to be powered +on manually. + +Seed VM +------- + +The seed VM (and any other service VM) should start automatically when the seed +hypervisor is powered on. If it does not, it can be started with: + +.. code-block:: console + + virsh start + +Full power on +------------- + +Follow the order in :ref:`full-shutdown`, but in reverse order. + +Shutting Down / Restarting Monitoring Services +---------------------------------------------- + +Shutting down ++++++++++++++ + +Log into the monitoring host(s): + +.. code-block:: console + + ssh stack@monitoring0 + +Stop all Docker containers: + +.. code-block:: console + + monitoring0# for i in `docker ps -a`; do systemctl stop kolla-$i-container; done + +Shut down the node: + +.. code-block:: console + + monitoring0# sudo shutdown -h + +Starting up ++++++++++++ + +The monitoring services containers will automatically start when the monitoring +node is powered back on. + +Software Updates +================ + +Sync local Pulp server with StackHPC Release Train +-------------------------------------------------- + +The host packages and Kolla container images are distributed from `StackHPC Release Train +`__ to ensure tested and reliable +software releases are provided. + +Syncing new StackHPC Release Train contents to local Pulp server is needed before updating +host packages and/or Kolla services. + +To sync host packages: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml + +If the system is production environment and want to use packages tested in test/staging +environment, you can promote them by: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml + +To sync container images: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + +For more information about StackHPC Release Train, see :ref:`stackhpc-release-train` documentation. + +Once sync with StackHPC Release Train is done, new contents will be accessible from local +Pulp server. + +Update Host Packages on Control Plane +------------------------------------- + +Host packages can be updated with: + +.. code-block:: console + + kayobe overcloud host package update --limit --packages '*' + kayobe seed host package update --packages '*' + +See https://docs.openstack.org/kayobe/latest/administration/overcloud.html#updating-packages + +Troubleshooting +=============== + +Deploying to a Specific Hypervisor +---------------------------------- + +To test creating an instance on a specific hypervisor, *as an admin-level user* +you can specify the hypervisor name. + +To see the list of hypervisor names: + +.. code-block:: console + + # From host that can reach Openstack + openstack hypervisor list + +To boot an instance on a specific hypervisor + +.. code-block:: console + + openstack server create --flavor --network --key-name --image --os-compute-api-version 2.74 --host + +OpenSearch indexes retention +============================= + +To alter default rotation values for OpenSearch, edit + +``$KAYOBE_CONFIG_PATH/kolla/globals.yml``: + +.. code-block:: console + + # Duration after which index is closed (default 30) + opensearch_soft_retention_period_days: 90 + # Duration after which index is deleted (default 60) + opensearch_hard_retention_period_days: 180 + +Reconfigure Opensearch with new values: + +.. code-block:: console + + kayobe overcloud service reconfigure --kolla-tags opensearch + +For more information see the `upstream documentation +`__. diff --git a/doc/source/operations/customising-horizon.rst b/doc/source/operations/customising-horizon.rst new file mode 100644 index 000000000..e86dd62f2 --- /dev/null +++ b/doc/source/operations/customising-horizon.rst @@ -0,0 +1,118 @@ +=================== +Customising Horizon +=================== + +Horizon is the most frequent site-specific container customisation required: +other customisations tend to be common across deployments, but personalisation +of Horizon is unique to each institution. + +This describes a simple process for customising the Horizon theme. + +Creating a custom Horizon theme +------------------------------- + +A simple custom theme for Horizon can be implemented as small modifications of +an existing theme, such as the `Default +`__ +one. + +A theme contains at least two files: ``static/_styles.scss``, which can be empty, and +``static/_variables.scss``, which can reference another theme like this: + +.. code-block:: scss + + @import "/themes/default/variables"; + @import "/themes/default/styles"; + +Some resources such as logos can be overridden by dropping SVG image files into +``static/img`` (since the Ocata release, files must be SVG instead of PNG). See +`the Horizon documentation +`__ +for more details. + +Content on some pages such as the splash (login) screen can be updated using +templates. + +See `our example horizon-theme `__ +which inherits from the default theme and includes: + +* a custom splash screen logo +* a custom top-left logo +* a custom message on the splash screen + +Further reading: + +* https://docs.openstack.org/horizon/latest/configuration/customizing.html +* https://docs.openstack.org/horizon/latest/configuration/themes.html +* https://docs.openstack.org/horizon/latest/configuration/branding.html + +Adding the custom theme +----------------------- + +Create a directory and transfer custom theme files to it ``$KAYOBE_CONFIG_PATH/kolla/config/horizon/themes/``. + +Define the custom theme in ``$KAYOBE_CONFIG_PATH/kolla/globals.yml`` + +.. code-block:: yaml + + horizon_custom_themes: + - name: + label: # This will be the visible name to users + +Deploy and use the custom theme +------------------------------- + +Configure Horizon to include the custom theme and use it by default: + +.. code-block:: console + + mkdir -p $KAYOBE_CONFIG_PATH/kolla/config/horizon/ + +Create file ``$KAYOBE_CONFIG_PATH/kolla/config/horizon/_9999-custom-settings.py`` and add followings + +.. code-block:: console + + AVAILABLE_THEMES = [ + ('default', 'Default', 'themes/default'), + ('material', 'Material', 'themes/material'), + ('', '', 'themes/'), + ] + DEFAULT_THEME = '' + +You can also set other customisations in this file, such as the HTML title of the page: + +.. code-block:: console + + SITE_BRANDING = "" + +Deploy with: + +.. code-block:: console + + kayobe overcloud service reconfigure --kolla-tags horizon + +Troubleshooting +--------------- + +If the theme is selected but the logo doesn’t load, try running these commands +inside the ``horizon`` container: + +.. code-block:: console + + /var/lib/kolla/venv/bin/python /var/lib/kolla/venv/bin/manage.py collectstatic --noinput --clear + /var/lib/kolla/venv/bin/python /var/lib/kolla/venv/bin/manage.py compress --force + settings_bundle | md5sum > /var/lib/kolla/.settings.md5sum.txt + +Alternatively, try changing anything in ``_9999-custom-settings.py`` and restarting +the ``horizon`` container. + +If the ``horizon`` container is restarting with the following error: + +.. code-block:: console + + /var/lib/kolla/venv/bin/python /var/lib/kolla/venv/bin/manage.py compress --force + CommandError: An error occurred during rendering /var/lib/kolla/venv/lib/python3.6/site-packages/openstack_dashboard/templates/horizon/_scripts.html: Couldn't find any precompiler in COMPRESS_PRECOMPILERS setting for mimetype '\'text/javascript\''. + +It can be resolved by dropping cached content with ``systemctl restart +kolla-memcached-container``. Note this will log out users from Horizon, as Django +sessions are stored in Memcached. diff --git a/doc/source/operations/gpu-in-openstack.rst b/doc/source/operations/gpu-in-openstack.rst new file mode 100644 index 000000000..502342692 --- /dev/null +++ b/doc/source/operations/gpu-in-openstack.rst @@ -0,0 +1,678 @@ +============================= +Support for GPUs in OpenStack +============================= + +PCI Passthrough +############### + +Prerequisite - BIOS Configuration +--------------------------------- + +On an Intel system: + +* Enable ``VT-x`` in the BIOS for virtualisation support. +* Enable ``VT-d`` in the BIOS for IOMMU support. + +On an AMD system: + +* Enable ``AMD-v`` in the BIOS for virtualisation support. +* Enable ``AMD-Vi`` (also just called ``IOMMU`` on older hardware) in the BIOS + for IOMMU support. + +It may be possible to configure passthrough without these settings, though +stability or performance may be affected. + +Host and Service Configuration +------------------------------ + +PCI passthrough GPU variables can be found in the +``etc/kayobe/stackhpc-compute.yml`` file. + +The ``gpu_group_map`` is a dictionary mapping inventory groups to GPU types. +This is used to determine which GPU types each compute node should pass through +to OpenStack. The keys are group names, the values are a list of GPU types. + +Possible GPU types are defined in the ``stackhpc_gpu_data`` dictionary. It +contains data for many common GPUs. If you have a GPU that is not included, +extend the dictionary following the same pattern. + +The ``resource_name`` is the name that will be used in the flavor extra specs. +These can be overridden e.g. ``a100_80_resource_name: "big_gpu"``. + +Example configuration for three groups containing A100s, V100s, and both: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/stackhpc-compute.yml + + gpu_group_map: + compute_a100: + - a100_80 + compute_v100: + - v100_32 + compute_multi_gpu: + - a100_80 + - v100_32 + +All groups in the ``gpu_group_map`` must also be added to +``kolla_overcloud_inventory_top_level_group_map`` in ``etc/kayobe/kolla.yml``. +Always include the Kayobe defaults unless you know what you are doing. + +When ``gpu_group_map`` is populated, the ``pci-passthrough.yml`` playbook will +be added as a pre-hook to ``kayobe overcloud host configure``. Either run host +configuration or trigger the playbook manually: + +.. code-block:: console + + kayobe overcloud host configure --limit compute_a100,compute_v100,compute_multi_gpu + # OR + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pci-passthrough.yml --limit compute_a100,compute_v100,compute_multi_gpu + +The playbook will apply the necessary configuraion and reboot the hosts if +required. + +Once host configuration is complete, deploy Nova: + +.. code-block:: console + + kayobe overcloud service deploy -kt nova + +Create a flavor +--------------- + +For example, to request two of the GPUs with alias **v100_32** + +.. code-block:: text + + openstack flavor set m1.medium-gpu --property "pci_passthrough:alias"="v100_32:2" + +This can be also defined in the openstack-config repository. + +Add extra_specs to flavor in etc/openstack-config/openstack-config.yml: + +.. code-block:: console + + cd src/openstack-config + vim etc/openstack-config/openstack-config.yml + + name: "m1.medium-gpu" + ram: 4096 + disk: 40 + vcpus: 2 + extra_specs: + "pci_passthrough:alias": "v100_32:2" + +Invoke configuration playbooks afterwards: + +.. code-block:: console + + source src/kayobe-config/etc/kolla/public-openrc.sh + source venvs/openstack/bin/activate + tools/openstack-config --vault-password-file + +Create instance with GPU passthrough +------------------------------------ + +.. code-block:: text + + openstack server create --flavor m1.medium-gpu --image ubuntu22.04 --wait test-pci + +Testing GPU in a Guest VM +------------------------- + +The Nvidia drivers must be installed first. For example, on an Ubuntu guest: + +.. code-block:: text + + sudo apt install nvidia-headless-440 nvidia-utils-440 nvidia-compute-utils-440 + +The ``nvidia-smi`` command will generate detailed output if the driver has +loaded successfully. + + +Virtual GPUs +############ + +BIOS configuration +------------------ + +See upstream documentation: `BIOS configuration `__ + +Obtain driver from NVIDIA licensing portal +------------------------------------------ + +See upstream documentation: `Obtain driver from NVIDIA licencing portal `__ + +.. _NVIDIA Pulp: + +Uploading the GRID driver to pulp +--------------------------------- + +Uploading the driver to pulp will make it possible to run kayobe from any host. This can be useful when +running in a CI environment. + +.. code:: shell + + pulp artifact upload --file ~/NVIDIA-GRID-Linux-KVM-525.105.14-525.105.17-528.89.zip + pulp file content create --relative-path "NVIDIA-GRID-Linux-KVM-525.105.14-525.105.17-528.89.zip" --sha256 c8e12c15b881df35e618bdee1f141cbfcc7e112358f0139ceaa95b48e20761e0 + pulp file repository create --name nvidia + pulp file repository content add --repository nvidia --sha256 c8e12c15b881df35e618bdee1f141cbfcc7e112358f0139ceaa95b48e20761e0 --relative-path "NVIDIA-GRID-Linux-KVM-525.105.14-525.105.17-528.89.zip" + pulp file publication create --repository nvidia + pulp file distribution create --name nvidia --base-path nvidia --repository nvidia + +The file will then be available at ``/pulp/content/nvidia/NVIDIA-GRID-Linux-KVM-525.105.14-525.105.17-528.89.zip``. You +will need to set the ``vgpu_driver_url`` configuration option to this value: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/vgpu.yml + + # URL of GRID driver in pulp + vgpu_driver_url: "{{ pulp_url }}/pulp/content/nvidia/NVIDIA-GRID-Linux-KVM-525.105.14-525.105.17-528.89.zip" + +See :ref:`NVIDIA Role Configuration`. + +.. _NVIDIA control host: + +Placing the GRID driver on the ansible control host +--------------------------------------------------- + +Copy the driver bundle to a known location on the ansible control host. Set the ``vgpu_driver_url`` configuration variable to reference this +path using ``file`` as the url scheme e.g: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/vgpu.yml + + # Location of NVIDIA GRID driver on localhost + vgpu_driver_url: "file://{{ lookup('env', 'HOME') }}/NVIDIA-GRID-Linux-KVM-525.105.14-525.105.17-528.89.zip" + +See :ref:`NVIDIA Role Configuration`. + +.. _NVIDIA OS Configuration: + +OS Configuration +---------------- + +Host OS configuration is done by using roles in the `stackhpc.linux `_ ansible collection. + +Create a new playbook or update an existing on to apply the roles: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/ansible/host-configure.yml + + --- + - hosts: iommu + tags: + - iommu + tasks: + - import_role: + name: stackhpc.linux.iommu + handlers: + - name: reboot + set_fact: + kayobe_needs_reboot: true + + - hosts: vgpu + tags: + - vgpu + tasks: + - import_role: + name: stackhpc.linux.vgpu + handlers: + - name: reboot + set_fact: + kayobe_needs_reboot: true + + - name: Reboot when required + hosts: iommu:vgpu + tags: + - reboot + tasks: + - name: Reboot + reboot: + reboot_timeout: 3600 + become: true + when: kayobe_needs_reboot | default(false) | bool + +Ansible Inventory Configuration +------------------------------- + +Add some hosts into the ``vgpu`` group. The example below maps two custom +compute groups, ``compute_multi_instance_gpu`` and ``compute_vgpu``, +into the ``vgpu`` group: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/inventory/custom + + [compute] + [compute_multi_instance_gpu] + [compute_vgpu] + + [vgpu:children] + compute_multi_instance_gpu + compute_vgpu + + [iommu:children] + vgpu + +Having multiple groups is useful if you want to be able to do conditional +templating in ``nova.conf`` (see :ref:`NVIDIA Kolla Ansible +Configuration`). Since the vgpu role requires iommu to be enabled, all of the +hosts in the ``vgpu`` group are also added to the ``iommu`` group. + +If using bifrost and the ``kayobe overcloud inventory discover`` mechanism, +hosts can automatically be mapped to these groups by configuring +``overcloud_group_hosts_map``: + +.. code-block:: yaml + :caption: ``$KAYOBE_CONFIG_PATH/overcloud.yml`` + + overcloud_group_hosts_map: + compute_vgpu: + - "computegpu000" + compute_mutli_instance_gpu: + - "computegpu001" + +.. _NVIDIA Role Configuration: + +Role Configuration +------------------ + +Configure the VGPU devices: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/inventory/group_vars/compute_vgpu/vgpu + + #nvidia-692 GRID A100D-4C + #nvidia-693 GRID A100D-8C + #nvidia-694 GRID A100D-10C + #nvidia-695 GRID A100D-16C + #nvidia-696 GRID A100D-20C + #nvidia-697 GRID A100D-40C + #nvidia-698 GRID A100D-80C + #nvidia-699 GRID A100D-1-10C + #nvidia-700 GRID A100D-2-20C + #nvidia-701 GRID A100D-3-40C + #nvidia-702 GRID A100D-4-40C + #nvidia-703 GRID A100D-7-80C + #nvidia-707 GRID A100D-1-10CME + vgpu_definitions: + # Configuring a MIG backed VGPU + - pci_address: "0000:17:00.0" + virtual_functions: + - mdev_type: nvidia-700 + index: 0 + - mdev_type: nvidia-700 + index: 1 + - mdev_type: nvidia-700 + index: 2 + - mdev_type: nvidia-699 + index: 3 + mig_devices: + "1g.10gb": 1 + "2g.20gb": 3 + # Configuring a card in a time-sliced configuration (non-MIG backed) + - pci_address: "0000:65:00.0" + virtual_functions: + - mdev_type: nvidia-697 + index: 0 + - mdev_type: nvidia-697 + index: 1 + +.. _NVIDIA Kolla Ansible Configuration: + +Kolla-Ansible configuration +--------------------------- + +See upstream documentation: `Kolla Ansible configuration `__ +then follow the rest. + +Map through the kayobe inventory groups into kolla: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/kolla.yml + + kolla_overcloud_inventory_top_level_group_map: + control: + groups: + - controllers + network: + groups: + - network + compute_cpu: + groups: + - compute_cpu + compute_gpu: + groups: + - compute_gpu + compute_multi_instance_gpu: + groups: + - compute_multi_instance_gpu + compute_vgpu: + groups: + - compute_vgpu + compute: + groups: + - compute + monitoring: + groups: + - monitoring + storage: + groups: + "{{ kolla_overcloud_inventory_storage_groups }}" + +Where the ``compute_`` groups have been added to the kayobe defaults. + +You will need to reconfigure nova for this change to be applied: + +.. code-block:: shell + + kayobe overcloud service deploy -kt nova --kolla-limit compute_vgpu + +Openstack flavors +----------------- + +See upstream documentation: `OpenStack flavors `__ + +NVIDIA License Server +--------------------- + +The Nvidia delegated license server is a virtual machine based appliance. You simply need to boot an instance +using the image supplied on the NVIDIA Licensing portal. This can be done on the OpenStack cloud itself. The +requirements are: + +* All tenants wishing to use GPU based instances must have network connectivity to this machine. (network licensing) + - It is possible to configure node locked licensing where tenants do not need access to the license server +* Satisfy minimum requirements detailed `here `__. + +The official documentation for configuring the instance +can be found `here `__. + +Below is a snippet of openstack-config for defining a project, and a security group that can be used for a non-HA deployment: + +.. code-block:: yaml + + secgroup_rules_nvidia_dls: + # Allow ICMP (for ping, etc.). + - ethertype: IPv4 + protocol: icmp + # Allow SSH. + - ethertype: IPv4 + protocol: tcp + port_range_min: 22 + port_range_max: 22 + # https://docs.nvidia.com/license-system/latest/nvidia-license-system-user-guide/index.html + - ethertype: IPv4 + protocol: tcp + port_range_min: 443 + port_range_max: 443 + - ethertype: IPv4 + protocol: tcp + port_range_min: 80 + port_range_max: 80 + - ethertype: IPv4 + protocol: tcp + port_range_min: 7070 + port_range_max: 7070 + + secgroup_nvidia_dls: + name: nvidia-dls + project: "{{ project_cloud_services.name }}" + rules: "{{ secgroup_rules_nvidia_dls }}" + + openstack_security_groups: + - "{{ secgroup_nvidia_dls }}" + + project_cloud_services: + name: "cloud-services" + description: "Internal Cloud services" + project_domain: default + user_domain: default + users: [] + quotas: "{{ quotas_project }}" + +Booting the VM: + +.. code-block:: shell + + # Uploading the image and making it available in the cloud services project + $ openstack image create --file nls-3.0.0-bios.qcow2 nls-3.0.0-bios --disk-format qcow2 + $ openstack image add project nls-3.0.0-bios cloud-services + $ openstack image set --accept nls-3.0.0-bios --project cloud-services + $ openstack image member list nls-3.0.0-bios + + # Booting a server as the admin user in the cloud-services project. We pre-create the port so that + # we can recreate it without changing the MAC address. + $ openstack port create --mac-address fa:16:3e:a3:fd:19 --network external nvidia-dls-1 --project cloud-services + $ openstack role add member --project cloud-services --user admin + $ export OS_PROJECT_NAME=cloud-services + $ openstack server group create nvidia-dls --policy anti-affinity + $ openstack server create --flavor 8cpu-8gbmem-30gbdisk --image nls-3.0.0-bios --port nvidia-dls-1 --hint group=179dfa59-0947-4925-a0ff-b803bc0e58b2 nvidia-dls-cci1-1 --security-group nvidia-dls + $ openstack server add security group nvidia-dls-1 nvidia-dls + + +Manual VM driver and licence configuration +------------------------------------------ + +vGPU client VMs need to be configured with Nvidia drivers to run GPU workloads. +The host drivers should already be applied to the hypervisor. + +GCP hosts compatible client drivers `here +`__. + +Find the correct version (when in doubt, use the same version as the host) and +download it to the VM. The exact dependencies will depend on the base image you +are using but at a minimum, you will need GCC installed. + +Ubuntu Jammy example: + +.. code-block:: bash + + sudo apt update + sudo apt install -y make gcc wget + wget https://storage.googleapis.com/nvidia-drivers-us-public/GRID/vGPU17.1/NVIDIA-Linux-x86_64-550.54.15-grid.run + sudo sh NVIDIA-Linux-x86_64-550.54.15-grid.run + +Check the ``nvidia-smi`` client is available: + +.. code-block:: bash + + nvidia-smi + +Generate a token from the licence server, and copy the token file to the client +VM. + +On the client, create an Nvidia grid config file from the template: + +.. code-block:: bash + + sudo cp /etc/nvidia/gridd.conf.template /etc/nvidia/gridd.conf + +Edit it to set ``FeatureType=1`` and leave the rest of the settings as default. + +Copy the client configuration token into the ``/etc/nvidia/ClientConfigToken`` +directory. + +Ensure the correct permissions are set: + +.. code-block:: bash + + sudo chmod 744 /etc/nvidia/ClientConfigToken/client_configuration_token_.tok + +Restart the ``nvidia-gridd`` service: + +.. code-block:: bash + + sudo systemctl restart nvidia-gridd + +Check that the token has been recognised: + +.. code-block:: bash + + nvidia-smi -q | grep 'License Status' + +If not, an error should appear in the journal: + +.. code-block:: bash + + sudo journalctl -xeu nvidia-gridd + +A successfully licenced VM can be snapshotted to create an image in Glance that +includes the drivers and licencing token. Alternatively, an image can be +created using Diskimage Builder. + +Disk image builder recipe to automatically license VGPU on boot +--------------------------------------------------------------- + +`stackhpc-image-elements `__ provides a ``nvidia-vgpu`` +element to configure the nvidia-gridd service in VGPU mode. This allows you to boot VMs that automatically license themselves. +Snippets of ``openstack-config`` that allow you to do this are shown below: + +.. code-block:: shell + + image_rocky9_nvidia: + name: "Rocky9-NVIDIA" + type: raw + elements: + - "rocky-container" + - "rpm" + - "nvidia-vgpu" + - "cloud-init" + - "epel" + - "cloud-init-growpart" + - "selinux-permissive" + - "dhcp-all-interfaces" + - "vm" + - "extra-repos" + - "grub2" + - "stable-interface-names" + - "openssh-server" + is_public: True + packages: + - "dkms" + - "git" + - "tmux" + - "cuda-minimal-build-12-1" + - "cuda-demo-suite-12-1" + - "cuda-libraries-12-1" + - "cuda-toolkit" + - "vim-enhanced" + env: + DIB_CONTAINERFILE_NETWORK_DRIVER: host + DIB_CONTAINERFILE_RUNTIME: docker + DIB_RPMS: "http://192.168.1.2:80/pulp/content/nvidia/nvidia-linux-grid-525-525.105.17-1.x86_64.rpm" + YUM: dnf + DIB_EXTRA_REPOS: "https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo" + DIB_NVIDIA_VGPU_CLIENT_TOKEN: "{{ lookup('file' , 'secrets/client_configuration_token_05-30-2023-12-41-40.tok') }}" + DIB_CLOUD_INIT_GROWPART_DEVICES: + - "/" + DIB_RELEASE: "9" + properties: + os_type: "linux" + os_distro: "rocky" + os_version: "9" + + openstack_images: + - "{{ image_rocky9_nvidia }}" + + openstack_image_git_elements: + - repo: "https://github.com/stackhpc/stackhpc-image-elements" + local: "{{ playbook_dir }}/stackhpc-image-elements" + version: master + elements_path: elements + +The gridd driver was uploaded pulp using the following procedure: + +.. code-block:: shell + + $ unzip NVIDIA-GRID-Linux-KVM-525.105.14-525.105.17-528.89.zip + $ pulp artifact upload --file ~/nvidia-linux-grid-525-525.105.17-1.x86_64.rpm + $ pulp file content create --relative-path "nvidia-linux-grid-525-525.105.17-1.x86_64.rpm" --sha256 58fda68d01f00ea76586c9fd5f161c9fbb907f627b7e4f4059a309d8112ec5f5 + $ pulp file repository add --name nvidia --sha256 58fda68d01f00ea76586c9fd5f161c9fbb907f627b7e4f4059a309d8112ec5f5 --relative-path "nvidia-linux-grid-525-525.105.17-1.x86_64.rpm" + $ pulp file publication create --repository nvidia + $ pulp file distribution update --name nvidia --base-path nvidia --repository nvidia + +This is the file we reference in ``DIB_RPMS``. It is important to keep the driver versions aligned between hypervisor and guest VM. + +The client token can be downloaded from the web interface of the licensing portal. Care should be taken +when copying the contents as it can contain invisible characters. It is best to copy the file directly +into your openstack-config repository and vault encrypt it. The ``file`` lookup plugin can be used to decrypt +the file (as shown in the example above). + +Testing vGPU VMs +---------------- + +vGPU VMs can be validated using the following test workload. The test should +succeed if the VM is correctly licenced and drivers are correctly installed for +both the host and client VM. + +Install ``cuda-toolkit`` using the instructions `here +`__. + +Ubuntu Jammy example: + +.. code-block:: bash + + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb + sudo apt update -y + sudo apt install -y cuda-toolkit make + +The VM may require a reboot at this point. + +Clone the ``cuda-samples`` repo: + +.. code-block:: bash + + git clone https://github.com/NVIDIA/cuda-samples.git + +Build and run a test workload: + +.. code-block:: bash + + cd cuda-samples/Samples/6_Performance/transpose + make + ./transpose + +Example output: + +.. code-block:: + + Transpose Starting... + + GPU Device 0: "Ampere" with compute capability 8.0 + + > Device 0: "GRID A100D-1-10C MIG 1g.10gb" + > SM Capability 8.0 detected: + > [GRID A100D-1-10C MIG 1g.10gb] has 14 MP(s) x 64 (Cores/MP) = 896 (Cores) + > Compute performance scaling factor = 1.00 + + Matrix size: 1024x1024 (64x64 tiles), tile size: 16x16, block size: 16x16 + + transpose simple copy , Throughput = 159.1779 GB/s, Time = 0.04908 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + transpose shared memory copy, Throughput = 152.1922 GB/s, Time = 0.05133 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + transpose naive , Throughput = 117.2670 GB/s, Time = 0.06662 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + transpose coalesced , Throughput = 135.0813 GB/s, Time = 0.05784 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + transpose optimized , Throughput = 145.4326 GB/s, Time = 0.05372 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + transpose coarse-grained , Throughput = 145.2941 GB/s, Time = 0.05377 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + transpose fine-grained , Throughput = 150.5703 GB/s, Time = 0.05189 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + transpose diagonal , Throughput = 117.6831 GB/s, Time = 0.06639 ms, Size = 1048576 fp32 elements, NumDevsUsed = 1, Workgroup = 256 + Test passed + +Changing VGPU device types +-------------------------- + +See upstream documentation: `Changing VGPU device types `__ + +Further Reference +----------------- + +For PCI Passthrough and GPUs in OpenStack: + +* Consumer-grade GPUs: https://gist.github.com/claudiok/890ab6dfe76fa45b30081e58038a9215 +* https://www.jimmdenton.com/gpu-offloading-openstack/ +* https://docs.openstack.org/nova/latest/admin/pci-passthrough.html +* https://docs.openstack.org/nova/latest/admin/virtual-gpu.html (vGPU only) +* Tesla models in OpenStack: https://egallen.com/openstack-nvidia-tesla-gpu-passthrough/ +* https://wiki.archlinux.org/index.php/PCI_passthrough_via_OVMF +* https://www.kernel.org/doc/Documentation/Intel-IOMMU.txt +* https://access.redhat.com/documentation/en-us/red_hat_virtualization/4.1/html/installation_guide/appe-configuring_a_hypervisor_host_for_pci_passthrough +* https://www.gresearch.co.uk/article/utilising-the-openstack-placement-service-to-schedule-gpu-and-nvme-workloads-alongside-general-purpose-instances/ diff --git a/doc/source/operations/hotfix-playbook.rst b/doc/source/operations/hotfix-playbook.rst new file mode 100644 index 000000000..8f7c6145e --- /dev/null +++ b/doc/source/operations/hotfix-playbook.rst @@ -0,0 +1,70 @@ +=============== +Hotfix Playbook +=============== + +Using the Container Hotfix Playbook +=================================== + +The StackHPC Kayobe configuration contains a playbook called +``hotfix-containers.yml`` which can be used to execute commands on, and copy +files into, a given set of containers. + +This playbook will first copy across any hotfix files, and then run the +hotfix command. If either of these are not specified, the corresponding step +will be skipped. + +This playbook is designed for use in high-severity hotfixes ONLY and should not +be used for regular operations. + +The playbook can be invoked with: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/hotfix-containers.yml + +Playbook variables: +------------------- + +* ``container_hotfix_command``: A command to run on each of the target + containers. Default is an empty string. + +* ``container_hotfix_files``: A list of files to copy into each target + container. Consists of a list of dicts with keys ``src`` and ``dest`` + (required), and ``mode`` (optional - default 400). Default is an empty list. + +* ``container_hotfix_container_regex``: Regex to match container names against. + Must match the entire name e.g. "nova" or "nova*" will result in only + matching a single container called "nova". To properly match every container + starting with "nova", the regex must be "nova.*" Default is an empty string. + +* ``container_hotfix_restart_containers``: Whether to restart containers after + applying the hotfix. Default is False. + +* ``container_hotfix_become``: Create files and exec as root in the target + containers. Default is False. + + +It is strongly recommended that you write your container_hotfix_* variables +to a file, then add them as an extra var. e.g: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/hotfix-containers.yml -e "@~/vars.yml" + + +Example Variables file +---------------------- + +.. code-block:: yaml + + --- + container_hotfix_command: "/tmp/quick-fix.sh" + container_hotfix_files: + - src: "~/quick-fix.sh" + dest: "/tmp/quick-fix.sh" + mode: "700" + - src: "/home/stackhpc/new_nova_conf.conf" + dest: "/etc/nova/nova.conf" + container_hotfix_container_regex: "nova.*" + container_hotfix_restart_containers: True + container_hotfix_become: True diff --git a/doc/source/operations/index.rst b/doc/source/operations/index.rst new file mode 100644 index 000000000..f8f733394 --- /dev/null +++ b/doc/source/operations/index.rst @@ -0,0 +1,25 @@ +============== +Operator Guide +============== + +This guide is for operators of the StackHPC Kayobe configuration project. + +.. toctree:: + :maxdepth: 1 + + ceph-management + control-plane-operation + customising-horizon + gpu-in-openstack + bifrost-hardware-inventory-management + hotfix-playbook + migrating-vm + nova-compute-ironic + octavia + openstack-projects-and-users-management + openstack-reconfiguration + secret-rotation + tempest + upgrading-openstack + upgrading-ceph + ubuntu-noble diff --git a/doc/source/operations/migrating-vm.rst b/doc/source/operations/migrating-vm.rst new file mode 100644 index 000000000..031df1b60 --- /dev/null +++ b/doc/source/operations/migrating-vm.rst @@ -0,0 +1,22 @@ +========================== +Migrating virtual machines +========================== + +To see where all virtual machines are running on the hypervisors: + +.. code-block:: console + + openstack server list --all-projects --long + +To move a virtual machine with shared storage or booted from volume from one hypervisor to another, for example to +hypervisor-01: + +.. code-block:: console + + openstack server migrate --live-migration --host hypervisor-01 + +To move a virtual machine with local disks: + +.. code-block:: console + + openstack server migrate --live-migration --block-migration --host hypervisor-01 diff --git a/doc/source/operations/nova-compute-ironic.rst b/doc/source/operations/nova-compute-ironic.rst new file mode 100644 index 000000000..9b9af76ff --- /dev/null +++ b/doc/source/operations/nova-compute-ironic.rst @@ -0,0 +1,307 @@ +=================== +Nova Compute Ironic +=================== + +This section describes the deployment of the OpenStack Nova Compute +Ironic service. The Nova Compute Ironic service is used to integrate +OpenStack Ironic into Nova as a 'hypervisor' driver. The end users of Nova +can then deploy and manage baremetal hardware, in a similar way to VMs. + +High Availability (HA) +====================== + +The OpenStack Nova Compute service is designed to be installed once on every +hypervisor in an OpenStack deployment. In this configuration, it makes little +sense to run additional service instances. Even if you wanted to, it's not +supported by design. This pattern breaks down with the Ironic baremetal +service, which must run on the OpenStack control plane. It is not feasible +to have a 1:1 mapping of Nova Compute Ironic services to baremetal nodes. + +The obvious HA solution is to run multiple instances of Nova Compute Ironic +on the control plane, so that if one fails, the others can take over. However, +due to assumptions long baked into the Nova source code, this is not trivial. +The HA feature provided by the Nova Compute Ironic service has proven to be +unstable, and the direction upstream is to switch to an active/passive +solution [1]. + +However, challenges still exist with the active/passive solution. Since the +Nova Compute Ironic HA feature is 'always on', one must ensure that only a +single instance (per Ironic conductor group) is ever running. It is not +possible to simply put multiple service instances behind HAProxy and use the +active/passive mode. + +Such problems are commonly solved with a technology such as Pacemaker, or in +the modern world, with a container orchestration engine such as Kubernetes. +Kolla Ansible provides neither, because in general it doesn't need to. Its +goal is simplicity. + +The interim solution is to therefore run a single Nova Compute Ironic +service. If the service goes down, remedial action must be taken before +Ironic nodes can be managed. In many environments the loss of the Ironic +API for short periods is acceptable, providing that it can be easily +resurrected. The purpose of this document is to faciliate that. + +.. note:: + + The new sharding mode is not covered here and it is assumed that you are + not using it. See [1] for further information. This will be updated in + the future. + +Optimal configuration of Nova Compute Ironic +============================================ + +Determine the current configuration for the site. How many Nova Compute +Ironic instances are running on the control plane? + +.. code-block:: console + + $ openstack compute service list + +Typically you will see either three or one. By default the host will +marked with a postfix, eg. ``controller1-ironic``. If you find more than +one, you will need to remove some instances. You must complete the +following section. + +Moving from multiple Nova Compute Instances to a single instance +---------------------------------------------------------------- + +1. Decide where the single instance should run. This should normally be + one of the three OpenStack control plane hosts. For convention, pick + the first one, unless you can think of a good reason not to. Once you + have chosen, set the following variable in ``$KAYOBE_CONFIG_PATH/nova.yml``. + Here we have picked ``controller1``. + + .. code-block:: yaml + + kolla_nova_compute_ironic_host: controller1 + +2. Ensure that you have organised a maintenance window, during which + there will be no Ironic operations. You will be breaking the Ironic + API. + +3. Perform a database backup. + + .. code-block:: console + + $ kayobe overcloud database backup -vvv + + Check the output of the command, and locate the backup files. + +4. Identify baremetal nodes associated with Nova Compute Ironic instances + that will be removed. You don't need to do anything with these + specifically, it's just for reference later. For example: + + .. code-block:: console + + $ openstack baremetal node list --long -c "Instance Info" | grep controller3-ironic | wc -l + 61 + $ openstack baremetal node list --long -c "Instance Info" | grep controller2-ironic | wc -l + 35 + $ openstack baremetal node list --long -c "Instance Info" | grep controller1-ironic | wc -l + 55 + +5. Disable the redundant Nova Compute Ironic services: + + .. code-block:: console + + $ openstack compute service set controller3-ironic nova-compute --disable + $ openstack compute service set controller2-ironic nova-compute --disable + +6. Delete the redundant Nova Compute Ironic services. You will need the service + ID. For example: + + .. code-block:: console + + $ ID=$(openstack compute service list | grep foo | awk '{print $2}') + $ openstack compute service delete --os-compute-api-version 2.53 $ID + + In older releases, you may hit a bug where the service can't be deleted if it + is not managing any instances. In this case just move on and leave the service + disabled. Eg. + + .. code-block:: console + + $ openstack compute service delete --os-compute-api-version 2.53 c993b57e-f60c-4652-8328-5fb0e17c99c0 + Failed to delete compute service with ID 'c993b57e-f60c-4652-8328-5fb0e17c99c0': HttpException: 500: Server Error for url: + https://acme.pl-2.internal.hpc.is:8774/v2.1/os-services/c993b57e-f60c-4652-8328-5fb0e17c99c0, Unexpected API Error. + Please report this at http://bugs.launchpad.net/nova/ and attach the Nova API log if possible. + +7. Remove the Docker containers for the redundant Nova Compute Ironic services: + + .. code-block:: console + + $ ssh controller2 sudo docker rm -f nova_compute_ironic + $ ssh controller3 sudo docker rm -f nova_compute_ironic + +8. Ensure that all Ironic nodes are using the single remaining Nova Compute + Ironic instance. Eg. Baremetal nodes in use by compute instances will not + fail over to the remaining Nova Compute Ironic service. Here, the active + service is running on ``controller1``: + + .. code-block:: console + + $ ssh controller1 + $ sudo docker exec -it mariadb mysql -u nova -p$(sudo grep 'mysql+pymysql://nova:' /etc/kolla/nova-api/nova.conf | awk -F'[:,@]' '{print $3}') + $ MariaDB [(none)]> use nova; + + Proceed with caution. It is good practise to update one record first: + + .. code-block:: console + + $ MariaDB [nova]> update instances set host='controller1-ironic' where uuid=0 and host='controller3-ironic' limit 1; + Query OK, 1 row affected (0.002 sec) + Rows matched: 1 Changed: 1 Warnings: 0 + + At this stage you should go back to step 4 and check that the numbers have + changed as expected. When you are happy, update remaining records for all + services which have been removed: + + .. code-block:: console + + $ MariaDB [nova]> update instances set host='controller1-ironic' where deleted=0 and host='controller3-ironic'; + Query OK, 59 rows affected (0.009 sec) + Rows matched: 59 Changed: 59 Warnings: 0 + $ MariaDB [nova]> update instances set host='controller1-ironic' where deleted=0 and host='controller2-ironic'; + Query OK, 35 rows affected (0.003 sec) + Rows matched: 35 Changed: 35 Warnings: 0 + +9. Repeat step 4. Verify that all Ironic nodes are using the single remaining + Nova Compute Ironic instance. + + +Making it easy to re-deploy Nova Compute Ironic +----------------------------------------------- + +In the previous section we saw that at any given time, a baremetal node is +associated with a single Nova Compute Ironic instance. At this stage, assuming +that you have diligently followed the instructions, you are in the situation +where all Ironic baremetal nodes are managed by a single Nova Compute Ironic +instance. If this service goes down, you will not be able to manage /any/ +baremetal nodes. + +By default, the single remaining Nova Compute Ironic instance will be named +after the host on which it is deployed. The host name is passed to the Nova +Compute Ironic instance via the default section of the ``nova.conf`` file, +using the field: ``host``. + +If you wish to re-deploy this instance, for example because the original host +was permanently mangled in the World Server Throwing Championship [2], you +must ensure that the new instance has the same name as the old one. Simply +setting ``kolla_nova_compute_ironic_host`` to another controller and +re-deploying the service is not enough; the new instance will be named after +the new host. + +To work around this you should set the ``host`` field in ``nova.conf`` to a +constant, such that the new Nova Compute Ironic instance comes up with the +same name as the one it replaces. + +For example, if the original instance resides on ``controller1``, then set the +following in ``$KAYOBE_CONFIG_PATH/nova.yml``: + +.. code-block:: yaml + + kolla_nova_compute_ironic_static_host_name: controller1-ironic + +Note that an ``-ironic`` postfix is added to the hostname. This comes from +a convention in Kolla Ansible. It is worth making this change ahead of time, +even if you don't need to immediately re-deploy the service. + +It is also possible to use an arbitrary ``host`` name, but you will need +to edit the database again. That is an optional exercise left for the reader. +See [1] for further details. + +.. note:: + + There is a bug when overriding the host name in Kolla Ansible, where it + is currently assumed that it will be set to the actual hostname + an + -ironic postfix. The service will come up correctly, but Kolla Ansible + will not detect it. See here: + https://bugs.launchpad.net/kolla-ansible/+bug/2056571 + +Re-deploying Nova Compute Ironic +-------------------------------- + +The decision to re-deploy Nova Compute Ironic to another host should only be +taken if there is a strong reason to do so. The objective is to minimise +the chance of the old instance starting up alongside the new one. If the +original host has been re-imaged, or physically replaced there is no risk. +However, if the original host has been taken down for non-destructive +maintenance, it is better to avoid re-deploying the service if the end users +can tolerate the wait. If you are forced to re-deploy the service, knowing +that the original instance may start when the host comes back online, you +must plan accordingly. For example, by booting the original host in maintenance +mode and removing the old service before it can start, or by stopping the +new instance before the original one comes back up, and then reverting the +config to move it to the new host. + +There are essentially two scenarios for re-deploying Nova Compute Ironic. +These are described in the following sub-sections: + +Current host is accessible +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Adjust the ``kolla_nova_compute_ironic_host`` variable to point to the +new host, eg. + +.. code-block:: diff + + +kolla_nova_compute_ironic_host: controller2 + -kolla_nova_compute_ironic_host: controller1 + +Remove the old container: + +.. code-block:: console + + $ ssh controller1 sudo docker rm -f nova_compute_ironic + +Deploy the new service: + +.. code-block:: console + + $ kayobe overcloud service deploy -kl controller2 -l controller2 -kt nova + +Verify that the new service appears as 'up' and 'enabled': + +.. code-block:: console + + $ openstack compute service list + +Current host is not accessible +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this case you will need to remove the inaccessible host from the inventory. +For example, in ``$KAYOBE_CONFIG_PATH/inventory/hosts``, remove ``controller1`` +from the ``controllers`` group. + +Adjust the ``kolla_nova_compute_ironic_host`` variable to point to the +new host, eg. + +.. code-block:: diff + + +kolla_nova_compute_ironic_host: controller2 + -kolla_nova_compute_ironic_host: controller1 + +Deploy the new service: + +.. code-block:: console + + $ kayobe overcloud service reconfigure -kl controller2 -l controller2 -kt nova + +Verify that the new service appears as 'up' and 'enabled': + +.. code-block:: console + + $ openstack compute service list + +.. note:: + + It is important to stop the original service from starting up again. It is + up to you to prevent this. + +.. note:: + + Once merged, the work on 'Kayobe reliability' may allow this step to run + without modifying the inventory to remove the broken host. + +[1] https://specs.openstack.org/openstack/nova-specs/specs/2024.1/approved/ironic-shards.html#migrate-from-peer-list-to-shard-key +[2] https://www.cloudfest.com/world-server-throwing-championship diff --git a/doc/source/operations/octavia.rst b/doc/source/operations/octavia.rst new file mode 100644 index 000000000..966f3302d --- /dev/null +++ b/doc/source/operations/octavia.rst @@ -0,0 +1,119 @@ +======= +Octavia +======= + +.. _Amphora image: + +Updating amphora images +======================= + +StackHPC kayobe config contains utility playbooks to update and build the amphora images. + +To update the image, first activate an openrc file containing the credentials +for the octavia service account, e.g: + +.. code-block:: console + + . $KOLLA_CONFIG_PATH/octavia-openrc.sh + +You can then run the playbook to upload the image: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/octavia-amphora-image-register.yml + +By default, this will download Amphora image corresponds to OpenStack release from +StackHPC Release Train. +Then it will rename the old image by adding a timestamp suffix, before uploading a +new image with the name, ``amphora-x64-haproxy``. Octavia should be configured +to discover the image by tag using the ``amp_image_tag`` config option. The +images are tagged with ``amphora`` to match the kolla-ansible default for +``octavia_amp_image_tag``. This prevents you needing to reconfigure octavia +when building new images. + +To rollback an image update, simply delete the newest image. The next newest image with +a tag matching ``amp_image_tag`` will be selected. + +Building amphora images locally +=============================== + +You can also build Amphora images locally. +With your kayobe environment activated, you can build a new amphora image with: + +.. code-block:: console + + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/octavia-amphora-image-build.yml + +The resultant image is based on Ubuntu. By default the image will be built on the +seed, but it is possible to change the group in the ansible inventory using the +``amphora_builder_group`` variable. + +To register locally built image, set ``download_amphora_from_ark`` to ``false`` in +``stackhpc.yml`` + +.. code-block:: yaml + :caption: ``stackhpc.yml`` + + # Whether or not to download Octavia Amphora image from Ark. Default is true. + download_amphora_from_ark: false + +Then copy the image to your first controller host and run the image register playbook. +The path to the image in the controller needs to be set as an extra variable. +The default image path is ``/tmp/amphora-x64-haproxy.qcow2``. + +.. code-block:: console + + kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/octavia-amphora-image-register.yml -e image_path="" + + +Manually deleting broken load balancers +======================================= + +Sometimes, a load balancer will get stuck in a broken state of ``PENDING_CREATE`` or ``PENDING_UPDATE``. +When in this state, the load balancer cannot be deleted; you will see the error ``Invalid state PENDING_CREATE of loadbalancer resource``. +To delete a load balancer in this state, you will need to manually update its provisioning status in the database. + +Find the database password: + +.. code-block:: console + + ansible-vault view --vault-password-file $KOLLA_CONFIG_PATH/passwords.yml + + # Search for database_password with: + /^database + +Access the database from a controller: + +.. code-block:: console + + docker exec -it mariadb bash + mysql -u root -p octavia + # Enter the database password when prompted. + +List the load balancers to find the ID of the broken one(s): + +.. code-block:: console + + SELECT * FROM load_balancer; + +Set the provisioning status to ERROR for any broken load balancer: + +.. code-block:: console + + UPDATE load_balancer SET provisioning_status='ERROR' WHERE id=''; + +Delete the load balancer from the OpenStack CLI, cascading if any stray +Amphorae are hanging around: + +.. code-block:: console + + openstack loadbalancer delete --cascade + + +Sometimes, Amphora may also fail to delete if they are stuck in state +``BOOTING``. These can be resolved entirely from the OpenStack CLI: + +.. code-block:: console + + openstack loadbalancer amphora configure + openstack loadbalancer amphora delete diff --git a/doc/source/operations/openstack-projects-and-users-management.rst b/doc/source/operations/openstack-projects-and-users-management.rst new file mode 100644 index 000000000..676db20ca --- /dev/null +++ b/doc/source/operations/openstack-projects-and-users-management.rst @@ -0,0 +1,38 @@ +======================================= +Openstack Projects and Users Management +======================================= + +Projects (in OpenStack) can be defined in the ``openstack-config`` repository + +To initialise the working environment for ``openstack-config``: + +.. code-block:: console + + git clone ~/src/openstack-config + python3 -m venv ~/venvs/openstack-config-venv + source ~/venvs/openstack-config-venv/bin/activate + cd ~/src/openstack-config + pip install -U pip + pip install -r requirements.txt + ansible-galaxy collection install \ + -p ansible/collections \ + -r requirements.yml + +To define a new project, add a new project to +``etc/openstack-config/openstack-config.yml``: + +Example invocation: + +.. code-block:: console + + source ~/src/kayobe-config/etc/kolla/public-openrc.sh + source ~/venvs/openstack-config-venv/bin/activate + cd ~/src/openstack-config + tools/openstack-config -- --vault-password-file + +Deleting Users and Projects +--------------------------- + +Ansible is designed for adding configuration that is not present; removing +state is less easy. To remove a project or user, the configuration should be +manually removed. diff --git a/doc/source/operations/openstack-reconfiguration.rst b/doc/source/operations/openstack-reconfiguration.rst new file mode 100644 index 000000000..392b92421 --- /dev/null +++ b/doc/source/operations/openstack-reconfiguration.rst @@ -0,0 +1,150 @@ +========================= +OpenStack Reconfiguration +========================= + +Disabling a Service +=================== + +Ansible is oriented towards adding or reconfiguring services, but removing a +service is handled less well, because of Ansible's imperative style. + +To remove a service, it is disabled in Kayobe's Kolla config, which prevents +other services from communicating with it. For example, to disable +``cinder-backup``, edit ``$KAYOBE_CONFIG_PATH/kolla.yml``: + +.. code-block:: diff + + -enable_cinder_backup: true + +enable_cinder_backup: false + +Then, reconfigure Cinder services with Kayobe: + +.. code-block:: console + + kayobe overcloud service reconfigure --kolla-tags cinder + +However, the service itself, no longer in Ansible's manifest of managed state, +must be manually stopped and prevented from restarting. + +On each controller: + +.. code-block:: console + + docker rm -f cinder_backup + +Some services may store data in a dedicated Docker volume, which can be removed +with ``docker volume rm``. + +.. _installing-external-tls-certificates: + +Installing External TLS Certificates +==================================== + +This section explains the process of deploying external TLS. +For internal and backend TLS, see `Hashicorp Vault for internal PKI +`__. + +To configure TLS for the first time, we write the contents of a PEM +file to the ``secrets.yml`` file as ``secrets_kolla_external_tls_cert``. +Use a command of this form: + +.. code-block:: console + + ansible-vault edit $KAYOBE_CONFIG_PATH/secrets.yml --vault-password-file= + +Concatenate the contents of the certificate and key files to create +``secrets_kolla_external_tls_cert``. The certificates should be installed in +this order: + +* TLS certificate for the public endpoint FQDN +* Any intermediate certificates +* The TLS certificate private key + +In ``$KAYOBE_CONFIG_PATH/kolla.yml``, set the following: + +.. code-block:: yaml + + kolla_enable_tls_external: True + kolla_external_tls_cert: "{{ secrets_kolla_external_tls_cert }}" + +To apply TLS configuration, we need to reconfigure all services, as endpoint URLs need to +be updated in Keystone: + +.. code-block:: console + + kayobe overcloud service reconfigure + +Alternative Configuration +------------------------- + +As an alternative to writing the certificates as a variable to +``secrets.yml``, it is also possible to write the same data to a file, +``$KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem``. The file should be +vault-encrypted in the same manner as secrets.yml. In this instance, +variable ``kolla_external_tls_cert`` does not need to be defined. + +See `Kolla-Ansible TLS guide +`__ for +further details. + +Updating External TLS Certificates +---------------------------------- + +Check the expiry date on an installed TLS certificate from a host that can +reach the OpenStack APIs: + +.. code-block:: console + + openssl s_client -connect :443 2> /dev/null | openssl x509 -noout -dates + +.. note:: + + Prometheus Blackbox monitoring can check certificates automatically + and alert when expiry is approaching. + +To update an existing certificate, for example when it has reached expiration, +change the value of ``secrets_kolla_external_tls_cert``, in the same order as +above. Run the following command: + +.. warning:: + + Services can be briefly unavailable during reconfiguring HAProxy. + +.. code-block:: console + + kayobe overcloud service reconfigure --kolla-tags haproxy + +.. _taking-a-hypervisor-out-of-service: + +Taking a Hypervisor out of Service +================================== + +To take a hypervisor out of Nova scheduling: + +.. code-block:: console + + openstack compute service set --disable nova-compute + +Running instances on the hypervisor will not be affected, but new instances +will not be deployed on it. + +A reason for disabling a hypervisor can be documented with the +``--disable-reason`` flag: + +.. code-block:: console + + openstack compute service set --disable \ + --disable-reason "Broken drive" nova-compute + +Details about all hypervisors and the reasons they are disabled can be +displayed with: + +.. code-block:: console + + openstack compute service list --long + +And then to enable a hypervisor again: + +.. code-block:: console + + openstack compute service set --enable nova-compute diff --git a/doc/source/operations/secret-rotation.rst b/doc/source/operations/secret-rotation.rst new file mode 100644 index 000000000..a5f7fa18d --- /dev/null +++ b/doc/source/operations/secret-rotation.rst @@ -0,0 +1,516 @@ +=============== +Secret Rotation +=============== + +General notes +============= + +This guide covers secret rotation in Kayobe and Kolla-Ansible for most services +in a standard deployment. It does not cover every secret. A full list of +passwords that have been successfully rotated is available at the bottom of +this page (:ref:`link`). + +Many of the secrets can simply be deleted from your ``passwords.yml`` and will +be automatically regenerated with a ``kayobe overcloud service deploy``. + +Some secrets require manual input from the operator to change. + +Following this process, there may be a few seconds of network downtime for +running VMs when Neutron is reconfigured when using ML2/OVS. + +There will be API downtime for all services. The main reason for the outage is +that RabbitMQ must be completely stopped to change the secrets it uses. The +services must all be reconfigured to use the new RabbitMQ cluster. Each service +will come back once it has been reconfigured. The outage time for each service +is therefore equal to the time between starting a ``kayobe overcloud service +deploy``, and that service being reconfigured. + +Some secrets currently have to be regenerated by hand. Make sure you use a +reliable tool and match the formatting (length, character set etc) of the +existing secret. ``pwgen`` is recommended and used as an example throughout +this guide. Installation: + +.. code:: bash + + sudo apt/dnf install pwgen + +Full method +=========== + +.. warning:: + + You **must** back up your ``passwords.yml`` before making changes. You will + need to refer back to it later + +1. Run a Tempest ``refstack`` & check Kibana/OpenSearch Dashboards to check + the state of the cloud before any changes are made + +2. Edit your Kolla-Ansible checkout to include changes not yet included + upstream. + +.. _kolla-change: + + 1. Add this line within the ``kolla_docker`` dict in + ``ansible/roles/nova/tasks/bootstrap_service.yml`` See `here + `__ + for an example. + + .. code:: + + command: bash -c 'sudo -E kolla_set_configs && nova-manage api_db sync && nova-manage db sync --local_cell' + + This change will break new deployments and should be reverted once this + process is complete + +.. _k-a-change: + + 2. Cherry-pick `this patch + `__ + + .. code:: bash + + git fetch https://review.opendev.org/openstack/kolla-ansible refs/changes/78/903178/2 && git cherry-pick FETCH_HEAD + + 3. Re-install Kolla-Ansible from source in your Kolla-Ansible Python + environment + + +3. Navigate to the directory containing your ``passwords.yml`` file + (``$KOLLA_CONFIG_PATH/passwords.yml`` OR + ``$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/passwords.yml``) + +4. Create a file called ``deletelist.txt`` and populate it with this content + (including all whitespace): + + .. code:: + + _keystone_password + _database_password + ^keystone_admin_password + ^memcache_secret_key + ^designate_rndc_key + ^docker_registry_password + ^keepalived_password + ^kibana_password + ^libvirt_sasl_password + ^metadata_secret + ^opensearch_dashboards_password + ^osprofiler_secret + ^prometheus_alertmanager_password + ^qdrouterd_password + ^redis_master_password + ^memcache_secret_key + _ssh_key + + private_key + public_key + ^$ + rabbitmq + ^haproxy_password + + +5. Decrypt your ``passwords.yml`` file with ``ansible-vault`` + +6. Delete all the passwords in the deletion list + + .. code:: bash + + grep -vf deletelist.txt passwords.yml > new-passwords.yml + +7. Check the new file for basic formatting errors. If it looks correct, + replace the existing ``passwords.yml`` file with ``new-passwords.yml`` + + .. code:: bash + + rm passwords.yml && mv new-passwords.yml passwords.yml + +8. Use the ``rekey-hosts.yml`` playbook to rotate your SSH keys for hosts + across the cloud. The playbook should exist under + ``kayobe-config/etc/kayobe/ansible/`` if not, merge the latest + ``stackhpc-kayobe-config`` + + 1. Run the playbook to generate a new keypair and add it to the authorised + keys of your hosts. + + .. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rekey-hosts.yml + + 2. Ensure you can SSH to other nodes using the new keypair + + 3. Re-run the playbook with arguments to remove the old keypair. + + .. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rekey-hosts.yml -t remove-key -e rekey_remove_existing_key=true + +9. Update the Pulp password + + 1. Generate a new Pulp password + + .. code:: bash + + pwgen -s 25 1 + + 2. Update ``secrets_pulp_password`` (usually found in ``secrets.yml``) + + 3. Deploy changes + + .. code:: bash + + kayobe seed service deploy -t seed-deploy-containers -kt none + + (note you will need to skip Docker registry login since the password will + now be ‘incorrect’ e.g. ``-e deploy_containers_registry_attempt_login=false``) + +10. Rotate ``horizon_secret_key`` + + 1. Generate a new secret: + + .. code:: bash + + pwgen -s 40 1 + + 2. Add it to the ``passwords.yml`` file, along with the old secret, in this + exact format (including quotes in the middle): + + .. code:: bash + + horizon_secret_key: newsecret' 'oldsecret + + This will allow both the old and new secrets to be used at the same + time, resulting in no interruption to service. The key is mainly used + for generating login and password reset tokens. The old secret can be + deleted & redeployed at a later date once all users have closed & + reopened their sessions. + +11. Update ``grafana_admin_password`` + + 1. Generate a new Grafana Admin password + + .. code:: bash + + pwgen -s 40 1 + + 2. Update the value of ``grafana_admin_password`` in ``passwords.yml`` + + 3. Exec into the MariaDB container on a controller then login to MariaDB + + .. code:: bash + + sudo docker exec -u 0 -it mariadb bash + (mariadb) mysql grafana -p + # Enter database password when prompted + + 4. Query for the ID of ``grafana_local_admin`` + + .. code:: sql + + SELECT id,login FROM user WHERE login = "grafana_local_admin"; + # Take a note of this ID + + 5. Exec into the Grafana container on a controller + + .. code:: bash + + sudo docker exec -it grafana bash + + 6. Run the password reset command, then enter the new password + + .. code:: bash + + grafana-cli admin reset-admin-password --user-id --password-from-stdin + +12. Update the MariaDB database password + + 1. Generate a new secret: + + .. code:: bash + + pwgen -s 40 1 + + 2. Update ``database_password`` in ``passwords.yml`` with your new + password. Make a note of the old password. + + 3. Exec into the MariaDB container on a controller + + .. code:: bash + + sudo docker exec -it mariadb bash + + 4. Log in to the database. You will be prompted for the password. Use the + old value of ``database_password`` + + .. code:: bash + + mysql -uroot -p + + 5. Check the current state of the ``root`` user + + .. code:: bash + + SELECT Host,User,Password FROM mysql.user WHERE User='root'; + + 6. Update the password for the ``root`` user + + .. code:: bash + + SET PASSWORD FOR 'root'@'%' = PASSWORD('newpassword'); + + 7. Check that the password hash has changed in the user list + + .. code:: bash + + SELECT Host,User,Password FROM mysql.user WHERE User='root'; + + 8. If there are any remaining root users with the old password e.g. + ``root@localhost``, change the password for them too + +.. _nova-change: + +13. Update the Nova Database password + + .. warning:: + + From this point onward, service may be disrupted + + #. Create a new ``nova_database_password`` and store it in + ``passwords.yml`` + + .. code:: bash + + pwgen -s 40 1 + + #. Exec into the ``nova_conductor`` container + + .. code:: bash + + sudo docker exec -it nova_conductor bash + + #. List the cells + + .. code:: bash + + nova-manage cell_v2 list_cells --verbose + + #. Find the entry for ``cell0``, copy the Database Connection value, + replace the password in the string with the new value, and update it + with the following command: + + .. code:: bash + + nova-manage cell_v2 update_cell --cell_uuid 00000000-0000-0000-0000-000000000000 --database_connection "CONNECTION WITH NEW PASSWORD HERE" --transport-url "none:///" + + (If the ``cell_uuid`` for cell0 is not + ``00000000-0000-0000-0000-000000000000``, change the above command + accordingly) + +14. Re-encrypt your ``passwords.yml`` file + +15. Stop all OpenStack services + + .. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/stop-openstack-services.yml + +16. Flush the Memcached data on all controllers (any old data will now be + inaccessible) + + #. Install Telnet (on one of the controllers) + + .. code:: bash + + sudo apt -y install telnet + + #. Check the config for the IP and port used by Memcached (on every + controller) + + .. code:: bash + + sudo grep command /etc/kolla/memcached/config.json + + The IP and port will be printed after ``-l`` and ``-p`` respectively + + #. For each controller start a Telnet session, clear all data, then + exit + + .. code:: bash + + telnet + flush_all + quit + +17. Nuke RabbitMQ + + .. code:: bash + + kayobe overcloud host command run -l controllers --become --command "systemctl stop kolla-rabbitmq-container.service && docker rm rabbitmq && docker volume rm rabbitmq" + +19. Reconfigure Overcloud services to apply changes + + .. warning:: + + VMs should continue running, but connections to them will briefly be + disrupted when Neutron is redeployed when using ML2/OVS + + .. code:: bash + + kayobe overcloud service deploy + +20. Manually update ``heat_domain_admin_password``, using the newly generated + OpenStack Admin credentials. + + .. code:: bash + + openstack user set --domain heat_user_domain --password heat_domain_admin + +21. Re-run Tempest to make sure everything has come back + +22. Inform other users of the steps they’ll need to take now that the secrets + have been rotated: + + 1. SSH keys have been rotated, so the new key will have to be distributed + if individual user accounts are used + + 2. Any existing ``openrc`` files generated by Kolla Ansible will need to be + re-generated or edited to use the new Keystone admin password + +23. Create a PR to merge the new secrets into your main Kayobe configuration + branch + + .. warning:: + + Unless you **really** enjoyed this process, RE-ENCRYPT + ``passwords.yml`` BEFORE COMMITTING + +24. Approximately 1 week after deploying, remove the old horizon secret key + from ``passwords.yml`` and reconfigure horizon + + +.. _full-password-list: + +Full password list +------------------- + +:: + + aodh_database_password + aodh_keystone_password + blazar_database_password + blazar_keystone_password + caso_keystone_password + ceilometer_database_password + ceilometer_keystone_password + cinder_database_password + cinder_keystone_password + barbican_database_password + barbican_keystone_password + cloudkitty_database_password + cloudkitty_keystone_password + congress_database_password + congress_keystone_password + cyborg_database_password + cyborg_keystone_password + designate_database_password + designate_keystone_password + freezer_database_password + freezer_keystone_password + glance_database_password + glance_keystone_password + gnocchi_database_password + gnocchi_keystone_password + heat_database_password + heat_keystone_password + horizon_database_password + ironic_database_password + ironic_inspector_database_password + ironic_inspector_keystone_password + ironic_keystone_password + karbor_database_password + karbor_keystone_password + keystone_database_password + magnum_database_password + manila_database_password + mariadb_backup_database_password + masakari_database_password + mistral_database_password + monasca_database_password + murano_database_password + neutron_database_password + nova_api_database_password + nova_database_password + octavia_database_password + panko_database_password + placement_database_password + prometheus_mysql_exporter_database_password + qinling_database_password + rally_database_password + sahara_database_password + senlin_database_password + solum_database_password + tacker_database_password + trove_database_password + vitrage_database_password + watcher_database_password + zun_database_password + keystone_admin_password + kuryr_keystone_password + magnum_keystone_password + manila_keystone_password + masakari_keystone_password + mistral_keystone_password + monasca_keystone_password + murano_keystone_password + neutron_keystone_password + nova_keystone_password + octavia_keystone_password + panko_keystone_password + rabbitmq_cluster_cookie + rabbitmq_monitoring_password + rabbitmq_password + database_password + heat_domain_admin_password + horizon_secret_key + placement_keystone_password + qinling_keystone_password + sahara_keystone_password + searchlight_keystone_password + senlin_keystone_password + solum_keystone_password + swift_keystone_password + tacker_keystone_password + trove_keystone_password + vitrage_keystone_password + watcher_keystone_password + zun_keystone_password + ceph_rgw_keystone_password + designate_rndc_key + keepalived_password + kibana_password + libvirt_sasl_password + metadata_secret + opensearch_dashboards_password + osprofiler_secret + prometheus_alertmanager_password + qdrouterd_password + grafana_admin_password + docker_registry_password + secrets_pulp_password + redis_master_password + haproxy_password + keystone_ssh_key + private_key + public_key + neutron_ssh_key + private_key + public_key + nova_ssh_key + private_key + public_key + octavia_amp_ssh_key + private_key + public_key + bifrost_ssh_key + private_key + public_key + diff --git a/doc/source/operations/tempest.rst b/doc/source/operations/tempest.rst new file mode 100644 index 000000000..1f6dd43cd --- /dev/null +++ b/doc/source/operations/tempest.rst @@ -0,0 +1,348 @@ +====================================== +Running Tempest with Kayobe Automation +====================================== + +Overview +======== + +This document describes how to configure and run `Tempest +`_ using `kayobe-automation +`_ from the ``.automation`` +submodule included with ``stackhpc-kayobe-config``. + +The best way of running Tempest is to use CI/CD workflows. Before proceeding, +consider whether it would be possible to use/set up a CI/CD workflow instead. +For more information, see the :doc:`CI/CD workflows page +`. + +The following guide will assume all commands are run from your +``kayobe-config`` root and the environment has been configured to run Kayobe +commands unless stated otherwise. + +Prerequisites +============= + +Installing Docker +----------------- + +``kayobe-automation`` runs in a container on the Ansible control host. This +means that Docker must be installed on the Ansible control host if it is not +already. + +.. warning:: + + Docker can cause networking issues when it is installed. By default, it + will create a bridge and change ``iptables`` rules. These can be disabled + by setting the following in ``/etc/docker/daemon.json``: + + .. code-block:: json + + { + "bridge": "none", + "iptables": false + } + + The bridge is the most common cause of issues and is *usually* safe to + disable. Disabling the ``iptables`` rules will break any GitHub actions + runners running on the host. + +To install Docker on Ubuntu: + +.. code-block:: bash + + # Add Docker's official GPG key: + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +Installing Docker on Rocky: + +.. code-block:: bash + + sudo dnf install -y dnf-utils + sudo dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + sudo dnf install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +Ensure Docker is running & enabled: + +.. code-block:: bash + + sudo systemctl start docker + sudo systemctl enable docker + +The Docker ``buildx`` plugin must be installed. If you are using an existing +installation of docker, you may need to install it with: + +.. code-block:: bash + + sudo dnf/apt install docker-buildx-plugin + sudo docker buildx install + # or if that fails: + sudo docker plugin install buildx + +Building a Kayobe container +--------------------------- + +Build a Kayobe automation image: + +.. code-block:: bash + + git submodule init + git submodule update + # If running on Ubuntu, the fact cache can confuse Kayobe in the Rocky-based container + mv etc/kayobe/facts{,-old} + sudo DOCKER_BUILDKIT=1 docker build --network host --build-arg BASE_IMAGE=rockylinux:9 --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest . + +Configuration +============= + +Kayobe automation configuration files are stored in the ``.automation.conf/`` +directory. It contains: + +- A script used to export environment variables for meta configuration of + Tempest - ``.automation.conf/config.sh``. +- Tempest configuration override files, stored in ``.automation.conf/tempest/`` + and conventionally named ``tempest.overrides.conf`` or + ``tempest-.overrides.conf``. +- Tempest load lists, stored in ``.automation.conf/tempest/load-lists``. +- Tempest skip lists, stored in ``.automation.conf/tempest/skip-lists``. + +config.sh +--------- + +``config.sh`` is a mandatory shell script, primarily used to export environment +variables for the meta configuration of Tempest. + +See: +https://github.com/stackhpc/docker-rally/blob/master/bin/rally-verify-wrapper.sh +for a full list of Tempest parameters that can be overridden. + +The most common variables to override are: + +- ``TEMPEST_CONCURRENCY`` - The maximum number of tests to run in parallel at + one time. Higher values are faster but increase the risk of timeouts. 1-2 is + safest in CI/Tenks/Multinode/AIO etc. 8-32 is typical in production. Default + value is 16. +- ``KAYOBE_AUTOMATION_TEMPEST_LOADLIST``: the filename of a load list in the + ``load-lists`` directory. Default value is ``default`` (symlink to refstack). +- ``KAYOBE_AUTOMATION_TEMPEST_SKIPLIST``: the filename of a load list in the + ``skip-lists`` directory. Default value is unset. +- ``TEMPEST_OPENRC``: The **contents** of an ``openrc.sh`` file, to be used by + Tempest to create resources on the cloud. Default is to read in the contents + of ``etc/kolla/public-openrc.sh``. + +tempest.overrides.conf +---------------------- + +Tempest uses a configuration file to define which tests are run and how to run +them. A full sample configuration file can be found `here +`_. Sensible +defaults exist for all values and in most situations, a blank +``*overrides.conf`` file will successfully run many tests. It will however also +skip many tests which may otherwise be appropriate to run. + +`Shakespeare `_ is a tool for +generating Tempest configuration files. It contains elements for different +cloud features, which can be combined to template out a detailed configuration +file. This is the best-practice approach. + +Below is an example of a manually generated file including many of the most +common overrides. It makes many assumptions about the environment, so make sure +you understand all the options before applying them. + +.. NOTE(upgrade): Microversions change for each release +.. code-block:: ini + + [openstack] + # Use a StackHPC-built image without a default password. + img_url=https://github.com/stackhpc/cirros/releases/download/20231206/cirros-d231206-x86_64-disk.img + + [auth] + # Expect unlimited quotas for CPU cores and RAM + compute_quotas = cores:-1,ram:-1 + + [compute] + # Required for migration testing + min_compute_nodes = 2 + # Required to test some API features + min_microversion = 2.1 + max_microversion = 2.96 + # Flavors for creating test servers and server resize. The ``alt`` flavor should be larger. + flavor_ref = + flavor_ref_alt = + volume_multiattach = true + + [compute-feature-enabled] + # Required for migration testing + resize = true + live_migration = true + block_migration_for_live_migration = false + volume_backed_live_migration = true + + [placement] + min_microversion = 1.0 + max_microversion = 1.39 + + [volume] + storage_protocol = ceph + # Required to test some API features + min_microversion = 3.0 + max_microversion = 3.71 + +Tempest configuration override files are stored in +``.automation.conf/tempest/``. The default file used is +``tempest.overrides.conf`` or ``tempest-.overrides.conf`` +depending on whether a Kayobe environment is enabled. This can be changed by +setting ``KAYOBE_AUTOMATION_TEMPEST_CONF_OVERRIDES`` to a different file path. +An ``overrides.conf`` file must be supplied, even if it is blank. + +Load Lists +---------- + +Load lists are a newline-separated list of tests to run. They are stored in +``.automation.conf/tempest/load-lists/``. The directory contains three objects +by default: + +- ``tempest-full`` - A complete list of all possible tests. +- ``platform.2022.11-test-list.txt`` - A reduced list of tests to match the + `Refstack `_ standard. +- ``default`` - A symlink to ``platform.2022.11-test-list.txt``. + +Test lists can be selected by changing ``KAYOBE_AUTOMATION_TEMPEST_LOADLIST`` +in ``config.sh``. The default value is ``default``, which symlinks to +``platform.2022.11-test-list.txt``. + +A common use case is to use the ``failed-tests`` list output from a previous +Tempest run as a load list, to retry the failed tests after making changes. + +Skip Lists +---------- + +Skip lists are a newline-separated list of tests to Skip. They are stored in +``.automation.conf/tempest/skip-lists/``. Each line consists of a pattern to +match against test names, and a string explaining why the test is being +skipped e.g. + +.. code-block:: + + tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_subnet_details.*: "Cirros image doesn't have /var/run/udhcpc.eth0.pid" + +There is no requirement for a skip list, and none is selected by default. A +skip list can be selected by setting ``KAYOBE_AUTOMATION_TEMPEST_SKIPLIST`` in +``config.sh``. + +Tempest runner +-------------- + +While the Kayobe automation container is always deployed to the ansible control +host, the Tempest container is deployed to the host in the ``tempest_runner`` +group, which can be any host in the Kayobe inventory. The group should only +ever contain one host. The seed is usually used as the tempest runner however +it is also common to use the Ansible control host or an infrastructure VM. The +main requirement of the host is that it can reach the OpenStack API. + +.. _tempest-cacert: + +Tempest CA certificate +---------------------- + +If your public OpenStack API uses TLS with a Certificate Authority (CA) that is +not trusted by the Python CA trust store, it may be necessary to add a CA +certificate to the trust store in the container that runs Tempest. This can be +done by defining a ``tempest_cacert`` Ansible variable to a path containing the +CA certificate. You may wish to use ``kayobe_config_path`` or +``kayobe_env_config_path`` to be agnostic to the path where kayobe-config is +mounted within the container. For example: + +.. code-block:: yaml + :caption: ``etc/kayobe/tempest.yml`` + + # Add the Vault CA certificate to the rally container when running tempest. + tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" + +Running Tempest +=============== + +Kayobe automation will need to SSH to the Tempest runner (even if they are on +the same host), so requires an SSH key exported as +``KAYOBE_AUTOMATION_SSH_PRIVATE_KEY`` e.g. + +.. code-block:: bash + + export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(cat ~/.ssh/id_rsa) + +Tempest outputs will be sent to the ``tempest-artifacts/`` directory. Create +one if it does not exist. + +.. code-block:: bash + + mkdir tempest-artifacts + +The contents of ``tempest-artifacts`` will be overwritten. Ensure any previous +test results have been copied away. + +The Tempest playbook is invoked through the Kayobe container using this +command from the base of the ``kayobe-config`` directory: + +.. code-block:: bash + + sudo -E docker run --name kayobe-automation --detach -it --rm --network host \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + +By default, ``no_log`` is set to stop credentials from leaking. This can be +disabled by adding ``-e rally_no_sensitive_log=false`` to the end. + +To follow the progress of the Kayobe automation container, either remove +``--detach`` from the above command, or follow the docker logs of the +``kayobe`` container. + +To follow the progress of the Tempest tests themselves, follow the logs of the +``tempest`` container on the ``tempest_runner`` host. + +.. code-block:: bash + + ssh + sudo docker logs -f tempest + +Tempest will keep running until completion if the ``kayobe`` container is +stopped. The ``tempest`` container must be stopped manually. Doing so will +however stop test resources (such as networks, images, and VMs) from being +automatically cleaned up. They must instead be manually removed. They should be +clearly labeled with either rally or tempest in the name, often alongside some +randomly generated string. + +Outputs +------- + +Tempest outputs will be sent to the ``tempest-artifacts/`` directory. It +contain the following artifacts: + +- ``docker.log`` - The logs from the ``tempest`` docker container +- ``failed-tests`` - A simple list of tests that failed +- ``rally-junit.xml`` - An XML file listing all tests in the test list and + their status (skipped/succeeded/failed). Usually not useful. +- ``rally-verify-report.html`` - An HTML page with all test results including + an error trace for failed tests. It is often best to ``scp`` this file back + to your local machine to view it. This is the most user-friendly way to view + the test results, however can be awkward to host. +- ``rally-verify-report.json`` - A JSON blob with all test results including an + error trace for failed tests. It contains all the same data as the HTML + report but without formatting. +- ``stderr.log`` - The stderr log. Usually not useful. +- ``stdout.log`` - The stdout log. Usually not useful. +- ``tempest-load-list`` - The load list that Tempest was invoked with. +- ``tempest.log`` - Detailed logs from Tempest. Contains more data than the + ``verify`` reports, but can be difficult to parse. Useful for tracing specific + errors. diff --git a/doc/source/operations/ubuntu-noble.rst b/doc/source/operations/ubuntu-noble.rst new file mode 100644 index 000000000..3df166809 --- /dev/null +++ b/doc/source/operations/ubuntu-noble.rst @@ -0,0 +1,556 @@ +========================= +Upgrading to Ubuntu Noble +========================= + +Overview +======== + +This document describes how to upgrade systems from Ubuntu Jammy 22.04 to +Ubuntu Noble 24.04. This procedure must be performed on Ubuntu Jammy 22.04 +OpenStack Caracal systems before it is possible to upgrade to OpenStack Epoxy. +It is possible to perform a rolling upgrade to ensure service is not disrupted. + +Upgrades are performed in-place with a script using the ``do-release-upgrade`` +tool provided by Canonical, rather than reprovisioning. The scripts are found +at ``tools/ubuntu-upgrade-*.sh``. For overcloud and infrastructure VM upgrades, +the script takes one argument - the host(s) to upgrade. The scripts execute a +playbook to upgrade the host, then run the appropriate ``kayobe * host +configure`` command. + +The guide assumes a local pulp instance is deployed and all hosts use it +to pull ``apt`` packages. To upgrade a host using upstream packages, see the +manual upgrade process at the bottom of this page. + +While it is technically possible to upgrade hosts in any order, it is +recommended that upgrades for one type of node be completed before moving on +to the next i.e. all compute node upgrades are performed before all storage +node upgrades. + +The order of node groups is less important however it is arguably safest to +perform controller node upgrades first, given that they are the most complex +and it is easiest to revert their state in the event of a failure. +This guide covers the following types of hosts: + +- Controllers +- Compute hosts +- Storage hosts +- Seed +- Other hosts not managed by Kayobe + +The following types of hosts will be covered in the future: + +- Ansible control host +- Seed hypervisor (an upgrade script exists but has not been tested) +- Infrastructure VMs (an upgrade script exists but has not been tested) + +.. warning:: + + Due to `Bug 66389 `__, do not upgrade + Ceph hosts to Noble until the Ceph cluster has been upgraded to at least + Reef v18.2.5. Upgrading a host prematurely will prevent its Ceph daemons + from starting, and it will not be able to rejoin the cluster. + +Prerequisites +============= + +Before starting the upgrade, ensure any appropriate prerequisites are +satisfied. These will be specific to each deployment, but here are some +suggestions: + +* Merge in the latest ``stackhpc-kayobe-config`` ``stackhpc/2024.1`` branch. +* Ensure that there is sufficient hypervisor capacity to drain + at least one node. +* If using Ironic for bare metal compute, ensure that at least one node is + available for testing provisioning. +* Ensure that expected test suites are passing, e.g. Tempest. +* Resolve any Prometheus alerts. +* Check for unexpected ``ERROR`` or ``CRITICAL`` messages in OpenSearch + Dashboard. +* Check Grafana dashboards. + +Sync Release Train artifacts +---------------------------- + +New `StackHPC Release Train <../configuration/release-train.html>`__ content +should be synced to the local Pulp server. This includes host packages +(Deb/RPM) and container images. + +To sync host packages: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml -e stackhpc_pulp_sync_ubuntu_jammy=true -e stackhpc_pulp_sync_ubuntu_noble=true + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml -e stackhpc_pulp_sync_ubuntu_jammy=true -e stackhpc_pulp_sync_ubuntu_noble=true + +Once the host package content has been tested in a test/staging environment, it +may be promoted to production: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml -e stackhpc_pulp_sync_ubuntu_jammy=true -e stackhpc_pulp_sync_ubuntu_noble=true + +To sync container images: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + +Build locally customised container images +----------------------------------------- + +.. note:: + + The container images provided by StackHPC Release Train are suitable for + most deployments. In this case, this step can be skipped. + +In some cases, it is necessary to build some or all images locally to apply +customisations. To do this, set +``stackhpc_pulp_sync_for_local_container_build`` to ``true`` before syncing +container images. + +To build the overcloud images locally and push them to the local Pulp server: + +.. code-block:: console + + kayobe overcloud container image build --push + +It is possible to build a specific set of images by supplying one or more +image name regular expressions: + +.. code-block:: console + + kayobe overcloud container image build --push ironic- nova-api + +Deploy the latest container images +---------------------------------- + +Make sure you deploy the latest containers before this upgrade: + +.. code-block:: console + + kayobe seed service deploy + kayobe overcloud service deploy + +Common issues for all host types +================================ + +- Disk names can change during upgrades. This can be resolved in kayobe-config + once the new name is known (i.e. after the first upgrade) and applied by + re-running ``host configure`` for the affected host. +- Timeouts can become an issue with some hardware. The host will reboot once + or twice depending on whether it needs to apply package updates. Edit the + timeouts in the upgrade playbook (``ubuntu-upgrade.yml``) where required. + +Controllers +=========== + +Upgrade controllers *one by one*, ideally upgrading the host with the Kolla +Virtual IP (VIP) last. Before upgrading a host with the VIP, stop the +``keepalived`` container for a few seconds to fail it over to another +controller (restarting the container does not always stop the container for +long enough). + +.. code-block:: bash + + sudo systemctl stop kolla-keepalived-container.service + sudo systemctl start kolla-keepalived-container.service + +Always back up the overcloud DB before starting: + +.. code-block:: bash + + kayobe overcloud database backup + +Potential issues +---------------- + +- If the system uses OVS as a network driver, there's a chance that kolla + services can struggle to find reply queues from RabbitMQ during the upgrade. + Currently this could be observed when rolling reboot of controllers are done + or deploying Ubuntu Noble based Kolla containers are deployed after all + hosts are upgraded to Ubuntu to Noble. + You can use the ``rabbitmq-reset.yml`` playbook but all messages that are + in progress will be lost: + + .. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rabbitmq-reset.yml + +- If you are using hyper-converged Ceph, please also note the potential issues + in the Storage section below. +- After controllers are rebooted, Hashicorp Vault can be sealed. Run the + ``vault-unseal-overcloud.yml`` playbook to unseal the vaults. + + .. code-block:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-unseal-overcloud.yml + +Full procedure for one controller +--------------------------------- + +1. Export the ``KAYOBE_PATH`` environment variable to be the source of Kayobe + e.g. + + .. code-block:: console + + export KAYOBE_PATH=~/venvs/kayobe/share/kayobe + + # or if you have a kayobe source locally + + export KAYOBE_PATH=~/src/kayobe + +2. If the controller is running Ceph services: + + 1. Set host in maintenance mode: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml --limit + + 2. Check nothing remains on the host: + + .. code-block:: console + + # From cephadm shell + ceph orch ps + +3. Run the upgrade script: + + .. code-block:: console + + $KAYOBE_CONFIG_PATH/../../tools/ubuntu-upgrade-overcloud.sh + +4. If the controller is running Ceph OSD services: + + 1. Make sure the cephadm public key is in ``authorized_keys`` for stack or + root user - depends on your setup. For example, your SSH key may + already be defined in ``users.yml``. If in doubt, run the cephadm + deploy playbook to copy the SSH key and install the cephadm binary. + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml + + 2. Take the host out of maintenance mode: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml --limit + + 3. Make sure that everything is back in working condition before moving + on to the next host: + + .. code-block:: console + + # From cephadm shell + ceph -s + ceph -w + +After each controller has been upgraded you may wish to perform some smoke +testing, run Tempest, check for alerts and errors etc. + +Compute +======= + +Compute nodes can be upgraded in batches. +The possible batches depend on: + +* willingness for instance reboots and downtime +* available spare hypervisor capacity +* sizes of groups of compatible hypervisors + +Potential issues +---------------- + +None so far! + +Full procedure for one batch of hosts +------------------------------------- + +1. Export the ``KAYOBE_PATH`` environment variable e.g. + + .. code-block:: console + + export KAYOBE_PATH=~/venvs/kayobe/share/kayobe + + # or if you have a kayobe source locally + + export KAYOBE_PATH=~/src/kayobe + +2. Disable the Nova compute service and drain it of VMs using live migration. + If any VMs fail to migrate, they may be cold migrated or powered off: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/nova-compute-{disable,drain}.yml --limit + +3. If the compute node is running Ceph OSD services: + + 1. Set host in maintenance mode: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml --limit + + 2. Check there's nothing remaining on the host: + + .. code-block:: console + + # From cephadm shell + ceph orch ps + +4. Run the upgrade script: + + .. code-block:: console + + $KAYOBE_CONFIG_PATH/../../tools/ubuntu-upgrade-overcloud.sh + +5. If the compute node is running Ceph OSD services: + + 1. Make sure the cephadm public key is in ``authorized_keys`` for stack or + root user - depends on your setup. For example, your SSH key may + already be defined in ``users.yml`` . If in doubt, run the cephadm + deploy playbook to copy the SSH key and install the cephadm binary. + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml + + 2. Take the host out of maintenance mode: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml --limit + + 3. Make sure that everything is back in working condition before moving + on to the next host: + + .. code-block:: console + + # From cephadm shell + ceph -s + ceph -w + +6. Restore the system to full health. + + 1. If any VMs were powered off, they may now be powered back on. + + 2. Wait for Prometheus alerts and errors in OpenSearch Dashboard to resolve, + or address them. + + 3. Once happy that the system has been restored to full health, enable the + hypervisor in Nova if it is still disabled and then move onto the next + host or batch or hosts. + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/nova-compute-enable.yml --limit + +Storage +======= + +Potential issues +---------------- + +- Ensure the Ceph cluster is running at least Reef v18.2.5. + Upgrading hosts with an older Ceph version will cause daemons to fail. +- It is recommended that you upgrade the bootstrap host last. +- Before upgrading the bootstrap host, it can be beneficial to backup + ``/etc/ceph`` and ``/var/lib/ceph``, as sometimes the keys, config, etc. + stored here will not be moved/recreated correctly. +- It has been seen that sometimes the Ceph containers do not come up after + upgrading. This seems to be related to having ``/var/lib/ceph`` persisted + through the reprovision (e.g. seen at a customer in a volume with software + RAID). Further investigation is needed for the root cause. When this + occurs, you will need to redeploy the daemons: + + List the daemons on the host: + + .. code-block:: console + + ceph orch ps + + Redeploy the daemons, one at a time. It is recommended that you start with + the crash daemon, as this will have the least impact if unexpected issues + occur. + + .. code-block:: console + + ceph orch daemon redeploy to redeploy a daemon. + +- Commands starting with ``ceph`` are all run on the cephadm bootstrap + host in a cephadm shell unless stated otherwise. + +Full procedure for a storage host +--------------------------------- + +1. Export the ``KAYOBE_PATH`` environment variable e.g. + + .. code-block:: console + + export KAYOBE_PATH=~/venvs/kayobe/share/kayobe + + # or if you have a kayobe source locally + + export KAYOBE_PATH=~/src/kayobe + +2. Set host in maintenance mode: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml --limit + +3. Check there's nothing remaining on the host: + + .. code-block:: console + + # From cephadm shell + ceph orch ps + +4. Run the upgrade script: + + .. code-block:: console + + $KAYOBE_CONFIG_PATH/../../tools/ubuntu-upgrade-overcloud.sh + +5. Make sure the cephadm public key is in ``authorized_keys`` for stack or + root user - depends on your setup. For example, your SSH key may + already be defined in ``users.yml``. If in doubt, run the cephadm + deploy playbook to copy the SSH key and install the cephadm binary. + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml + +6. Take the host out of maintenance mode: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml --limit + +7. Make sure that everything is back in working condition before moving + on to the next host: + + .. code-block:: console + + # From cephadm shell + ceph -s + ceph -w + +Seed +==== + +Potential issues +---------------- + +- The process has not been tested as well as for other hosts. Proceed with + caution. +- The Seed can take significantly longer to upgrade than other hosts. + ``do-release-upgrade`` has been observed taking more than 45 minutes to + complete. + +Full procedure +-------------- + +1. Export the ``KAYOBE_PATH`` environment variable e.g. + + .. code-block:: console + + export KAYOBE_PATH=~/venvs/kayobe/share/kayobe + + # or if you have a kayobe source locally + + export KAYOBE_PATH=~/src/kayobe + +2. Run the upgrade script: + + .. code-block:: console + + $KAYOBE_CONFIG_PATH/../../tools/ubuntu-upgrade-seed.sh + +Wazuh manager +============= + +TODO + +Seed hypervisor +=============== + +TODO + +Ansible control host +==================== + +TODO + +Manual Process +============== + +Sometimes it is necessary to upgrade a system that is not managed by Kayobe +(and therefore does not use packages from pulp). Below is a set of instructions +to manually execute the upgrade process. + +Full procedure +-------------- + +1. Update all packages to the latest available versions + + .. code-block:: console + + sudo apt update -y && sudo apt upgrade -y + +2. Install the upgrade tool + + .. code-block:: console + + sudo apt install ubuntu-release-upgrader-core + +3. Check whether a reboot is required + + .. code-block:: console + + cat /var/run/reboot-required + +4. Where required, reboot to apply updates + + .. code-block:: console + + sudo reboot + +5. Run ``do-release-upgrade`` + + .. code-block:: console + + do-release-upgrade -f DistUpgradeViewNonInteractive + +6. Reboot to apply the upgrade + + .. code-block:: console + + sudo reboot + + +Post Upgrade works +================== + +Deploy Ubuntu Noble Kolla containers +------------------------------------ + +Once all hosts are upgraded to Ubuntu Noble and stable, Kolla containers built +with Ubuntu Noble base image need to be deployed. + +.. code-block:: console + + kayobe overcloud service upgrade + +System verification +------------------- + +After new Kolla containers are deployed, check the system status with + +- Opensearch Dashboards +- Grafana +- Prometheus + +and run appropriate test suites. e.g. Tempest. diff --git a/doc/source/operations/upgrading-ceph.rst b/doc/source/operations/upgrading-ceph.rst new file mode 100644 index 000000000..5ce711011 --- /dev/null +++ b/doc/source/operations/upgrading-ceph.rst @@ -0,0 +1,175 @@ +============== +Upgrading Ceph +============== + +This section describes show to upgrade from one version of Ceph to another. +The Ceph upgrade procedure is described :ceph-doc:`here `. + +The Ceph release series is not strictly dependent upon the StackHPC OpenStack +release, however this configuration does define a default Ceph release series +and container image tag. The default release series is currently |ceph_series|. + +Prerequisites +============= + +Before starting the upgrade, ensure any appropriate prerequisites are +satisfied. These will be specific to each deployment, but here are some +suggestions: + +* Ensure that expected test suites are passing, e.g. Tempest. +* Resolve any Prometheus alerts. +* Check for unexpected ``ERROR`` or ``CRITICAL`` messages in OpenSearch + Dashboard. +* Check Grafana dashboards. + +Consider whether the Ceph cluster needs to be upgraded within or outside of a +maintenance/change window. + +Preparation +=========== + +Ensure that the local Kayobe configuration environment is up to date. + +If you wish to use a different Ceph release series, set +``cephadm_ceph_release``. + +If you wish to use different Ceph container image tags, set the following +variables: + +* ``cephadm_image_tag`` (`tags `__) +* ``cephadm_haproxy_image_tag`` (`tags `__) +* ``cephadm_keepalived_image_tag`` (`tags `__) + +Be sure to use a tag that `matches the release series +`__. + +Upgrading Host Packages +======================= + +Prior to upgrading the Ceph storage cluster, it may be desirable to upgrade +system packages on the hosts. + +Note that these commands do not affect packages installed in containers, only +those installed on the host. + +In order to avoid downtime, it is important to control how package updates are +rolled out. In general, Ceph monitor hosts should be updated *one by one*. For +Ceph OSD hosts it may be possible to update packages in batches of hosts, +provided there is sufficient capacity to maintain data availability. + +For each host or batch of hosts, perform the following steps. + +Place the host or batch of hosts into maintenance mode: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml -l + +To update all eligible packages, use ``*``, escaping if necessary: + +.. code-block:: console + + kayobe overcloud host package update --packages "*" --limit + +If the kernel has been upgraded, reboot the host or batch of hosts to pick up +the change. While running this playbook, consider setting ``ANSIBLE_SERIAL`` to +the maximum number of hosts that can safely reboot concurrently. + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml -l + +Remove the host or batch of hosts from maintenance mode: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml -l + +Wait for Ceph health to return to ``HEALTH_OK``: + +.. code-block:: console + + ceph -s + +Wait for Prometheus alerts and errors in OpenSearch Dashboard to resolve, or +address them. + +Once happy that the system has been restored to full health, move onto the next +host or batch or hosts. + +Sync container images +===================== + +If using the local Pulp server to host Ceph images +(``stackhpc_sync_ceph_images`` is ``true``), sync the new Ceph images into the +local Pulp: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-{sync,publish}.yml -e stackhpc_pulp_images_kolla_filter=none + +Upgrade Ceph services +===================== + +Start the upgrade. If using the local Pulp server to host Ceph images: + +.. code-block:: console + + sudo cephadm shell -- ceph orch upgrade start --image /ceph/ceph: + +Otherwise: + +.. code-block:: console + + sudo cephadm shell -- ceph orch upgrade start --image quay.io/ceph/ceph: + +The tag should match the ``cephadm_image_tag`` variable set in `preparation +<#preparation>`_. The registry should be the address and port of the local Pulp +server. + +Check the update status: + +.. code-block:: console + + ceph orch upgrade status + +Wait for Ceph health to return to ``HEALTH_OK``: + +.. code-block:: console + + ceph -s + +Watch the cephadm logs: + +.. code-block:: console + + ceph -W cephadm + +Upgrade Cephadm +=============== + +Update the Cephadm package: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml -e cephadm_package_update=true + +Testing +======= + +At this point it is recommended to perform a thorough test of the system to +catch any unexpected issues. This may include: + +* Check Prometheus, OpenSearch Dashboards and Grafana +* Smoke tests +* All applicable tempest tests +* Horizon UI inspection + +Cleaning up +=========== + +Prune unused container images: + +.. code-block:: console + + kayobe overcloud host command run -b --command "docker image prune -a -f" -l ceph diff --git a/doc/source/operations/upgrading-openstack.rst b/doc/source/operations/upgrading-openstack.rst new file mode 100644 index 000000000..73010b61b --- /dev/null +++ b/doc/source/operations/upgrading-openstack.rst @@ -0,0 +1,1188 @@ +=================== +Upgrading OpenStack +=================== + +This section describes how to upgrade from the |previous_release| OpenStack +release series to |current_release|. It is based on the :kayobe-doc:`upstream +Kayobe documentation ` with additional considerations for using +StackHPC Kayobe Configuration. + +Overview +======== + +A StackHPC OpenStack upgrade is broken down into several phases. + +* Prerequisites +* Preparation +* Upgrading the Seed Hypervisor +* Upgrading the Seed +* Upgrading Wazuh Manager +* Upgrading Wazuh Agents +* Upgrading the Overcloud +* Cleaning up + +After preparation is complete, the remaining phases may be completed in any +order, however the order specified above allows for completing as much as +possible before the user-facing overcloud upgrade. It is not recommended to +keep different parts of the system on different releases for extended periods +due to the need to maintain and use separate local Kayobe environments. + +.. NOTE(upgrade): Update these notable changes for the current release. + +Notable changes in the |current_release| Release +================================================ + +There are many changes in the OpenStack |current_release| release described in +the release notes for each project. Here are some notable ones. + +Heat disabled by default +------------------------ + +The Heat OpenStack service is no longer enabled by default. + +This behavior can be overridden manually: + +.. code-block:: yaml + :caption: ``kolla.yml`` + + kolla_enable_heat: true + +Wherever possible, Magnum deployments should be migrated to the CAPI Helm +driver. Instructions for enabling the driver can be found `here +<../configuration/magnum-capi.html>`_. Enable the driver, recreate any clusters +using Heat, and disable the service. + +After the upgrade (so that alerts don't fire) you can remove Heat with the +following: + +.. code-block:: console + + kayobe overcloud host command run --command "rm /etc/kolla/haproxy/services.d/heat-api.cfg" -l network -b + kayobe overcloud host command run --command "rm /etc/kolla/haproxy/services.d/heat-api-cfn.cfg" -l network -b + + kayobe overcloud host command run --command "systemctl restart kolla-haproxy-container.service" -l network[0] -b + kayobe overcloud host command run --command "systemctl restart kolla-haproxy-container.service" -l network[1] -b + kayobe overcloud host command run --command "systemctl restart kolla-haproxy-container.service" -l network[2] -b + + kayobe overcloud host command run --command "systemctl stop kolla-heat_api-container.service kolla-heat_api_cfn-container.service kolla-heat_engine-container.service" -l controllers -b + kayobe overcloud host command run --command "systemctl disable kolla-heat_api-container.service kolla-heat_api_cfn-container.service kolla-heat_engine-container.service" -l controllers -b + kayobe overcloud host command run --command "rm /etc/systemd/system/kolla-heat_api-container.service" -l controllers -b + kayobe overcloud host command run --command "rm /etc/systemd/system/kolla-heat_api_cfn-container.service" -l controllers -b + kayobe overcloud host command run --command "rm /etc/systemd/system/kolla-heat_engine-container.service" -l controllers -b + + kayobe overcloud host command run --command "docker rm heat_api heat_api_cfn heat_engine" -l controllers + + kayobe overcloud host command run --command "rm -rf /etc/kolla/heat-api /etc/kolla/heat-api-cfn /etc/kolla/heat-engine" --limit controllers -b + +Then from the OpenStack CLI: + +.. code-block:: console + + openstack service delete heat + openstack service delete heat-cfn + openstack user delete heat + openstack domain set --disable heat_user_domain + openstack domain delete heat_user_domain + +You can drop the ``heat`` database too, unless you want to keep historical content. + +.. code-block:: console + + docker exec -it mariadb mysql -u root -p + Enter the database password when prompted. + drop database heat; + +Designate sink disabled by default +---------------------------------- + +Designate sink is an optional Designate service which listens for event +notifications, primarily from Nova and Neutron. It is disabled by default (when +designate is enabled) in Caracal. It is not required for Designate to function. + +If you still wish to use it, you should set the flag manually: + +.. code-block:: yaml + :caption: ``kolla/globals.yml`` + + designate_enable_notifications_sink: true + +If you are using Designate and do not make this change, the Antelope +``designate-sink`` container will remain on the controllers after the upgrade. +It must be removed manually. + +Grafana Volume +-------------- +The Grafana container volume is no longer used. If you wish to automatically +remove the old volume, set ``grafana_remove_old_volume`` to ``true`` in +``kolla/globals.yml``. Note that doing this will lose any plugins installed via +the CLI directly and not through Kolla. If you have previously installed +Grafana plugins via the Grafana UI or CLI, you must change to installing them +at image build time. The Grafana volume, which contains existing custom +plugins, will be automatically removed in the next release. + +Prometheus HAproxy Exporter +--------------------------- +Due to the change from using the ``prometheus-haproxy-exporter`` to using the +native support for Prometheus which is now built into HAProxy, metric names may +have been replaced and/or removed, and in some cases the metric names may have +remained the same but the labels may have changed. Alerts and dashboards may +also need to be updated to use the new metrics. Please review any configuration +that references the old metrics as this is not a backwards compatible change. + +Horizon configuration +--------------------- +The Horizon role has been reworked to the preferred ``local_settings.d`` +configuration model. Files ``local_settings`` and ``custom_local_settings`` +have been renamed to ``_9998-kolla-settings.py`` and +``_9999-custom-settings.py`` respectively. Users who use Horizon's custom +configuration must change the names of those files in +``etc/kolla/config/horizon`` as well. + +Neutron DNS Domain +------------------ +When Designate is enabled and the default Neutron DNS integration has not been +disabled, ``neutron_dns_domain`` must be configured manually in +``kolla/globals.yml``. + +The ``neutron_dns_domain`` must end with a period ``.`` e.g. ``example.com.``. +The domain set should be something that is not use anywhere else such as +``internal.compute.example.com.`` + +The Neutron DNS integration can be disabled by setting +``neutron_dns_integration: false`` in ``kolla/globals.yml`` + +Redis Default User +------------------ + +The ``redis_connection_string`` has changed the username used from ``admin`` +to ``default``. Whilst this does not have any negative impact on services +that utilise Redis it will feature prominently in any preview of the overcloud +configuration. + +AvailabilityZoneFilter removal +------------------------------ + +Support for the ``AvailabilityZoneFilter`` filter has been dropped in Nova. +Remove it from any Nova config files before upgrading. It will cause errors in +Caracal and halt the Nova scheduler. + +Keystone LDAP TLS configuration +------------------------------- + +Either ``[ldap] tls_cacertfile`` or ``[ldap] tls_cacertdir`` must be configured +if ``[ldap] use_tls`` is true or LDAP URL uses the ``ldaps://`` scheme. LDAP +authentication will fail if this configuration is absent. See `upstream +Keystone change `__ +for more details. + +OS Capacity exporter and dashboard enabled by default +----------------------------------------------------- + +The OS Capacity exporter will automatically be deployed after the upgrade. +During the upgrade, HAProxy config, Prometheus config and Grafana dashboards +will also be updated to use the exporter. If you want to disable this, change +the following in ``kayobe-config/etc/kayobe/stackhpc-monitoring.yml``: + +.. code-block:: yaml + + # Whether the OpenStack Capacity exporter is enabled. + # Enabling this flag will result in HAProxy configuration and Prometheus scrape + # targets being templated during deployment. + stackhpc_enable_os_capacity: false + +Known issues +============ + +* Due to an incorrect default value NGS will attempt to use v3alpha for the api + path when communicating with etcd3. This isn't possible as in Caracal etcd is + running a newer version that has dropped support for v3alpha. You can work + around this in custom config, see the SMS PR for an example: + https://github.com/stackhpc/smslab-kayobe-config/pull/354 + +* Due to a `security-related change in the GRUB package on Rocky Linux 9 + `__, the operating + system can become unbootable (boot will stop at a ``grub>`` prompt). Remove + the ``--root-dev-only`` option from ``/boot/efi/EFI/rocky/grub.cfg`` after + applying package updates. This will happen automatically as a post hook when + running the ``kayobe overcloud host package update`` command. + +* After upgrading OpenSearch to the latest 2023.1 container image, we have seen + cluster routing allocation be disabled on some systems. See bug for details: + https://bugs.launchpad.net/kolla-ansible/+bug/2085943. + This will cause the "Perform a flush" handler to fail during the 2024.1 + OpenSearch upgrade. To workaround this, you can run the following PUT request + to enable allocation again: + + .. code-block:: console + + curl -X PUT "https://:9200/_cluster/settings?pretty" -H 'Content-Type: application/json' -d '{ "transient" : { "cluster.routing.allocation.enable" : "all" } } ' + +* Cinder database migrations fail during the upgrade process when the + ``use_quota`` column is set to ``NULL``, which can be the case on deleted + volumes and snapshots if OpenStack has been in operation for several + releases. See `Launchpad bug 2070475 + `__ for details. Until the + `database migrations are fixed + `__, the data can be + fixed with the following MySQL queries: + + .. code-block:: mysql + + UPDATE volumes SET use_quota = 1 WHERE use_quota IS NULL AND deleted_at IS NOT NULL; + UPDATE snapshots SET use_quota = 1 WHERE use_quota IS NULL AND deleted_at IS NOT NULL; + +Security baseline +================= + +As part of the Caracal release we are looking to improve the security +baseline of StackHPC OpenStack deployments. If any of the following have not +been done, they should be completed before the upgrade begins. + +.. TODO: Add these when docs exist + + * Enable `host firewalling `_ + +* Enable `Center for Internet Security (CIS) compliance <../configuration/security-hardening.html>`_ +* Enable TLS on the :kayobe-doc:`public API network + ` +* Enable TLS on the `internal API network <../configuration/vault.html>`_ +* Configure `walled garden networking <../configuration/walled-garden.html>`_ +* Use `LVM-based host images <../configuration/lvm.html>`_ +* Deploy `Wazuh <../configuration/wazuh.html>`_ + +Prerequisites +============= + +Before starting the upgrade, ensure any appropriate prerequisites are +satisfied. These will be specific to each deployment, but here are some +suggestions: + +* If hypervisors will be rebooted, e.g. to pick up a new kernel, or + reprovisioned, ensure that there is sufficient hypervisor capacity to drain + at least one node. +* If using Ironic for bare metal compute, ensure that at least one node is + available for testing provisioning. +* Ensure that expected test suites are passing, e.g. Tempest. +* Resolve any Prometheus alerts. +* Check for unexpected ``ERROR`` or ``CRITICAL`` messages in OpenSearch + Dashboard. +* Check Grafana dashboards. +* Update the deployment to use the latest |previous_release| images and + configuration. +* If your customer has overriden any policies, check to see if they need + updating to align with new defaults. These will be written to files + ``kolla/config//policy.yaml``. Policy reference documentation can + generally be found in the documentation of each project. For example, Nova + policy: https://docs.openstack.org/nova/latest/configuration/policy.html + +RabbitMQ SLURP upgrade +---------------------- + +.. note:: + The upgrade is reliant on recent changes. Make sure you have updated to + the latest version of kolla ansible and deployed the latest kolla containers + before proceeding. + +Because this is a SLURP upgrade, RabbitMQ must be upgraded manually from 3.11, +to 3.12, then to 3.13 on Antelope before the Caracal upgrade. This upgrade +should not cause an API outage (though it should still be considered "at +risk"). + +Some errors have been observed in testing when the upgrades are performed +back-to-back. A 200s delay eliminates this issue. On particularly large or slow +deployments, consider increasing this timeout. + +Additionally errors have been observed at sites with OVS networking where after +the upgrade, tenant networking is broken and requires a reset of RabbitMQ. This +can be done by running the rabbitmq-reset playbook. + +.. code-block:: bash + + kayobe overcloud service configuration generate --node-config-dir /tmp/ignore -kt none + kayobe kolla ansible run "rabbitmq-upgrade 3.12" + sleep 200 + kayobe kolla ansible run "rabbitmq-upgrade 3.13" + +RabbitMQ quorum queues +---------------------- + +In Caracal, quorum queues are enabled by default for RabbitMQ. This is +different to Antelope which used HA queues. Before upgrading to Caracal, it is +strongly recommended that you migrate from HA to quorum queues. The migration +is automated using a script. + +.. warning:: + This migration will stop all services using RabbitMQ and cause an + extended API outage while queues are migrated. It should only be + performed in a pre-agreed maintenance window. + +Set the following variables in your kolla globals file (i.e. +``$KAYOBE_CONFIG_PATH/kolla/globals.yml`` or +``$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml``): + +.. code-block:: yaml + + om_enable_rabbitmq_high_availability: false + om_enable_rabbitmq_quorum_queues: true + +Then execute the migration script: + +.. code-block:: bash + + $KAYOBE_CONFIG_PATH/../../tools/rabbitmq-quorum-migration.sh + +Preparation +=========== + +Preparation is crucial for a successful upgrade. It allows for a minimal +maintenance/change window and ensures we are ready if unexpected issues arise. + +Upgrade plan +------------ + +The less you need to think on upgrade day, the better. Save your brain for +solving any issues that arise. Write an upgrade plan detailing: + +* the predicted schedule +* a checklist of prerequisites +* a set of smoke tests to perform after significant changes +* a list of steps to perform during the preparation phase +* a list of steps to perform during the upgrade maintenance/change window phase +* a list of steps to perform during the follow up phase +* a set of full system tests to perform after the upgrade is complete +* space to make notes of progress and any issues/solutions/workarounds that + arise + +Ideally all steps will include the exact commands to execute that can be +copy/pasted, or links to appropriate CI/CD workflows to run. + +Backing up +---------- + +Before you start, be sure to back up any local changes, configuration, and +data. + +See the :kayobe-doc:`Kayobe documentation +` for information on +backing up the overcloud MariaDB database. It may be prudent to take backups at +various stages of the upgrade since the database state will change over time. + +Updating code forks +------------------- + +If the deployment uses any source code forks (other than the StackHPC ones), +update them to use the |current_release| release. + +Migrating Kayobe Configuration +------------------------------ + +Kayobe configuration options may be changed between releases of Kayobe. Ensure +that all site local configuration is migrated to the target version format. +See the :skc-doc:`StackHPC Kayobe Configuration release notes +`, :kayobe-renos:`Kayobe release notes <>` and +:kolla-ansible-renos:`Kolla Ansible release notes <>`. In particular, the +*Upgrade Notes* and *Deprecation Notes* sections provide information that might +affect the configuration migration. + +In the following example we assume a branch naming scheme of +``example/``. + +Create a branch for the new release: + +.. code-block:: console + :substitutions: + + git fetch origin + git checkout example/|previous_release| + git checkout -b example/|current_release| + git push origin example/|current_release| + +Merge in the new branch of StackHPC Kayobe Configuration: + +.. code-block:: console + :substitutions: + + git remote add stackhpc https://github.com/stackhpc/stackhpc-kayobe-config + git fetch stackhpc + git fetch origin + git checkout -b example/|current_release|-sync origin/example/|current_release| + git merge stackhpc/|current_release_git_branch_name| + +There may be conflicts to resolve. The configuration should be manually +inspected after the merge to ensure that it is correct. Once complete, push the +branch and create a pull request with the changes: + +.. code-block:: console + :substitutions: + + git push origin example/|current_release|-sync + +Once approved and merged, update the configuration to adapt to the new release. +This may involve e.g. adding, removing or renaming variables to allow for +upstream changes. Note that configuration in the base environment +(``etc/kayobe/``) will be merged with upstream changes, but anything in a +deployment-specific environment directory (``etc/kayobe/environments/`` may +require manual inspection. + +If using the ``kayobe-env`` environment file in ``kayobe-config``, this should +also be inspected for changes and modified to suit the local Ansible control +host environment if necessary. When ready, source the environment file: + +.. code-block:: console + + source kayobe-env + +Create one or more pull requests with these changes. + +Once the configuration has been migrated, it is possible to view the global +variables for all hosts: + +.. code-block:: console + + kayobe configuration dump + +The output of this command is a JSON object mapping hosts to their +configuration. The output of the command may be restricted using the +``--host``, ``--hosts``, ``--var-name`` and ``--dump-facts`` options. + +Upgrading local Kayobe environment +---------------------------------- + +The local Kayobe environment should be either recreated or upgraded to use the +new release. It may be beneficial to keep a Kayobe environment for the old +release in case it is necessary before the uprade begins. + +In general it is safer to rebuild an environment than upgrade, but for +completeness the following shows how to upgrade an existing local Kayobe +environment. + +Change to the Kayobe configuration directory: + +.. code-block:: console + + cd /path/to/src/kayobe-config + +Check the status: + +.. code-block:: console + + git status + +Pull down the new branch: + +.. code-block:: console + :substitutions: + + git checkout example/|current_release| + git pull origin example/|current_release| + +Activate the Kayobe virtual environment: + +.. code-block:: console + + source /path/to/venvs/kayobe/bin/activate + +Reinstall Kayobe and other dependencies: + +.. code-block:: console + + pip install --force-reinstall -r requirements.txt + +Source the ``kayobe-env`` script: + +.. code-block:: console + + source kayobe-env [--environment ] + +Export the Ansible Vault password: + +.. code-block:: console + + export KAYOBE_VAULT_PASSWORD=$(cat /path/to/vault/password/file) + +Next we must upgrade the Ansible control host. Tasks performed here include: + +- Install updated Ansible collection and role dependencies from Ansible Galaxy. +- Generate an SSH key if necessary and add it to the current user's authorised + keys. +- Upgrade Kolla Ansible locally to the configured version. + +To upgrade the Ansible control host: + +.. code-block:: console + + kayobe control host upgrade + +Syncing Release Train artifacts +------------------------------- + +New :ref:`stackhpc-release-train` content should be synced to the local Pulp +server. This includes host packages (Deb/RPM) and container images. + +.. _sync-rt-package-repos: + +To sync host packages: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml + +Once the host package content has been tested in a test/staging environment, it +may be promoted to production: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-promote-production.yml + +To sync container images: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + +Build locally customised container images +----------------------------------------- + +.. note:: + + The container images are provided by StackHPC Release Train are + suitable for most deployments. In this case, this step can be skipped. + +In some cases it is necessary to build some or all images locally to apply +customisations. In order to do this it is necessary to set +``stackhpc_pulp_sync_for_local_container_build`` to ``true`` before +:ref:`syncing container images `. + +To build the overcloud images locally and push them to the local Pulp server: + +.. code-block:: console + + kayobe overcloud container image build --push + +It is possible to build a specific set of images by supplying one or more +image name regular expressions: + +.. code-block:: console + + kayobe overcloud container image build --push ironic- nova-api + +Pull container images to hosts +------------------------------ + +Pulling container images from the local Pulp server to the control plane hosts +can take a considerable time, because images are only synced from Ark to the +local Pulp on demand, and there is potentially a large fan-out. Pulling images +in advance of the upgrade moves this step out of the maintenance/change window. +Consider checking available disk space before pulling: + +.. code-block:: console + + kayobe overcloud host command run --command "df -h" --show-output --limit controllers[0],compute[0],storage[0] + +Then pull the images: + +.. code-block:: console + + kayobe overcloud container image pull + +Preview overcloud service configuration changes +----------------------------------------------- + +Kayobe allows us to generate overcloud service configuration in advance, and +compare it with the running configuration. This allows us to check for any +unexpected changes. + +This can take a significant time, and it may be advisable to limit these +commands to one of each type of host (controller, compute, storage, etc.). +The following commands use a limit including the first host in each of these +groups. + +Save the old configuration locally. + +.. code-block:: console + + kayobe overcloud service configuration save --node-config-dir /etc/kolla --output-dir ~/kolla-diff/old --limit controllers[0],compute[0],storage[0] --exclude ironic-agent.initramfs,ironic-agent.kernel + +Generate the new configuration to a tmpdir. + +.. code-block:: console + + kayobe overcloud service configuration generate --node-config-dir /tmp/kolla --kolla-limit controllers[0],compute[0],storage[0] + +Save the new configuration locally. + +.. code-block:: console + + kayobe overcloud service configuration save --node-config-dir /tmp/kolla --output-dir ~/kolla-diff/new --limit controllers[0],compute[0],storage[0] --exclude ironic-agent.initramfs,ironic-agent.kernel + +The old and new configuration will be saved to ``~/kolla-diff/old`` and +``~/kolla-diff/new`` respectively on the Ansible control host. + +Fix up the paths: + +.. code-block:: console + + cd ~/kolla-diff/new + for i in *; do mv $i/tmp $i/etc; done + cd - + +Compare the old & new configuration: + +.. code-block:: console + + diff -ru ~/kolla-diff/{old,new} > ~/kolla-diff.diff + less ~/kolla-diff.diff + +Upgrading the Seed Hypervisor +============================= + +Currently, upgrading the seed hypervisor services is not supported. It may +however be necessary to upgrade host packages and some host services. + +Consider whether the seed hypervisor needs to be upgraded within or outside of +a maintenance/change window. + +Upgrading Host Packages +----------------------- + +.. note:: + + In case of issues booting up, consider alternative access methods if the + hypervisor is also used as the Ansible control host (or runs it in a VM). + +Prior to upgrading the seed hypervisor, it may be desirable to upgrade system +packages on the seed hypervisor host. + +To update all eligible packages, use ``*``, escaping if necessary: + +.. code-block:: console + + kayobe seed hypervisor host package update --packages "*" + +If the kernel has been upgraded, reboot the seed hypervisor to pick up the +change: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml -l seed-hypervisor + +Upgrading Host Services +----------------------- + +It may be necessary to upgrade some host services: + +.. code-block:: console + + kayobe seed hypervisor host upgrade + +Note that this will not perform full configuration of the host, and will +instead perform a targeted upgrade of specific services where necessary. + +Configuring hosts +----------------- + +Performing host configuration is not a formal part of the upgrade process, but +it is possible for host configuration to drift over time as new features and +other changes are added to Kayobe. + +Host configuration, particularly around networking, can lead to loss of network +connectivity and other issues if the configuration is not correct. For this +reason it is sensible to first run Ansible in "check mode" to see what changes +would be applied: + +.. code-block:: console + + kayobe seed hypervisor host configure --check --diff + +When ready to apply the changes: + +.. code-block:: console + + kayobe seed hypervisor host configure + +Upgrading the Seed +================== + +Consider whether the seed needs to be upgraded within or outside of a +maintenance/change window. + +Upgrading Host Packages +----------------------- + +.. note:: + + In case of issues booting up, consider alternative access methods if the + seed is also used as the Ansible control host. + +Prior to upgrading the seed, it may be desirable to upgrade system packages on +the seed host. + +Note that these commands do not affect packages installed in containers, only +those installed on the host. + +To update all eligible packages, use ``*``, escaping if necessary: + +.. code-block:: console + + kayobe seed host package update --packages "*" + +If the kernel has been upgraded, reboot the seed to pick up the change: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml -l seed + +Verify that Bifrost, Ironic and Inspector are running as expected: + +.. code-block:: console + + ssh stack@ + sudo docker exec -it bifrost_deploy bash + systemctl + export OS_CLOUD=bifrost + baremetal node list + baremetal introspection list + exit + exit + +Building Ironic Deployment Images +--------------------------------- + +.. note:: + + It is possible to use prebuilt deployment images. In this case, this step + can be skipped. + +It is possible to use prebuilt deployment images from the `OpenStack hosted +tarballs `_ or another +source. In some cases it may be necessary to build images locally either to +apply local image customisation or to use a downstream version of Ironic Python +Agent (IPA). In order to build IPA images, the ``ipa_build_images`` variable +should be set to ``True``. To build images locally: + +.. code-block:: console + + kayobe seed deployment image build + +To overwrite existing images, add the ``--force-rebuild`` argument. + +Upgrading Host Services +----------------------- + +It may be necessary to upgrade some host services: + +.. code-block:: console + + kayobe seed host upgrade + +Note that this will not perform full configuration of the host, and will +instead perform a targeted upgrade of specific services where necessary. + +Configuring hosts +----------------- + +Performing host configuration is not a formal part of the upgrade process, but +it is possible for host configuration to drift over time as new features and +other changes are added to Kayobe. + +Host configuration, particularly around networking, can lead to loss of network +connectivity and other issues if the configuration is not correct. For this +reason it is sensible to first run Ansible in "check mode" to see what changes +would be applied: + +.. code-block:: console + + kayobe seed host configure --check --diff + +When ready to apply the changes: + +.. code-block:: console + + kayobe seed host configure + +Building Container Images +------------------------- + +.. note:: + + The container images are provided by StackHPC Release Train are + suitable for most deployments. In this case, this step can be skipped. + +In some cases it is necessary to build some or all images locally to apply +customisations. In order to do this it is necessary to set +``stackhpc_pulp_sync_for_local_container_build`` to ``true`` before +:ref:`syncing container images `. + +To build the seed images locally and push them to the local Pulp server: + +.. code-block:: console + + kayobe seed container image build --push + +Upgrading Containerised Services +-------------------------------- + +Containerised seed services may be upgraded by replacing existing containers +with new containers using updated images which have been pulled from the local +Pulp registry. + +To upgrade the containerised seed services: + +.. code-block:: console + + kayobe seed service upgrade + +Verify that Bifrost, Ironic and Inspector are running as expected: + +.. code-block:: console + + ssh stack@ + sudo docker exec -it bifrost_deploy bash + systemctl + export OS_CLOUD=bifrost + baremetal node list + baremetal introspection list + exit + exit + +Upgrading Wazuh Manager +======================= + +Consider whether Wazuh Manager needs to be upgraded within or outside of a +maintenance/change window. + +Upgrading Host Packages +----------------------- + +Prior to upgrading the Wazuh manager services, it may be desirable to upgrade +system packages on the Wazuh manager host. + +To update all eligible packages, use ``*``, escaping if necessary: + +.. code-block:: console + + kayobe infra vm host package update --packages "*" -l wazuh-manager + +If the kernel has been upgraded, reboot the Wazuh Manager to pick up the +change: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml -l wazuh-manager + +Verify that Wazuh Manager is functioning correctly by :ref:`logging into the +Wazuh UI `. + +Configuring hosts +----------------- + +Performing host configuration is not a formal part of the upgrade process, but +it is possible for host configuration to drift over time as new features and +other changes are added to Kayobe. + +Host configuration, particularly around networking, can lead to loss of network +connectivity and other issues if the configuration is not correct. For this +reason it is sensible to first run Ansible in "check mode" to see what changes +would be applied: + +.. code-block:: console + + kayobe infra vm host configure --check --diff -l wazuh-manager + +When ready to apply the changes: + +.. code-block:: console + + kayobe infra vm host configure -l wazuh-manager + +Upgrade Wazuh Manager services +------------------------------ + +.. todo + + Is this the correct way to update Wazuh Manager? + +Run the following playbook to update Wazuh Manager services and configuration: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml + +Verify that Wazuh Manager is functioning correctly by :ref:`logging into the +Wazuh UI `. + +Upgrading Wazuh Agents +====================== + +Consider whether Wazuh Agents need to be upgraded within or outside of a +maintenance/change window. + +Upgrade Wazuh Agent services +---------------------------- + +.. todo + + Is this the correct way to update Wazuh Agents? + +Run the following playbook to update Wazuh Agent services and configuration: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml + +Verify that the agents have conncted to Wazuh Manager correctly by +:ref:`logging into the Wazuh UI `. + +Upgrading the Overcloud +======================= + +Consider which of the overcloud upgrade steps need to be performed within or +outside of a maintenance/change window. + +Upgrading Host Packages +----------------------- + +Prior to upgrading the OpenStack control plane, it may be desirable to upgrade +system packages on the overcloud hosts. + +Note that these commands do not affect packages installed in containers, only +those installed on the host. + +In order to avoid downtime, it is important to control how package updates are +rolled out. In general, controllers and network hosts should be updated *one by +one*, ideally updating the host with the Virtual IP (VIP) last. For hypervisors +it may be possible to update packages in batches of hosts, provided there is +sufficient capacity to migrate VMs to other hypervisors. + +For each host or batch of hosts, perform the following steps. + +If the host is a hypervisor, disable the Nova compute service and drain it of +VMs using live migration. If any VMs fail to migrate, they may be cold migrated +or powered off: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/nova-compute-{disable,drain}.yml --limit + +To update all eligible packages, use ``*``, escaping if necessary: + +.. code-block:: console + + kayobe overcloud host package update --packages "*" --limit + +.. note:: + + Due to a `security-related change in the GRUB package on Rocky Linux 9 + `__, the operating + system can become unbootable (boot will stop at a ``grub>`` prompt). Remove + the ``--root-dev-only`` option from ``/boot/efi/EFI/rocky/grub.cfg`` after + applying package updates. This will happen automatically as a post hook when + running the ``kayobe overcloud host package update`` command. + +If the kernel has been upgraded, reboot the host or batch of hosts to pick up +the change: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml -l + +.. warning:: + + Take extra care when updating packages on Ceph hosts. Docker live-restore + does not work until the Squid version of Ceph, so a reload of docker will + restart all Ceph containers. Set the hosts to maintenance mode before + updating packages, and unset when done: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml --limit + kayobe overcloud host package update --packages "*" --limit + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/reboot.yml -l + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml --limit + + **Always** reconfigure hosts in small batches or one-by-one. Check the Ceph + state after each host configuration. Ensure all warnings and errors are + resolved before moving on. + +If the host is a hypervisor, enable the Nova compute service. + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/nova-compute-enable.yml --limit + +If any VMs were powered off, they may now be powered back on. + +Wait for Prometheus alerts and errors in OpenSearch Dashboard to resolve, or +address them. + +After updating controllers or network hosts, run any appropriate smoke tests. + +Once happy that the system has been restored to full health, move onto the next +host or batch or hosts. + +Upgrading Host Services +----------------------- + +Prior to upgrading the OpenStack control plane, the overcloud host services +should be upgraded: + +.. code-block:: console + + kayobe overcloud host upgrade + +Note that this will not perform full configuration of the host, and will +instead perform a targeted upgrade of specific services where necessary. + +Configuring hosts +----------------- + +Performing host configuration is not a formal part of the upgrade process, but +it is possible for host configuration to drift over time as new features and +other changes are added to Kayobe. + +Host configuration, particularly around networking, can lead to loss of network +connectivity and other issues if the configuration is not correct. For this +reason it is sensible to first run Ansible in "check mode" to see what changes +would be applied: + +.. code-block:: console + + kayobe overcloud host configure --check --diff + +When ready to apply the changes, it may be advisable to do so in batches, or at +least start with a small number of hosts: + +.. code-block:: console + + kayobe overcloud host configure --limit + + +.. warning:: + + Take extra care when configuring Ceph hosts. Set the hosts to maintenance + mode before reconfiguring them, and unset when done: + + .. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-enter-maintenance.yml --limit + kayobe overcloud host configure --limit + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/ceph-exit-maintenance.yml --limit + + **Always** reconfigure hosts in small batches or one-by-one. Check the Ceph + state after each host configuration. Ensure all warnings and errors are + resolved before moving on. + +.. _building_ironic_deployment_images: + +Building Ironic Deployment Images +--------------------------------- + +.. note:: + + It is possible to use prebuilt deployment images. In this case, this step + can be skipped. + +It is possible to use prebuilt deployment images from the `OpenStack hosted +tarballs `_ or another +source. In some cases it may be necessary to build images locally either to +apply local image customisation or to use a downstream version of Ironic Python +Agent (IPA). In order to build IPA images, the ``ipa_build_images`` variable +should be set to ``True``. To build images locally: + +.. code-block:: console + + kayobe overcloud deployment image build + +To overwrite existing images, add the ``--force-rebuild`` argument. + +Upgrading Ironic Deployment Images +---------------------------------- + +Prior to upgrading the OpenStack control plane you should upgrade +the deployment images. If you are using prebuilt images, update +the following variables in ``etc/kayobe/ipa.yml`` accordingly: + +* ``ipa_kernel_upstream_url`` +* ``ipa_kernel_checksum_url`` +* ``ipa_kernel_checksum_algorithm`` +* ``ipa_ramdisk_upstream_url`` +* ``ipa_ramdisk_checksum_url`` +* ``ipa_ramdisk_checksum_algorithm`` + +Alternatively, you can update the files that the URLs point to. If building the +images locally, follow the process outlined in +:ref:`building_ironic_deployment_images`. + +To get Ironic to use an updated set of overcloud deployment images, you can run: + +.. code-block:: console + + kayobe baremetal compute update deployment image + +This will register the images in Glance and update the ``deploy_ramdisk`` +and ``deploy_kernel`` properties of the Ironic nodes. + +Before rolling out the update to all nodes, it can be useful to test the image +on a limited subset. To do this, you can use the ``--baremetal-compute-limit`` +option. The argument should take the form of an `ansible host pattern +`_ +which is matched against the Ironic node name. + +Upgrading Containerised Services +-------------------------------- + +Containerised control plane services may be upgraded by replacing existing +containers with new containers using updated images which have been pulled from +a registry or built locally. + +If using overcloud Ironic, check whether any ironic nodes are in a wait state: + +.. code-block:: console + + baremetal node list | grep wait + +This will block the upgrade, but may be overridden by setting +``ironic_upgrade_skip_wait_check`` to ``true`` in +``etc/kayobe/kolla/globals.yml`` or +``etc/kayobe/environments//kolla/globals.yml``. + +To upgrade the containerised control plane services: + +.. code-block:: console + + kayobe overcloud service upgrade + +It is possible to specify tags for Kayobe and/or kolla-ansible to restrict the +scope of the upgrade: + +.. code-block:: console + + kayobe overcloud service upgrade --tags config --kolla-tags keystone + +Updating the Octavia Amphora Image +---------------------------------- + +If using Octavia with the Amphora driver, you should :ref:`update the amphora +image `. + +Testing +------- + +At this point it is recommended to perform a thorough test of the system to +catch any unexpected issues. This may include: + +* Check Prometheus, OpenSearch Dashboards and Grafana +* Smoke tests +* All applicable tempest tests +* Horizon UI inspection + +Cleaning up +=========== + +Prune unused container images: + +.. code-block:: console + + kayobe overcloud host command run -b --command "docker image prune -a -f" diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst new file mode 100644 index 000000000..efff02ca4 --- /dev/null +++ b/doc/source/release-notes.rst @@ -0,0 +1,6 @@ +==================================== +2024.1 Caracal Series Release Notes +==================================== + +.. release-notes:: + :branch: stackhpc/2024.1 diff --git a/doc/source/usage.rst b/doc/source/usage.rst new file mode 100644 index 000000000..06e54ba0f --- /dev/null +++ b/doc/source/usage.rst @@ -0,0 +1,107 @@ +===== +Usage +===== + +How to consume this configuration +================================= + +This configuration is not a complete Kayobe configuration, rather it should be +treated as a base, in place of the `upstream kayobe-config +`__. Indeed, this repository is +based on the upstream kayobe-config, with some opinionated configuration +changes applied. + +Since this repository makes changes to the base configuration, it works best +when used with Kayobe's :kayobe-doc:`multiple environments +` feature. + +This configuration should be consumed using the `StackHPC Kayobe fork +`__, which includes +backported support for Ansible collections. + +New deployments +--------------- + +If starting a new deployment, clone this repository as the starting point for +your configuration: + +.. parsed-literal:: + + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b |current_release_git_branch_name| + +Existing deployments +-------------------- + +If migrating an existing deployment to StackHPC Kayobe configuration, you will +need to merge the changes in this repository into your repository: + +.. parsed-literal:: + + git remote add stackhpc https://github.com/stackhpc/stackhpc-kayobe-config + git fetch stackhpc + git merge stackhpc/|current_release_git_branch_name| + +Updating +-------- + +This base configuration will be updated over time, to update repository +versions, container image tags, and other configuration. Deployments may +consume these updates by merging in the changes with their local +configuration: + +.. parsed-literal:: + + git fetch stackhpc + git merge stackhpc/|current_release_git_branch_name| + +The intention is to avoid merge conflicts where possible, but there may be +cases where this is difficult. We are open to discussion on how best to +approach this on both sides. + +Beokay +------ + +`Beokay ` is a tool to manage Kayobe +environments. This can create new StackHPC Kayobe environments and +ensure StackHPC Kayobe Configuration dependencies are from the correct repositories and +are up-to-date: + +To create a Beokay environment using the base configuration, for the latest release: + +.. code-block:: console + + beokay.py create \ + --base-path skc-environment \ + --kayobe-config-repo https://github.com/stackhpc/stackhpc-kayobe-config.git \ + --kayobe-config-branch |current_release_git_branch_name| \ + --kayobe-in-requirements + +Kayobe environments can also be specified, for example, to create an AIO environment: + +.. code-block:: console + + beokay.py create \ + --base-path skc-aio-environment \ + --kayobe-config-repo https://github.com/stackhpc/stackhpc-kayobe-config.git \ + --kayobe-config-branch |current_release_git_branch_name| \ + --kayobe-config-env-name ci-aio \ + --vault-password-file ~/vault-pw \ + --kayobe-in-requirements + +When Beokay environments are no longer required, they can be deleted by running: + +.. code-block:: console + + beokay.py destroy \ + --base-path skc-environment + +Specific Kayobe commands can also be run via Beokay, for example, to run a Kolla +service deployment on overcloud hosts: + +.. code-block:: console + + beokay.py run \ + 'kayobe overcloud service deploy' \ + --base-path skc-aio-environment \ + --kayobe-config-env-name ci-aio \ + --vault-password-file ~/vault-pw diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg index b38cb8239..36a3f8ff2 100644 --- a/etc/kayobe/ansible.cfg +++ b/etc/kayobe/ansible.cfg @@ -1,8 +1,8 @@ [defaults] forks = 20 -# Use the YAML stdout callback plugin. -stdout_callback = yaml -# Use the stdout_callback when running ad-hoc commands. +# Use the YAML callback to format output. +callback_result_format = yaml +# Use the callback plugin when running ad-hoc commands bin_ansible_callbacks = True # Disable fact variable injection to improve performance. inject_facts_as_vars = False @@ -17,3 +17,4 @@ any_unparsed_is_failed = True [ssh_connection] pipelining = True +ssh_extra_args = -o ControlPersist=1h diff --git a/etc/kayobe/ansible/advise-run.yml b/etc/kayobe/ansible/advise-run.yml new file mode 100644 index 000000000..f6be804bf --- /dev/null +++ b/etc/kayobe/ansible/advise-run.yml @@ -0,0 +1,47 @@ +--- +- name: ADVise run + hosts: localhost + gather_facts: false + tags: + - advise + vars: + venv: ~/venvs/advise-review + input_dir: "{{ lookup('env', 'PWD') }}/overcloud-introspection-data" + output_dir: "{{ lookup('env', 'PWD') }}/review" + advise_pattern: .*.eval # Uses regex + tasks: + - name: Install dependencies + ansible.builtin.pip: + virtualenv: "{{ venv }}" + name: + - git+https://github.com/stackhpc/ADVise + state: latest + virtualenv_command: python3 -m venv + + - name: Create data directory + ansible.builtin.file: + path: "{{ output_dir }}/data" + state: directory + + - name: Extract data + ansible.builtin.shell: + cmd: > + {{ venv }}/bin/m2-extract {{ input_dir }}/*.json --output_dir {{ output_dir }}/data + + - name: Create review directory + ansible.builtin.file: + path: "{{ output_dir }}/results" + state: directory + + - name: Process data + ansible.builtin.shell: + cmd: > + {{ venv }}/bin/advise-process + -I ipmi + -p '{{ output_dir }}/data/extra-hardware/{{ advise_pattern }}' + -o '{{ output_dir }}' + + - name: Visualise data + ansible.builtin.command: > + {{ venv }}/bin/advise-visualise + --output_dir '{{ output_dir }}' diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml new file mode 100644 index 000000000..343183078 --- /dev/null +++ b/etc/kayobe/ansible/build-ofed-rocky.yml @@ -0,0 +1,45 @@ +--- +- name: Build OFED kernel modules + become: true + hosts: ofed-builder + gather_facts: false + tasks: + - name: Check whether noexec is enabled for /var/tmp + ansible.builtin.lineinfile: + path: /etc/fstab + regexp: noexec + state: absent + changed_when: false + check_mode: true + register: result + failed_when: result.found + + - name: Install package dependencies + ansible.builtin.dnf: + name: + - autoconf + - automake + - createrepo + - cmake-filesystem + - doca-extra + - gcc-gfortran + - kernel-devel + - kernel-rpm-macros + - kernel-modules-extra + - kpartx + - libtool + - libnl3-devel + - lsof + - patch + - pciutils + - perl + - python3-devel + - rpm-build + - tcl + - tk + state: latest + update_cache: true + + - name: Build OFED kernel modules + ansible.builtin.command: + cmd: /opt/mellanox/doca/tools/doca-kernel-support diff --git a/etc/kayobe/ansible/ceph-enter-maintenance.yml b/etc/kayobe/ansible/ceph-enter-maintenance.yml new file mode 100644 index 000000000..c119583e1 --- /dev/null +++ b/etc/kayobe/ansible/ceph-enter-maintenance.yml @@ -0,0 +1,13 @@ +--- +- name: Ensure a Ceph host has entered maintenance + gather_facts: true + any_errors_fatal: true + # We need to check whether it is OK to stop hosts after previous hosts have + # entered maintenance. + serial: 1 + hosts: ceph + become: true + tasks: + - name: Ensure a Ceph host has entered maintenance + ansible.builtin.import_role: + name: stackhpc.cephadm.enter_maintenance diff --git a/etc/kayobe/ansible/ceph-exit-maintenance.yml b/etc/kayobe/ansible/ceph-exit-maintenance.yml new file mode 100644 index 000000000..a16f338d9 --- /dev/null +++ b/etc/kayobe/ansible/ceph-exit-maintenance.yml @@ -0,0 +1,12 @@ +--- +- name: Ensure a Ceph host has exited maintenance + gather_facts: true + any_errors_fatal: true + hosts: ceph + # The role currently requires hosts to exit maintenance serially. + serial: 1 + become: true + tasks: + - name: Ensure a Ceph host has exited maintenance + ansible.builtin.import_role: + name: stackhpc.cephadm.exit_maintenance diff --git a/etc/kayobe/ansible/cephadm-commands-post.yml b/etc/kayobe/ansible/cephadm-commands-post.yml index d60a135a3..6bd083321 100644 --- a/etc/kayobe/ansible/cephadm-commands-post.yml +++ b/etc/kayobe/ansible/cephadm-commands-post.yml @@ -7,7 +7,8 @@ - cephadm - cephadm-commands tasks: - - import_role: + - name: Apply Cephadm commands role + ansible.builtin.import_role: name: stackhpc.cephadm.commands vars: cephadm_commands: "{{ cephadm_commands_post | default([]) }}" diff --git a/etc/kayobe/ansible/cephadm-commands-pre.yml b/etc/kayobe/ansible/cephadm-commands-pre.yml index abe58caa5..01969ed73 100644 --- a/etc/kayobe/ansible/cephadm-commands-pre.yml +++ b/etc/kayobe/ansible/cephadm-commands-pre.yml @@ -7,7 +7,8 @@ - cephadm - cephadm-commands tasks: - - import_role: + - name: Apply Cephadm commands role + ansible.builtin.import_role: name: stackhpc.cephadm.commands vars: cephadm_commands: "{{ cephadm_commands_pre | default([]) }}" diff --git a/etc/kayobe/ansible/cephadm-crush-rules.yml b/etc/kayobe/ansible/cephadm-crush-rules.yml index b95b9f960..7bc9c9fbe 100644 --- a/etc/kayobe/ansible/cephadm-crush-rules.yml +++ b/etc/kayobe/ansible/cephadm-crush-rules.yml @@ -7,5 +7,6 @@ - cephadm - cephadm-crush-rules tasks: - - import_role: + - name: Apply Cephadm crush rule role + ansible.builtin.import_role: name: stackhpc.cephadm.crush_rules diff --git a/etc/kayobe/ansible/cephadm-deploy.yml b/etc/kayobe/ansible/cephadm-deploy.yml index 9cb0f712b..471d6037d 100644 --- a/etc/kayobe/ansible/cephadm-deploy.yml +++ b/etc/kayobe/ansible/cephadm-deploy.yml @@ -7,5 +7,6 @@ - cephadm - cephadm-deploy tasks: - - import_role: + - name: Apply Cephadm role + ansible.builtin.import_role: name: stackhpc.cephadm.cephadm diff --git a/etc/kayobe/ansible/cephadm-ec-profiles.yml b/etc/kayobe/ansible/cephadm-ec-profiles.yml index 5fb0f3bdc..9ba7f90bb 100644 --- a/etc/kayobe/ansible/cephadm-ec-profiles.yml +++ b/etc/kayobe/ansible/cephadm-ec-profiles.yml @@ -7,5 +7,6 @@ - cephadm - cephadm-ec-profiles tasks: - - import_role: + - name: Apply Cephadm EC profiles role + ansible.builtin.import_role: name: stackhpc.cephadm.ec_profiles diff --git a/etc/kayobe/ansible/cephadm-gather-keys.yml b/etc/kayobe/ansible/cephadm-gather-keys.yml index d2acf92d0..1b36f8bdd 100644 --- a/etc/kayobe/ansible/cephadm-gather-keys.yml +++ b/etc/kayobe/ansible/cephadm-gather-keys.yml @@ -32,13 +32,13 @@ loop: "{{ kolla_ceph_services | selectattr('required') | map(attribute='keys') | flatten | unique }}" - name: Generate ceph.conf - command: "cephadm shell -- ceph config generate-minimal-conf" + ansible.builtin.command: cephadm shell -- ceph config generate-minimal-conf become: true register: cephadm_ceph_conf changed_when: false - name: Ensure Kolla config directories are present - file: + ansible.builtin.file: state: directory path: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_key_dir[item.name] }}" loop: "{{ kolla_ceph_services | selectattr('required') }}" @@ -51,7 +51,7 @@ key_info: "{{ cephadm_key_info.results | selectattr('item', 'equalto', item.1) | first }}" cephadm_key: "{{ key_info.stdout }}" cephadm_user: "{{ item.1 }}" - copy: + ansible.builtin.copy: # Include a trailing newline. content: | {{ cephadm_key }} @@ -63,11 +63,12 @@ notify: Please add and commit the Kayobe configuration - name: Save ceph.conf to Kayobe configuration - copy: + ansible.builtin.copy: # Include a trailing newline. # Kolla Ansible's merge_configs module does not like the leading tabs in ceph.conf. content: | {{ cephadm_ceph_conf.stdout | regex_replace('\t') }} + {{ kolla_ceph_conf_append if kolla_ceph_conf_append is defined }} dest: "{{ kayobe_env_config_path }}/kolla/config/{{ kolla_service_to_conf_dir[item.0.name] }}/ceph.conf" loop: "{{ query('subelements', kolla_ceph_services | selectattr('required'), 'keys') }}" loop_control: @@ -77,7 +78,7 @@ handlers: - name: Please add and commit the Kayobe configuration - debug: + ansible.builtin.debug: msg: >- Please add and commit the Ceph configuration files and keys in Kayobe configuration. Remember to encrypt the keys using Ansible Vault. diff --git a/etc/kayobe/ansible/cephadm-keys.yml b/etc/kayobe/ansible/cephadm-keys.yml index 6f632f4ca..6b2604519 100644 --- a/etc/kayobe/ansible/cephadm-keys.yml +++ b/etc/kayobe/ansible/cephadm-keys.yml @@ -7,5 +7,6 @@ - cephadm - cephadm-keys tasks: - - import_role: + - name: Apply Cephadm keys role + ansible.builtin.import_role: name: stackhpc.cephadm.keys diff --git a/etc/kayobe/ansible/cephadm-pools.yml b/etc/kayobe/ansible/cephadm-pools.yml index 938e6de19..bb61f9efb 100644 --- a/etc/kayobe/ansible/cephadm-pools.yml +++ b/etc/kayobe/ansible/cephadm-pools.yml @@ -7,5 +7,6 @@ - cephadm - cephadm-keys tasks: - - import_role: + - name: Apply Cephadm pools role + ansible.builtin.import_role: name: stackhpc.cephadm.pools diff --git a/etc/kayobe/ansible/cephadm.yml b/etc/kayobe/ansible/cephadm.yml index b699df153..b3c02654b 100644 --- a/etc/kayobe/ansible/cephadm.yml +++ b/etc/kayobe/ansible/cephadm.yml @@ -1,9 +1,22 @@ --- # Deploy Ceph via Cephadm. Create EC profiles, CRUSH rules, pools and keys. -- import_playbook: cephadm-deploy.yml -- import_playbook: cephadm-commands-pre.yml -- import_playbook: cephadm-ec-profiles.yml -- import_playbook: cephadm-crush-rules.yml -- import_playbook: cephadm-pools.yml -- import_playbook: cephadm-keys.yml -- import_playbook: cephadm-commands-post.yml +- name: Import Cephadm deploy playbook + import_playbook: cephadm-deploy.yml + +- name: Import Cephadm commands pre playbook + import_playbook: cephadm-commands-pre.yml + +- name: Import Cephadm EC profiles playbook + import_playbook: cephadm-ec-profiles.yml + +- name: Import Cephadm crush rules playbook + import_playbook: cephadm-crush-rules.yml + +- name: Import Cephadm pools playbook + import_playbook: cephadm-pools.yml + +- name: Import Cephadm keys playbook + import_playbook: cephadm-keys.yml + +- name: Import Cephadm commands post playbook + import_playbook: cephadm-commands-post.yml diff --git a/etc/kayobe/ansible/check-kayobe-version.yml b/etc/kayobe/ansible/check-kayobe-version.yml new file mode 100644 index 000000000..b893d806f --- /dev/null +++ b/etc/kayobe/ansible/check-kayobe-version.yml @@ -0,0 +1,72 @@ +--- +- name: Check Kayobe version + tags: kayobe-version-check + hosts: localhost + gather_facts: false + vars: + requirements_path: "{{ kayobe_config_path }}/../../requirements.txt" + tasks: + - name: Check version + when: stackhpc_enable_kayobe_check + check_mode: false + block: + - name: Get package info + community.general.pip_package_info: + register: packages + + - name: Check if pip is version 24.0 or newer + ansible.builtin.assert: + that: "{{ packages.packages.pip.pip[0].version is version('24.0', '>=') }}" + fail_msg: | + Pip must be 24.0 or newer to run this check. Upgrade pip by running + pip install -U pip and reinstall Kayobe by running: + pip install --force-reinstall -r {{ requirements_path }} + + - name: Get installed Kayobe commit + ansible.builtin.shell: + cmd: set -o pipefail && pip freeze | grep kayobe | cut -d @ -f 3 + executable: /usr/bin/bash + register: kayobe_git_commit + failed_when: kayobe_git_commit.stdout == "" + + - name: Create a temporary directory to clone Kayobe into + ansible.builtin.tempfile: + state: directory + register: kayobe_temp_dir + + - name: Clone Kayobe + ansible.builtin.git: + repo: https://github.com/stackhpc/kayobe.git + dest: "{{ kayobe_temp_dir.path }}/kayobe-git" + version: stackhpc/{{ openstack_release }} + + - name: Get tag from Kayobe commit + ansible.builtin.command: + cmd: git describe --tags {{ kayobe_git_commit.stdout }} + chdir: "{{ kayobe_temp_dir.path }}/kayobe-git" + register: kayobe_current_version + + - name: Clean up temporary directory + ansible.builtin.file: + state: absent + path: "{{ kayobe_temp_dir.path }}" + + - name: Get latest Kayobe version + ansible.builtin.shell: + cmd: set -o pipefail && grep -o kayobe@stackhpc\/.*$ {{ requirements_path }} | cut -d @ -f 2 + executable: /usr/bin/bash + register: kayobe_latest_version + + - name: Check installed Kayobe version is the latest + ansible.builtin.assert: + that: "kayobe_latest_version.stdout in kayobe_current_version.stdout" + fail_msg: | + Kayobe must use the expected version before continuing. + + Current Kayobe version: {{ kayobe_current_version.stdout }} + Expected Kayobe version: {{ kayobe_latest_version.stdout }} + + Recreate the Kayobe environment, or install the expected version + by running: pip install --force-reinstall -r {{ requirements_path }} + success_msg: | + Kayobe running at version: {{ kayobe_current_version.stdout }} diff --git a/etc/kayobe/ansible/check-kolla-ansible-version.yml b/etc/kayobe/ansible/check-kolla-ansible-version.yml new file mode 100644 index 000000000..e6bfd92fd --- /dev/null +++ b/etc/kayobe/ansible/check-kolla-ansible-version.yml @@ -0,0 +1,28 @@ +--- +- name: Check Kolla-Ansible version + tags: kolla-ansible-version-check + hosts: localhost + gather_facts: false + tasks: + - name: Check version + when: stackhpc_enable_kolla_ansible_check + check_mode: false + block: + - name: Get current Kolla-Ansible tag + ansible.builtin.command: + cmd: git describe --tags + chdir: "{{ lookup('ansible.builtin.env', 'KOLLA_SOURCE_PATH') }}" + register: kolla_ansible_current_version + + - name: Check installed Kolla-Ansible version is the expected version + ansible.builtin.assert: + that: "stackhpc_kolla_ansible_source_version in kolla_ansible_current_version.stdout" + fail_msg: | + Kolla-Ansible must use the expected version before continuing. + + Current Kolla-Ansible version: {{ kolla_ansible_current_version.stdout }} + Expected Kolla-Ansible version: {{ stackhpc_kolla_ansible_source_version }} + + Upgrade Kolla-Ansible by running: kayobe control host upgrade + success_msg: | + Kolla-Ansible running at version: {{ kolla_ansible_current_version.stdout }} diff --git a/etc/kayobe/ansible/check-kolla-images-py.yml b/etc/kayobe/ansible/check-kolla-images-py.yml new file mode 100644 index 000000000..ba934a222 --- /dev/null +++ b/etc/kayobe/ansible/check-kolla-images-py.yml @@ -0,0 +1,35 @@ +--- +- name: Check kolla-images.py image map and tag hierarchy + hosts: localhost + gather_facts: false + tasks: + - name: Create a temporary directory + ansible.builtin.tempfile: + state: directory + suffix: kolla-ansible + register: tempdir_result + + - name: Clone Kolla Ansible repository + ansible.builtin.git: + repo: "{{ stackhpc_kolla_ansible_source_url }}" + version: "{{ stackhpc_kolla_ansible_source_version }}" + dest: "{{ tempdir_result.path }}" + + - name: Check image mapping + ansible.builtin.command: + cmd: >- + {{ kayobe_config_path }}/../../tools/kolla-images.py + check-image-map + --kolla-ansible-path {{ tempdir_result.path }} + + - name: Check tag hierarchy + ansible.builtin.command: + cmd: >- + {{ kayobe_config_path }}/../../tools/kolla-images.py + check-hierarchy + --kolla-ansible-path {{ tempdir_result.path }} + + - name: Remove temporary directory + ansible.builtin.file: + path: "{{ tempdir_result.path }}" + state: absent diff --git a/etc/kayobe/ansible/check-tags.yml b/etc/kayobe/ansible/check-tags.yml new file mode 100644 index 000000000..f01d0b5f1 --- /dev/null +++ b/etc/kayobe/ansible/check-tags.yml @@ -0,0 +1,38 @@ +--- +# This playbook queries the Pulp server to check that all image tags specified +# in kolla_image_tags are present. + +- name: Check whether tags exist in Pulp container registry + hosts: localhost + gather_facts: false + tasks: + - name: Query images and tags + ansible.builtin.command: + cmd: >- + {{ kayobe_config_path }}/../../tools/kolla-images.py list-tags + register: kolla_images_result + changed_when: false + + - name: Set a fact about images and tags + ansible.builtin.set_fact: + kolla_images: "{{ kolla_images_result.stdout | from_yaml }}" + + # Use state=read and allow_missing=false to check for missing tags in test pulp. + - name: Check for missing tags + ansible.builtin.import_role: + name: stackhpc.pulp.pulp_container_content + vars: + pulp_container_content: >- + {%- set contents = [] -%} + {%- for image, tags in kolla_images.items() -%} + {%- set repository = kolla_docker_namespace ~ "/" ~ image -%} + {%- set content = { + "allow_missing": False, + "is_push": pulp_url == stackhpc_release_pulp_url, + "repository": repository, + "state": "read", + "tags": tags, + } -%} + {%- set _ = contents.append(content) -%} + {%- endfor -%} + {{ contents }} diff --git a/etc/kayobe/ansible/cis.yml b/etc/kayobe/ansible/cis.yml new file mode 100644 index 000000000..656a87563 --- /dev/null +++ b/etc/kayobe/ansible/cis.yml @@ -0,0 +1,42 @@ +--- +- name: Security hardening + hosts: cis-hardening + become: true + tags: + - cis + tasks: + # TODO: Remove this when Red Hat FIPS policy has been updated to allow ed25519 keys. + # https://gitlab.com/gitlab-org/gitlab/-/issues/367429#note_1840422075 + - name: Assert that we are using a supported SSH key + ansible.builtin.assert: + that: + - ssh_key_type != 'ed25519' + fail_msg: FIPS policy does not currently support ed25519 SSH keys on RHEL family systems + when: ansible_facts.os_family == 'RedHat' and rhel9cis_crypto_policy == 'FIPS' + + - name: Ensure the cron package is installed on ubuntu + ansible.builtin.package: + name: cron + state: present + when: ansible_facts.distribution == 'Ubuntu' + + - name: Ensure service accounts have no expiry options set + # This is to workaround an issue where we set the expiry to 365 days on kayobe + # service accounts in a previous iteration of the CIS benchmark hardening + # defaults. This should restore the defaults and can eventually be removed. + ansible.builtin.command: chage -m 0 -M 99999 -W 7 -I -1 {{ item }} + become: true + changed_when: false + with_items: + - "{{ kayobe_ansible_user }}" + - "{{ kolla_ansible_user }}" + + - name: Run CIS hardening role (RHEL 9) + ansible.builtin.include_role: + name: ansible-lockdown.rhel9_cis + when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '9' + + - name: Run CIS hardening role (Ubuntu 22) + ansible.builtin.include_role: + name: ansible-lockdown.ubuntu22_cis + when: ansible_facts.distribution == 'Ubuntu' and ansible_facts.distribution_major_version == '22' diff --git a/etc/kayobe/ansible/configure-aio-resources.yml b/etc/kayobe/ansible/configure-aio-resources.yml new file mode 100644 index 000000000..42123f225 --- /dev/null +++ b/etc/kayobe/ansible/configure-aio-resources.yml @@ -0,0 +1,61 @@ +--- +- name: Ensure dependencies are installed + hosts: controllers[0] + gather_facts: true + vars: + venv: "{{ virtualenv_path }}/openstack" + tasks: + - name: Install python build dependencies + ansible.builtin.package: + name: "{{ packages | select | list }}" + cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" + update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" + state: present + become: true + vars: + packages: + - "{% if ansible_facts.os_family == 'RedHat' %}gcc{% else %}build-essential{% endif %}" + - python3-dev{% if ansible_facts.os_family == 'RedHat' %}el{% endif %} + - "{% if ansible_facts.os_family == 'Debian' %}python3-venv{% endif %}" + + - name: Ensure latest version of pip is installed + ansible.builtin.pip: + name: pip + state: latest + virtualenv: "{{ venv }}" + virtualenv_command: /usr/bin/python3 -m venv + + - name: Ensure python openstack client is installed + ansible.builtin.pip: + name: python-openstackclient + virtualenv: "{{ venv }}" + extra_args: -c "{{ pip_upper_constraints_file }}" + + - name: Include kolla secrets + ansible.builtin.include_vars: + dir: "{{ kayobe_env_config_path }}/kolla/" + files_matching: passwords.yml + name: kolla_passwords + + - name: Run init-run-once + ansible.builtin.script: + cmd: scripts/aio-init.sh + creates: /tmp/.init-runonce + environment: + KOLLA_OPENSTACK_COMMAND: "{{ venv }}/bin/openstack" + EXT_NET_CIDR: "{{ aio_cidr }}" + EXT_NET_RANGE: "start={{ aio_neutron_allocation_pool_start }},end={{ aio_neutron_allocation_pool_end }}" + EXT_NET_GATEWAY: "{{ external_net_names | first | net_ip(inventory_hostname=groups['controllers'][0]) }}" + OS_PROJECT_DOMAIN_NAME: Default + OS_USER_DOMAIN_NAME: Default + OS_PROJECT_NAME: admin + OS_TENANT_NAME: admin + OS_USERNAME: admin + OS_PASSWORD: "{{ kolla_passwords.keystone_admin_password | mandatory('Could not find keystone_admin_password in passwords.yml') }}" + # Use kolla_external_fqdn in wallaby + OS_AUTH_URL: http://{{ kolla_external_fqdn | default(public_net_name | net_fqdn) | default(public_net_name | net_vip_address, true) }}:5000 + OS_INTERFACE: public + OS_ENDPOINT_TYPE: publicURL + OS_IDENTITY_API_VERSION: "3" + OS_REGION_NAME: RegionOne + OS_AUTH_PLUGIN: password diff --git a/etc/kayobe/ansible/configure-vxlan.yml b/etc/kayobe/ansible/configure-vxlan.yml new file mode 100644 index 000000000..0101b1ce1 --- /dev/null +++ b/etc/kayobe/ansible/configure-vxlan.yml @@ -0,0 +1,13 @@ +--- +- name: Configure VXLAN + hosts: controllers,compute,infra-vms,seed,storage + gather_facts: true + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: -o StrictHostKeyChecking=no + roles: + - role: stackhpc.vxlan diff --git a/etc/kayobe/ansible/deploy-github-runner.yml b/etc/kayobe/ansible/deploy-github-runner.yml new file mode 100644 index 000000000..8f69d8345 --- /dev/null +++ b/etc/kayobe/ansible/deploy-github-runner.yml @@ -0,0 +1,32 @@ +--- +- name: Deploy GitHub Runner + hosts: github-runners + become: true + roles: + - role: geerlingguy.pip + - role: geerlingguy.docker + tasks: + - name: Deploy runners + ansible.builtin.include_role: + name: monolithprojects.github_actions_runner + vars: + runner_name: "{{ ansible_facts.nodename }}-{{ runner.key }}" + runner_dir: "{{ base_runner_dir }}/{{ runner.key }}" + runner_labels: "{{ runner.value.labels | default(default_runner_labels) }}" + runner_state: "{{ runner.value.state | default('started') }}" + with_dict: "{{ github_runners }}" + loop_control: + loop_var: runner + + # FIXME: Sometimes the runner service is not running at the end of the role. + # Start the service manually. + - name: Ensure runner service is running + ansible.builtin.service: + name: actions.runner.{{ github_account }}-{{ github_repo }}.{{ ansible_facts.nodename }}-{{ runner.key }}.service + state: started + enabled: true + become: true + when: runner_state | default('started') == 'started' + with_dict: "{{ github_runners }}" + loop_control: + loop_var: runner diff --git a/etc/kayobe/ansible/deploy-gitlab-runner.yml b/etc/kayobe/ansible/deploy-gitlab-runner.yml new file mode 100644 index 000000000..44a1002b8 --- /dev/null +++ b/etc/kayobe/ansible/deploy-gitlab-runner.yml @@ -0,0 +1,24 @@ +--- +- name: Deploy GitLab runners + hosts: gitlab-runners + become: true + pre_tasks: + - name: Ensure /opt/.docker folder exists + ansible.builtin.file: + path: /opt/.docker + state: directory + + - name: Ensure docker/config.json exists for runner + ansible.builtin.copy: + content: | + { + "auths": { + "{{ pulp_url | regex_replace('^https?://|^http?://', '') }}": { + "auth": "{{ (pulp_username + ':' + pulp_password) | b64encode }}" + } + } + } + dest: /opt/.docker/config.json + mode: "0600" + roles: + - role: riemers.gitlab-runner diff --git a/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml new file mode 100644 index 000000000..195f23add --- /dev/null +++ b/etc/kayobe/ansible/deploy-openbao-kayobe-automation.yml @@ -0,0 +1,78 @@ +--- +- name: Deploy OpenBao on the runners + any_errors_fatal: true + gather_facts: true + hosts: github-runners,gitlab-runners + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure /opt/kayobe/vault exists + ansible.builtin.file: + path: /opt/kayobe/vault + state: directory + become: true + + - name: Ensure vault directory exists in environment + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/vault" + state: directory + become: true + + - name: Import OpenBao role + ansible.builtin.import_role: + name: stackhpc.hashicorp.openbao + vars: + openbao_config_dir: "/opt/kayobe/vault" + openbao_cluster_name: "kayobe-automation" + copy_self_signed_ca: false + openbao_write_keys_file: true + openbao_write_keys_file_path: "{{ kayobe_env_config_path }}/vault/kayobe-automation-keys.json" + + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/vault/kayobe-automation-keys.json" + name: openbao_keys + tags: always + + - name: Import Vault unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: "{{ openbao_api_addr }}" + vault_unseal_token: "{{ openbao_keys.root_token }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + vault_unseal_verify: false + environment: + https_proxy: '' + + - name: Create secret store + ansible.legacy.hashivault_secret_engine: + name: kayobe-automation + backend: kv + url: "{{ openbao_api_addr }}" + token: "{{ openbao_keys.root_token }}" + + - name: Ensure secret store is present + community.hashi_vault.vault_write: + url: "{{ openbao_api_addr }}" + token: "{{ openbao_keys.root_token }}" + path: kayobe-automation/{{ kayobe_environment }} + data: + kayobe_vault_password: "{{ kolla_ansible_vault_password }}" + kayobe_automation_ssh_private_key: "{{ lookup('ansible.builtin.file', '{{ ssh_private_key_path }}') }}" + kayobe_public_openrc: "{{ lookup('ansible.builtin.file', '{{ kolla_config_path }}/public-openrc.sh') }}" + tags: add_secrets diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml new file mode 100644 index 000000000..8ef4673e1 --- /dev/null +++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml @@ -0,0 +1,76 @@ +--- +- name: Remove legacy os_exporter.cfg file + hosts: network + tags: os_capacity + gather_facts: false + tasks: + - name: Ensure legacy os_exporter.cfg config file is deleted + ansible.builtin.file: + path: /etc/kolla/haproxy/services.d/os_exporter.cfg + state: absent + become: true + +- name: Deploy os-capacity exporter + hosts: monitoring + tags: os_capacity + gather_facts: false + tasks: + - name: Check if admin-openrc.sh exists + ansible.builtin.stat: + path: "{{ lookup('ansible.builtin.env', 'KOLLA_CONFIG_PATH') }}/admin-openrc.sh" + delegate_to: localhost + register: openrc_file_stat + run_once: true + + - name: If admin-openrc.sh exists, deploy os-capacity exporter + when: stackhpc_enable_os_capacity and openrc_file_stat.stat.exists + block: + - name: Ensure os-capacity directory exists + ansible.builtin.file: + path: /opt/kayobe/os-capacity/ + state: directory + + - name: Read admin-openrc credential file + ansible.builtin.command: + cmd: "cat {{ lookup('ansible.builtin.env', 'KOLLA_CONFIG_PATH') }}/admin-openrc.sh" + delegate_to: localhost + register: credential + changed_when: false + + - name: Set facts for admin credentials + ansible.builtin.set_fact: + stackhpc_os_capacity_auth_url: "{{ credential.stdout_lines | select('match', '.*OS_AUTH_URL*.') | first | split('=') | last | replace(\"'\", '') }}" + stackhpc_os_capacity_project_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_NAME*.') | first | split('=') | last | replace(\"'\", '') }}" + stackhpc_os_capacity_domain_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_DOMAIN_NAME*.') | first | split('=') | last | replace(\"'\", '') }}" + stackhpc_os_capacity_openstack_region_name: "{{ credential.stdout_lines | select('match', '.*OS_REGION_NAME*.') | first | split('=') | last | replace(\"'\", '') }}" + stackhpc_os_capacity_username: "{{ credential.stdout_lines | select('match', '.*OS_USERNAME*.') | first | split('=') | last | replace(\"'\", '') }}" + stackhpc_os_capacity_password: "{{ credential.stdout_lines | select('match', '.*OS_PASSWORD*.') | first | split('=') | last | replace(\"'\", '') }}" + + - name: Template clouds.yml + ansible.builtin.template: + src: templates/os_capacity-clouds.yml.j2 + dest: /opt/kayobe/os-capacity/clouds.yaml + register: clouds_yaml_result + + - name: Copy CA certificate to OpenStack Capacity nodes + ansible.builtin.copy: + src: "{{ stackhpc_os_capacity_openstack_cacert }}" + dest: /opt/kayobe/os-capacity/cacert.pem + when: stackhpc_os_capacity_openstack_cacert | length > 0 + register: cacert_result + + - name: Ensure os_capacity container is running + community.docker.docker_container: + name: os_capacity + image: ghcr.io/stackhpc/os-capacity:{{ stackhpc_os_capacity_version }} + env: + OS_CLOUD: openstack + OS_CLIENT_CONFIG_FILE: /etc/openstack/clouds.yaml + mounts: + - type: bind + source: /opt/kayobe/os-capacity/ + target: /etc/openstack/ + network_mode: host + restart: "{{ clouds_yaml_result is changed or cacert_result is changed }}" + restart_policy: unless-stopped + become: true diff --git a/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml new file mode 100644 index 000000000..d9db106bf --- /dev/null +++ b/etc/kayobe/ansible/deploy-radosgw-usage-exporter.yml @@ -0,0 +1,152 @@ +--- +- name: Deploy RADOS gateway usage exporter + hosts: monitoring + gather_facts: false + tags: radosgw_usage_exporter + vars: + venv: "{{ virtualenv_path }}/openstack" + tasks: + - name: Deploy RADOS gateway usage exporter + when: stackhpc_enable_radosgw_usage_exporter + block: + - name: Set up openstack cli virtualenv + ansible.builtin.pip: + virtualenv: "{{ venv }}" + virtualenv_command: "/usr/bin/python3 -m venv" + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + delegate_to: "{{ groups['controllers'][0] }}" + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Read admin-openrc credential file + ansible.builtin.command: + cmd: "cat {{ lookup('ansible.builtin.env', 'KOLLA_CONFIG_PATH') }}/admin-openrc.sh" + delegate_to: localhost + register: credential + changed_when: false + + - name: Set facts for admin credentials + ansible.builtin.set_fact: + openstack_auth_env: + OS_PROJECT_DOMAIN_NAME: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_DOMAIN_NAME*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_USER_DOMAIN_NAME: "{{ credential.stdout_lines | select('match', '.*OS_USER_DOMAIN_NAME*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_PROJECT_NAME: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_NAME*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_USERNAME: "{{ credential.stdout_lines | select('match', '.*OS_USERNAME*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_PASSWORD: "{{ credential.stdout_lines | select('match', '.*OS_PASSWORD*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_AUTH_URL: "{{ credential.stdout_lines | select('match', '.*OS_AUTH_URL*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_INTERFACE: "{{ credential.stdout_lines | select('match', '.*OS_INTERFACE*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_IDENTITY_API_VERSION: "{{ credential.stdout_lines | select('match', '.*OS_IDENTITY_API_VERSION*.') | first | split('=') | last | replace(\"'\", '') }}" + OS_CACERT: "{{ '/etc/ssl/certs/ca-certificates.crt' if os_distribution == 'ubuntu' else '/etc/pki/tls/certs/ca-bundle.crt' }}" + + - name: Check ec2 credential for ceph_rgw + ansible.builtin.command: > + {{ venv }}/bin/openstack + ec2 credentials list --user ceph_rgw + --format json + environment: "{{ openstack_auth_env }}" + register: credential_check + delegate_to: "{{ groups['controllers'][0] }}" + changed_when: false + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + run_once: true + + - name: Create ec2 credential if there's none + ansible.builtin.command: > + {{ venv }}/bin/openstack + ec2 credentials create --user ceph_rgw --project service + --format json + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + changed_when: true + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + run_once: true + when: credential_check.stdout | from_json == [] + + - name: Query ec2 credential for ceph_rgw + ansible.builtin.command: > + {{ venv }}/bin/openstack + ec2 credentials list --user ceph_rgw + --format json + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + changed_when: false + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + register: credential + run_once: true + + - name: Get object storage endpoint + ansible.builtin.command: > + {{ venv }}/bin/openstack + endpoint list --service object-store --interface internal + --format json + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + changed_when: false + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + register: endpoint + run_once: true + + - name: Process object storage endpoint + ansible.builtin.set_fact: + radosgw_server: "{{ scheme + '://' + hostname + ':' + radosgw_port }}" + vars: + swift: "{{ endpoint.stdout | from_json | first }}" + hostname: "{{ swift.URL | urlsplit('hostname') }}" + scheme: "{{ swift.URL | urlsplit('scheme') }}" + radosgw_port: "{{ swift.URL | urlsplit('port') }}" + run_once: true + + - name: Ensure radosgw_usage_exporter container is running + community.docker.docker_container: + name: radosgw_usage_exporter + image: ghcr.io/stackhpc/radosgw_usage_exporter:v0.1.3 + network_mode: host + env: + RADOSGW_SERVER: "{{ radosgw_server }}" + ADMIN_ENTRY: admin + ACCESS_KEY: "{{ ec2.Access }}" + SECRET_KEY: "{{ ec2.Secret }}" + VIRTUAL_PORT: "{{ stackhpc_radosgw_usage_exporter_backend_port | string }}" + REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt" + entrypoint: "{{ ['python', '-u', './radosgw_usage_exporter.py', '--insecure'] if not stackhpc_radosgw_usage_exporter_verify else omit }}" + vars: + ec2: "{{ credential.stdout | from_json | first }}" + become: true + + - name: Ensure that the internal TLS certificate is trusted by the exporter + when: stackhpc_radosgw_usage_exporter_cacert | length > 0 + block: + - name: Create radosgw-usage-exporter directory + ansible.builtin.file: + path: /opt/kayobe/radosgw-usage-exporter/ + state: directory + mode: 0755 + + - name: Copy CA certificate to RADOS gateway usage exporter nodes + ansible.builtin.copy: + src: "{{ stackhpc_radosgw_usage_exporter_cacert }}" + dest: "/opt/kayobe/radosgw-usage-exporter/{{ stackhpc_radosgw_usage_exporter_cacert | basename }}" + mode: 0644 + register: copy_to_node_result + + - name: Copy CA certificate to RADOS gateway usage exporter container + community.docker.docker_container_copy_into: + container: radosgw_usage_exporter + path: "{{ copy_to_node_result.dest }}" + container_path: "/usr/local/share/ca-certificates/{{ copy_to_node_result.dest | basename }}" + become: true + + - name: Update CA certificate of RADOS gateway usage exporter container + community.docker.docker_container_exec: + container: radosgw_usage_exporter + command: update-ca-certificates + user: root + become: true diff --git a/etc/kayobe/ansible/diagnostics.yml b/etc/kayobe/ansible/diagnostics.yml new file mode 100644 index 000000000..edb5800b6 --- /dev/null +++ b/etc/kayobe/ansible/diagnostics.yml @@ -0,0 +1,71 @@ +--- +# This playbook runs a script that collects diagnostic information from hosts. +# The diagnostics are aggregated to a directory +# (diagnostics_path_local/inventory_hostname) on localhost. +# +# NOTE: The diagnostic information contains sensitive information such as +# passwords in configuration files. + +- name: Collect diagnostic information + hosts: seed-hypervisor:seed:overcloud:infra-vms + vars: + diagnostics_path_local: "{{ lookup('env', 'PWD') }}/diagnostics" + tasks: + - name: Run diagnostics + block: + - name: Create a temporary directory for diagnostics + ansible.builtin.tempfile: + state: directory + suffix: diagnostics + register: diagnostics_tmpdir + + - name: Write host variables to a file + ansible.builtin.copy: + content: "{{ hostvars[inventory_hostname].ansible_facts | to_nice_json }}" + dest: "{{ diagnostics_tmpdir.path }}/facts.json" + + - name: Run diagnostics script + ansible.builtin.script: "{{ kayobe_config_path }}/../../tools/diagnostics.sh" + become: true + failed_when: diagnostics_result.rc is not defined + register: diagnostics_result + environment: + LOG_DIR: "{{ diagnostics_tmpdir.path }}" + CONFIG_DIR: "{{ kayobe_config_path }}/../.." + + - name: Download diagnostic logs to localhost + ansible.posix.synchronize: + src: "{{ diagnostics_tmpdir.path }}/" + dest: "{{ diagnostics_path_local }}/{{ inventory_hostname }}" + mode: pull + archive: false + recursive: true + copy_links: true + verify_host: true + # For jump host + use_ssh_args: true + always: + - name: Clean up temporary directory + ansible.builtin.file: + path: "{{ diagnostics_tmpdir.path }}" + state: absent + + - name: Display diagnostics collection stdout + ansible.builtin.debug: + msg: "{{ diagnostics_result.stdout }}" + when: diagnostics_result.stdout is defined + + - name: Display diagnostics collection stderr + ansible.builtin.debug: + msg: "{{ diagnostics_result.stderr }}" + when: diagnostics_result.stderr is defined + + - name: Fail if diagnostics collection failed + ansible.builtin.fail: + msg: Diagnostics collection failed + when: diagnostics_result.rc != 0 + + - name: Display location of diagnostics archive + ansible.builtin.debug: + msg: >- + Wrote diagnostics to {{ diagnostics_path_local }} on localhost diff --git a/etc/kayobe/ansible/docker-registry-login.yml b/etc/kayobe/ansible/docker-registry-login.yml new file mode 100644 index 000000000..5db2ed826 --- /dev/null +++ b/etc/kayobe/ansible/docker-registry-login.yml @@ -0,0 +1,11 @@ +--- +- name: Login to docker registry + gather_facts: false + hosts: container-image-builders + tasks: + - name: Login to docker registry + community.docker.docker_login: + registry_url: "{{ kolla_docker_registry or omit }}" + username: "{{ kolla_docker_registry_username }}" + password: "{{ kolla_docker_registry_password }}" + reauthorize: true diff --git a/etc/kayobe/ansible/firewalld-watchdog.yml b/etc/kayobe/ansible/firewalld-watchdog.yml new file mode 100644 index 000000000..c74edd358 --- /dev/null +++ b/etc/kayobe/ansible/firewalld-watchdog.yml @@ -0,0 +1,70 @@ +--- +# This playbook can be applied in advance of rolling out a firewall +# configuration. It sets up a timer that disables the firewalld service after a +# period of time (default 600s). It should be used as follows: +# 1. Enable firewalld-watchdog +# kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l +# 2. Apply firewall config +# kayobe host configure -l -t network,firewall +# 3. Disable watchdog +# kayobe playbook run etc/kayobe/ansible/firewalld-watchdog.yml -l -e firewalld_watchdog_state=absent +# If the firewall changes result in being locked out of the system, the +# watchdog will disable the firewall after the timeout. +# Remember to disable the watchdog, otherwise the firewall will be disabled! + +- name: Create a systemd timer to stop firewalld + hosts: seed:seed-hypervisor:overcloud:infra-vms + tags: + - firewalld-watchdog + vars: + # Watchdog state: present or absent. + firewalld_watchdog_state: present + # Watchdog timeout in seconds. + firewalld_watchdog_timeout_s: 600 + become: true + tasks: + - name: Ensure watchdog is present + when: firewalld_watchdog_state == 'present' + block: + - name: Create firewalld-watchdog service unit file + ansible.builtin.copy: + dest: /etc/systemd/system/firewalld-watchdog.service + content: | + [Unit] + Description=Firewalld watchdog service + + [Service] + Type=oneshot + ExecStart=/usr/bin/systemctl stop firewalld + register: service_result + + - name: Create firewalld-watchdog timer unit file + ansible.builtin.copy: + dest: /etc/systemd/system/firewalld-watchdog.timer + content: | + [Unit] + Description=Firewalld watchdog timer + + [Timer] + OnActiveSec={{ firewalld_watchdog_timeout_s }} + Unit=firewalld-watchdog.service + + [Install] + WantedBy=timers.target + register: timer_result + + - name: Enable or disable firewalld-watchdog timer + ansible.builtin.systemd_service: + name: firewalld-watchdog.timer + daemon_reload: "{{ service_result is changed or timer_result is changed }}" + enabled: false + state: "{{ 'started' if firewalld_watchdog_state == 'present' else 'stopped' }}" + + - name: Remove firewalld-watchdog unit files + ansible.builtin.file: + path: /etc/systemd/system/{{ item }} + state: absent + loop: + - firewalld-watchdog.service + - firewalld-watchdog.timer + when: firewalld_watchdog_state == 'absent' diff --git a/etc/kayobe/ansible/fix-grub-rl9.yml b/etc/kayobe/ansible/fix-grub-rl9.yml new file mode 100644 index 000000000..6d81f137d --- /dev/null +++ b/etc/kayobe/ansible/fix-grub-rl9.yml @@ -0,0 +1,22 @@ +--- +- name: Remove "--root-dev-only" from grub.cfg if OS is Rocky Linux 9 + hosts: overcloud + become: true + gather_facts: true + tasks: + - name: Remove "--root-dev-only" from grub.cfg if OS is Rocky Linux 9 + when: + - ansible_facts['distribution'] == 'Rocky' + - ansible_facts['distribution_major_version'] == '9' + block: + - name: Check that /boot/efi/EFI/rocky/grub.cfg exists + ansible.builtin.stat: + path: /boot/efi/EFI/rocky/grub.cfg + register: stat_result + + - name: Remove "--root-dev-only" from /boot/efi/EFI/rocky/grub.cfg + ansible.builtin.replace: + path: /boot/efi/EFI/rocky/grub.cfg + regexp: --root-dev-only\s? + replace: "" + when: stat_result.stat.exists diff --git a/etc/kayobe/ansible/fix-hostname.yml b/etc/kayobe/ansible/fix-hostname.yml new file mode 100644 index 000000000..539d47e62 --- /dev/null +++ b/etc/kayobe/ansible/fix-hostname.yml @@ -0,0 +1,30 @@ +--- +- name: Ensure hostnames match inventory hostnames + hosts: fix-hostname + gather_facts: false + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: -o StrictHostKeyChecking=no + tasks: + - name: Get current hostname + ansible.builtin.command: + cmd: hostname + changed_when: false + register: current_hostname + + - name: Set hostname to a non FQDN + ansible.builtin.command: + cmd: hostnamectl set-hostname "{{ inventory_hostname }}" + when: current_hostname.stdout != inventory_hostname + become: true + +- name: Reboot hosts + import_playbook: "{{ playbook_dir | realpath }}/reboot.yml" + vars: + reboot_hosts: fix-hostname + reboot_with_bootstrap_user: true + when: current_hostname.stdout != inventory_hostname diff --git a/etc/kayobe/ansible/fix-houston.yml b/etc/kayobe/ansible/fix-houston.yml new file mode 100644 index 000000000..9dfdc924f --- /dev/null +++ b/etc/kayobe/ansible/fix-houston.yml @@ -0,0 +1,44 @@ +--- +# When OVS HW offloading is enabled - typically in conjunction with VF-LAG and ASAP^2 +# the DMESG log reports frequent errors on the internal OVS Bridge interface: +# "tc mirred to Houston: device bond0-ovs is down". +# This interface is down by default. The errors are mitigated by bringing the interface up. +# For further context, see: +# https://bugs.launchpad.net/charm-neutron-openvswitch/+bug/1899364 +# https://patchwork.kernel.org/project/netdevbpf/patch/c2ef23da1d9a4eb62f4e7b7c4540f9bafb553c15.1658420239.git.dcaratti@redhat.com/ +# To deploy this playbook, use the following commands: +# kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/fix-houston.yml +# Enable with Kayobe Hooks by running: +# mkdir -p ${KAYOBE_CONFIG_PATH}/hooks/overcloud-service-deploy/post.d +# cd ${KAYOBE_CONFIG_PATH}/hooks/overcloud-service-deploy/post.d +# ln -s ../../../ansible/fix-houston.yml 90-fix-houston.yml + +- name: Create Systemd Unit to workaround 'tc mirred to Houston' error + hosts: network,compute + become: true + + tasks: + - name: Include kolla-ansible host vars + ansible.builtin.include_vars: "{{ kolla_config_path }}/inventory/overcloud/host_vars/{{ inventory_hostname }}" + + - name: Create systemd service for -ovs network interface + ansible.builtin.template: + src: fix-houston-interface.service.j2 + dest: /etc/systemd/system/fix-houston-{{ item }}.service + loop: "{{ neutron_bridge_name.split(',') }}" + vars: + interface_name: "{{ item }}" + when: neutron_bridge_name | length > 0 + notify: Reload systemd + + - name: Enable and start systemd service for -ovs network interface + ansible.builtin.systemd: + name: fix-houston-{{ item }} + enabled: true + state: started + when: neutron_bridge_name | length > 0 + loop: "{{ neutron_bridge_name.split(',') }}" + + handlers: + - name: Reload systemd + ansible.builtin.command: systemctl daemon-reload # noqa command-instead-of-module diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml new file mode 100644 index 000000000..d4d291d0e --- /dev/null +++ b/etc/kayobe/ansible/fix-networking.yml @@ -0,0 +1,23 @@ +--- +- name: Fix networking + hosts: controllers,compute,infra-vms,seed,storage + gather_facts: false + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: -o StrictHostKeyChecking=no + tasks: + - name: Ensure hosts are reachable + ansible.builtin.wait_for_connection: + + - name: Ensure `hosts` file contains pulp and API entries + ansible.builtin.blockinfile: + path: /etc/hosts + marker: "# {mark} Kayobe entries" + block: | + 192.168.37.2 internal.infra.mos.{{ root_domain }} + 192.168.39.2 public.infra.mos.{{ root_domain }} + become: true diff --git a/etc/kayobe/ansible/get-cloud-facts.yml b/etc/kayobe/ansible/get-cloud-facts.yml new file mode 100644 index 000000000..e966f8acc --- /dev/null +++ b/etc/kayobe/ansible/get-cloud-facts.yml @@ -0,0 +1,87 @@ +--- +- name: Gather Cloud Facts + hosts: localhost + gather_facts: true + tasks: + - name: Write facts to file + vars: + cloud_facts: + ansible_control_host_distribution: "{{ ansible_facts.distribution }}" + ansible_control_host_distribution_release: "{{ ansible_facts.distribution_release }}" + openstack_release: "{{ openstack_release }}" + openstack_release_name: "{{ openstack_release_codename }}" + ansible_control_host_is_vm: "{{ ansible_facts.virtualization_role == 'guest' }}" + controller_count: "{{ groups['controllers'] | length }}" + hypervisor_count: "{{ groups['hypervisors'] | length }}" + monitoring_count: "{{ groups['monitoring'] | length }}" + osd_count: "{{ groups['osds'] | length }}" + compute_count: "{{ groups['compute'] | length }}" + baremetal_count: "{{ groups['baremetal-compute'] | length }}" + ceph_deployed: "{{ groups['ceph'] | length > 0 | bool }}" + ceph_count: "{{ groups['ceph'] | length }}" + ceph_release: "{{ cephadm_ceph_release }}" + storage_hyperconverged: "{{ groups['controllers'] | intersect(groups['osds']) | length > 0 | bool }}" + wazuh_enabled: "{{ groups['wazuh-agent'] | length > 0 | bool }}" + kayobe_managed_switches: "{{ groups['switches'] | length > 0 | bool }}" + proxy_configured: "{{ http_proxy | bool or https_proxy | bool }}" + bifrost_version: "{{ kolla_bifrost_source_version }}" + barbican_enabled: "{{ kolla_enable_barbican }}" + nova_enabled: "{{ kolla_enable_nova }}" + neutron_enabled: "{{ kolla_enable_neutron }}" + ovs_enabled: "{{ kolla_enable_openvswitch }}" + ovn_enabled: "{{ kolla_enable_ovn }}" + glance_enabled: "{{ kolla_enable_glance }}" + cinder_enabled: "{{ kolla_enable_cinder }}" + keystone_enabled: "{{ kolla_enable_keystone }}" + horizon_enabled: "{{ kolla_enable_horizon }}" + fluentd_enabled: "{{ kolla_enable_fluentd }}" + rabbitmq_enabled: "{{ kolla_enable_rabbitmq }}" + mariadb_enabled: "{{ kolla_enable_mariadb }}" + mariabackup_enabled: "{{ kolla_enable_mariabackup }}" + memcached_enabled: "{{ kolla_enable_memcached }}" + haproxy_enabled: "{{ kolla_enable_haproxy }}" + keepalived_enabled: "{{ kolla_enable_keepalived }}" + octavia_enabled: "{{ kolla_enable_octavia }}" + designate_enabled: "{{ kolla_enable_designate }}" + manila_enabled: "{{ kolla_enable_manila }}" + magnum_enabled: "{{ kolla_enable_magnum }}" + heat_enabled: "{{ kolla_enable_heat }}" + ironic_enabled: "{{ kolla_enable_ironic }}" + skyline_enabled: "{{ kolla_enable_skyline }}" + blazar_enabled: "{{ kolla_enable_blazar }}" + pulp_enabled: "{{ seed_pulp_container_enabled }}" + opensearch_enabled: "{{ kolla_enable_opensearch }}" + opensearch_dashboards_enabled: "{{ kolla_enable_opensearch_dashboards }}" + influxdb_enabled: "{{ kolla_enable_influxdb }}" + grafana_enabled: "{{ kolla_enable_grafana }}" + prometheus_enabled: "{{ kolla_enable_prometheus }}" + cloudkitty_enabled: "{{ kolla_enable_cloudkitty }}" + telegraf_enabled: "{{ kolla_enable_telegraf }}" + internal_tls_enabled: "{{ kolla_enable_tls_internal }}" + external_tls_enabled: "{{ kolla_enable_tls_external }}" + firewalld_enabled_all: >- + {{ + controller_firewalld_enabled and + compute_firewalld_enabled and + storage_firewalld_enabled and + monitoring_firewalld_enabled and + infra_vm_firewalld_enabled and + seed_firewalld_enabled and + seed_hypervisor_firewalld_enabled + }} + firewalld_enabled_any: >- + {{ + controller_firewalld_enabled or + compute_firewalld_enabled or + storage_firewalld_enabled or + monitoring_firewalld_enabled or + infra_vm_firewalld_enabled or + seed_firewalld_enabled or + seed_hypervisor_firewalld_enabled + }} + stackhpc_package_repos_enabled: "{{ stackhpc_repos_enabled }}" + pulp_tls_enabled: "{{ pulp_enable_tls }}" + kolla_image_tags: "{{ kolla_image_tags }}" + ansible.builtin.copy: + content: "{{ cloud_facts | to_nice_json(sort_keys=false) }}" + dest: ~/cloud-facts.json diff --git a/etc/kayobe/ansible/get-nvme-drives.yml b/etc/kayobe/ansible/get-nvme-drives.yml new file mode 100644 index 000000000..1d2404d80 --- /dev/null +++ b/etc/kayobe/ansible/get-nvme-drives.yml @@ -0,0 +1,96 @@ +--- +- name: Gather unique NVMe disk models on all hosts + hosts: overcloud + gather_facts: no + tasks: + - name: Retrieve NVMe device information + ansible.builtin.command: "nvme list -o json" + register: nvme_list + changed_when: false + become: true + + - name: Parse NVMe device model names + ansible.builtin.set_fact: + nvme_models: "{{ nvme_models | default([]) + [item.ModelNumber] }}" + loop: "{{ nvme_list.stdout | from_json | json_query('Devices[].{ModelNumber: ModelNumber}') }}" + changed_when: false + + - name: Set unique NVMe models as host facts + ansible.builtin.set_fact: + unique_nvme_models: "{{ (nvme_models | default([])) | unique }}" + + - name: Show unique NVMe models per host + ansible.builtin.debug: + var: unique_nvme_models + +- name: Aggregate all unique NVMe models from all hosts + hosts: localhost + gather_facts: no + tasks: + - name: Aggregate unique NVMe models from all overcloud hosts + ansible.builtin.set_fact: + all_nvme_models: "{{ groups['overcloud'] | map('extract', hostvars, 'unique_nvme_models') | select('defined') | sum(start=[]) | unique }}" + + - name: Show all unique NVMe models + ansible.builtin.debug: + var: all_nvme_models + + - name: Ensure dwpd-ratings.yml exists + ansible.builtin.stat: + path: "{{ kayobe_env_config_path }}/dwpd-ratings.yml" + register: dwpd_ratings_stat + run_once: true + + - name: Load existing dwpd-ratings.yml + ansible.builtin.set_fact: + existing_dwpd_yml: "{{ lookup('file', kayobe_env_config_path ~ '/dwpd-ratings.yml') | from_yaml }}" + when: dwpd_ratings_stat.stat.exists + run_once: true + + - name: Convert existing YAML array into a dictionary + ansible.builtin.set_fact: + dwpd_lookup: "{{ dwpd_lookup | default({}) | combine({item.model_name: item.rated_dwpd}) }}" + loop: "{{ existing_dwpd_yml.stackhpc_dwpd_ratings | default([]) }}" + loop_control: + label: "{{ item.model_name }}" + run_once: true + + - name: Get list of existing model names + ansible.builtin.set_fact: + existing_model_names: "{{ existing_dwpd_yml.stackhpc_dwpd_ratings | default([]) | map(attribute='model_name') | list }}" + run_once: true + + - name: Identify new models not already in the configuration + ansible.builtin.set_fact: + new_models: "{{ all_nvme_models | default([]) | reject('in', existing_model_names | default([])) | list }}" + run_once: true + + - name: Create entry dictionary for new models + ansible.builtin.set_fact: + new_entries: "{{ new_entries | default([]) + [{'model_name': item, 'rated_dwpd': 1}] }}" + loop: "{{ new_models }}" + run_once: true + when: new_models | length > 0 + + - name: Build updated list for stackhpc_dwpd_ratings + ansible.builtin.set_fact: + new_dwpd_list: "{{ existing_dwpd_yml.stackhpc_dwpd_ratings | default([]) + (new_entries | default([])) }}" + run_once: true + + - name: Write updated dwpd-ratings.yml + ansible.builtin.copy: + content: "---\nstackhpc_dwpd_ratings:\n{% for item in new_dwpd_list %} - model_name: \"{{ item.model_name }}\"\n rated_dwpd: {{ item.rated_dwpd }}\n{% endfor %}" + dest: "{{ kayobe_env_config_path }}/dwpd-ratings.yml" + run_once: true + notify: Show updated dwpd-ratings.yml contents + when: new_dwpd_list is defined and new_dwpd_list | length > 0 + + handlers: + - name: Show updated dwpd-ratings.yml contents + ansible.builtin.debug: + msg: + - "Updated local dwpd-ratings.yml contents" + - "{{ {'stackhpc_dwpd_ratings': new_dwpd_list} | to_nice_yaml }}" + - "PLEASE REVIEW AND COMMIT {{ kayobe_env_config_path }}/dwpd-ratings.yml TO VERSION CONTROL." + run_once: true + changed_when: true diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml new file mode 100644 index 000000000..6525c2107 --- /dev/null +++ b/etc/kayobe/ansible/growroot.yml @@ -0,0 +1,97 @@ +--- +# Custom playbook to grow the partition and LVM PV of the root VG. This allows +# for expansion of LVs in that VG. This may be used as a pre-overcloud host +# configure hook. +# +# Variables: +# * growroot_group: Host pattern against which to target the playbook. Default +# is 'overcloud'. +# * growroot_vg: Name of the VG containing the PV to grow. Default is 'rootvg'. + +- name: Grow root PV + hosts: "{{ growroot_group | default('overcloud') }}" + # Avoid using facts because this may be used as a pre overcloud host + # configure hook, and we don't want to populate the fact cache (if one is in + # use) with the bootstrap user's context. + gather_facts: false + tags: + - growroot + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: -o StrictHostKeyChecking=no + # Don't assume facts are present. + os_family: "{{ ansible_facts.os_family | default('Debian' if os_distribution == 'ubuntu' else 'RedHat') }}" + # Ignore LVM check + growroot_ignore_lvm_check: false + + tasks: + - name: Check LVM status + ansible.builtin.shell: + executable: "/bin/bash" + cmd: set -o pipefail && vgdisplay | grep lvm2 >> /dev/null + changed_when: false + failed_when: false + check_mode: false + register: lvm_check + become: true + + - name: Grow root PV + when: lvm_check.rc == 0 or growroot_ignore_lvm_check + # when: "'NOCHANGE' not in growpart.stdout" + # Commenting out the conditional because growpart is already triggered by cloud-init - hence it emits NOCHANGE + # Cloud-Inits growpart implementation has a bug https://bugzilla.redhat.com/show_bug.cgi?id=2122575 + # PVresize is not being triggered + block: + - name: Check if growpart is installed + ansible.builtin.command: + cmd: type growpart + changed_when: false + failed_when: false + check_mode: false + register: growpart_check + become: true + + - name: Ensure growpart is installed + ansible.builtin.package: + name: "{% if os_family == 'RedHat' %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" + state: present + cache_valid_time: "{{ apt_cache_valid_time if os_family == 'Debian' else omit }}" + update_cache: "{{ true if os_family == 'Debian' else omit }}" + become: true + when: growpart_check.rc != 0 + + - name: Get root PV device + ansible.builtin.command: pvs --select vg_name={{ growroot_vg }} --reportformat json + register: pvs + become: true + changed_when: false + check_mode: false + + - name: Fail if root PV device not found + ansible.builtin.fail: + msg: > + Expected LVM physical volume devices not found in volume group {{ growroot_vg }} + when: (pvs.stdout | from_json).report[0].pv | length == 0 + + - name: Grow partition + ansible.builtin.command: growpart {{ disk }} {{ part_num }} + vars: + pv: "{{ pvs.stdout | from_json }}" + disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}" + disk: "{{ disk_tmp[:-1] if pv.report[0].pv[0].pv_name | regex_search('[a-z0-9]+[0-9]+p[0-9]+') else disk_tmp }}" + part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}" + become: true + failed_when: growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout + changed_when: "'NOCHANGE' not in growpart.stdout" + register: growpart + + - name: Grow LVM PV + ansible.builtin.command: pvresize {{ disk }} + vars: + pv: "{{ pvs.stdout | from_json }}" + disk: "{{ pv.report[0].pv[0].pv_name }}" + become: true diff --git a/etc/kayobe/ansible/hotfix-containers.yml b/etc/kayobe/ansible/hotfix-containers.yml new file mode 100644 index 000000000..6e152f4d6 --- /dev/null +++ b/etc/kayobe/ansible/hotfix-containers.yml @@ -0,0 +1,69 @@ +--- +# NOTE: This playbook is designed for use in high-severity hotfixes ONLY. +# If you're considering using this for regular operations, please consider +# developing a more suitable solution instead. +# +# See https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-yoga/operations/hotfix-containers.html +# for more information. + +- name: Hotfix containers + hosts: overcloud + tags: + - hotfix-containers + vars: + container_hotfix_command: "" + container_hotfix_files: [] + container_hotfix_container_regex: "" + container_hotfix_restart_containers: false + container_hotfix_become: false + tasks: + - name: Ensure inputs are valid + ansible.builtin.fail: + msg: Invalid input. Container list cannot be empty. Either container_hotfix_command or container_hotfix_files must be populated. + when: + - container_hotfix_container_regex == "" or container_hotfix_command == "" and container_hotfix_files == [] + + - name: Get list of containers to hotfix + ansible.builtin.command: '{{ kolla_container_engine | default("docker") }} ps --format {% raw %}"{{ .Names }}"{% endraw %}' + register: host_containers + + - name: Set fact for containers list + ansible.builtin.set_fact: + containers_list: "{{ host_containers.stdout }}" + + - name: Fail if no containers match given regex + vars: + hotfix_containers: "{{ containers_list | split('\n') | regex_search(container_hotfix_container_regex) }}" + ansible.builtin.fail: + msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers.stdout_lines }}" + when: hotfix_containers == "" + + - name: Ensure hotfix-files directory exists on the remote host + ansible.builtin.file: + path: /tmp/hotfix-files + state: directory + + - name: Ensure container hotfix file(s) exist on host + ansible.builtin.copy: + src: "{{ item.src }}" + dest: /tmp/hotfix-files/{{ index }} + loop: "{{ container_hotfix_files }}" + loop_control: + index_var: index + when: container_hotfix_files != [] + + - name: Apply hotfix + ansible.builtin.include_tasks: run-container-hotfix.yml + loop: "{{ containers_list | regex_findall(container_hotfix_container_regex, multiline=True) | list | unique }}" + loop_control: + loop_var: hotfix_container + + - name: Cleanup temporary files + ansible.builtin.file: + path: /tmp/hotfix-files + state: absent + + - name: Restart containers if requested + ansible.builtin.command: "{{ kolla_container_engine | default('docker') }} restart {{ item }}" + loop: "{{ containers_list | regex_findall(container_hotfix_container_regex, multiline=True) | list | unique }}" + when: container_hotfix_restart_containers diff --git a/etc/kayobe/ansible/install-doca.yml b/etc/kayobe/ansible/install-doca.yml new file mode 100644 index 000000000..0d7009e51 --- /dev/null +++ b/etc/kayobe/ansible/install-doca.yml @@ -0,0 +1,52 @@ +--- +- name: Install DOCA + become: true + hosts: mlnx + gather_facts: true + vars: + # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built, + # but not 5.14.0-611.20.1 + doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] }}" + tasks: + - name: Install kernel repo + ansible.builtin.dnf: + name: doca-kernel-repo + state: latest + update_cache: true + + # not the same as doca_kernel_version: some dots changed to underscore or dash + - name: Discover kernel repo filename + ansible.builtin.shell: | + set -o pipefail + rpm -ql doca-kernel-repo | grep /etc/yum.repos.d/ + register: kernel_repo_filename + changed_when: false + + - name: Ensure correct priority for DOCA modules + ansible.builtin.lineinfile: + line: "priority=-2" + insertafter: EOF + path: "{{ kernel_repo_filename.stdout }}" + + # This is required by mlnx-ofa_kernel, and comes from the doca repository. + # It is already present when doca-ofed is installed, but will be upgraded. + - name: Ensure mlnx-tools is installed + ansible.builtin.dnf: + name: mlnx-tools + state: latest + update_cache: true + become: true + + - name: Ensure appropriate doca-kernel is installed + ansible.builtin.dnf: + name: "doca-kernel-{{ doca_kernel_version }}" + state: latest + disablerepo: doca + become: true + + - name: Ensure DOCA OFED userspace is installed + ansible.builtin.dnf: + name: + - doca-ofed-userspace + state: latest + update_cache: true diff --git a/etc/kayobe/ansible/install-pre-commit-hooks.yml b/etc/kayobe/ansible/install-pre-commit-hooks.yml new file mode 100644 index 000000000..6059d3bdd --- /dev/null +++ b/etc/kayobe/ansible/install-pre-commit-hooks.yml @@ -0,0 +1,21 @@ +--- +- name: Install pre-commit hooks + hosts: localhost + gather_facts: false + vars: + pre_commit_version: 3.5.0 + tasks: + - name: Install pre-commit hooks + block: + - name: Install pre-commit hooks into kayobe virtual env + ansible.builtin.pip: + name: pre-commit + version: "{{ pre_commit_version }}" + virtualenv: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(omit, true) }}" + register: pip_install + + - name: Register pre-commit hooks with git + ansible.builtin.command: + cmd: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') | default(lookup('ansible.builtin.env', 'HOME') ~ '/.local', true) }}/bin/pre-commit install" + args: + chdir: "{{ playbook_dir | dirname | dirname | dirname }}" diff --git a/etc/kayobe/ansible/nova-compute-disable.yml b/etc/kayobe/ansible/nova-compute-disable.yml new file mode 100644 index 000000000..497fa7ea3 --- /dev/null +++ b/etc/kayobe/ansible/nova-compute-disable.yml @@ -0,0 +1,48 @@ +--- +- name: Disable nova compute service + hosts: compute + gather_facts: true + tags: + - nova-compute-disable + vars: + venv: "{{ virtualenv_path }}/openstack" + disabled_reason: Down for maintenance by nova-compute-disable.yml + tasks: + - name: Set up openstack cli virtualenv + ansible.builtin.pip: + virtualenv: "{{ venv }}" + virtualenv_command: /usr/bin/python3 -m venv + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + delegate_to: "{{ groups['controllers'][0] }}" + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Query nova compute services + ansible.builtin.command: > + {{ venv }}/bin/openstack + compute service list + --format json + --service nova-compute --host {{ ansible_facts.nodename }} + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + register: compute_services + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Disable nova compute service + ansible.builtin.command: > + {{ venv }}/bin/openstack + compute service set + {{ ansible_facts.nodename }} nova-compute + --disable --disable-reason '{{ disabled_reason }}' + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + vars: + service: "{{ compute_services.stdout | from_json | first }}" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + when: + - service.Status != 'disabled' diff --git a/etc/kayobe/ansible/nova-compute-drain.yml b/etc/kayobe/ansible/nova-compute-drain.yml new file mode 100644 index 000000000..6aaa8cad5 --- /dev/null +++ b/etc/kayobe/ansible/nova-compute-drain.yml @@ -0,0 +1,75 @@ +--- +- name: Drain a nova compute host of instances + hosts: compute + gather_facts: true + tags: + - nova-compute-drain + vars: + venv: "{{ virtualenv_path }}/openstack" + live_migration_fatal: true + tasks: + - name: Set up openstack cli virtualenv + ansible.builtin.pip: + virtualenv: "{{ venv }}" + virtualenv_command: /usr/bin/python3 -m venv + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + delegate_to: "{{ groups['controllers'][0] }}" + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Migrate instances + delegate_to: "{{ groups['controllers'][0] }}" + environment: "{{ openstack_auth_env }}" + when: + - "'compute' in group_names" + - groups['compute'] | length > 1 + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + block: + - name: Query instances + ansible.builtin.command: > + {{ venv }}/bin/openstack + server list --host {{ ansible_facts.nodename }} + --all-projects + --status ACTIVE + --format json + register: instances + + - name: Live migrate instances + ansible.builtin.command: > + {{ venv }}/bin/openstack + --os-compute-api-version 2.25 + server migrate + {{ instance_uuid }} + --live-migration + --wait + loop: "{{ instances.stdout | from_json }}" + loop_control: + label: "{{ instance_uuid }}" + vars: + instance_uuid: "{{ item.ID | default }}" + register: result + failed_when: + - live_migration_fatal | bool + - result is failed + + - name: Query instances + ansible.builtin.command: > + {{ venv }}/bin/openstack + server list --host {{ ansible_facts.nodename }} + --all-projects + --status ACTIVE + --format json + register: instances + + - name: Fail if there are instances still on the host + ansible.builtin.fail: + msg: > + Instances still on {{ inventory_hostname }}: {{ instances.stdout | from_json }} + when: + - live_migration_fatal | bool + - instances.stdout | from_json | length > 0 diff --git a/etc/kayobe/ansible/nova-compute-enable.yml b/etc/kayobe/ansible/nova-compute-enable.yml new file mode 100644 index 000000000..9efe15106 --- /dev/null +++ b/etc/kayobe/ansible/nova-compute-enable.yml @@ -0,0 +1,51 @@ +--- +- name: Enable nova compute service + hosts: compute + gather_facts: true + tags: + - nova-compute-enable + vars: + venv: "{{ virtualenv_path }}/openstack" + disabled_reason: Down for maintenance by nova-compute-disable.yml + tasks: + - name: Set up openstack cli virtualenv + ansible.builtin.pip: + virtualenv: "{{ venv }}" + virtualenv_command: /usr/bin/python3 -m venv + name: + - python-openstackclient + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + run_once: true + delegate_to: "{{ groups['controllers'][0] }}" + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Query nova compute services + ansible.builtin.command: > + {{ venv }}/bin/openstack + compute service list + --format json + --service nova-compute --host {{ ansible_facts.nodename }} --long + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + register: compute_services + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + + - name: Enable nova compute service + ansible.builtin.command: > + {{ venv }}/bin/openstack + compute service set + {{ ansible_facts.nodename }} nova-compute + --enable + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + vars: + service: "{{ compute_services.stdout | from_json | first }}" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + when: + # Don't enable the compute service if it was not disabled by + # nova-compute-disable.yml + - service.Status == 'disabled' + - service['Disabled Reason'] == disabled_reason diff --git a/etc/kayobe/ansible/octavia-amphora-image-build.yml b/etc/kayobe/ansible/octavia-amphora-image-build.yml new file mode 100644 index 000000000..93dc1e568 --- /dev/null +++ b/etc/kayobe/ansible/octavia-amphora-image-build.yml @@ -0,0 +1,108 @@ +--- +- name: Build an Octavia Amphora image + hosts: "{{ amphora_builder_group | default('seed') }}" + vars: + amphora_dib_upper_constraints_file: "{{ pip_upper_constraints_file }}" + amphora_image_dest: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2" + tasks: + - name: Install EPEL + ansible.builtin.package: + name: epel-release + become: true + when: + - ansible_facts.os_family == "RedHat" + - dnf_custom_repos is falsy + + - name: Ensure packages are installed + become: true + vars: + packages_common: + - debootstrap + - git + - python3-venv + packages_for_os_family: + RedHat: + - qemu-img + - e2fsprogs + - policycoreutils-python-utils + - yum-utils + Debian: + - qemu-utils + - kpartx + ansible.builtin.package: + name: "{{ packages_common + packages_for_os_family[ansible_facts.os_family] }}" + + - name: Create a temporary directory + ansible.builtin.tempfile: + state: directory + register: tempfile_result + + - name: Build Amphora image + block: + - name: Check whether the image cache directory exists + ansible.builtin.stat: + path: "{{ image_cache_path }}" + get_checksum: false + mime: false + register: image_cache_stat + + - name: Ensure the image cache directory exists + ansible.builtin.file: + path: "{{ image_cache_path }}" + state: directory + owner: "{{ ansible_facts.user_uid }}" + group: "{{ ansible_facts.user_gid }}" + become: true + when: >- + not image_cache_stat.stat.exists or + not image_cache_stat.stat.writeable + + - name: Set path facts + vars: + work_path: "{{ tempfile_result.path }}" + ansible.builtin.set_fact: + src_path: "{{ work_path }}/octavia" + venv_path: "{{ work_path }}/venv" + work_path: "{{ work_path }}" + + - name: Clone Octavia source code + ansible.builtin.git: + depth: 1 + dest: "{{ src_path }}" + repo: https://opendev.org/openstack/octavia + version: "{{ openstack_branch }}" + + - name: Install diskimage-builder in a virtual environment + ansible.builtin.pip: + name: diskimage-builder + extra_args: "{% if amphora_dib_upper_constraints_file %}-c {{ amphora_dib_upper_constraints_file }}{% endif %}" + virtualenv: "{{ venv_path }}" + virtualenv_command: python3.{{ ansible_facts.python.version.minor }} -m venv + + - name: Create build log file (/var/log/octavia-amphora-image-build.log) + ansible.builtin.file: + path: /var/log/octavia-amphora-image-build.log + state: touch + owner: "{{ ansible_facts.user_uid }}" + group: "{{ ansible_facts.user_gid }}" + become: true + + - name: Create the Amphora image + ansible.builtin.shell: + executable: /bin/bash + cmd: >- + source {{ venv_path }}/bin/activate && + ./diskimage-create.sh -i ubuntu-minimal -s 3 -g {{ openstack_branch }} >> /var/log/octavia-amphora-image-build.log 2>&1 + chdir: "{{ src_path }}/diskimage-create" + changed_when: true + + - name: Copy image to image store + ansible.builtin.copy: + src: "{{ src_path }}/diskimage-create/amphora-x64-haproxy.qcow2" + dest: "{{ amphora_image_dest }}" + remote_src: true + always: + - name: Remove temporary files + ansible.builtin.file: + path: "{{ work_path }}" + state: absent diff --git a/etc/kayobe/ansible/octavia-amphora-image-register.yml b/etc/kayobe/ansible/octavia-amphora-image-register.yml new file mode 100644 index 000000000..3435397a3 --- /dev/null +++ b/etc/kayobe/ansible/octavia-amphora-image-register.yml @@ -0,0 +1,122 @@ +--- +- name: Octavia credentials precheck + gather_facts: yes + hosts: localhost + tasks: + - name: Fail if not using octavia user and service project + ansible.builtin.fail: + msg: >- + Source the octavia-openrc.sh file before executing this playbook + when: >- + lookup('env', 'OS_USERNAME') != 'octavia' or + lookup('env', 'OS_PROJECT_NAME') != 'service' + +- name: Download Amphora image from Ark + ansible.builtin.import_playbook: pulp-amphora-image-download.yml + when: download_amphora_from_ark | bool + +- name: Register an Octavia Amphora image in Glance + gather_facts: yes + hosts: localhost + vars: + venv: "{{ virtualenv_path }}/octavia-amphora" + image_path: "/tmp/amphora-x64-haproxy.qcow2" + tasks: + - name: Get image checksum + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + ansible.builtin.stat: + path: "{{ image_path }}" + checksum_algorithm: md5 + changed_when: false + register: image_checksum + delegate_to: "{{ groups['controllers'][0] }}" + become: true + + - name: Assert that Amphora image exists + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + ansible.builtin.assert: + that: image_checksum.stat.exists + fail_msg: | + The amphora image: {{ image_path }} + does not exist. Did you download or build the image? + delegate_to: "{{ groups['controllers'][0] }}" + + - name: Ensure packages for python-openstackclient are installed + become: true + vars: + os_family_required_packages: + Debian: + - python3-venv + - python3-dev + RedHat: [] + common_required_packages: + - gcc + required_packages: "{{ common_required_packages + os_family_required_packages[ansible_facts.os_family] }}" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + ansible.builtin.package: + name: "{{ required_packages }}" + delegate_to: "{{ groups['controllers'][0] }}" + + - name: Set up openstack virtualenv + vars: + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + ansible.builtin.pip: + virtualenv: "{{ venv }}" + virtualenv_command: python3 -m venv + name: + - openstacksdk + - python-openstackclient + state: latest + extra_args: "{% if openstacksdk_upper_constraints_file %}-c {{ openstacksdk_upper_constraints_file }}{% endif %}" + delegate_to: "{{ groups['controllers'][0] }}" + + - name: Query Octavia Amphora image + vars: + ansible_python_interpreter: "{{ venv }}/bin/python" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + openstack.cloud.image_info: + auth_type: password + auth: "{{ openstack_auth }}" + ca_cert: "{{ openstack_cacert }}" + interface: "{{ openstack_interface }}" + image: amphora-x64-haproxy + register: image_info + delegate_to: "{{ groups['controllers'][0] }}" + + - name: Ensure old Octavia Amphora image is renamed + vars: + ansible_python_interpreter: "{{ venv }}/bin/python" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + ansible.builtin.command: + cmd: >- + {{ venv }}/bin/openstack image set amphora-x64-haproxy --name amphora-x64-haproxy-{{ ansible_facts.date_time.iso8601_basic_short }} + when: + - image_info.images | length != 0 + - image_info.images[0].checksum != image_checksum.stat.checksum + changed_when: true + environment: "{{ openstack_auth_env }}" + delegate_to: "{{ groups['controllers'][0] }}" + + - name: Ensure new Octavia Amphora image is registered + vars: + ansible_python_interpreter: "{{ venv }}/bin/python" + ansible_host: "{{ hostvars[groups['controllers'][0]].ansible_host }}" + openstack.cloud.image: + auth_type: password + auth: "{{ openstack_auth }}" + ca_cert: "{{ openstack_cacert }}" + interface: "{{ openstack_interface }}" + name: amphora-x64-haproxy + tags: ["amphora"] + container_format: bare + disk_format: qcow2 + is_public: false + filename: "{{ image_path }}" + properties: + hw_architecture: x86_64 + hw_rng_model: virtio + when: image_info.images | length == 0 or (image_info.images | length != 0 and image_info.images[0].checksum != image_checksum.stat.checksum) + become: true + delegate_to: "{{ groups['controllers'][0] }}" diff --git a/etc/kayobe/ansible/openbao-deploy-barbican.yml b/etc/kayobe/ansible/openbao-deploy-barbican.yml new file mode 100644 index 000000000..50d55aa22 --- /dev/null +++ b/etc/kayobe/ansible/openbao-deploy-barbican.yml @@ -0,0 +1,107 @@ +--- +- name: Configure AppRole + any_errors_fatal: true + gather_facts: true + hosts: controllers[0] + vars: + openbao_api_addr: https://{{ internal_net_name | net_ip }}:8200 + openbao_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + tasks: + - name: Assert that secrets_barbican_approle_secret_id is defined + ansible.builtin.assert: + that: + - secrets_barbican_approle_secret_id is defined + fail_msg: Please define secrets_barbican_approle_secret_id in your secrets.yml + + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + name: openbao_keys + + - name: Ensure hvac is installed + ansible.builtin.pip: + name: hvac + state: present + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv_path }}/kayobe" + + - name: Ensure AppRole is configured + environment: + https_proxy: "" + block: + - name: Enable AppRole auth module + hashivault_auth_method: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ openbao_ca_cert }}" + token: "{{ openbao_keys.root_token }}" + method_type: approle + state: enabled + + - name: Enable barbican kv store + hashivault_secret_engine: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ openbao_ca_cert }}" + token: "{{ openbao_keys.root_token }}" + name: barbican + backend: kv + description: Barbican kv store + + - name: Ensure barbican policy is defined + hashivault_policy: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ openbao_ca_cert }}" + token: "{{ openbao_keys.root_token }}" + name: barbican-policy + state: present + rules: | + path "barbican/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } + + - name: Ensure barbican AppRole is defined + hashivault_approle_role: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ openbao_ca_cert }}" + token: "{{ openbao_keys.root_token }}" + bind_secret_id: true + secret_id_bound_cidrs: "{{ internal_net_name | net_cidr }}" + secret_id_ttl: 0 + token_policies: barbican-policy + name: barbican + + - name: Get barbican Approle ID + hashivault_approle_role_id: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ openbao_ca_cert }}" + token: "{{ openbao_keys.root_token }}" + name: barbican + register: barbican_role_id + + - name: Print barbican Approle ID + ansible.builtin.debug: + msg: barbican role id is {{ barbican_role_id.id }} + + - name: Write barbican Approle ID to file if requested + delegate_to: localhost + ansible.builtin.copy: + content: "{{ barbican_role_id.id }}" + dest: "{{ stackhpc_barbican_role_id_file_path | default('~/barbican-role-id') }}" + when: stackhpc_write_barbican_role_id_to_file | default(false) | bool + + - name: Check if barbican Approle Secret ID is defined + hashivault_approle_role_secret_get: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ openbao_ca_cert }}" + token: "{{ openbao_keys.root_token }}" + secret: "{{ secrets_barbican_approle_secret_id }}" + name: barbican + register: barbican_approle_secret_get + + - name: Ensure barbican AppRole Secret ID is defined + hashivault_approle_role_secret: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ openbao_ca_cert }}" + token: "{{ openbao_keys.root_token }}" + secret: "{{ secrets_barbican_approle_secret_id }}" + name: barbican + when: barbican_approle_secret_get.status == "absent" diff --git a/etc/kayobe/ansible/openbao-deploy-overcloud.yml b/etc/kayobe/ansible/openbao-deploy-overcloud.yml new file mode 100644 index 000000000..8b2a59860 --- /dev/null +++ b/etc/kayobe/ansible/openbao-deploy-overcloud.yml @@ -0,0 +1,145 @@ +--- +# Required for uri module to work with self-signed certificates and for systems to trust +# the self-signed CA +- name: Install CA on controllers + hosts: controllers + tasks: + - name: Copy the intermediate CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-ROOT.pem" + dest: "{{ '/etc/pki/ca-trust/source/anchors/OS-TLS-ROOT.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' + }}" + mode: "0644" + become: true + + - name: Update system CA + become: true + ansible.builtin.command: "{{ 'update-ca-trust' if ansible_facts.os_family == 'RedHat' else 'update-ca-certificates' }}" + +- name: Deploy OpenBao on the overcloud + any_errors_fatal: true + gather_facts: true + hosts: controllers + vars: + openbao_bind_addr: "{{ internal_net_name | net_ip }}" + # This is the IP address of the first controller and therefore the leader within + # OpenBao. This could be replaced with the VIP address of the internal network if + # HAProxy has been configured to load balance the OpenBao API. + openbao_raft_leaders: + - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][0]) }}" + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure /opt/kayobe/openbao exists + ansible.builtin.file: + path: /opt/kayobe/openbao + state: directory + + - name: Template out TLS key and cert + ansible.builtin.copy: + # Within the OpenBao container these uids & gids map to the openbao user + src: "{{ kayobe_env_config_path }}/openbao/{{ item }}" + dest: /opt/kayobe/openbao/{{ item }} + owner: 100 + group: 1000 + mode: "0600" + loop: + - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" + - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + - "OS-TLS-INT.crt" + become: true + + - name: Apply OpenBao role + ansible.builtin.import_role: + name: stackhpc.hashicorp.openbao + vars: + openbao_registry_url: "{{ overcloud_openbao_registry_url }}" + openbao_registry_username: "{{ overcloud_openbao_registry_username }}" + openbao_registry_password: "{{ overcloud_openbao_registry_password }}" + openbao_config_dir: /opt/kayobe/openbao + openbao_cluster_name: overcloud + openbao_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + openbao_docker_image: "{{ overcloud_openbao_docker_image }}" + openbao_docker_tag: "{{ overcloud_openbao_docker_tag }}" + openbao_tls_cert: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" + openbao_tls_key: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + openbao_tls_ca: "OS-TLS-INT.crt" + copy_self_signed_ca: true + openbao_api_addr: https://{{ internal_net_name | net_ip }}:8200 + openbao_write_keys_file: true + openbao_write_keys_file_path: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + name: openbao_keys + + - name: Unseal first OpenBao instance + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ openbao_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + environment: + https_proxy: "" + run_once: true + + # As the first instance is now unsealed the other instances will now need some + # time to connect before we can proceed. + - name: Wait for OpenBao Raft peers to connect + ansible.builtin.wait_for: + timeout: 30 + delegate_to: localhost + + # Raft peers take few seconds before they report an unsealed state therefore + # we must wait. + - name: Unseal all OpenBao instances + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ openbao_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + vault_unseal_timeout: 10 + environment: + https_proxy: "" + +- name: Configure PKI + any_errors_fatal: true + gather_facts: true + hosts: controllers[0] + tasks: + - name: Apply OpenBao pki role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_pki + vars: + vault_token: "{{ openbao_keys.root_token }}" + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_pki_root_create: false + vault_pki_intermediate_import: true + vault_pki_intermediate_ca_name: OS-TLS-INT + vault_pki_intermediate_ca_bundle: "{{ lookup('file', kayobe_env_config_path + '/openbao/OS-TLS-INT.pem') }}" + vault_pki_intermediate_ca_cert: "{{ lookup('file', kayobe_env_config_path + '/openbao/OS-TLS-INT.crt') }}" + vault_pki_intermediate_roles: "{{ overcloud_openbao_pki_roles }}" + vault_pki_write_certificate_files: true + vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/openbao" + environment: + https_proxy: "" diff --git a/etc/kayobe/ansible/openbao-deploy-seed.yml b/etc/kayobe/ansible/openbao-deploy-seed.yml new file mode 100644 index 000000000..42972b35e --- /dev/null +++ b/etc/kayobe/ansible/openbao-deploy-seed.yml @@ -0,0 +1,78 @@ +--- +- name: Deploy OpenBao on the seed + any_errors_fatal: true + gather_facts: true + hosts: seed + vars: + openbao_bind_addr: "{{ ansible_facts['lo'].ipv4.address }}" + openbao_api_addr: "http://{{ openbao_bind_addr }}:8200" + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python PyYAML and hvac modules are installed + ansible.builtin.pip: + name: + - PyYAML + - hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure OpenBao directory exists in Kayobe configuration + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/openbao/" + state: directory + delegate_to: localhost + run_once: true + + - name: Apply OpenBao role + ansible.builtin.import_role: + name: stackhpc.hashicorp.openbao + vars: + openbao_registry_url: "{{ seed_openbao_registry_url }}" + openbao_registry_username: "{{ seed_openbao_registry_username }}" + openbao_registry_password: "{{ seed_openbao_registry_password }}" + openbao_config_dir: /opt/kayobe/openbao + openbao_cluster_name: seed + openbao_docker_image: "{{ seed_openbao_docker_image }}" + openbao_docker_tag: "{{ seed_openbao_docker_tag }}" + openbao_write_keys_file: true + openbao_write_keys_file_path: "{{ kayobe_env_config_path }}/openbao/seed-openbao-keys.json" + + - name: Include Vault keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/seed-openbao-keys.json" + name: openbao_keys + + - name: Unseal OpenBao + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: "{{ openbao_api_addr }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + + - name: Apply PKI role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_pki + vars: + vault_api_addr: "{{ openbao_api_addr }}" + vault_token: "{{ openbao_keys.root_token }}" + vault_pki_root_ca_name: OS-TLS-ROOT + vault_pki_write_root_ca_to_file: true + vault_pki_intermediate_ca_name: OS-TLS-INT + vault_pki_intermediate_export: true + vault_pki_intermediate_roles: "{{ seed_openbao_pki_roles }}" + vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/openbao" + vault_pki_generate_certificates: true + vault_pki_write_certificates: true + vault_pki_certificate_subject: "{{ seed_openbao_pki_certificate_subject }}" + vault_pki_write_certificate_files: true + vault_pki_write_pem_bundle: false + vault_pki_write_int_ca_to_file: true diff --git a/etc/kayobe/ansible/openbao-generate-backend-tls.yml b/etc/kayobe/ansible/openbao-generate-backend-tls.yml new file mode 100644 index 000000000..f43513ff1 --- /dev/null +++ b/etc/kayobe/ansible/openbao-generate-backend-tls.yml @@ -0,0 +1,83 @@ +--- +# Required for uri module to work with self-signed certificates and for systems to trust +# the self-signed CA +- name: Install CA + hosts: controllers:network + tasks: + - name: Copy the intermediate CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-ROOT.pem" + dest: "{{ '/etc/pki/ca-trust/source/anchors/OS-TLS-ROOT.crt' if ansible_facts.os_family == 'RedHat' \ + else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + mode: "0644" + become: true + + - name: Update system CA + become: true + ansible.builtin.command: "{{ 'update-ca-trust' if ansible_facts.os_family == 'RedHat' else 'update-ca-certificates' }}" + +- name: Generate backend API certificates + hosts: controllers:network + vars: + openbao_api_addr: https://{{ internal_net_name | net_ip(groups['controllers'][0]) }}:8200 + openbao_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + name: openbao_keys + + - name: Issue a certificate for backend TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ openbao_keys.root_token }}" + mount_point: "{{ openbao_intermediate_ca_name }}" + role: "{{ overcloud_openbao_pki_backend_tls_role_name }}" + common_name: "" + extra_params: + ip_sans: "{{ internal_net_name | net_ip }}" + register: backend_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Copy backend cert + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/{{ inventory_hostname }}-cert.pem" + content: | + {{ backend_cert.data.certificate }} + {{ backend_cert.data.issuing_ca }} + mode: "0600" + delegate_to: localhost + + - name: Copy backend key + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/{{ inventory_hostname }}-key.pem" + content: | + {{ backend_cert.data.private_key }} + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/openbao-generate-internal-tls.yml b/etc/kayobe/ansible/openbao-generate-internal-tls.yml new file mode 100644 index 000000000..2cc9e841a --- /dev/null +++ b/etc/kayobe/ansible/openbao-generate-internal-tls.yml @@ -0,0 +1,56 @@ +--- +- name: Generate internal API certificate + hosts: controllers + run_once: true + vars: + openbao_api_addr: https://{{ internal_net_name | net_ip }}:8200 + openbao_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + name: openbao_keys + + - name: Issue a certificate for internal TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ openbao_keys.root_token }}" + mount_point: "{{ openbao_intermediate_ca_name }}" + role: "{{ overcloud_openbao_pki_internal_tls_role_name }}" + common_name: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% endif %}" + extra_params: + ip_sans: "{{ kolla_internal_vip_address }}" + register: internal_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Ensure CA certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates/ca" + state: directory + delegate_to: localhost + + - name: Copy internal API PEM bundle + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy-internal.pem" + content: | + {{ internal_cert.data.certificate }} + {{ internal_cert.data.issuing_ca }} + {{ internal_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + + - name: Copy root CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/openbao-generate-test-external-tls.yml b/etc/kayobe/ansible/openbao-generate-test-external-tls.yml new file mode 100644 index 000000000..e7150fed1 --- /dev/null +++ b/etc/kayobe/ansible/openbao-generate-test-external-tls.yml @@ -0,0 +1,57 @@ +--- +- name: Generate external API certificate (for testing only) + hosts: controllers + run_once: true + vars: + openbao_api_addr: https://{{ internal_net_name | net_ip }}:8200 + # NOTE: Using the same CA as internal TLS. + openbao_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + name: openbao_keys + + - name: Issue a certificate for external TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ openbao_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ openbao_keys.root_token }}" + mount_point: "{{ openbao_intermediate_ca_name }}" + role: "{{ overcloud_openbao_pki_external_tls_role_name }}" + common_name: "{% if kolla_external_fqdn != kolla_external_vip_address %}{{ kolla_external_fqdn }}{% endif %}" + extra_params: + ip_sans: "{{ kolla_external_vip_address }}" + register: external_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Ensure CA certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates/ca" + state: directory + delegate_to: localhost + + - name: Copy external API PEM bundle + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy.pem" + content: | + {{ external_cert.data.certificate }} + {{ external_cert.data.issuing_ca }} + {{ external_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + + - name: Copy root CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/openbao/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/openbao.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/openbao-unseal-overcloud.yml b/etc/kayobe/ansible/openbao-unseal-overcloud.yml new file mode 100644 index 000000000..0a631c598 --- /dev/null +++ b/etc/kayobe/ansible/openbao-unseal-overcloud.yml @@ -0,0 +1,37 @@ +--- +- name: Unseal OpenBao on the overcloud + any_errors_fatal: true + gather_facts: true + hosts: controllers + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + name: openbao_keys + + - name: Apply OpenBao unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ openbao_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" + environment: + https_proxy: "" diff --git a/etc/kayobe/ansible/openbao-unseal-seed.yml b/etc/kayobe/ansible/openbao-unseal-seed.yml new file mode 100644 index 000000000..82f9b8fa7 --- /dev/null +++ b/etc/kayobe/ansible/openbao-unseal-seed.yml @@ -0,0 +1,34 @@ +--- +- name: Unseal OpenBao on the seed + any_errors_fatal: true + gather_facts: true + hosts: seed + vars: + vault_api_addr: http://127.0.0.1:8200 + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include OpenBao keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/openbao/seed-openbao-keys.json" + name: openbao_keys + + - name: Apply OpenBao unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_unseal_keys: "{{ openbao_keys.keys_base64 }}" diff --git a/etc/kayobe/ansible/openstack-host-image-upload.yml b/etc/kayobe/ansible/openstack-host-image-upload.yml new file mode 100644 index 000000000..daff0549d --- /dev/null +++ b/etc/kayobe/ansible/openstack-host-image-upload.yml @@ -0,0 +1,55 @@ +--- +# This playbook is designed to be used by the overcloud-host-image-build.yml +# GitHub workflow to upload newly-built images to a development cloud for +# testing and use in CI. +- name: Upload an OS image to Glance + hosts: seed + vars: + local_image_path: /opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2 + image_name: overcloud-{{ os_distribution }}-{{ os_release }} + tasks: + - name: Upload image to Glance + block: + - name: Write out clouds.yaml + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.env', 'CLOUDS_YAML') }}" + dest: clouds.yaml + mode: "0600" + + - name: Write out secure.yaml + no_log: true + vars: + os_secrets: + clouds: + openstack: + auth: + application_credential_id: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}" + application_credential_secret: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" + ansible.builtin.copy: + content: "{{ os_secrets | to_nice_yaml }}" + dest: secure.yaml + mode: "0600" + + - name: Ensure dependencies are installed + ansible.builtin.pip: + name: openstacksdk + + - name: Upload an image to Glance + openstack.cloud.image: + cloud: openstack + name: "{{ image_name }}" + container_format: bare + disk_format: qcow2 + state: present + filename: "{{ local_image_path }}" + + always: + - name: Remove clouds.yaml + ansible.builtin.file: + path: clouds.yaml + state: absent + + - name: Remove secure.yaml + ansible.builtin.file: + path: secure.yaml + state: absent diff --git a/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml b/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml new file mode 100644 index 000000000..0b83b472c --- /dev/null +++ b/etc/kayobe/ansible/ovn-fix-chassis-priorities.yml @@ -0,0 +1,164 @@ +--- +# Sometimes, typically after restarting OVN services, the priorities of entries +# in the ha_chassis and gateway_chassis tables in the OVN northbound database +# can become misaligned. This results in broken routing for external (bare +# metal/SR-IOV) ports. + +# This playbook can be used to fix the issue by realigning the priorities of +# the table entries. It executes a small inline shell script against the +# OVN northbound database to ensure that, for each router, the HA chassis +# backing its internal networks is aligned with the chassis currently hosting +# the router's external gateway interface. + +# The playbook can be run as follows: +# kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/fixes/ovn-fix-chassis-priorities.yml +# By default this runs in dry-run mode; pass '-e apply=yes' to perform the updates. + +# If the 'controllers' group does not align with the group used to deploy the +# OVN NB DB, this can be overridden by passing the following: +# '-e ovn_nb_db_group=some_other_group' + +- name: Find OVN NB DB Leader + hosts: "{{ ovn_nb_db_group | default('controllers') }}" + tasks: + - name: Find OVN NB DB Leader + when: kolla_enable_ovn | bool + block: + - name: Find the OVN NB DB leader + ansible.builtin.command: >- + docker exec ovn_nb_db + ovs-appctl -t /var/run/ovn/ovnnb_db.ctl + cluster/status OVN_Northbound + changed_when: false + failed_when: false + register: ovn_cluster_status + check_mode: false + + - name: Group hosts by leader/follower role + ansible.builtin.group_by: + key: "{{ 'ovn_nb_leader' if 'Role: leader' in ovn_cluster_status.stdout else 'ovn_nb_follower' }}" + changed_when: false + + - name: Assert one leader exists + ansible.builtin.assert: + that: + - groups['ovn_nb_leader'] | default([]) | length == 1 + +- name: Fix OVN chassis priorities + hosts: ovn_nb_leader + gather_facts: false + vars: + apply_updates: "{{ apply | default(false) | bool }}" + tasks: + - name: Realign HA chassis priorities with active gateways + when: kolla_enable_ovn | bool + ansible.builtin.shell: | + docker exec -i ovn_nb_db bash -s <<'EOF' + set -euo pipefail + + MAX_PRIORITY=32767 + APPLY="{{ 'yes' if apply_updates else 'no' }}" + + if [ "$APPLY" = "yes" ]; then + echo "APPLY MODE: Updating OVN HA priorities" + else + echo "DRY-RUN MODE: Showing proposed changes only" + echo "Re-run with -e apply=yes to apply changes" + fi + echo "" + + # Get all external gateway ports + ext_ports=$(ovn-nbctl --data=bare --no-headings --columns=name find logical_router_port 'external_ids:"neutron:is_ext_gw"="True"') + + for ext_port in $ext_ports; do + # Get router name + router=$(ovn-nbctl --data=bare --no-headings get logical_router_port "$ext_port" 'external_ids:"neutron:router_name"' | tr -d '"') + + if [ -z "$router" ]; then + echo "Skipping $ext_port: no router name found" + continue + fi + + # Get gateway chassis list (ordered by priority) + gateway_chassis="" + gateway_info=$(ovn-nbctl lrp-get-gateway-chassis "$ext_port" 2>/dev/null || true) + + while IFS= read -r line; do + # Strip prefix, allowing '-' or '_' separator + chassis=$(echo "$line" | awk '{print $1}' | sed "s/^${ext_port}[-_]//") + gateway_chassis="$gateway_chassis $chassis" + done <<< "$gateway_info" + + gateway_chassis=${gateway_chassis# } + + if [ -z "$gateway_chassis" ]; then + echo "Router $router: no gateway chassis configured" + continue + fi + + # The first chassis in the list is the active gateway + active_gateway=$(echo "$gateway_chassis" | awk '{print $1}') + echo "Router: $router | Port: $ext_port | Active Gateway: $active_gateway" + + # Process all internal ports on this router + router_ports=$(ovn-nbctl --data=bare --no-headings --columns=name \ + find logical_router_port "external_ids:\"neutron:router_name\"=\"$router\"") + + for port in $router_ports; do + + # Skip external gateway ports + is_external=$(ovn-nbctl --data=bare --no-headings get logical_router_port "$port" 'external_ids:"neutron:is_ext_gw"' 2>/dev/null) + [ "$is_external" = "True" ] && continue + + # Get network name and HA chassis group + network=$(ovn-nbctl --data=bare --no-headings get logical_router_port "$port" 'external_ids:"neutron:network_name"' 2>/dev/null) + ha_group=$(ovn-nbctl --data=bare --no-headings --columns=_uuid find ha_chassis_group name="$network") + + if [ -z "$ha_group" ]; then + echo " Port $port: no HA group found for network '$network'" + continue + fi + + echo " Port: $port | Network: $network" + + # Update priorities for each chassis in the HA group + ha_chassis_list=$(ovn-nbctl --data=bare --no-headings get ha_chassis_group "$ha_group" ha_chassis | tr -d '[],') + + for uuid in $ha_chassis_list; do + chassis_name=$(ovn-nbctl --data=bare --no-headings get ha_chassis "$uuid" chassis_name) + current_priority=$(ovn-nbctl --data=bare --no-headings get ha_chassis "$uuid" priority) + + # Calculate desired priority + desired_priority="" + index=0 + for gw in $gateway_chassis; do + if [ "$chassis_name" = "$gw" ]; then + desired_priority=$((MAX_PRIORITY - index)) + break + fi + index=$((index + 1)) + done + [ -z "$desired_priority" ] && continue + + # Apply or report change + if [ "$current_priority" -ne "$desired_priority" ]; then + if [ "$APPLY" = "yes" ]; then + ovn-nbctl set ha_chassis "$uuid" priority=$desired_priority + echo " $chassis_name: updated priority $current_priority to $desired_priority" + else + echo " $chassis_name: would update priority $current_priority to $desired_priority" + fi + else + echo " $chassis_name: priority $current_priority (no change needed)" + fi + done + done + echo "" + done + EOF + register: fix_output + changed_when: apply_updates and ('updated priority' in (fix_output.stdout | default(''))) + + - name: Display results + ansible.builtin.debug: + msg: "{{ fix_output.stdout }}" diff --git a/etc/kayobe/ansible/pci-passthrough.yml b/etc/kayobe/ansible/pci-passthrough.yml new file mode 100644 index 000000000..392713c35 --- /dev/null +++ b/etc/kayobe/ansible/pci-passthrough.yml @@ -0,0 +1,142 @@ +--- +- name: Enable GPU passthough + hosts: "{{ (gpu_group_map | default({})).keys() }}" + vars: + # This playbook will execute after nodes are deployed + # and before overcloud host configure - we can't assume + # users and venvs exist. + ansible_user: "{{ bootstrap_user }}" + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + ansible_python_interpreter: "/usr/bin/python3" + vfio_pci_ids: |- + {% set gpu_list = [] %} + {% set output = [] %} + {% for gpu_group in (gpu_group_map | default({})) | dict2items %} + {% if gpu_group.key in group_names %} + {% set _ = gpu_list.append(gpu_group.value) %} + {% endif %} + {% endfor %} + {% for item in gpu_list | flatten | unique %} + {% set _ = output.append(stackhpc_gpu_data[item]['vendor_id'] + ':' + stackhpc_gpu_data[item]['product_id']) %} + {% endfor %} + {{ output | join(',') }} + reboot_timeout_s: "{{ 20 * 60 }}" + tasks: + - name: Template dracut config + ansible.builtin.blockinfile: + path: /etc/dracut.conf.d/gpu-vfio.conf + block: | + add_drivers+="vfio vfio_iommu_type1 vfio_pci vfio_virqfd" + owner: root + group: root + mode: 0660 + create: true + become: true + notify: + - Regenerate initramfs + - reboot + + - name: Add vfio to modules-load.d + ansible.builtin.blockinfile: + path: /etc/modules-load.d/vfio.conf + block: | + vfio + vfio_iommu_type1 + vfio_pci + vfio_virqfd + owner: root + group: root + mode: 0664 + create: true + become: true + notify: reboot + + - name: Blacklist nouveau + ansible.builtin.blockinfile: + path: /etc/modprobe.d/blacklist-nouveau.conf + block: | + blacklist nouveau + options nouveau modeset=0 + mode: 0664 + owner: root + group: root + create: true + become: true + notify: + - reboot + - Regenerate initramfs + + - name: Ignore unsupported model specific registers + # Occasionally, applications running in the VM may crash unexpectedly, + # whereas they would run normally on a physical machine. If, while + # running dmesg -wH, you encounter an error mentioning MSR, the reason + # for those crashes is that KVM injects a General protection fault (GPF) + # when the guest tries to access unsupported Model-specific registers + # (MSRs) - this often results in guest applications/OS crashing. A + # number of those issues can be solved by passing the ignore_msrs=1 + # option to the KVM module, which will ignore unimplemented MSRs. + # source: https://wiki.archlinux.org/index.php/QEMU + ansible.builtin.blockinfile: + path: /etc/modprobe.d/kvm.conf + block: | + options kvm ignore_msrs=Y + # This option is not available in centos 7 as the kernel is too old, + # but it can help with dmesg spam in newer kernels (centos8?). Sample + # dmesg log message: + # [ +0.000002] kvm [8348]: vcpu0, guest rIP: 0xffffffffb0a767fa ignored rdmsr: 0x619 + # options kvm report_ignored_msrs=N + mode: 0664 + owner: root + group: root + create: true + become: true + notify: reboot + + - name: Add vfio-pci.ids kernel args + ansible.builtin.include_role: + name: stackhpc.linux.grubcmdline + vars: + kernel_cmdline: + - intel_iommu=on + - iommu=pt + - "vfio-pci.ids={{ vfio_pci_ids }}" + kernel_cmdline_remove: + - iommu + - intel_iommu + - vfio-pci.ids + + handlers: + - name: Regenerate initramfs (RedHat) + listen: Regenerate initramfs + ansible.builtin.shell: |- + #!/bin/bash + set -eux + dracut -v -f /boot/initramfs-$(uname -r).img $(uname -r) + become: true + changed_when: true + when: ansible_facts.os_family == 'RedHat' + + - name: Regenerate initramfs (Debian) + listen: Regenerate initramfs + ansible.builtin.shell: |- + #!/bin/bash + set -eux + update-initramfs -u -k $(uname -r) + become: true + changed_when: true + when: ansible_facts.os_family == 'Debian' + + - name: Reboot + listen: reboot + become: true + ansible.builtin.reboot: + reboot_timeout: "{{ reboot_timeout_s }}" + search_paths: + # Systems running molly-guard hang waiting for confirmation before rebooting without this. + - /lib/molly-guard + # Default list: + - /sbin + - /bin + - /usr/sbin + - /usr/bin + - /usr/local/sbin diff --git a/etc/kayobe/ansible/prometheus-network-names.yml b/etc/kayobe/ansible/prometheus-network-names.yml new file mode 100644 index 000000000..73d3c449f --- /dev/null +++ b/etc/kayobe/ansible/prometheus-network-names.yml @@ -0,0 +1,40 @@ +--- +- name: Prometheus friendly network names + hosts: overcloud + gather_facts: false + tasks: + - name: Gather network maps from each host with unique identifiers + ansible.builtin.set_fact: + host_network_maps: >- + {%- set if_list = [] -%} + {%- for i in network_interfaces -%} + {%- set device_name = hostvars[inventory_hostname][i ~ '_interface'] -%} + {%- set friendly_name = i -%} + {%- set unique_id = device_name ~ '_' ~ friendly_name -%} + {%- set _ = if_list.append({ + 'unique_id': unique_id, + 'device_name': device_name, + 'friendly_name': friendly_name + }) -%} + {%- endfor -%} + {{ if_list }} + + - name: Aggregate network maps from all hosts + ansible.builtin.set_fact: + prometheus_network_maps_aggregated: "{{ groups['overcloud'] | map('extract', hostvars, 'host_network_maps') | flatten }}" + run_once: true + delegate_to: localhost + + - name: Deduplicate the aggregated list based on unique IDs + ansible.builtin.set_fact: + prometheus_network_maps_blob: "{{ prometheus_network_maps_aggregated | unique(attribute='unique_id') }}" + run_once: true + delegate_to: localhost + + - name: Apply template with deduplicated network maps + ansible.builtin.template: + src: prometheus.yml.j2 + dest: "{{ kayobe_env_config_path }}/kolla/config/prometheus/prometheus.yml" + become: true + run_once: true + delegate_to: localhost diff --git a/etc/kayobe/ansible/prometheus.yml.j2 b/etc/kayobe/ansible/prometheus.yml.j2 new file mode 100644 index 000000000..c40b0cdcb --- /dev/null +++ b/etc/kayobe/ansible/prometheus.yml.j2 @@ -0,0 +1,287 @@ +{{ '{%' }} raw {{ '%}' }} +{% raw %} +global: + scrape_interval: {{ prometheus_scrape_interval }} + scrape_timeout: 10s + evaluation_interval: 15s +{% if prometheus_external_labels %} + external_labels: +{% for label, value in prometheus_external_labels.items() %} + {{ label }}: {{ value }} +{% endfor %} +{% endif %} + +{% if prometheus_alert_rules.files is defined and prometheus_alert_rules.files | length > 0 %} +rule_files: +{% for rule in prometheus_alert_rules.files %} + - "/etc/prometheus/{{ rule.path | basename }}" +{% endfor %} +{% endif %} + +scrape_configs: + - job_name: prometheus + basic_auth: + username: admin + password: "{{ prometheus_password }}" + static_configs: +{% for host in groups['prometheus'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_port }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} + +{% if enable_prometheus_node_exporter | bool %} + - job_name: node + static_configs: +{% for host in groups['prometheus-node-exporter'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_node_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endraw %} + metric_relabel_configs: + - replacement: $1 + source_labels: [device] + target_label: 'original_device' +{% for net_map in prometheus_network_maps_blob %} + - source_labels: [__name__, device] + regex: '(.*);(.*{{ net_map.device_name }}$)' + target_label: 'device' + replacement: '{{ net_map.friendly_name }}.$2' +{% endfor %} +{% raw %} +{% endif %} + +{% if enable_prometheus_mysqld_exporter | bool %} + - job_name: mysqld + static_configs: +{% for host in groups['prometheus-mysqld-exporter'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_mysqld_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_haproxy_exporter | bool %} + - job_name: haproxy + static_configs: +{% for host in groups['loadbalancer'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ prometheus_haproxy_exporter_port }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_rabbitmq_exporter | bool %} + - job_name: rabbitmq + static_configs: +{% for host in groups['rabbitmq'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_rabbitmq_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_memcached_exporter | bool %} + - job_name: memcached + static_configs: +{% for host in groups['prometheus-memcached-exporter'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_memcached_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_cadvisor | bool %} + - job_name: cadvisor + static_configs: +{% for host in groups["prometheus-cadvisor"] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_cadvisor_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endraw %} + metric_relabel_configs: + - replacement: $1 + source_labels: [interface] + target_label: 'original_interface' +{% for net_map in prometheus_network_maps_blob %} + - source_labels: [__name__, image, interface] + regex: '(.*);(.*{{ net_map.device_name }}$)' + target_label: 'interface' + replacement: '{{ net_map.friendly_name }}' +{% endfor %} +{% raw %} +{% endif %} + +{% if enable_prometheus_fluentd_integration | bool %} + - job_name: fluentd + static_configs: +{% for host in groups['fluentd'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_fluentd_integration_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_ceph_mgr_exporter | bool %} + - job_name: ceph_mgr_exporter + honor_labels: true + scrape_interval: {{ prometheus_ceph_exporter_interval }} + static_configs: + - targets: +{% for exporter in prometheus_ceph_mgr_exporter_endpoints %} + - '{{ exporter }}' +{% endfor %} +{% endif %} + +{% if enable_prometheus_openstack_exporter | bool %} + - job_name: openstack_exporter + scrape_interval: {{ prometheus_openstack_exporter_interval }} + scrape_timeout: {{ prometheus_openstack_exporter_timeout }} +{% if kolla_enable_tls_internal | bool %} + scheme: https +{% endif %} + honor_labels: true + static_configs: + - targets: + - '{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ prometheus_openstack_exporter_port }}' +{% endif %} + +{% if enable_prometheus_elasticsearch_exporter | bool %} + - job_name: elasticsearch_exporter + scrape_interval: {{ prometheus_elasticsearch_exporter_interval }} + static_configs: +{% for host in groups["prometheus-elasticsearch-exporter"] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_elasticsearch_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_blackbox_exporter | bool and prometheus_blackbox_exporter_endpoints | length > 0 | bool %} + - job_name: blackbox_exporter + metrics_path: /probe + honor_labels: true + static_configs: + - targets: +{% for target in prometheus_blackbox_exporter_endpoints %} + - '{{ target }}' +{% endfor %} + relabel_configs: + - source_labels: [__address__] + regex: (\w+):(\w+):(.+) + target_label: service + replacement: ${1} + - source_labels: [__address__] + regex: (\w+):(\w+):(.+) + target_label: __param_module + replacement: ${2} + - source_labels: [__param_module] + target_label: module + - source_labels: [__address__] + regex: (\w+):(\w+):(.+) + target_label: __param_target + replacement: ${3} + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: '{{ api_interface_address | put_address_in_context('url') }}:{{ prometheus_blackbox_exporter_port }}' +{% endif %} + +{% if enable_prometheus_libvirt_exporter | bool %} + - job_name: libvirt_exporter + scrape_interval: {{ prometheus_libvirt_exporter_interval }} + honor_labels: true + static_configs: +{% for host in groups["prometheus-libvirt-exporter"] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_libvirt_exporter_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_etcd_integration | bool %} + - job_name: etcd +{% if etcd_enable_tls | bool %} + scheme: https +{% endif %} + static_configs: +{% for host in groups["etcd"] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_etcd_integration_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_ironic_prometheus_exporter | bool %} + - job_name: ironic_prometheus_exporter + static_configs: +{% for host in groups['ironic-conductor'] %} + - targets: ["{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['ironic_prometheus_exporter_port'] }}"] +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} + +{% if enable_prometheus_alertmanager | bool %} + - job_name: alertmanager + static_configs: +{% for host in groups['prometheus-alertmanager'] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} + +alerting: + alertmanagers: + - static_configs: +{% for host in groups["prometheus-alertmanager"] %} + - targets: + - '{{ 'api' | kolla_address(host) | put_address_in_context('url') }}:{{ hostvars[host]['prometheus_alertmanager_port'] }}' +{% if hostvars[host].prometheus_instance_label | default(false, true) %} + labels: + instance: "{{ hostvars[host].prometheus_instance_label }}" +{% endif %} +{% endfor %} +{% endif %} +{% endraw %} +{{ '{%' }} endraw {{ '%}' }} diff --git a/etc/kayobe/ansible/pulp-amphora-image-download.yml b/etc/kayobe/ansible/pulp-amphora-image-download.yml new file mode 100644 index 000000000..ee2c8ee82 --- /dev/null +++ b/etc/kayobe/ansible/pulp-amphora-image-download.yml @@ -0,0 +1,35 @@ +--- +- name: Download an amphora image from Ark + hosts: controllers[0] + vars: + # The auth is handled by username and + # password in the get_url task of this playbook + stackhpc_amphora_image_url_no_auth: "{{ stackhpc_release_pulp_content_url }}/amphora-images/\ + {{ openstack_release }}/\ + {{ stackhpc_amphora_image_version }}/\ + amphora-x64-haproxy.qcow2" + download_destination: /tmp/amphora-x64-haproxy.qcow2 + + tasks: + - name: Print image information + ansible.builtin.debug: + msg: | + Image tag: {{ stackhpc_amphora_image_version }} + + # TODO: Add checksum support + - name: Download image artifact + ansible.builtin.get_url: + url: "{{ stackhpc_amphora_image_url_no_auth }}" + username: "{{ stackhpc_release_pulp_username }}" + password: "{{ stackhpc_release_pulp_password }}" + force_basic_auth: true + unredirected_headers: + - "Authorization" + dest: "{{ download_destination }}" + mode: "0644" + owner: stack + group: stack + register: image_download_result + until: image_download_result.status_code >= 200 and image_download_result.status_code <= 399 + retries: 3 + delay: 60 diff --git a/etc/kayobe/ansible/pulp-artifact-promote.yml b/etc/kayobe/ansible/pulp-artifact-promote.yml new file mode 100644 index 000000000..f62a76ce3 --- /dev/null +++ b/etc/kayobe/ansible/pulp-artifact-promote.yml @@ -0,0 +1,46 @@ +--- +# Tag is the version of the artifact to promote, stored in an env variable named ARTIFACT_TAG +- name: Promote an artifact to production + hosts: localhost + vars: + remote_pulp_url: "{{ stackhpc_release_pulp_url }}" + remote_pulp_username: "{{ stackhpc_release_pulp_username }}" + remote_pulp_password: "{{ stackhpc_release_pulp_password }}" + repository_name: "{{ artifact_type }}-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" + pulp_base_path: "{{ artifact_type }}/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + promotion_tag: "{{ lookup('env', 'ARTIFACT_TAG') }}" + tasks: + - name: Check whether the artifact exists + pulp.squeezer.file_distribution: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}_{{ promotion_tag }}" + base_path: "{{ pulp_base_path }}/{{ promotion_tag }}" + register: distribution_details + until: distribution_details is success + retries: 3 + delay: 5 + + - name: Fail if the artifact does not exist + ansible.builtin.fail: + msg: "artifact {{ promotion_tag }} does not exist" + when: distribution_details.distribution is none + + - name: Ensure production content guard is set + pulp.squeezer.file_distribution: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}_{{ promotion_tag }}" + base_path: "{{ pulp_base_path }}/{{ promotion_tag }}" + content_guard: release + state: present + register: content_guard_result + until: content_guard_result is success + retries: 3 + delay: 5 + + - name: Print version tag and os + ansible.builtin.debug: + msg: "Promoted tag: {{ promotion_tag }}" diff --git a/etc/kayobe/ansible/pulp-artifact-upload.yml b/etc/kayobe/ansible/pulp-artifact-upload.yml new file mode 100644 index 000000000..450d5348a --- /dev/null +++ b/etc/kayobe/ansible/pulp-artifact-upload.yml @@ -0,0 +1,181 @@ +--- +- name: Upload and create a distribution for an artifact + hosts: seed + vars: + remote_pulp_url: "{{ stackhpc_release_pulp_url }}" + remote_pulp_username: "{{ stackhpc_release_pulp_username }}" + remote_pulp_password: "{{ stackhpc_release_pulp_password }}" + repository_name: "{{ artifact_type }}-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" + pulp_base_path: "{{ artifact_type }}/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + upload_checksum: false + tasks: + - name: Print artifact tag + ansible.builtin.debug: + msg: "artifact tag: {{ artifact_tag }}" + + - name: Get filename + ansible.builtin.find: + paths: "{{ artifact_path }}" + patterns: "{{ file_regex }}" + register: found_files + + - name: Install pip packages + ansible.builtin.pip: + name: + - pyopenssl + - ndg-httpsclient + - pyasn1 + - urllib3 + state: present + + - name: Upload an artifact + pulp.squeezer.artifact: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + file: "{{ found_files.files[0].path }}" + state: present + register: upload_result + until: upload_result is success + retries: 3 + delay: 60 + + - name: Get sha256 hash + ansible.builtin.stat: + path: "{{ found_files.files[0].path }}" + checksum_algorithm: sha256 + register: file_stats + + - name: Write checksum to file + ansible.builtin.copy: + content: "{{ file_stats.stat.checksum }}" + dest: "/tmp/{{ found_files.files[0].path | basename }}.sha256" + when: upload_checksum + + - name: Get sha256 hash for checksum + ansible.builtin.stat: + path: "/tmp/{{ found_files.files[0].path | basename }}.sha256" + checksum_algorithm: sha256 + register: checksum_stats + + - name: Upload checksum artifact + pulp.squeezer.artifact: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + file: "/tmp/{{ found_files.files[0].path | basename }}.sha256" + state: present + register: checksum_upload_result + until: checksum_upload_result is success + retries: 3 + delay: 60 + when: upload_checksum + + - name: Create file content from artifact + pulp.squeezer.file_content: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + sha256: "{{ file_stats.stat.checksum }}" + relative_path: "{{ found_files.files[0].path | basename }}" + state: present + register: file_content_result + until: file_content_result is success + retries: 3 + delay: 5 + + - name: Create checksum content from artifact + pulp.squeezer.file_content: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + sha256: "{{ checksum_stats.stat.checksum }}" + relative_path: "{{ found_files.files[0].path | basename }}.sha256" + state: present + register: checksum_content_result + until: checksum_content_result is success + retries: 3 + delay: 5 + when: upload_checksum + + - name: Ensure file repo exists + pulp.squeezer.file_repository: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}" + state: present + register: file_repo_result + until: file_repo_result is success + retries: 3 + delay: 5 + + - name: Add content to file repo + pulp.squeezer.file_repository_content: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + repository: "{{ repository_name }}" + present_content: + - relative_path: "{{ found_files.files[0].path | basename }}" + sha256: "{{ file_stats.stat.checksum }}" + register: file_repo_content_result + until: file_repo_content_result is success + retries: 3 + delay: 5 + + - name: Add checksum content to file repo + pulp.squeezer.file_repository_content: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + repository: "{{ repository_name }}" + present_content: + - relative_path: "{{ found_files.files[0].path | basename }}.sha256" + sha256: "{{ checksum_stats.stat.checksum }}" + register: checksum_repo_content_result + until: checksum_repo_content_result is success + retries: 3 + delay: 5 + when: upload_checksum + + - name: Create a new publication to point to this version + pulp.squeezer.file_publication: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + repository: "{{ repository_name }}" + state: present + register: publication_details + until: publication_details is success + retries: 3 + delay: 5 + + - name: Create distribution for given version + pulp.squeezer.file_distribution: + pulp_url: "{{ remote_pulp_url }}" + username: "{{ remote_pulp_username }}" + password: "{{ remote_pulp_password }}" + name: "{{ repository_name }}_{{ artifact_tag }}" + base_path: "{{ pulp_base_path }}/{{ artifact_tag }}" + publication: "{{ publication_details.publication.pulp_href }}" + content_guard: development + state: present + register: distribution_result + until: distribution_result is success + retries: 3 + delay: 5 + + - name: Update new artifacts file with versioned path + ansible.builtin.lineinfile: + path: /tmp/updated_artifacts.txt + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/{{ artifact_tag }}/{{ found_files.files[0].path | basename }}" + create: true + + - name: Print versioned path + ansible.builtin.debug: + msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/{{ artifact_tag }}/{{ found_files.files[0].path | basename }}" + + - name: Print version tag + ansible.builtin.debug: + msg: "New tag: {{ artifact_tag }}" diff --git a/etc/kayobe/ansible/pulp-auth-proxy.yml b/etc/kayobe/ansible/pulp-auth-proxy.yml new file mode 100644 index 000000000..728963858 --- /dev/null +++ b/etc/kayobe/ansible/pulp-auth-proxy.yml @@ -0,0 +1,15 @@ +--- +# See roles/pulp_auth_proxy/README.md for details. + +- name: Deploy Pulp auth proxy + hosts: container-image-builders + gather_facts: false + tasks: + - name: Deploy Pulp auth proxy + ansible.builtin.import_role: + name: pulp_auth_proxy + vars: + pulp_auth_proxy_url: "{{ stackhpc_release_pulp_url }}" + pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}" + pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}" + pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy" diff --git a/etc/kayobe/ansible/pulp-container-publish.yml b/etc/kayobe/ansible/pulp-container-publish.yml new file mode 100644 index 000000000..e98acf601 --- /dev/null +++ b/etc/kayobe/ansible/pulp-container-publish.yml @@ -0,0 +1,11 @@ +--- +- name: Publish Pulp container images + hosts: localhost + gather_facts: true + tasks: + - name: Ensure repository publications exist + ansible.builtin.import_role: + name: stackhpc.pulp.pulp_distribution + # NOTE: use intermediate variable to avoid distributing RPMs. + vars: + pulp_distribution_container: "{{ stackhpc_pulp_distribution_container }}" diff --git a/etc/kayobe/ansible/pulp-container-sync.yml b/etc/kayobe/ansible/pulp-container-sync.yml new file mode 100644 index 000000000..285387778 --- /dev/null +++ b/etc/kayobe/ansible/pulp-container-sync.yml @@ -0,0 +1,11 @@ +--- +- name: Sync client Pulp container repositories + hosts: localhost + gather_facts: false + tasks: + - name: Sync Pulp container repositories + ansible.builtin.import_role: + name: stackhpc.pulp.pulp_repository + # NOTE: use intermediate variable to avoid syncing RPMs. + vars: + pulp_repository_container_repos: "{{ stackhpc_pulp_repository_container_repos }}" diff --git a/etc/kayobe/ansible/pulp-host-image-download.yml b/etc/kayobe/ansible/pulp-host-image-download.yml new file mode 100644 index 000000000..54fd36ae4 --- /dev/null +++ b/etc/kayobe/ansible/pulp-host-image-download.yml @@ -0,0 +1,34 @@ +--- +- name: Download an overcloud host image from Ark + hosts: localhost + vars: + # This var is an edited version of stackhpc_overcloud_host_image_url + # without the auth credentials in it. Auth is handled by username and + # password in the get_url task of this playbook + stackhpc_overcloud_host_image_url_no_auth: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/\ + {{ openstack_release }}/{{ os_distribution }}/{{ os_release }}/\ + {{ stackhpc_overcloud_host_image_version }}/\ + overcloud-{{ os_distribution }}-{{ os_release }}.qcow2" + tasks: + - name: Print image information + ansible.builtin.debug: + msg: | + OS Distribution: {{ os_distribution }} + OS Release: {{ os_release }} + Image tag: {{ stackhpc_overcloud_host_image_version }} + + # TODO: Add checksum support + - name: Download image artifact + ansible.builtin.get_url: + url: "{{ stackhpc_overcloud_host_image_url_no_auth }}" + username: "{{ stackhpc_release_pulp_username }}" + password: "{{ stackhpc_release_pulp_password }}" + force_basic_auth: true + unredirected_headers: + - Authorization + dest: /tmp/{{ os_distribution }}-{{ os_release }}.qcow2 + mode: "0644" + register: image_download_result + until: image_download_result.status_code == 200 + retries: 3 + delay: 60 diff --git a/etc/kayobe/ansible/pulp-repo-promote-production.yml b/etc/kayobe/ansible/pulp-repo-promote-production.yml new file mode 100644 index 000000000..653716687 --- /dev/null +++ b/etc/kayobe/ansible/pulp-repo-promote-production.yml @@ -0,0 +1,11 @@ +--- +- name: Promote development Pulp repositories to production + hosts: localhost + gather_facts: true + tasks: + - name: Ensure repository distributions exist + ansible.builtin.import_role: + name: stackhpc.pulp.pulp_distribution + vars: + pulp_distribution_deb: "{{ stackhpc_pulp_distribution_deb_production | selectattr('required') }}" + pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_production | selectattr('required') }}" diff --git a/etc/kayobe/ansible/pulp-repo-publish.yml b/etc/kayobe/ansible/pulp-repo-publish.yml new file mode 100644 index 000000000..1b8ca1f2d --- /dev/null +++ b/etc/kayobe/ansible/pulp-repo-publish.yml @@ -0,0 +1,19 @@ +--- +- name: Publish Pulp repositories + hosts: localhost + gather_facts: false + tasks: + - name: Ensure repository publications exist + ansible.builtin.import_role: + name: stackhpc.pulp.pulp_publication + # NOTE: use intermediate variable to avoid publishing containers. + vars: + pulp_publication_deb: "{{ stackhpc_pulp_publication_deb_development | selectattr('required') }}" + pulp_publication_rpm: "{{ stackhpc_pulp_publication_rpm_development | selectattr('required') }}" + + - name: Ensure repository distributions exist + ansible.builtin.import_role: + name: stackhpc.pulp.pulp_distribution + vars: + pulp_distribution_deb: "{{ stackhpc_pulp_distribution_deb_development | selectattr('required') }}" + pulp_distribution_rpm: "{{ stackhpc_pulp_distribution_rpm_development | selectattr('required') }}" diff --git a/etc/kayobe/ansible/pulp-repo-sync.yml b/etc/kayobe/ansible/pulp-repo-sync.yml new file mode 100644 index 000000000..e47a2f572 --- /dev/null +++ b/etc/kayobe/ansible/pulp-repo-sync.yml @@ -0,0 +1,12 @@ +--- +- name: Sync Pulp repositories + hosts: localhost + gather_facts: false + tasks: + - name: Sync Pulp repositories + ansible.builtin.import_role: + name: stackhpc.pulp.pulp_repository + # NOTE: use intermediate variable to avoid syncing containers. + vars: + pulp_repository_deb_repos: "{{ stackhpc_pulp_repository_deb_repos | selectattr('required') }}" + pulp_repository_rpm_repos: "{{ stackhpc_pulp_repository_rpm_repos | selectattr('required') }}" diff --git a/etc/kayobe/ansible/pulp-sync-publish-promote.yml b/etc/kayobe/ansible/pulp-sync-publish-promote.yml new file mode 100644 index 000000000..a2e460662 --- /dev/null +++ b/etc/kayobe/ansible/pulp-sync-publish-promote.yml @@ -0,0 +1,16 @@ +--- +- name: "Import pulp-repo-sync" + import_playbook: pulp-repo-sync.yml + +- name: "Import pulp-repo-publish" + import_playbook: pulp-repo-publish.yml + +- name: "Import pulp-repo-promote-production" + import_playbook: pulp-repo-promote-production.yml + when: repo_promote_production | default(true) + +- name: "Import pulp-container-sync" + import_playbook: pulp-container-sync.yml + +- name: "Import pulp-container-publish" + import_playbook: pulp-container-publish.yml diff --git a/etc/kayobe/ansible/purge-command-not-found.yml b/etc/kayobe/ansible/purge-command-not-found.yml new file mode 100644 index 000000000..855bfbc10 --- /dev/null +++ b/etc/kayobe/ansible/purge-command-not-found.yml @@ -0,0 +1,33 @@ +--- +# Currently the pulp_deb plugin in Pulp does not support certain types of +# content, including i18n files and command-not-found indices. This breaks APT +# when the command-not-found is installed. This playbook can be used to +# uninstall the package, prior to running any other APT commands. It may be +# installed as a hook to the 'host configure' commands. +# See https://github.com/pulp/pulp_deb/issues/419 +# FIXME: If used as a hook, this playbook matches all hosts, so will run +# against the seed, even when running 'overcloud host configure'. Depending on +# the stage of deployment, some hosts may be unreachable. This could be fixed +# by implementing this playbook separately for each group. +- name: Purge command-not-found package + hosts: seed-hypervisor:seed:overcloud:infra-vms + gather_facts: false + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: -o StrictHostKeyChecking=no + tasks: + - name: Purge command-not-found package + ansible.builtin.package: + name: + - command-not-found + - python3-command-not-found + purge: true + state: absent + become: true + failed_when: false + ignore_unreachable: true + when: os_distribution == "ubuntu" diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml new file mode 100644 index 000000000..b686c7414 --- /dev/null +++ b/etc/kayobe/ansible/push-ofed.yml @@ -0,0 +1,81 @@ +--- +- name: Push OFED packages + hosts: ofed-builder + vars: + venv: "/opt/kayobe/venvs/kayobe" + doca_extract_path: "/home/stack/doca" + tasks: + - name: Get OFED module repo variables + ansible.builtin.set_fact: + doca_modules_repo_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='name') | join('') }}" + doca_modules_repo_base_path: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='base_path') | join('') }}" + doca_modules_repo_distribution_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='distribution_name') | join('') }}" + + - name: Install python dependencies + ansible.builtin.pip: + name: pulp-cli + + - name: Ensure Pulp configuration directory exists + ansible.builtin.file: + path: /home/stack/.config/pulp/ + state: directory + recurse: true + + - name: Setup Pulp credentials + ansible.builtin.blockinfile: + path: /home/stack/.config/pulp/cli.toml + create: true + block: | + [cli] + base_url = '{{ stackhpc_release_pulp_url }}' + verify_ssl = true + format = "json" + username = '{{ stackhpc_release_pulp_username }}' + password = '{{ stackhpc_release_pulp_password }}' + no_log: true + + - name: Find DOCA kernel repo package + ansible.builtin.shell: + cmd: 'find /tmp/DOCA.* -name doca-kernel-repo-*' + register: doca_kernel_repo + changed_when: false + + - name: Create Pulp repository for DOCA kernel modules + pulp.squeezer.rpm_repository: + pulp_url: "{{ stackhpc_release_pulp_url }}" + username: "{{ stackhpc_release_pulp_username }}" + password: "{{ stackhpc_release_pulp_password }}" + name: "{{ doca_modules_repo_name }}" + state: present + retries: "{{ pulp_timeout_retries | default(3) }}" + + - name: Upload OFED RPMs to Pulp + ansible.builtin.shell: + cmd: | + {{ venv }}/bin/pulp \ + rpm content \ + --type package upload \ + --repository '{{ doca_modules_repo_name }}' \ + --file {{ item }} \ + --no-publish \ + with_items: "{{ doca_kernel_repo.stdout_lines }}" + + - name: Create Pulp publication for OFED + pulp.squeezer.rpm_publication: + pulp_url: "{{ stackhpc_release_pulp_url }}" + username: "{{ stackhpc_release_pulp_username }}" + password: "{{ stackhpc_release_pulp_password }}" + repository: "{{ doca_modules_repo_name }}" + state: present + register: publication + + - name: Create Pulp distribution for OFED + pulp.squeezer.rpm_distribution: + pulp_url: "{{ stackhpc_release_pulp_url }}" + username: "{{ stackhpc_release_pulp_username }}" + password: "{{ stackhpc_release_pulp_password }}" + name: "{{ doca_modules_repo_distribution_name + ofed_tag }}" + publication: "{{ publication.publication.pulp_href }}" + content_guard: release + base_path: "{{ doca_modules_repo_base_path + ofed_tag }}" + state: present diff --git a/etc/kayobe/ansible/rabbitmq-reset.yml b/etc/kayobe/ansible/rabbitmq-reset.yml new file mode 100644 index 000000000..ef013217b --- /dev/null +++ b/etc/kayobe/ansible/rabbitmq-reset.yml @@ -0,0 +1,77 @@ +--- +# Reset a broken RabbitMQ cluster. +# Also restarts all OpenStack services using RabbitMQ. + +- name: Reset RabbitMQ + hosts: controllers + become: true + gather_facts: false + tags: + - rabbitmq-reset + vars: + container_name: rabbitmq + tasks: + - name: Checking timedatectl status + ansible.builtin.command: timedatectl status + register: timedatectl_status + changed_when: false + + - name: Fail if the clock is not synchronized + ansible.builtin.fail: + msg: >- + timedatectl sees the system clock as unsynchronized. + You may need to force synchronisation using `chronyc makestep`. + Otherwise, please wait for synchronization. + when: + - "'synchronized: yes' not in timedatectl_status.stdout" + + - name: Inspect the {{ container_name }} container + ansible.builtin.shell: + cmd: docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ container_name }} + register: inspection + + - name: Ensure the {{ container_name }} container is running + ansible.builtin.command: systemctl start kolla-{{ container_name }}-container.service # noqa command-instead-of-module + when: inspection.stdout == 'false' + + - name: Wait for the {{ container_name }} container to reach state 'Running' + ansible.builtin.shell: + cmd: docker container inspect --format '{{ '{{' }} .State.Running {{ '}}' }}' {{ container_name }} + register: result + until: result.stdout == 'true' + retries: 10 + delay: 6 + + - name: Wait for the rabbitmq node to automatically start on container start + ansible.builtin.command: docker exec {{ container_name }} /bin/bash -c 'rabbitmqctl wait /var/lib/rabbitmq/mnesia/rabbitmq.pid --timeout 60' + when: inspection.stdout == 'false' + + - name: Stop app + ansible.builtin.command: docker exec {{ container_name }} /bin/bash -c 'rabbitmqctl stop_app' + + - name: Force reset app + ansible.builtin.command: docker exec {{ container_name }} /bin/bash -c 'rabbitmqctl force_reset' + + - name: Start app + ansible.builtin.command: docker exec {{ container_name }} /bin/bash -c 'rabbitmqctl start_app' + + - name: Wait for all nodes to join the cluster + ansible.builtin.command: docker exec {{ container_name }} /bin/bash -c 'rabbitmqctl await_online_nodes {{ groups['controllers'] | length }}' + +- name: Restart OpenStack services + hosts: controllers:compute + become: true + gather_facts: false + tags: + - restart-openstack + tasks: + # The following services use RabbitMQ. + - name: Restart OpenStack services + ansible.builtin.shell: + cmd: >- + set -o pipefail && + systemctl list-units --type=service --all --no-legend --plain | + egrep 'kolla-(barbican|blazar|cinder|cloudkitty|designate|heat|ironic|keystone|magnum|manila|neutron|nova|octavia)' | + awk '{ print $1 }' | + xargs -r systemctl restart + executable: "/bin/bash" diff --git a/etc/kayobe/ansible/reboot.yml b/etc/kayobe/ansible/reboot.yml new file mode 100644 index 000000000..63d5ce9ae --- /dev/null +++ b/etc/kayobe/ansible/reboot.yml @@ -0,0 +1,46 @@ +--- +- name: Reboot the host + hosts: "{{ reboot_hosts | default('seed-hypervisor:seed:overcloud:infra-vms') }}" + serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}" + gather_facts: false + vars: + reboot_timeout_s: "{{ 20 * 60 }}" + post_reboot_delay_s: 5 # Extra delay to ensure SSH is working reliably + reboot_with_bootstrap_user: false + ansible_user: "{{ bootstrap_user if reboot_with_bootstrap_user | bool else kayobe_ansible_user }}" + ansible_ssh_common_args: "{{ '-o StrictHostKeyChecking=no' if reboot_with_bootstrap_user | bool else '' }}" + ansible_python_interpreter: /usr/bin/python3 + confirm_reboot: false + tags: + - reboot + tasks: + - name: Prompt to confirm reboot + ansible.builtin.pause: + prompt: > + The following hosts will be rebooted: + {{ play_hosts | join(', ') }} + If you want to proceed type: yes + register: pause_prompt + when: not confirm_reboot + + - name: Fail if reboot is not confirmed + ansible.builtin.assert: + that: confirm_reboot | bool or pause_prompt.user_input == 'yes' + msg: > + Reboot has not been confirmed. You must either type 'yes' when + prompted, or set ``confirm_reboot: true``. + + - name: Reboot and wait + become: true + ansible.builtin.reboot: + post_reboot_delay: "{{ post_reboot_delay_s | int }}" + reboot_timeout: "{{ reboot_timeout_s | int }}" + search_paths: + # Systems running molly-guard hang waiting for confirmation before rebooting without this. + - /lib/molly-guard + # Default list: + - /sbin + - /bin + - /usr/sbin + - /usr/bin + - /usr/local/sbin diff --git a/etc/kayobe/ansible/rekey-hosts.yml b/etc/kayobe/ansible/rekey-hosts.yml new file mode 100644 index 000000000..eee15ead4 --- /dev/null +++ b/etc/kayobe/ansible/rekey-hosts.yml @@ -0,0 +1,118 @@ +--- +# Playbook to rotate SSH keys across the cloud. By default it will rotate the +# standard keys used by kayobe/kolla-ansible, but it can be configured for any +# keys. + +- name: Rekey hosts + hosts: overcloud,seed,seed-hypervisor,infra-vms + gather_facts: false + vars: + # The existing key is the key that is currently used to access overcloud hosts + existing_private_key_path: "{{ ssh_private_key_path }}" + existing_public_key_path: "{{ ssh_public_key_path }}" + # The new key is the key that will be generated by this playbook + new_private_key_path: "{{ ssh_private_key_path }}" + new_public_key_path: "{{ ssh_public_key_path }}" + new_key_type: "{{ ssh_key_type }}" + # The existing key will locally be moved to deprecated_key_path once it is replaced + deprecated_key_path: ~/old_ssh_key + rekey_users: + - stack + - kolla + rekey_remove_existing_key: false + tasks: + - name: Stat existing private key file + ansible.builtin.stat: + path: "{{ existing_private_key_path }}" + register: stat_result + delegate_to: localhost + run_once: true + + - name: Fail when existing private key does not exist + ansible.builtin.fail: + msg: No existing private key file found. Check existing_private_key_path is set correctly. + when: + - not stat_result.stat.exists + delegate_to: localhost + run_once: true + + - name: Stat existing public key file + ansible.builtin.stat: + path: "{{ existing_public_key_path }}" + register: stat_result + delegate_to: localhost + run_once: true + + - name: Fail when existing public key does not exist + ansible.builtin.fail: + msg: No existing public key file found. Check existing_public_key_path is set correctly. + when: + - not stat_result.stat.exists + delegate_to: localhost + run_once: true + + - name: Generate a new SSH key + community.crypto.openssh_keypair: + path: "{{ existing_private_key_path }}_new" + type: "{{ new_key_type }}" + delegate_to: localhost + run_once: true + + - name: Set new authorized keys + vars: + lookup_path: "{{ existing_private_key_path }}_new.pub" + ansible.posix.authorized_key: + user: "{{ item }}" + state: present + key: "{{ lookup('file', lookup_path) }}" + loop: "{{ rekey_users }}" + become: true + + - name: Locally deprecate existing key (private) + ansible.builtin.command: mv {{ existing_private_key_path }} {{ deprecated_key_path }} + delegate_to: localhost + run_once: true + + - name: Locally deprecate existing key (public) + ansible.builtin.command: mv {{ existing_public_key_path }} {{ deprecated_key_path }}.pub + delegate_to: localhost + run_once: true + + - name: Locally promote new key (private) + ansible.builtin.command: mv {{ existing_private_key_path }}_new {{ new_private_key_path }} + delegate_to: localhost + run_once: true + + - name: Locally promote new key (public) + ansible.builtin.command: mv {{ existing_private_key_path }}_new.pub {{ new_public_key_path }} + delegate_to: localhost + run_once: true + + - name: Remove old key + tags: remove-key + when: rekey_remove_existing_key | bool + block: + - name: Stat old key file + ansible.builtin.stat: + path: "{{ deprecated_key_path }}.pub" + register: stat_result + delegate_to: localhost + run_once: true + + - name: Fail when deprecated public key does not exist + ansible.builtin.fail: + msg: No deprecated public key file found. Check deprecated_key_path is set correctly. + when: + - not stat_result.stat.exists + delegate_to: localhost + run_once: true + + - name: Remove old key from hosts + vars: + lookup_path: "{{ deprecated_key_path }}.pub" + ansible.posix.authorized_key: + user: "{{ item }}" + state: absent + key: "{{ lookup('file', lookup_path) }}" + loop: "{{ rekey_users }}" + become: true diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 09bce9565..cc73b867d 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,8 +1,37 @@ --- collections: - - name: stackhpc.linux - version: 1.2.3 - name: stackhpc.cephadm version: 1.19.1 + # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the + # pulp_glue Python library being installed. + - name: pulp.squeezer + version: 0.0.13 + - name: stackhpc.pulp + version: 0.5.5 - name: stackhpc.hashicorp - version: 2.6.1 + version: 2.7.1 + - name: stackhpc.kayobe_workflows + version: 1.2.0 +roles: + - src: stackhpc.vxlan + version: 1.1.0 + - name: ansible-lockdown.ubuntu22_cis + src: https://github.com/ansible-lockdown/UBUNTU22-CIS + version: 1.4.1 + - name: ansible-lockdown.rhel9_cis + src: https://github.com/ansible-lockdown/RHEL9-CIS + version: 1.3.1 + - name: wazuh-ansible + src: https://github.com/stackhpc/wazuh-ansible + version: stackhpc-v4.10.0 + - name: geerlingguy.pip + version: 2.2.0 + - name: monolithprojects.github_actions_runner + version: 1.18.5 + - src: https://github.com/stackhpc/ansible-role-docker.git + name: geerlingguy.docker + version: stackhpc/7.0.1.1 + # (jackhodgkiss) Update once patch is merged and released upstream. + - src: https://github.com/stackhpc/ansible-gitlab-runner + name: riemers.gitlab-runner + version: use-ansible-facts diff --git a/etc/kayobe/ansible/reset-bls-entries.yml b/etc/kayobe/ansible/reset-bls-entries.yml new file mode 100644 index 000000000..e26593b70 --- /dev/null +++ b/etc/kayobe/ansible/reset-bls-entries.yml @@ -0,0 +1,39 @@ +--- +# Custom playbook to reset Boot Loader Specification (BLS) entries to resolve +# an issue with GRUB defaulting to an old kernel. This is adapted from a Bash +# script in diskimage-builder: +# https://opendev.org/openstack/diskimage-builder/src/branch/master/diskimage_builder/elements/rhel/post-install.d/03-reset-bls-entries + +- name: Reset BLS entries + hosts: "{{ reset_bls_host | default('overcloud') }}" + become: true + tags: + - reset-bls-entries + tasks: + - name: Get machine ID + ansible.builtin.command: cat /etc/machine-id + register: machine_id + check_mode: false + + - name: Find entries with wrong machine ID + ansible.builtin.find: + paths: /boot/loader/entries + patterns: "*.conf" + register: bls_entries + check_mode: false + + # We set force to false to avoid replacing an existing BLS entry with the + # correct machine ID. + - name: Rename entries with wrong machine ID + ansible.builtin.copy: + src: /boot/loader/entries/{{ item }} + dest: /boot/loader/entries/{{ item | ansible.builtin.regex_replace('^[a-f0-9]*', machine_id.stdout) }} + force: false + remote_src: true + with_items: "{{ bls_entries.files | map(attribute='path') | reject('search', machine_id.stdout) | map('basename') }}" + + - name: Remove entries with wrong machine ID + ansible.builtin.file: + path: /boot/loader/entries/{{ item }} + state: absent + with_items: "{{ bls_entries.files | map(attribute='path') | reject('search', machine_id.stdout) | map('basename') }}" diff --git a/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml b/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml new file mode 100644 index 000000000..570512017 --- /dev/null +++ b/etc/kayobe/ansible/rocky-97-ofed-upgrade.yml @@ -0,0 +1,106 @@ +--- +# This playbook performs the system upgrade from RockyLinux 9.6 to 9.7 on hosts +# using doca ofed kernel modules and utilities. +# It prevents the install of the dkms toolchain and dkms modules that would be +# used instead or our precompiled modules if a direct `dnf upgrade` was run. +# +# It must be run after new package snapshots have been merged and +# `pulp-repo-sync.yml` and `pulp-repo-publish.yml` have been run. +# Also `kayobe overcloud host configure -t dnf` must have been run for the new +# `doca.repo` to be present (the doca version is in the url) on the mlnx hosts. + +- name: Prepare upgrade from Rocky Linux 9.6 to 9.7 + hosts: mlnx + serial: "{{ lookup('env', 'ANSIBLE_SERIAL') | default(1, true) }}" + gather_facts: true + tags: pre + vars: + # we don't build kernel modules for each version, eg 5.14.0-611.13.1 has been built, + # but not 5.14.0-611.20.1. + doca_kernel_version: "{{ stackhpc_doca_kernel_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] }}" + tasks: + - name: Assert that hosts are running Rocky Linux 9.6 + ansible.builtin.assert: + that: + - ansible_facts.distribution == 'Rocky' + - ansible_facts.distribution_version == '9.6' + - os_distribution == 'rocky' + fail_msg: >- + This playbook is only designed for Rocky Linux 9.6 hosts. Ensure + that you are limiting it to only run on Rocky Linux 9.6 hosts and + os_distribution is set to rocky. This can be bypassed by setting the + skip_rocky_96_version_check variable to true. + when: not skip_rocky_96_version_check | default(false) | bool + + - name: Ensure doca kernel repo is up to date + ansible.builtin.dnf: + name: doca-kernel-repo + state: latest + update_cache: true + become: true + + # This is required by mlnx-ofa_kernel, and comes from the doca repository. + # It is already present when doca-ofed is installed, but will be upgraded. + - name: Ensure mlnx-tools is installed + ansible.builtin.dnf: + name: mlnx-tools + state: latest + update_cache: true + become: true + + - name: Ensure appropriate doca-kernel is installed + ansible.builtin.dnf: + name: "doca-kernel-{{ doca_kernel_version }}" + state: latest + disablerepo: doca + become: true + + # doca-ofed 3.2 starts to depend on the dkms modules. It was not the case + # in doca-ofed 2.9.3. + - name: Ensure doca-ofed is not present (upgrading it brings dkms) + ansible.builtin.dnf: + name: doca-ofed + state: absent + autoremove: false + become: true + + - name: Ensure latest doca-ofed-userspace instead of doca-ofed + ansible.builtin.dnf: + name: doca-ofed-userspace + state: latest + become: true + + - name: Upgrade all + ansible.builtin.dnf: + name: "*" + state: latest + become: true + +- name: Fix potential grub config preventing new kernel from being used + ansible.builtin.import_playbook: reset-bls-entries.yml + vars: + reset_bls_hosts: mlnx + +- name: Reboot to apply updates + ansible.builtin.import_playbook: reboot.yml + vars: + reboot_hosts: mlnx + +- name: Confirm the host is upgraded to Rocky Linux 9.7 + hosts: mlnx + tags: post + tasks: + - name: Update distribution facts + ansible.builtin.setup: + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" + + # Can fail (eg in multinode) when there are bad entries in grub config + # Fixed by `kayobe playbook run ansible/maintenance/reset-bls-entries.yml` + # and manual reboot + - name: Assert that hosts are now using Rocky Linux 9.7 + ansible.builtin.assert: + that: + - ansible_facts.distribution == 'Rocky' + - ansible_facts.distribution_version == '9.7' + - os_distribution == 'rocky' diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md new file mode 100644 index 000000000..66568a640 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/README.md @@ -0,0 +1,26 @@ +# Pulp Auth Proxy + +There is currently no practical, secure way to provide credentials for +accessing Ark's authenticated package repositories from within a Kolla build. +Docker provides [build +secrets](https://docs.docker.com/build/building/secrets/), but these must be +explicitly requested for each RUN statement, making them challenging to use in +Kolla. + +This role deploys an Nginx container that runs as a reverse proxy, injecting an +HTTP basic authentication header into requests. + +Because this proxy bypasses Pulp's authentication, it must not be exposed to +any untrusted environment. + +## Role variables + +* `pulp_auth_proxy_url`: URL of the Pulp server to proxy requests to. +* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to. +* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to. +* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx + configuration. +* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to + listen. Default is `127.0.0.1`. +* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen. + Default is 80. diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml new file mode 100644 index 000000000..bd13e071f --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml @@ -0,0 +1,8 @@ +--- +pulp_auth_proxy_url: +pulp_auth_proxy_username: +pulp_auth_proxy_password: +pulp_auth_proxy_conf_path: +pulp_auth_proxy_listen_ip: 127.0.0.1 +pulp_auth_proxy_listen_port: 80 +pulp_auth_proxy_network_mode: diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml new file mode 100644 index 000000000..66550b894 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml @@ -0,0 +1,56 @@ +--- +- name: Set facts + when: pulp_auth_proxy_network_mode is none + block: + - name: Check if Docker bridge network exists + community.docker.docker_host_info: + networks: true + register: pulp_auth_proxy_docker_host_info + + - name: Set a fact about the network mode + ansible.builtin.set_fact: + pulp_auth_proxy_network_mode: "{{ 'host' if pulp_auth_proxy_docker_host_info.networks | selectattr('Driver', 'equalto', 'bridge') | list | length == 0 else 'bridge' }}" + +- name: Assert that localhost is resolvable when using host networking + ansible.builtin.assert: + that: + - "'localhost' is ansible.utils.resolvable" + fail_msg: >- + localhost must be resolvable when using Docker host networking with this container. + Consider adding '127.0.0.1 localhost' to /etc/hosts. + when: pulp_auth_proxy_network_mode == 'host' + +- name: Ensure {{ pulp_auth_proxy_conf_path }} exists + ansible.builtin.file: + path: "{{ pulp_auth_proxy_conf_path }}" + state: directory + mode: "0700" + become: true + +- name: Ensure pulp_proxy.conf is templated + ansible.builtin.template: + src: pulp_proxy.conf.j2 + dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf" + mode: "0600" + become: true + register: pulp_auth_proxy_conf + +- name: Ensure pulp_proxy container is running + community.docker.docker_container: + name: pulp_proxy + image: nginx:stable-alpine + network_mode: "{{ pulp_auth_proxy_network_mode }}" + ports: + - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80" + restart_policy: "no" + restart: "{{ pulp_auth_proxy_conf is changed }}" + volumes: + - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro" + +- name: Wait for pulp_proxy container to become accessible + ansible.builtin.uri: + url: http://localhost/pulp/api/v3/status/ + register: pulp_auth_proxy_uri_result + until: pulp_auth_proxy_uri_result is success + retries: 30 + delay: 2 diff --git a/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 new file mode 100644 index 000000000..3d5a87ae7 --- /dev/null +++ b/etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2 @@ -0,0 +1,17 @@ +server { + listen {{ pulp_auth_proxy_listen_port }}; + server_name pulp_proxy; + location / { + proxy_pass {{ pulp_auth_proxy_url }}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }}; + # The important part: add basic auth header + proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}"; + proxy_pass_header Authorization; + # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027 + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2; + } +} diff --git a/etc/kayobe/ansible/rsyslog.yml b/etc/kayobe/ansible/rsyslog.yml new file mode 100644 index 000000000..79c04b82f --- /dev/null +++ b/etc/kayobe/ansible/rsyslog.yml @@ -0,0 +1,36 @@ +--- +# Custom playbook to configure rsyslog to forward operating system logs to +# fluentd, eventually reaching Kolla's OpenSearch cluster. +# +# Variables: +# * rsyslog_group: Host pattern against which to target the playbook. Default +# is 'overcloud'. + +- name: Configure rsyslog to forward messages + hosts: "{{ rsyslog_group | default('overcloud') }}" + become: true + + tasks: + - name: Ensure rsyslog is installed + ansible.builtin.package: + name: rsyslog + state: present + + - name: Ensure rsyslog is started and enabled + ansible.builtin.systemd: + state: started + enabled: true + name: rsyslog + + - name: Update rsyslog configuration + ansible.builtin.lineinfile: + path: /etc/rsyslog.conf + insertafter: ^#*.* @@remote-host:514 + line: "*.* @{{ internal_net_name | net_ip }}:5140" + notify: Restart rsyslog + + handlers: + - name: Restart rsyslog + ansible.builtin.systemd: + state: restarted + name: rsyslog diff --git a/etc/kayobe/ansible/run-container-hotfix.yml b/etc/kayobe/ansible/run-container-hotfix.yml new file mode 100644 index 000000000..ca175aff8 --- /dev/null +++ b/etc/kayobe/ansible/run-container-hotfix.yml @@ -0,0 +1,25 @@ +--- +- name: Run container_hotfix + when: container_hotfix_files != [] + block: + - name: Ensure any required directories exist in container(s) + ansible.builtin.command: "{{ kolla_container_engine | default('docker') }} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} mkdir + -p {{ item.dest | dirname }}" + loop: "{{ container_hotfix_files }}" + + - name: Copy file into container(s) + ansible.builtin.command: "{{ kolla_container_engine | default('docker') }} cp /tmp/hotfix-files/{{ index }} {{ hotfix_container }}:{{ item.dest }}" + loop: "{{ container_hotfix_files }}" + loop_control: + index_var: index + + - name: Set mode for copied files + ansible.builtin.command: "{{ kolla_container_engine | default('docker') }} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} chmod + {{ item.mode | default('400') }} {{ item.dest }}" + loop: "{{ container_hotfix_files }}" + loop_control: + index_var: index + +- name: Run container_hotfix_command + ansible.builtin.command: "{{ kolla_container_engine | default('docker') }} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} {{ container_hotfix_command }}" + when: container_hotfix_command diff --git a/etc/kayobe/ansible/scripts/aio-init.sh b/etc/kayobe/ansible/scripts/aio-init.sh new file mode 100644 index 000000000..b50ccfd38 --- /dev/null +++ b/etc/kayobe/ansible/scripts/aio-init.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +#!/bin/bash + +set -o errexit +set -o pipefail + + +KOLLA_DEBUG=${KOLLA_DEBUG:-0} + +KOLLA_OPENSTACK_COMMAND=${KOLLA_OPENSTACK_COMMAND:-openstack} + +if [[ $KOLLA_DEBUG -eq 1 ]]; then + set -o xtrace + KOLLA_OPENSTACK_COMMAND="$KOLLA_OPENSTACK_COMMAND --debug" +fi + +# This script is meant to be run once after running start for the first +# time. This script downloads a cirros image and registers it. Then it +# configures networking and nova quotas to allow 40 m1.small instances +# to be created. + +ARCH=$(uname -m) +IMAGE_PATH=/opt/cache/files/ +IMAGE_URL=https://github.com/cirros-dev/cirros/releases/download/0.6.2/ +IMAGE=cirros-0.6.2-${ARCH}-disk.img +IMAGE_NAME=cirros +IMAGE_TYPE=linux + +# This EXT_NET_CIDR is your public network,that you want to connect to the internet via. +ENABLE_EXT_NET=${ENABLE_EXT_NET:-1} +EXT_NET_CIDR=${EXT_NET_CIDR:-'10.0.2.0/24'} +EXT_NET_RANGE=${EXT_NET_RANGE:-'start=10.0.2.150,end=10.0.2.199'} +EXT_NET_GATEWAY=${EXT_NET_GATEWAY:-'10.0.2.1'} + +# Sanitize language settings to avoid commands bailing out +# with "unsupported locale setting" errors. +unset LANG +unset LANGUAGE +LC_ALL=C +export LC_ALL +for i in curl "$KOLLA_OPENSTACK_COMMAND"; do + if [[ ! $(type ${i} 2>/dev/null) ]]; then + if [ "${i}" == 'curl' ]; then + echo "Please install ${i} before proceeding" + else + echo "Please install python-${i}client before proceeding" + fi + exit + fi +done + +# Test for credentials set +if [[ "${OS_USERNAME}" == "" ]]; then + echo "No Keystone credentials specified. Try running source /etc/kolla/admin-openrc.sh command" + exit +fi + +# Test to ensure configure script is run only once +if $KOLLA_OPENSTACK_COMMAND image list | grep -q cirros; then + echo "This tool should only be run once per deployment." + exit +fi + +echo Checking for locally available cirros image. +# Let's first try to see if the image is available locally +# nodepool nodes caches them in $IMAGE_PATH +if ! [ -f "${IMAGE_PATH}/${IMAGE}" ]; then + IMAGE_PATH='./' + if ! [ -f "${IMAGE_PATH}/${IMAGE}" ]; then + echo None found, downloading cirros image. + curl --fail -L -o ${IMAGE_PATH}/${IMAGE} ${IMAGE_URL}/${IMAGE} + fi +else + echo Using cached cirros image from the nodepool node. +fi + +echo Creating glance image. +$KOLLA_OPENSTACK_COMMAND image create --disk-format qcow2 --container-format bare --public \ + --property os_type=${IMAGE_TYPE} --file ${IMAGE_PATH}/${IMAGE} ${IMAGE_NAME} + +echo Configuring neutron. + +$KOLLA_OPENSTACK_COMMAND router create demo-router + +$KOLLA_OPENSTACK_COMMAND network create demo-net +$KOLLA_OPENSTACK_COMMAND subnet create --subnet-range 10.0.0.0/24 --network demo-net \ + --gateway 10.0.0.1 --dns-nameserver 8.8.8.8 demo-subnet +$KOLLA_OPENSTACK_COMMAND router add subnet demo-router demo-subnet + +if [[ $ENABLE_EXT_NET -eq 1 ]]; then + $KOLLA_OPENSTACK_COMMAND network create --external --provider-physical-network physnet1 \ + --provider-network-type flat public1 + $KOLLA_OPENSTACK_COMMAND subnet create --no-dhcp \ + --allocation-pool ${EXT_NET_RANGE} --network public1 \ + --subnet-range ${EXT_NET_CIDR} --gateway ${EXT_NET_GATEWAY} public1-subnet + $KOLLA_OPENSTACK_COMMAND router set --external-gateway public1 demo-router +fi + +# Get admin user and tenant IDs +ADMIN_USER_ID=$($KOLLA_OPENSTACK_COMMAND user list | awk '/ admin / {print $2}') +ADMIN_PROJECT_ID=$($KOLLA_OPENSTACK_COMMAND project list | awk '/ admin / {print $2}') +ADMIN_SEC_GROUP=$($KOLLA_OPENSTACK_COMMAND security group list --project ${ADMIN_PROJECT_ID} | awk '/ default / {print $2}') + +# Sec Group Config +$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \ + --protocol icmp ${ADMIN_SEC_GROUP} +$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \ + --protocol tcp --dst-port 22 ${ADMIN_SEC_GROUP} +# Open heat-cfn so it can run on a different host +$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \ + --protocol tcp --dst-port 8000 ${ADMIN_SEC_GROUP} +$KOLLA_OPENSTACK_COMMAND security group rule create --ingress --ethertype IPv4 \ + --protocol tcp --dst-port 8080 ${ADMIN_SEC_GROUP} + +if [ ! -f ~/.ssh/id_rsa.pub ]; then + echo Generating ssh key. + ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa +fi +if [ -r ~/.ssh/id_rsa.pub ]; then + echo Configuring nova public key and quotas. + $KOLLA_OPENSTACK_COMMAND keypair create --public-key ~/.ssh/id_rsa.pub mykey +fi + +# Increase the quota to allow 40 m1.small instances to be created + +# 40 instances +$KOLLA_OPENSTACK_COMMAND quota set --instances 40 ${ADMIN_PROJECT_ID} + +# 40 cores +$KOLLA_OPENSTACK_COMMAND quota set --cores 40 ${ADMIN_PROJECT_ID} + +# 96GB ram +$KOLLA_OPENSTACK_COMMAND quota set --ram 96000 ${ADMIN_PROJECT_ID} + +# add default flavors, if they don't already exist +if ! $KOLLA_OPENSTACK_COMMAND flavor list | grep -q m1.tiny; then + $KOLLA_OPENSTACK_COMMAND flavor create --id 1 --ram 512 --disk 1 --vcpus 1 m1.tiny + $KOLLA_OPENSTACK_COMMAND flavor create --id 2 --ram 2048 --disk 20 --vcpus 1 m1.small + $KOLLA_OPENSTACK_COMMAND flavor create --id 3 --ram 4096 --disk 40 --vcpus 2 m1.medium + $KOLLA_OPENSTACK_COMMAND flavor create --id 4 --ram 8192 --disk 80 --vcpus 4 m1.large + $KOLLA_OPENSTACK_COMMAND flavor create --id 5 --ram 16384 --disk 160 --vcpus 8 m1.xlarge +fi + +touch /tmp/.init-runonce diff --git a/etc/kayobe/ansible/scripts/generate_fixtures.py b/etc/kayobe/ansible/scripts/generate_fixtures.py new file mode 100644 index 000000000..5f8f7cc64 --- /dev/null +++ b/etc/kayobe/ansible/scripts/generate_fixtures.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +import json +import re +from pySMART import DeviceList + +SMARTMON_ATTRS = { + "airflow_temperature_cel", + "command_timeout", + "current_pending_sector", + "end_to_end_error", + "erase_fail_count", + "g_sense_error_rate", + "hardware_ecc_recovered", + "host_reads_32mib", + "host_reads_mib", + "host_writes_32mib", + "host_writes_mib", + "load_cycle_count", + "media_wearout_indicator", + "nand_writes_1gib", + "offline_uncorrectable", + "power_cycle_count", + "power_on_hours", + "program_fail_cnt_total", + "program_fail_count", + "raw_read_error_rate", + "reallocated_event_count", + "reallocated_sector_ct", + "reported_uncorrect", + "runtime_bad_block", + "sata_downshift_count", + "seek_error_rate", + "spin_retry_count", + "spin_up_time", + "start_stop_count", + "temperature_case", + "temperature_celsius", + "temperature_internal", + "total_lbas_read", + "total_lbas_written", + "udma_crc_error_count", + "unsafe_shutdown_count", + "unused_rsvd_blk_cnt_tot", + "wear_leveling_count", + "workld_host_reads_perc", + "workld_media_wear_indic", + "workload_minutes", + "critical_warning", + "temperature", + "available_spare", + "available_spare_threshold", + "percentage_used", + "data_units_read", + "data_units_written", + "host_reads", + "host_writes", + "controller_busy_time", + "power_cycles", + "unsafe_shutdowns", + "media_errors", + "num_err_log_entries", + "warning_temp_time", + "critical_comp_time", +} + +DISK_INFO = { + "name", + "interface", + "vendor", + "family", + "model", + "serial", + "firmware", + "smart_capable", + "smart_enabled", + "assessment", +} + +def camel_to_snake(name): + """ + Convert a CamelCase string to snake_case. + + Reference: https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case + """ + return re.sub(r'(? + +# Check if we are root +if [ "$EUID" -ne 0 ]; then + echo "${0##*/}: Please run as root!" >&2 + exit 1 +fi + +# Check if programs are installed +if ! command -v nvme >/dev/null 2>&1; then + echo "${0##*/}: nvme is not installed. Aborting." >&2 + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "${0##*/}: jq is required but not installed. Aborting." >&2 + exit 1 +fi + +# Path to the DWPD ratings JSON file +dwpd_file="/opt/kayobe/etc/monitoring/dwpd_ratings.json" + +declare -A rated_dwpd + +load_dwpd_ratings() { + if [[ -f "$dwpd_file" ]]; then + # Read the JSON; if it fails, default to empty array + dwpd_json="$(cat "$dwpd_file" 2>/dev/null | jq '.' || echo '[]')" + + # We iterate over each array element in dwpd_json + while IFS= read -r line; do + key="$(echo "$line" | jq -r '.model_name')" + value="$(echo "$line" | jq -r '.rated_dwpd')" + + # Clean up trailing whitespace + key="${key%%[[:space:]]*}" + value="${value%%[[:space:]]*}" + + # If we have a valid key, store it in the dictionary + if [[ -n "$key" && "$key" != "null" ]]; then + rated_dwpd["$key"]="$value" + fi + done < <(echo "$dwpd_json" | jq -c '.[]') + else + echo "Warning: DWPD ratings file not found at '$dwpd_file'. Defaulting to rated_dwpd=1." >&2 + fi +} + + +load_dwpd_ratings + +output_format_awk="$( + cat <<'OUTPUTAWK' +BEGIN { v = "" } +v != $1 { + print "# HELP nvme_" $1 " SMART metric " $1; + if ($1 ~ /_total$/) + print "# TYPE nvme_" $1 " counter"; + else + print "# TYPE nvme_" $1 " gauge"; + v = $1 +} +{print "nvme_" $0} +OUTPUTAWK +)" + +format_output() { + sort | awk -F'{' "${output_format_awk}" +} + +# Get the nvme-cli version +nvme_version="$(nvme version | awk '$1 == "nvme" {print $3}')" +echo "nvmecli{version=\"${nvme_version}\"} 1" | format_output + +# Get devices (DevicePath, PhysicalSize and ModelNumber) +device_info="$(nvme list -o json | jq -c '.Devices[] | {DevicePath, PhysicalSize, ModelNumber, SerialNumber}')" + +# Convert device_info to an array +device_info_array=() +while IFS= read -r line; do + device_info_array+=("$line") +done <<< "$device_info" + +# Loop through the NVMe devices +for device_data in "${device_info_array[@]}"; do + device="$(echo "$device_data" | jq -r '.DevicePath')" + json_check="$(nvme smart-log -o json "${device}")" + disk="${device##*/}" + model_name="$(echo "$device_data" | jq -r '.ModelNumber')" + serial_number="$(echo "$device_data" | jq -r '.SerialNumber')" + + physical_size="$(echo "$device_data" | jq -r '.PhysicalSize')" + echo "physical_size_bytes{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${physical_size}" + + # The temperature value in JSON is in Kelvin, we want Celsius + value_temperature="$(echo "$json_check" | jq '.temperature - 273')" + echo "temperature_celsius{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_temperature}" + + # Get the rated DWPD from the dictionary or default to 1 if not found + value_rated_dwpd="${rated_dwpd[$model_name]:-1}" + echo "rated_dwpd{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_rated_dwpd}" + + value_available_spare="$(echo "$json_check" | jq '.avail_spare / 100')" + echo "available_spare_ratio{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_available_spare}" + + value_available_spare_threshold="$(echo "$json_check" | jq '.spare_thresh / 100')" + echo "available_spare_threshold_ratio{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_available_spare_threshold}" + + value_percentage_used="$(echo "$json_check" | jq '.percent_used / 100')" + echo "percentage_used_ratio{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_percentage_used}" + + value_critical_warning="$(echo "$json_check" | jq '.critical_warning')" + echo "critical_warning_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_critical_warning}" + + value_media_errors="$(echo "$json_check" | jq '.media_errors')" + echo "media_errors_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_media_errors}" + + value_num_err_log_entries="$(echo "$json_check" | jq '.num_err_log_entries')" + echo "num_err_log_entries_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_num_err_log_entries}" + + value_power_cycles="$(echo "$json_check" | jq '.power_cycles')" + echo "power_cycles_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_power_cycles}" + + value_power_on_hours="$(echo "$json_check" | jq '.power_on_hours')" + echo "power_on_hours_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_power_on_hours}" + + value_controller_busy_time="$(echo "$json_check" | jq '.controller_busy_time')" + echo "controller_busy_time_seconds{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_controller_busy_time}" + + value_data_units_written="$(echo "$json_check" | jq '.data_units_written')" + echo "data_units_written_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_data_units_written}" + + value_data_units_read="$(echo "$json_check" | jq '.data_units_read')" + echo "data_units_read_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_data_units_read}" + + value_host_read_commands="$(echo "$json_check" | jq '.host_read_commands')" + echo "host_read_commands_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_host_read_commands}" + + value_host_write_commands="$(echo "$json_check" | jq '.host_write_commands')" + echo "host_write_commands_total{device=\"${disk}\",model=\"${model_name}\",serial_number=\"${serial_number}\"} ${value_host_write_commands}" +done | format_output diff --git a/etc/kayobe/ansible/scripts/smartmon.py b/etc/kayobe/ansible/scripts/smartmon.py new file mode 100644 index 000000000..033ddbb94 --- /dev/null +++ b/etc/kayobe/ansible/scripts/smartmon.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 + +import subprocess +import json +import re +import datetime +import os + +from prometheus_client import CollectorRegistry, Gauge, write_to_textfile +from pySMART import DeviceList + +SMARTCTL_PATH = "/usr/sbin/smartctl" + +SMARTMON_ATTRS = { + "airflow_temperature_cel", + "command_timeout", + "current_pending_sector", + "end_to_end_error", + "erase_fail_count", + "g_sense_error_rate", + "hardware_ecc_recovered", + "host_reads_32mib", + "host_reads_mib", + "host_writes_32mib", + "host_writes_mib", + "load_cycle_count", + "media_wearout_indicator", + "nand_writes_1gib", + "offline_uncorrectable", + "power_cycle_count", + "power_on_hours", + "program_fail_cnt_total", + "program_fail_count", + "raw_read_error_rate", + "reallocated_event_count", + "reallocated_sector_ct", + "reported_uncorrect", + "runtime_bad_block", + "sata_downshift_count", + "seek_error_rate", + "spin_retry_count", + "spin_up_time", + "start_stop_count", + "temperature_case", + "temperature_celsius", + "temperature_internal", + "total_lbas_read", + "total_lbas_written", + "udma_crc_error_count", + "unsafe_shutdown_count", + "unused_rsvd_blk_cnt_tot", + "wear_leveling_count", + "workld_host_reads_perc", + "workld_media_wear_indic", + "workload_minutes", + "critical_warning", + "temperature", + "available_spare", + "available_spare_threshold", + "percentage_used", + "data_units_read", + "data_units_written", + "host_reads", + "host_writes", + "controller_busy_time", + "power_cycles", + "unsafe_shutdowns", + "media_errors", + "num_err_log_entries", + "warning_temp_time", + "critical_comp_time", +} + +def run_command(command, parse_json=False): + """ + Helper to run a subprocess command and optionally parse JSON output. + """ + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if parse_json: + return json.loads(result.stdout) + return result.stdout.strip() + +def camel_to_snake(name): + """ + Convert a CamelCase string to snake_case. + + Reference: https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case + """ + return re.sub(r'(? 1, otherwise => 0 + assessment = device_info.get("assessment", "").upper() + if assessment in ["PASS", "WARN", "FAIL"]: + expected_val = float(1) if assessment == "PASS" else float(0) + smart_healthy_found = any( + line.startswith("smartmon_device_smart_healthy{") and + f'disk="{dev_name}"' in line and + line.endswith(f" {expected_val}") + for line in metrics + ) + self.assertTrue( + smart_healthy_found, + f"Expected smartmon_device_smart_healthy={expected_val} for {dev_name}, not found." + ) + + def test_parse_device_info(self): + """ + Test parse_device_info() for every JSON fixture in ./tests/. + Each fixture is tested individually with clear error reporting. + """ + for fixture_path in self.fixture_files: + fixture_name = os.path.basename(fixture_path) + with self.subTest(fixture=fixture_name): + self._test_parse_device_info(fixture_name) + + def _test_parse_if_attributes(self, fixture_name): + """ + Helper method to test parse_if_attributes() for a single JSON fixture. + """ + data = load_json_fixture(fixture_name) + device_info = data["device_info"] + if_attrs = data.get("if_attributes", {}) + + device = self.create_mock_device_from_json(device_info, if_attrs) + metrics = parse_if_attributes(device) + + dev_name = device_info["name"] + dev_iface = device_info["interface"] + dev_serial = device_info["serial"].lower() + + # For each numeric attribute in JSON, if it's in SMARTMON_ATTRS, + # we expect a line in the script's output. + for attr_key, attr_val in if_attrs.items(): + snake_key = camel_to_snake(attr_key) + + if isinstance(attr_val, (int, float)) and snake_key in SMARTMON_ATTRS: + expected_line = ( + f"smartmon_{snake_key}{{disk=\"{dev_name}\",serial_number=\"{dev_serial}\",type=\"{dev_iface}\"}} {float(attr_val)}" + ) + self.assertIn( + expected_line, + metrics, + f"Expected metric '{expected_line}' for attribute '{attr_key}' not found." + ) + else: + # If it's not in SMARTMON_ATTRS or not numeric, + # we do NOT expect a line with that name+value + unexpected_line = ( + f"smartmon_{snake_key}{{disk=\"{dev_name}\",serial_number=\"{dev_serial}\",type=\"{dev_iface}\"}} {float(attr_val)}" + ) + self.assertNotIn( + unexpected_line, + metrics, + f"Unexpected metric '{unexpected_line}' found for {attr_key}." + ) + + # Also ensure that non-numeric or disallowed attributes do not appear + # For instance "notInSmartmonAttrs" should never appear. + for line in metrics: + self.assertNotIn( + "not_in_smartmon_attrs", + line, + f"'notInSmartmonAttrs' attribute unexpectedly found in metric line: {line}" + ) + + def test_parse_if_attributes(self): + """ + Test parse_if_attributes() for every JSON fixture in ./tests/. + Each fixture is tested individually with clear error reporting. + """ + for fixture_path in self.fixture_files: + fixture_name = os.path.basename(fixture_path) + with self.subTest(fixture=fixture_name): + self._test_parse_if_attributes(fixture_name) + + @patch("smartmon.run_command") + @patch("smartmon.DeviceList") + @patch("smartmon.write_metrics_to_textfile", wraps=write_metrics_to_textfile) + def test_main(self, mock_write_metrics, mock_devicelist_class, mock_run_cmd): + """ + End-to-end test of main() for every JSON fixture in ./tests/. + This ensures we can handle multiple disks (multiple fixture files). + Checks metrics written to a temp file, and that write_metrics_to_textfile is called once. + """ + + # Patch run_command to return a version & "active" power_mode + def run_command_side_effect(cmd, parse_json=False): + if "--version" in cmd: + return "smartctl 7.3 5422 [x86_64-linux-5.15.0]\n..." + if "-n" in cmd and "standby" in cmd and parse_json: + return {"power_mode": "active"} + return "" + + mock_run_cmd.side_effect = run_command_side_effect + + for fixture_path in self.fixture_files: + fixture_name = os.path.basename(fixture_path) + with self.subTest(msg=f"Testing main() with {fixture_name}"): + mock_write_metrics.reset_mock() + data = load_json_fixture(fixture_name) + device_info = data["device_info"] + if_attrs = data.get("if_attributes", {}) + + # Mock a single device from the fixture + device_mock = self.create_mock_device_from_json(device_info, if_attrs) + + # Make DeviceList() return our single mock device + mock_dev_list = MagicMock() + mock_dev_list.devices = [device_mock] + mock_devicelist_class.return_value = mock_dev_list + + with tempfile.NamedTemporaryFile(mode="r+", delete_on_close=False) as tmpfile: + path= tmpfile.name + main(output_path=path) + tmpfile.close() + + # Ensure write_metrics_to_textfile was called once + self.assertEqual(mock_write_metrics.call_count, 1) + + with open(path, "r") as f: + # Read the metrics from the file + metrics_lines = [line.strip() for line in f.readlines() if line.strip() and not line.startswith('#')] + print(f"Metrics lines: {metrics_lines}") + + # Generate expected metrics using the parse functions + expected_metrics = [] + expected_metrics.extend(parse_device_info(device_mock)) + expected_metrics.extend(parse_if_attributes(device_mock)) + + # Check that all expected metrics are present in the file + for expected in expected_metrics: + exp_metric, exp_val_str = expected.rsplit(" ", 1) + exp_val = float(exp_val_str) + found = any( + (exp_metric in line) and + math.isclose(float(line.rsplit(" ", 1)[1]), exp_val) + for line in metrics_lines + ) + self.assertTrue(found, f"Expected metric '{expected}' not found") + + # Check that smartctl_version metric is present + version_found = any(line.startswith("smartmon_smartctl_version{") for line in metrics_lines) + self.assertTrue(version_found, "Expected 'smartmon_smartctl_version' metric not found in output file.") + + # Check that the output file is not empty + self.assertTrue(metrics_lines, "Metrics output file is empty.") + +if __name__ == "__main__": + unittest.main() diff --git a/etc/kayobe/ansible/scripts/tests/Dell_ENT_NVMe_CM6.json b/etc/kayobe/ansible/scripts/tests/Dell_ENT_NVMe_CM6.json new file mode 100644 index 000000000..d867910ae --- /dev/null +++ b/etc/kayobe/ansible/scripts/tests/Dell_ENT_NVMe_CM6.json @@ -0,0 +1,26 @@ +{ + "device_info": { + "assessment": "PASS", + "firmware": "2.1.8", + "interface": "nvme", + "model": "Dell Ent NVMe CM6 RI 7.68TB", + "name": "nvme8", + "serial": "Y2Q0A0BPTCF8", + "smart_capable": true, + "smart_enabled": true, + "vendor": "Dell" + }, + "if_attributes": { + "availableSpare": 100, + "availableSpareThreshold": 10, + "controllerBusyTime": 2478, + "criticalWarning": 0, + "dataUnitsRead": 177817765, + "dataUnitsWritten": 127992843, + "percentageUsed": 1, + "powerCycles": 750, + "powerOnHours": 17427, + "temperature": 36, + "unsafeShutdowns": 37 + } +} diff --git a/etc/kayobe/ansible/scripts/tests/nvme.json b/etc/kayobe/ansible/scripts/tests/nvme.json new file mode 100644 index 000000000..bbff19ec0 --- /dev/null +++ b/etc/kayobe/ansible/scripts/tests/nvme.json @@ -0,0 +1,24 @@ +{ + "device_info": { + "name": "/dev/nvme0", + "interface": "nvme", + "vendor": "AcmeCorp", + "family": "Acme NVMe Family", + "model": "Acme NVMe 1TB", + "serial": "ABCD1234", + "firmware": "3.0.1", + "smart_capable": true, + "smart_enabled": true, + "assessment": "PASS" + }, + "if_attributes": { + "criticalWarning": 0, + "temperature": 36, + "availableSpare": 100, + "availableSpareThreshold": 10, + "percentageUsed": 0, + "dataUnitsRead": 117446405, + "dataUnitsWritten": 84630284, + "notInSmartmonAttrs": 999 + } +} diff --git a/etc/kayobe/ansible/secret-store-deploy-barbican.yml b/etc/kayobe/ansible/secret-store-deploy-barbican.yml new file mode 100644 index 000000000..4324ff2ef --- /dev/null +++ b/etc/kayobe/ansible/secret-store-deploy-barbican.yml @@ -0,0 +1,107 @@ +--- +- name: Configure AppRole + any_errors_fatal: true + gather_facts: true + hosts: controllers[0] + vars: + secret_store_api_address: https://{{ internal_net_name | net_ip }}:8200 + secret_store_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + tasks: + - name: Assert that secrets_barbican_approle_secret_id is defined + ansible.builtin.assert: + that: + - secrets_barbican_approle_secret_id is defined + fail_msg: Please define secrets_barbican_approle_secret_id in your secrets.yml + + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Ensure hvac is installed + ansible.builtin.pip: + name: hvac + state: present + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv_path }}/kayobe" + + - name: Ensure AppRole is configured + environment: + https_proxy: "" + block: + - name: Enable AppRole auth module + hashivault_auth_method: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ secret_store_ca_cert }}" + token: "{{ secret_store_keys.root_token }}" + method_type: approle + state: enabled + + - name: Enable barbican kv store + hashivault_secret_engine: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ secret_store_ca_cert }}" + token: "{{ secret_store_keys.root_token }}" + name: barbican + backend: kv + description: Barbican kv store + + - name: Ensure barbican policy is defined + hashivault_policy: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ secret_store_ca_cert }}" + token: "{{ secret_store_keys.root_token }}" + name: barbican-policy + state: present + rules: | + path "barbican/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } + + - name: Ensure barbican AppRole is defined + hashivault_approle_role: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ secret_store_ca_cert }}" + token: "{{ secret_store_keys.root_token }}" + bind_secret_id: true + secret_id_bound_cidrs: "{{ internal_net_name | net_cidr }}" + secret_id_ttl: 0 + token_policies: barbican-policy + name: barbican + + - name: Get barbican Approle ID + hashivault_approle_role_id: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ secret_store_ca_cert }}" + token: "{{ secret_store_keys.root_token }}" + name: barbican + register: barbican_role_id + + - name: Print barbican Approle ID + ansible.builtin.debug: + msg: barbican role id is {{ barbican_role_id.id }} + + - name: Write barbican Approle ID to file if requested + delegate_to: localhost + ansible.builtin.copy: + content: "{{ barbican_role_id.id }}" + dest: "{{ stackhpc_barbican_role_id_file_path | default('~/barbican-role-id') }}" + when: stackhpc_write_barbican_role_id_to_file | default(false) | bool + + - name: Check if barbican Approle Secret ID is defined + hashivault_approle_role_secret_get: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ secret_store_ca_cert }}" + token: "{{ secret_store_keys.root_token }}" + secret: "{{ secrets_barbican_approle_secret_id }}" + name: barbican + register: barbican_approle_secret_get + + - name: Ensure barbican AppRole Secret ID is defined + hashivault_approle_role_secret: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ secret_store_ca_cert }}" + token: "{{ secret_store_keys.root_token }}" + secret: "{{ secrets_barbican_approle_secret_id }}" + name: barbican + when: barbican_approle_secret_get.status == "absent" diff --git a/etc/kayobe/ansible/secret-store-deploy-overcloud.yml b/etc/kayobe/ansible/secret-store-deploy-overcloud.yml new file mode 100644 index 000000000..aa9a200d1 --- /dev/null +++ b/etc/kayobe/ansible/secret-store-deploy-overcloud.yml @@ -0,0 +1,172 @@ +--- +# Required for uri module to work with self-signed certificates and for systems to trust +# the self-signed CA +- name: Install CA on controllers + hosts: controllers + tasks: + - name: Copy the intermediate CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/OS-TLS-ROOT.pem" + dest: "{{ '/etc/pki/ca-trust/source/anchors/OS-TLS-ROOT.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' + }}" + mode: "0644" + become: true + + - name: Update system CA + become: true + ansible.builtin.command: "{{ 'update-ca-trust' if ansible_facts.os_family == 'RedHat' else 'update-ca-certificates' }}" + +- name: Deploy Secret store on the overcloud + any_errors_fatal: true + gather_facts: true + hosts: controllers + vars: + secret_store_bind_interface: "{{ internal_net_name | net_interface }}" + secret_store_bind_address: "{{ internal_net_name | net_ip }}" + # This is the IP address of the first controller and therefore the leader within + # OpenBao. This could be replaced with the VIP address of the internal network if + # HAProxy has been configured to load balance the OpenBao API. + openbao_raft_leaders: + - "{{ internal_net_name | net_ip(inventory_hostname=groups['controllers'][0]) }}" + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure /opt/kayobe/{{ stackhpc_ca_secret_store }} exists + ansible.builtin.file: + path: /opt/kayobe/{{ stackhpc_ca_secret_store }} + state: directory + + - name: Template out TLS key and cert + ansible.builtin.copy: + # Within the OpenBao container these uids & gids map to the openbao user + src: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/{{ item }}" + dest: /opt/kayobe/{{ stackhpc_ca_secret_store }}/{{ item }} + owner: 100 + group: 1000 + mode: "0600" + loop: + - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" + - "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + - "OS-TLS-INT.crt" + become: true + + - name: Apply vault role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault + vars: + hashicorp_registry_url: "{{ overcloud_hashicorp_registry_url }}" + hashicorp_registry_username: "{{ overcloud_hashicorp_registry_username }}" + hashicorp_registry_password: "{{ overcloud_hashicorp_registry_password }}" + consul_docker_image: "{{ overcloud_consul_docker_image }}" + consul_docker_tag: "{{ overcloud_consul_docker_tag }}" + consul_bind_interface: "{{ secret_store_bind_interface }}" + vault_bind_address: "{{ secret_store_bind_address }}" + vault_config_dir: /opt/kayobe/vault + vault_cluster_name: overcloud + vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_docker_image: "{{ overcloud_vault_docker_image }}" + vault_docker_tag: "{{ overcloud_vault_docker_tag }}" + vault_tls_cert: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" + vault_tls_key: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + copy_self_signed_ca: true + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_write_keys_file: true + vault_write_keys_file_path: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + when: stackhpc_ca_secret_store == "vault" + + - name: Apply OpenBao role + ansible.builtin.import_role: + name: stackhpc.hashicorp.openbao + vars: + openbao_bind_addr: "{{ secret_store_bind_address }}" + openbao_registry_url: "{{ overcloud_openbao_registry_url }}" + openbao_registry_username: "{{ overcloud_openbao_registry_username }}" + openbao_registry_password: "{{ overcloud_openbao_registry_password }}" + openbao_config_dir: /opt/kayobe/openbao + openbao_cluster_name: overcloud + openbao_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + openbao_docker_image: "{{ overcloud_openbao_docker_image }}" + openbao_docker_tag: "{{ overcloud_openbao_docker_tag }}" + openbao_tls_cert: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.crt" + openbao_tls_key: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}.key" + openbao_tls_ca: "OS-TLS-INT.crt" + copy_self_signed_ca: true + openbao_api_addr: https://{{ internal_net_name | net_ip }}:8200 + openbao_write_keys_file: true + openbao_write_keys_file_path: "{{ kayobe_env_config_path }}/openbao/overcloud-openbao-keys.json" + when: stackhpc_ca_secret_store == "openbao" + + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Unseal first secret store instance + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ secret_store_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}" + environment: + https_proxy: "" + run_once: true + + # As the first instance is now unsealed the other instances will now need some + # time to connect before we can proceed. + - name: Wait for OpenBao Raft peers to connect + ansible.builtin.wait_for: + timeout: 30 + delegate_to: localhost + + # Raft peers take few seconds before they report an unsealed state therefore + # we must wait. + - name: Unseal all secret store instances + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ secret_store_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}" + vault_unseal_timeout: 10 + environment: + https_proxy: "" + +- name: Configure PKI + any_errors_fatal: true + gather_facts: true + hosts: controllers[0] + tasks: + - name: Apply pki role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_pki + vars: + vault_token: "{{ secret_store_keys.root_token }}" + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_pki_root_create: false + vault_pki_intermediate_import: true + vault_pki_intermediate_ca_name: OS-TLS-INT + vault_pki_intermediate_ca_bundle: "{{ lookup('file', kayobe_env_config_path + '/' + stackhpc_ca_secret_store + '/OS-TLS-INT.pem') }}" + vault_pki_intermediate_ca_cert: "{{ lookup('file', kayobe_env_config_path + '/' + stackhpc_ca_secret_store + '/OS-TLS-INT.crt') }}" + vault_pki_intermediate_roles: "{{ overcloud_vault_pki_roles if stackhpc_ca_secret_store == 'vault' else overcloud_openbao_pki_roles }}" + vault_pki_write_certificate_files: true + vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}" + environment: + https_proxy: "" diff --git a/etc/kayobe/ansible/secret-store-deploy-seed.yml b/etc/kayobe/ansible/secret-store-deploy-seed.yml new file mode 100644 index 000000000..44073edf6 --- /dev/null +++ b/etc/kayobe/ansible/secret-store-deploy-seed.yml @@ -0,0 +1,102 @@ +--- +- name: Deploy CA secret store on the seed + any_errors_fatal: true + gather_facts: true + hosts: seed + vars: + secret_store_bind_interface: lo + secret_store_bind_address: "{{ ansible_facts[secret_store_bind_interface].ipv4.address }}" + secret_store_api_address: "http://{{ secret_store_bind_address }}:8200" + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python PyYAML and hvac modules are installed + ansible.builtin.pip: + name: + - PyYAML + - hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Ensure secret store directory exists in Kayobe configuration + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/" + state: directory + delegate_to: localhost + run_once: true + + - name: Apply vault role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault + vars: + hashicorp_registry_url: "{{ seed_hashicorp_registry_url }}" + hashicorp_registry_username: "{{ seed_hashicorp_registry_username }}" + hashicorp_registry_password: "{{ seed_hashicorp_registry_password }}" + consul_docker_image: "{{ seed_consul_docker_image }}" + consul_docker_tag: "{{ seed_consul_docker_tag }}" + consul_bind_interface: "{{ secret_store_bind_interface }}" + vault_bind_address: "{{ secret_store_bind_address }}" + vault_api_addr: "{{ secret_store_api_address }}" + vault_config_dir: /opt/kayobe/vault + vault_cluster_name: seed + vault_docker_image: "{{ seed_vault_docker_image }}" + vault_docker_tag: "{{ seed_vault_docker_tag }}" + vault_write_keys_file: true + vault_write_keys_file_path: "{{ kayobe_env_config_path }}/vault/seed-vault-keys.json" + when: stackhpc_ca_secret_store == "vault" + + - name: Apply OpenBao role + ansible.builtin.import_role: + name: stackhpc.hashicorp.openbao + vars: + openbao_bind_addr: "{{ secret_store_bind_address }}" + openbao_api_addr: "{{ secret_store_api_address }}" + openbao_registry_url: "{{ seed_openbao_registry_url }}" + openbao_registry_username: "{{ seed_openbao_registry_username }}" + openbao_registry_password: "{{ seed_openbao_registry_password }}" + openbao_config_dir: /opt/kayobe/openbao + openbao_cluster_name: seed + openbao_docker_image: "{{ seed_openbao_docker_image }}" + openbao_docker_tag: "{{ seed_openbao_docker_tag }}" + openbao_write_keys_file: true + openbao_write_keys_file_path: "{{ kayobe_env_config_path }}/openbao/seed-openbao-keys.json" + when: stackhpc_ca_secret_store == "openbao" + + - name: Include {{ stackhpc_ca_secret_store }} keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/seed-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Unseal {{ stackhpc_ca_secret_store }} + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: "{{ secret_store_api_address }}" + vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}" + + - name: Apply PKI role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_pki + vars: + vault_api_addr: "{{ secret_store_api_address }}" + vault_token: "{{ secret_store_keys.root_token }}" + vault_pki_root_ca_name: OS-TLS-ROOT + vault_pki_write_root_ca_to_file: true + vault_pki_intermediate_ca_name: OS-TLS-INT + vault_pki_intermediate_export: true + vault_pki_intermediate_roles: "{{ seed_vault_pki_roles if stackhpc_ca_secret_store == 'vault' else seed_openbao_pki_roles }}" + vault_pki_certificates_directory: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}" + vault_pki_generate_certificates: true + vault_pki_write_certificates: true + vault_pki_certificate_subject: "{{ seed_vault_pki_certificate_subject if stackhpc_ca_secret_store == 'vault' else seed_openbao_pki_certificate_subject }}" + vault_pki_write_certificate_files: true + vault_pki_write_pem_bundle: false + vault_pki_write_int_ca_to_file: true diff --git a/etc/kayobe/ansible/secret-store-generate-backend-tls.yml b/etc/kayobe/ansible/secret-store-generate-backend-tls.yml new file mode 100644 index 000000000..4b0eb87ee --- /dev/null +++ b/etc/kayobe/ansible/secret-store-generate-backend-tls.yml @@ -0,0 +1,83 @@ +--- +# Required for uri module to work with self-signed certificates and for systems to trust +# the self-signed CA +- name: Install CA + hosts: controllers:network + tasks: + - name: Copy the intermediate CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/OS-TLS-ROOT.pem" + dest: "{{ '/etc/pki/ca-trust/source/anchors/OS-TLS-ROOT.crt' if ansible_facts.os_family == 'RedHat' \ + else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + mode: "0644" + become: true + + - name: Update system CA + become: true + ansible.builtin.command: "{{ 'update-ca-trust' if ansible_facts.os_family == 'RedHat' else 'update-ca-certificates' }}" + +- name: Generate backend API certificates + hosts: controllers:network + vars: + secret_store_api_address: https://{{ internal_net_name | net_ip(groups['controllers'][0]) }}:8200 + secret_store_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Issue a certificate for backend TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ secret_store_keys.root_token }}" + mount_point: "{{ secret_store_intermediate_ca_name }}" + role: "{{ overcloud_vault_pki_backend_tls_role_name if stackhpc_ca_secret_store == 'vault' else overcloud_openbao_pki_backend_tls_role_name }}" + common_name: "" + extra_params: + ip_sans: "{{ internal_net_name | net_ip }}" + register: backend_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Copy backend cert + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/{{ inventory_hostname }}-cert.pem" + content: | + {{ backend_cert.data.certificate }} + {{ backend_cert.data.issuing_ca }} + mode: "0600" + delegate_to: localhost + + - name: Copy backend key + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/{{ inventory_hostname }}-key.pem" + content: | + {{ backend_cert.data.private_key }} + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/secret-store-generate-internal-tls.yml b/etc/kayobe/ansible/secret-store-generate-internal-tls.yml new file mode 100644 index 000000000..ea9dfd929 --- /dev/null +++ b/etc/kayobe/ansible/secret-store-generate-internal-tls.yml @@ -0,0 +1,56 @@ +--- +- name: Generate internal API certificate + hosts: controllers + run_once: true + vars: + secret_store_api_address: https://{{ internal_net_name | net_ip }}:8200 + secret_store_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Issue a certificate for internal TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ secret_store_keys.root_token }}" + mount_point: "{{ secret_store_intermediate_ca_name }}" + role: "{{ overcloud_vault_pki_internal_tls_role_name if stackhpc_ca_secret_store == 'vault' else overcloud_openbao_pki_internal_tls_role_name }}" + common_name: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% endif %}" + extra_params: + ip_sans: "{{ kolla_internal_vip_address }}" + register: internal_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Ensure CA certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates/ca" + state: directory + delegate_to: localhost + + - name: Copy internal API PEM bundle + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy-internal.pem" + content: | + {{ internal_cert.data.certificate }} + {{ internal_cert.data.issuing_ca }} + {{ internal_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + + - name: Copy root CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/{{ stackhpc_ca_secret_store }}.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/secret-store-generate-test-external-tls.yml b/etc/kayobe/ansible/secret-store-generate-test-external-tls.yml new file mode 100644 index 000000000..206bf2b7a --- /dev/null +++ b/etc/kayobe/ansible/secret-store-generate-test-external-tls.yml @@ -0,0 +1,57 @@ +--- +- name: Generate external API certificate (for testing only) + hosts: controllers + run_once: true + vars: + secret_store_api_address: https://{{ internal_net_name | net_ip }}:8200 + # NOTE: Using the same CA as internal TLS. + secret_store_intermediate_ca_name: OS-TLS-INT + tasks: + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Issue a certificate for external TLS + hashivault_pki_cert_issue: # noqa: fqcn + url: "{{ secret_store_api_address }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ secret_store_keys.root_token }}" + mount_point: "{{ secret_store_intermediate_ca_name }}" + role: "{{ overcloud_vault_pki_external_tls_role_name if stackhpc_ca_secret_store == 'vault' else overcloud_openbao_pki_external_tls_role_name }}" + common_name: "{% if kolla_external_fqdn != kolla_external_vip_address %}{{ kolla_external_fqdn }}{% endif %}" + extra_params: + ip_sans: "{{ kolla_external_vip_address }}" + register: external_cert + environment: + https_proxy: "" + + - name: Ensure certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Ensure CA certificates directory exists + ansible.builtin.file: + path: "{{ kayobe_env_config_path }}/kolla/certificates/ca" + state: directory + delegate_to: localhost + + - name: Copy external API PEM bundle + no_log: true + ansible.builtin.copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy.pem" + content: | + {{ external_cert.data.certificate }} + {{ external_cert.data.issuing_ca }} + {{ external_cert.data.private_key }} + mode: "0600" + delegate_to: localhost + + - name: Copy root CA + ansible.builtin.copy: + src: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/{{ stackhpc_ca_secret_store }}.crt" + mode: "0600" + delegate_to: localhost diff --git a/etc/kayobe/ansible/secret-store-unseal-overcloud.yml b/etc/kayobe/ansible/secret-store-unseal-overcloud.yml new file mode 100644 index 000000000..634e7e029 --- /dev/null +++ b/etc/kayobe/ansible/secret-store-unseal-overcloud.yml @@ -0,0 +1,37 @@ +--- +- name: Unseal secret store on the overcloud + any_errors_fatal: true + gather_facts: true + hosts: controllers + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/overcloud-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Apply secret store unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_api_addr: https://{{ internal_net_name | net_ip }}:8200 + vault_unseal_token: "{{ secret_store_keys.root_token }}" + vault_unseal_ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}" + environment: + https_proxy: "" diff --git a/etc/kayobe/ansible/secret-store-unseal-seed.yml b/etc/kayobe/ansible/secret-store-unseal-seed.yml new file mode 100644 index 000000000..a404101ed --- /dev/null +++ b/etc/kayobe/ansible/secret-store-unseal-seed.yml @@ -0,0 +1,34 @@ +--- +- name: Unseal secret store on the seed + any_errors_fatal: true + gather_facts: true + hosts: seed + vars: + vault_api_addr: http://127.0.0.1:8200 + tasks: + - name: Set a fact about the virtualenv on the remote system + ansible.builtin.set_fact: + virtualenv: "{{ ansible_python_interpreter | dirname | dirname }}" + when: + - ansible_python_interpreter is defined + - not ansible_python_interpreter.startswith('/bin/') + - not ansible_python_interpreter.startswith('/usr/bin/') + + - name: Ensure Python hvac module is installed + ansible.builtin.pip: + name: hvac + state: latest + extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}" + virtualenv: "{{ virtualenv is defined | ternary(virtualenv, omit) }}" + become: "{{ virtualenv is not defined }}" + + - name: Include secret store keys + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/{{ stackhpc_ca_secret_store }}/seed-{{ stackhpc_ca_secret_store }}-keys.json" + name: secret_store_keys + + - name: Apply unseal role + ansible.builtin.import_role: + name: stackhpc.hashicorp.vault_unseal + vars: + vault_unseal_keys: "{{ secret_store_keys.keys_base64 }}" diff --git a/etc/kayobe/ansible/smartmon-tools.yml b/etc/kayobe/ansible/smartmon-tools.yml new file mode 100644 index 000000000..4c69c7cbb --- /dev/null +++ b/etc/kayobe/ansible/smartmon-tools.yml @@ -0,0 +1,144 @@ +--- +- name: Install and set up SMART monitoring tools + hosts: overcloud + tasks: + - name: Ensure smartmontools, jq, nvme-cli and cron/cronie are installed + ansible.builtin.package: + name: + - smartmontools + - nvme-cli + - jq + - "{{ 'cron' if ansible_facts['distribution'] == 'Ubuntu' else 'cronie' }}" + state: present + become: true + + - name: Ensure Python 3, venv, and pip are installed on Debian/Ubuntu + ansible.builtin.apt: + name: + - python3 + - python3-venv + - python3-pip + state: present + when: ansible_facts.os_family == 'Debian' + become: true + + - name: Ensure Python 3, and pip are installed on RedHat/CentOS + ansible.builtin.yum: + name: + - python3 + - python3-pip + state: present + when: ansible_facts.os_family == 'RedHat' + become: true + + - name: Create smartmon Python virtual environment + ansible.builtin.command: + cmd: python3 -m venv /opt/smartmon-venv + creates: /opt/smartmon-venv/bin/activate + become: true + + - name: Install prometheus_client and pySMART in venv + ansible.builtin.pip: + name: + - prometheus_client + - pySMART + state: present + virtualenv: /opt/smartmon-venv + virtualenv_python: python3 + become: true + + - name: Ensure the cron/crond service is running + ansible.builtin.service: + name: "{{ 'cron' if ansible_facts['distribution'] == 'Ubuntu' else 'crond' }}" + state: started + enabled: true + become: true + + - name: Copy smartmon.py and nvmemon.sh from scripts folder + ansible.builtin.copy: + src: scripts/{{ item }} + dest: /usr/local/bin/{{ item }} + owner: root + group: root + mode: "0700" + loop: + - smartmon.py + - nvmemon.sh + become: true + + - name: Set PATH Variable for cron + ansible.builtin.cron: + name: PATH + user: root + env: true + job: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + become: true + + - name: Schedule cronjob to run smartmon.py every 5 minutes and save output to file + ansible.builtin.cron: + name: SMART metrics for drive monitoring using smartmon.py + user: root + minute: "*/5" + job: >- + umask 0022 && /opt/smartmon-venv/bin/python /usr/local/bin/smartmon.py --output /var/lib/docker/volumes/textfile/_data/smartmon.prom.temp && + mv -f /var/lib/docker/volumes/textfile/_data/smartmon.prom.temp /var/lib/docker/volumes/textfile/_data/smartmon.prom + become: true + + - name: Schedule cronjob to run nvmemon.sh every 5 minutes and save output to file + ansible.builtin.cron: + name: SMART metrics for drive monitoring using nvmemon.sh + user: root + minute: "*/5" + job: >- + umask 0022 && /usr/local/bin/nvmemon.sh > + /var/lib/docker/volumes/textfile/_data/nvmemon.prom.temp && + mv -f /var/lib/docker/volumes/textfile/_data/nvmemon.prom.temp /var/lib/docker/volumes/textfile/_data/nvmemon.prom + become: true + + - name: Remove old cronjobs if present + ansible.builtin.cron: + name: SMART metrics for drive monitoring using {{ item }} + user: root + state: absent + become: true + loop: + - smartmon + - nvmemon + + - name: Remove old smartmon.sh if present + ansible.builtin.file: + path: /usr/local/bin/smartmon.sh + state: absent + become: true + +- name: Gather NVMe drives and generate dwpd ratings + import_playbook: get-nvme-drives.yml + when: create_dwpd_ratings | default(false) + +- name: Copy DWPD ratings to overcloud hosts + hosts: overcloud + gather_facts: false + tasks: + - name: Convert the stackhpc_dwpd_ratings variable to JSON + ansible.builtin.set_fact: + dwpd_ratings_json: "{{ stackhpc_dwpd_ratings | default([]) | to_json }}" + run_once: true + when: stackhpc_dwpd_ratings is defined + + - name: Ensure /opt/kayobe/etc/monitoring directory exists + ansible.builtin.file: + path: /opt/kayobe/etc/monitoring + state: directory + mode: '0755' + become: true + when: stackhpc_dwpd_ratings is defined + + - name: Copy JSON file to remote + ansible.builtin.copy: + content: "{{ dwpd_ratings_json }}" + dest: "/opt/kayobe/etc/monitoring/dwpd_ratings.json" + owner: root + group: root + mode: '0644' + become: true + when: stackhpc_dwpd_ratings is defined diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml new file mode 100644 index 000000000..f13912ff8 --- /dev/null +++ b/etc/kayobe/ansible/stackhpc-cloud-tests.yml @@ -0,0 +1,197 @@ +--- +- name: Run StackHPC Cloud tests + hosts: tempest_runner:overcloud + tags: + - stackhpc-cloud-tests + vars: + sct_venv: "{{ virtualenv_path }}/sct-venv" + sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests + # Define the version of SCT used for testing, the github workflow overrides this with + # stackhpc_cloud_tests_version so this is only used if running "locally". + sct_version: main + sct_timeout: 30 + results_path_local: "{{ lookup('env', 'HOME') }}/sct-results" + tasks: + - name: Stackhpc Cloud tests + block: + - name: Assert that there is only one host in the tempest_runner group + ansible.builtin.assert: + that: groups.get('tempest_runner', []) | length == 1 + fail_msg: The tempest_runner group should contain exactly one host + + - name: Create a temporary directory for tests repo + ansible.builtin.tempfile: + state: directory + suffix: sct-repo + register: repo_tmpdir + + - name: Create a temporary directory for results + ansible.builtin.tempfile: + state: directory + suffix: sct-results + register: results_tmpdir + + - name: Clone the StackHPC Cloud tests repository + ansible.builtin.git: + repo: "{{ sct_repo }}" + version: "{{ sct_version }}" + dest: "{{ repo_tmpdir.path }}" + depth: 1 + single_branch: true + + - name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest + ansible.builtin.pip: + name: "{{ item.name }}" + state: latest + virtualenv: "{{ sct_venv }}" + virtualenv_command: python3 -m venv + with_items: + - { name: pip } + - { name: setuptools } + + - name: Ensure required individual Python packages are installed + ansible.builtin.pip: + name: + - pytest-html + - pytest-timeout + virtualenv: "{{ sct_venv }}" + + - name: Ensure Python requirements file packages are installed + ansible.builtin.pip: + requirements: "{{ repo_tmpdir.path }}/requirements.txt" + virtualenv: "{{ sct_venv }}" + + - name: Include Kolla Ansible passwords + ansible.builtin.include_vars: + file: "{{ kayobe_env_config_path }}/kolla/passwords.yml" + name: kolla_passwords + + # Monitoring tests should run once, executed on the host in the + # tempest_runner group. + - name: Check for StackHPC Cloud monitoring tests + ansible.builtin.stat: + path: "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/monitoring" + register: stackhpc_cloud_monitoring_tests + + - name: Run StackHPC Cloud monitoring tests + ansible.builtin.command: + cmd: > + {{ sct_venv }}/bin/py.test + --html={{ results_tmpdir.path }}/monitoring.html + --self-contained-html + --timeout {{ sct_timeout }} + -rfEx + -vv + "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/monitoring" + environment: + GRAFANA_URL: "{{ sct_grafana_url }}" + GRAFANA_USERNAME: "{{ sct_grafana_username }}" + GRAFANA_PASSWORD: "{{ sct_grafana_password }}" + OPENSEARCH_HOSTS: "{{ sct_opensearch_hosts }}" + OPENSEARCH_PORT: "{{ sct_opensearch_port }}" + OPENSEARCH_TLS: "{{ sct_opensearch_tls }}" + OPENSEARCH_DASHBOARDS_URL: "{{ sct_opensearch_dashboards_url }}" + OPENSEARCH_DASHBOARDS_USERNAME: "{{ sct_opensearch_dashboards_username }}" + OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sct_opensearch_dashboards_password }}" + PROMETHEUS_URL: "{{ sct_prometheus_url }}" + PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}" + PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}" + vars: + kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}" + kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}" + sct_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000" + sct_grafana_username: "grafana_local_admin" + sct_grafana_password: "{{ kolla_passwords.grafana_admin_password }}" + sct_opensearch_hosts: "{{ kolla_internal_fqdn }}" + sct_opensearch_port: 9200 + sct_opensearch_tls: "{{ kolla_enable_tls_internal | bool }}" + sct_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601" + sct_opensearch_dashboards_username: "opensearch" + sct_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}" + sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091" + sct_prometheus_username: admin + sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}" + failed_when: monitoring_results.rc not in [0, 1] + register: monitoring_results + when: "'tempest_runner' in group_names and stackhpc_cloud_monitoring_tests.stat.exists" + + # Host tests should run on every host in the overcloud group. + # TODO: Use TestInfra's native Ansible or SSH connection plugins for + # remote test execution? That would place all results in a single file + # and allow us to execute all tests from a single host. + # https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends + - name: Check for StackHPC Cloud host tests + ansible.builtin.stat: + path: "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host" + register: stackhpc_cloud_host_tests + + - name: Run StackHPC Cloud host tests + ansible.builtin.command: + cmd: > + {{ sct_venv }}/bin/py.test + --html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html + --self-contained-html + --timeout {{ sct_timeout }} + -vv + "{{ repo_tmpdir.path }}/stackhpc_cloud_tests/host" + environment: + DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}" + DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}" + SELINUX_STATE: "{{ sct_selinux_state }}" + vars: + # Inclusive min + sct_docker_version_min: "24.0.0" + # Exclusive max + sct_docker_version_max: "30.0.0" + sct_selinux_state: "{{ selinux_state }}" + failed_when: host_results.rc not in [0, 1] + register: host_results + # Some host checks may need to run as root + become: true + when: "'overcloud' in group_names and stackhpc_cloud_host_tests.stat.exists" + + # Host test results will be owned by root - we need to read and delete them + - name: Change permissions on SCT host test results + ansible.builtin.command: + cmd: chmod 666 {{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html + become: true + when: "'overcloud' in group_names and stackhpc_cloud_host_tests.stat.exists" + + always: + - name: Synchronize results + ansible.posix.synchronize: + src: "{{ results_tmpdir.path }}/" + dest: "{{ results_path_local }}/" + mode: pull + archive: no + recursive: true + # For jump host + use_ssh_args: true + + - name: Write a file containing failed test runs + ansible.builtin.copy: + content: |- + {% for host in ansible_play_hosts_all %} + {% if host not in ansible_play_hosts %} + {{ host }}: Host failure + {% endif %} + {% if hostvars[host].monitoring_results.rc | default(0) != 0 %} + monitoring.html + {% endif %} + {% if hostvars[host].host_results.rc | default(0) != 0 %} + host-{{ host }}.html + {% endif %} + {% endfor %} + dest: "{{ results_path_local }}/failed-tests" + delegate_to: localhost + run_once: true + + - name: Clean up temporary directory + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ repo_tmpdir.path }}" + - "{{ results_tmpdir.path }}" + # Some files used by host tests may now be owned by root + become: true diff --git a/etc/kayobe/ansible/stop-openstack-services.yml b/etc/kayobe/ansible/stop-openstack-services.yml new file mode 100644 index 000000000..1f34b6e90 --- /dev/null +++ b/etc/kayobe/ansible/stop-openstack-services.yml @@ -0,0 +1,32 @@ +--- +# Stops containers running OpenStack services + +- name: Stop OpenStack services + hosts: overcloud + become: true + gather_facts: false + vars: + stop_service_list: + - blazar + - barbican + - cinder + - cloudkitty + - designate + - glance + - heat + - horizon + - ironic + - keystone + - magnum + - manila + - neutron + - nova + - octavia + - placement + tasks: + - name: Stop OpenStack services + ansible.builtin.shell: + executable: "/bin/bash" + cmd: >- + set -o pipefail && + systemctl -a | egrep 'kolla-({{ stop_service_list | join('|') }})' | awk '{ print $1 }' | xargs systemctl stop diff --git a/etc/kayobe/ansible/templates/fix-houston-interface.service.j2 b/etc/kayobe/ansible/templates/fix-houston-interface.service.j2 new file mode 100644 index 000000000..24696b13e --- /dev/null +++ b/etc/kayobe/ansible/templates/fix-houston-interface.service.j2 @@ -0,0 +1,20 @@ +[Unit] +# This service addresses a specific issue when OVS HW offloading is enabled +# typically in conjunction with VF-LAG and ASAP^2 +# the DMESG log reports frequent errors on the internal OVS Bridge interface: +# "tc mirred to Houston: device bond0-ovs is down". +# This interface is down by default. The errors are mitigated by bringing the interface up. +# For further context, see: +# https://bugs.launchpad.net/charm-neutron-openvswitch/+bug/1899364 +# https://patchwork.kernel.org/project/netdevbpf/patch/c2ef23da1d9a4eb62f4e7b7c4540f9bafb553c15.1658420239.git.dcaratti@redhat.com/ +Description=Bring up {{ interface_name }} interface +After=kolla-openvswitch_vswitchd-container.service + +[Service] +Type=oneshot +ExecStartPre=/usr/bin/timeout 60s /bin/bash -c 'until ip link show {{ interface_name }}; do sleep 1; done' +ExecStart=/sbin/ip link set {{ interface_name }} up +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 new file mode 100644 index 000000000..6475848ba --- /dev/null +++ b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 @@ -0,0 +1,18 @@ +clouds: + openstack: + auth: + auth_url: "{{ stackhpc_os_capacity_auth_url }}" + project_name: "{{ stackhpc_os_capacity_project_name }}" + domain_name: "{{ stackhpc_os_capacity_domain_name }}" + username: "{{ stackhpc_os_capacity_username }}" + password: "{{ stackhpc_os_capacity_password }}" + region_name: "{{ stackhpc_os_capacity_openstack_region_name }}" + interface: "internal" + identity_api_version: 3 + auth_type: "password" +{% if stackhpc_os_capacity_openstack_cacert | length > 0 %} + cacert: /etc/openstack/cacert.pem +{% endif %} +{% if not stackhpc_os_capacity_openstack_verify | bool %} + verify: False +{% endif %} diff --git a/etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 b/etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 new file mode 100644 index 000000000..8294edec7 --- /dev/null +++ b/etc/kayobe/ansible/templates/wazuh-secrets.yml.j2 @@ -0,0 +1,14 @@ +--- +# Secrets used by Wazuh managers and agents +# Store these securely and use lookups here +secrets_wazuh: + # Wazuh agent authd pass + authd_pass: '{{ secrets_wazuh.authd_pass | default(lookup("password", "/dev/null"), true) }}' + # Strengthen default wazuh api user pass + wazuh_api_users: + - username: "wazuh" + password: '{{ secrets_wazuh.wazuh_api_users[0].password | default(lookup("community.general.random_string", min_lower=1, min_upper=1, min_special=1, min_numeric=1, length=30, override_special=override_special_characters)) }}' + # OpenSearch 'admin' user pass + opendistro_admin_password: '{{ secrets_wazuh.opendistro_admin_password | default(lookup("password", "/dev/null"), true) }}' + # OpenSearch 'kibanaserver' user pass + opendistro_kibana_password: '{{ secrets_wazuh.opendistro_kibana_password | default(lookup("password", "/dev/null"), true) }}' diff --git a/etc/kayobe/ansible/ubuntu-upgrade.yml b/etc/kayobe/ansible/ubuntu-upgrade.yml new file mode 100644 index 000000000..ead0dbda5 --- /dev/null +++ b/etc/kayobe/ansible/ubuntu-upgrade.yml @@ -0,0 +1,267 @@ +--- +# To prevent Ansible role dependency errors, this playbook requires that environment variable +# ANSIBLE_ROLES_PATH is defined and includes '$KAYOBE_PATH/ansible/roles' on the Ansible control host. +# Where KAYOBE_PATH is the path to the source of kayobe that the environment uses. +- name: Prepare upgrade from Ubuntu Jammy 22.04 to Noble 24.04 + hosts: overcloud:infra-vms:seed:seed-hypervisor + vars: + ansible_python_interpreter: /usr/bin/python3 + reboot_timeout_s: "{{ 20 * 60 }}" + tags: pre + tasks: + - name: Assert that hosts are running Ubuntu Jammy + ansible.builtin.assert: + that: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - ansible_facts.distribution_release == 'jammy' + - os_distribution == 'ubuntu' + fail_msg: >- + This playbook is only designed for Ubuntu Jammy 22.04 hosts. Ensure + that you are limiting it to only run on Jammy hosts and + os_distribution is set to ubuntu. + + - name: Ensure apt packages are up to date + ansible.builtin.apt: + update_cache: true + upgrade: true + become: true + + - name: Ensure do-release-upgrade is installed + ansible.builtin.package: + name: ubuntu-release-upgrader-core + state: latest + become: true + + - name: Check whether a reboot is required + ansible.builtin.stat: + path: /var/run/reboot-required + register: file_status + + - name: Reboot to apply updates + ansible.builtin.reboot: + reboot_timeout: "{{ reboot_timeout_s }}" + connect_timeout: 600 + search_paths: + # Systems running molly-guard hang waiting for confirmation before rebooting without this. + - /lib/molly-guard + # Default list: + - /sbin + - /bin + - /usr/sbin + - /usr/bin + - /usr/local/sbin + become: true + when: file_status.stat.exists + + - name: Ensure /tmp is mounted with exec + ansible.posix.mount: + path: /tmp + opts: exec + state: remounted + become: true + + - name: Ensure /var/lib/cephadm directory exists + ansible.builtin.file: + path: /var/lib/cephadm + state: directory + owner: root + group: root + become: true + when: inventory_hostname in groups['ceph'] + + # NOTE: We cannot use apt_repository here because definitions must exist within the standard repos.list + - name: Ensure Noble repo definitions exist in sources.list + ansible.builtin.blockinfile: + path: /etc/apt/sources.list + block: | + deb {{ stackhpc_repo_ubuntu_noble_url }} noble main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_noble_url }} noble-updates main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_noble_url }} noble-backports main restricted universe multiverse + deb {{ stackhpc_repo_ubuntu_noble_security_url }} noble-security main restricted universe multiverse + become: true + +- name: Upgrade hosts from Ubuntu Jammy 22.04 to Noble 24.04 + hosts: overcloud:infra-vms:seed:seed-hypervisor + vars: + ansible_python_interpreter: /usr/bin/python3 + reboot_timeout_s: "{{ 20 * 60 }}" + tags: upgrade + tasks: + - name: Perform in-place Ubuntu upgrade + block: + - name: Run do-release-upgrade + ansible.builtin.command: do-release-upgrade -f DistUpgradeViewNonInteractive + become: true + rescue: + - name: Ensure Noble repo definitions do not exist in sources.list + ansible.builtin.blockinfile: + path: /etc/apt/sources.list + state: absent + become: true + + - name: Ensure Kolla Ansible Docker repo definition does not exist + ansible.builtin.file: + path: /etc/apt/sources.list.d/docker.list + state: absent + become: true + when: apt_repositories | selectattr('url', 'match', '.*docker-ce.*') | list | length > 0 + + - name: Display recommanded action on upgrade failure + ansible.builtin.debug: + msg: > + Ubuntu upgrade failed. You can check the upgrade logs at /var/log/dist-upgrade + on the failed host. + It is likely due to packages with broken dependency. You can find broken packages + by running following command from the host. + cat /var/log/dist-upgrade/apt.log | grep "Holding Back" | awk '{print $3}' + + - name: Fail fast when upgrade fails + ansible.builtin.meta: end_host + +- name: Post upgrade of Ubuntu Jammy 22.04 to Noble 24.04 + hosts: overcloud:infra-vms:seed:seed-hypervisor + vars: + ansible_python_interpreter: /usr/bin/python3 + reboot_timeout_s: "{{ 20 * 60 }}" + tags: post + tasks: + - name: Ensure old venvs do not exist + ansible.builtin.file: + path: /opt/kayobe/venvs/{{ item }} + state: absent + loop: + - kayobe + - kolla-ansible + become: true + + - name: Update Python and current user facts before re-creating Kayobe venv + ansible.builtin.setup: + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" + vars: + ansible_python_interpreter: /usr/bin/python3 + +- name: Run the Kayobe network configuration playbook, to ensure definitions are not lost on reboot + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/network.yml" + vars: + ansible_python_interpreter: /usr/bin/python3 + tags: post + +- name: Run the Kayobe kayobe-target-venv playbook to ensure kayobe venv exists on remote host + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/kayobe-target-venv.yml" + tags: post + +- name: Clean up old repos + hosts: overcloud:infra-vms:seed:seed-hypervisor + vars: + ansible_python_interpreter: /usr/bin/python3 + tags: + - post + - cleanup-repos + tasks: + - name: Ensure Noble repo definitions do not exist in sources.list + ansible.builtin.blockinfile: + path: /etc/apt/sources.list + state: absent + become: true + + - name: Ensure Kolla Ansible Docker repo definition does not exist + ansible.builtin.file: + path: /etc/apt/sources.list.d/docker.list + state: absent + become: true + when: apt_repositories | selectattr('url', 'match', '.*docker-ce.*') | list | length > 0 + + # Make a backup, in case of having broken apt configuration. + - name: Backup upstream ubuntu.sources + ansible.builtin.copy: + src: /etc/apt/sources.list.d/ubuntu.sources + dest: /etc/apt/ubuntu.sources.bak + backup: true + remote_src: true + become: true + when: hostvars[inventory_hostname].stackhpc_repos_enabled + + - name: Ensure only Kayobe defined apt repositories are defined + ansible.builtin.file: + path: "/etc/apt/{{ item }}" + state: absent + loop: + - sources.list.distUpgrade + - sources.list.d/third-party.sources + - sources.list.d/ubuntu.sources + become: true + when: hostvars[inventory_hostname].stackhpc_repos_enabled + +- name: Run the Kayobe apt playbook to ensure Noble repositories are set on remote host + import_playbook: "{{ lookup('ansible.builtin.env', 'VIRTUAL_ENV') }}/share/kayobe/ansible/apt.yml" + tags: + - post + - post-cleanup + +- name: Fix broken packages after upgrade + hosts: overcloud:infra-vms:seed:seed-hypervisor + tags: + - post + - post-cleanup + tasks: + - name: Ensure iproute2 is installed + ansible.builtin.apt: + name: iproute2 + state: present + become: true + - name: Ensure cephadm dependencies are installed + ansible.builtin.apt: + name: + - python3-yaml + - python3-jinja2 + state: present + become: true + when: inventory_hostname in groups['ceph'] + + - name: Update Python and current user facts before running Kayobe network playbook + ansible.builtin.setup: + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" + +- name: Reboot and confirm the host is upgraded to Noble 24.04 + hosts: overcloud:infra-vms:seed:seed-hypervisor + vars: + ansible_python_interpreter: /usr/bin/python3 + reboot_timeout_s: "{{ 20 * 60 }}" + tags: + - post + - post-cleanup + tasks: + - name: Reboot and wait + ansible.builtin.reboot: + reboot_timeout: "{{ reboot_timeout_s }}" + connect_timeout: 600 + search_paths: + # Systems running molly-guard hang waiting for confirmation before rebooting without this. + - /lib/molly-guard + # Default list: + - /sbin + - /bin + - /usr/sbin + - /usr/bin + - /usr/local/sbin + become: true + + - name: Ensure all packages are in Noble version + ansible.builtin.apt: + upgrade: full + update_cache: true + become: true + + - name: Update distribution facts + ansible.builtin.setup: + filter: "{{ kayobe_ansible_setup_filter }}" + gather_subset: "{{ kayobe_ansible_setup_gather_subset }}" + + - name: Assert that hosts are now using Ubuntu 24.04 + ansible.builtin.assert: + that: + - ansible_facts.distribution_major_version == '24' + - ansible_facts.distribution_release == 'noble' diff --git a/etc/kayobe/ansible/vault-deploy-overcloud.yml b/etc/kayobe/ansible/vault-deploy-overcloud.yml index d1ad7eaff..31ccf2d2e 100644 --- a/etc/kayobe/ansible/vault-deploy-overcloud.yml +++ b/etc/kayobe/ansible/vault-deploy-overcloud.yml @@ -62,6 +62,8 @@ name: stackhpc.hashicorp.vault vars: hashicorp_registry_url: "{{ overcloud_hashicorp_registry_url }}" + hashicorp_registry_username: "{{ overcloud_hashicorp_registry_username }}" + hashicorp_registry_password: "{{ overcloud_hashicorp_registry_password }}" consul_docker_image: "{{ overcloud_consul_docker_image }}" consul_docker_tag: "{{ overcloud_consul_docker_tag }}" vault_config_dir: /opt/kayobe/vault diff --git a/etc/kayobe/ansible/vault-deploy-seed.yml b/etc/kayobe/ansible/vault-deploy-seed.yml index 356ea1970..9617aa905 100644 --- a/etc/kayobe/ansible/vault-deploy-seed.yml +++ b/etc/kayobe/ansible/vault-deploy-seed.yml @@ -38,6 +38,8 @@ name: stackhpc.hashicorp.vault vars: hashicorp_registry_url: "{{ seed_hashicorp_registry_url }}" + hashicorp_registry_username: "{{ seed_hashicorp_registry_username }}" + hashicorp_registry_password: "{{ seed_hashicorp_registry_password }}" consul_docker_image: "{{ seed_consul_docker_image }}" consul_docker_tag: "{{ seed_consul_docker_tag }}" vault_config_dir: /opt/kayobe/vault diff --git a/etc/kayobe/ansible/wazuh-agent.yml b/etc/kayobe/ansible/wazuh-agent.yml new file mode 100644 index 000000000..489191262 --- /dev/null +++ b/etc/kayobe/ansible/wazuh-agent.yml @@ -0,0 +1,42 @@ +--- +- name: Deploy Wazuh agent + hosts: wazuh-agent + become: true + tasks: + - name: Apply Wazuh agent role + ansible.builtin.import_role: + name: wazuh-ansible/wazuh-ansible/roles/wazuh/ansible-wazuh-agent + post_tasks: + - name: Check if custom SCA policies directory exists + ansible.builtin.stat: + path: "{{ local_custom_sca_policies_path }}" + register: custom_sca_policies_folder + delegate_to: localhost + + - name: Gather list of custom SCA policies + ansible.builtin.find: + paths: "{{ local_custom_sca_policies_path }}" + patterns: "*.yml" + delegate_to: localhost + register: custom_sca_policies + when: custom_sca_policies_folder.stat.exists + + - name: Allow Wazuh agents to execute commands in SCA policies sent from the Wazuh manager + become: true + ansible.builtin.blockinfile: + path: /var/ossec/etc/local_internal_options.conf + state: present + owner: wazuh + group: wazuh + block: sca.remote_commands=1 + when: + - custom_sca_policies_folder.stat.exists + - custom_sca_policies.files | length > 0 + notify: + - Restart wazuh-agent + + handlers: + - name: Restart wazuh-agent + ansible.builtin.service: + name: wazuh-agent + state: restarted diff --git a/etc/kayobe/ansible/wazuh-manager.yml b/etc/kayobe/ansible/wazuh-manager.yml new file mode 100644 index 000000000..21c560d5d --- /dev/null +++ b/etc/kayobe/ansible/wazuh-manager.yml @@ -0,0 +1,147 @@ +--- +- name: Pre-checks + gather_facts: false + hosts: localhost + tags: + - prechecks + tasks: + - name: Generate certificates + when: + - groups["wazuh-manager"] | length > 0 + + block: + - name: Fail if using old path for Wazuh certificates + ansible.builtin.fail: + msg: >- + The path used for Wazuh SSL certificates was changed in a previous release. The certificates + were found in the wrong location. Please move all files and directories in + {{ playbook_dir }}/wazuh/certificates to {{ kayobe_env_config_path }}/wazuh/ and remove the + empty directory. + when: + - (playbook_dir ~ '/wazuh/certificates') is exists + + - name: Fail if using old path for custom certificates + ansible.builtin.fail: + msg: >- + Wazuh custom SSL certificates have been merged with regular certificates. The certificates + were found in the wrong location. Please move them from {{ playbook_dir }}/wazuh/custom_certificates + to {{ kayobe_env_config_path }}/wazuh/wazuh-certificates and remove the empty directory. + when: + - (playbook_dir ~ '/wazuh/custom_certificates') is exists + + - name: Check that removed variable, local_custom_certs_path, is not set + ansible.builtin.assert: + that: local_custom_certs_path is not defined + fail_msg: The variable, `local_custom_certs_path`, is no longer used. Please remove this variable. + +# Certificates generation +- name: Apply Wazuh indexer role + hosts: localhost + roles: + - role: wazuh-ansible/wazuh-ansible/roles/wazuh/wazuh-indexer + perform_installation: false + become: false + tags: + - generate-certs +# Single node +- name: Install Wazuh manager + hosts: wazuh-manager + become: true + become_user: root + roles: + - role: wazuh-ansible/wazuh-ansible/roles/wazuh/wazuh-indexer + - role: wazuh-ansible/wazuh-ansible/roles/wazuh/ansible-wazuh-manager + - role: wazuh-ansible/wazuh-ansible/roles/wazuh/ansible-filebeat-oss + - role: wazuh-ansible/wazuh-ansible/roles/wazuh/wazuh-dashboard + post_tasks: + - name: Use custom SCA policies + notify: + - Restart wazuh + + block: + - name: Check if custom SCA policies directory exists + ansible.builtin.stat: + path: "{{ local_custom_sca_policies_path }}" + register: custom_sca_policies_folder + delegate_to: localhost + become: false + + - name: Gather list of custom SCA policies + ansible.builtin.find: + paths: "{{ local_custom_sca_policies_path }}" + patterns: "*.yml" + delegate_to: localhost + register: custom_sca_policies + when: custom_sca_policies_folder.stat.exists + become: false + + - name: Copy custom SCA policy files to Wazuh manager + ansible.builtin.copy: + # Note the trailing slash to copy directory contents + src: "{{ local_custom_sca_policies_path }}/" + dest: /var/ossec/etc/shared/default/ + owner: wazuh + group: wazuh + when: + - custom_sca_policies_folder.stat.exists + - custom_sca_policies.files | length > 0 + + - name: Add custom policy definition(s) to the shared Agent config + ansible.builtin.blockinfile: + path: /var/ossec/etc/shared/default/agent.conf + state: present + owner: wazuh + group: wazuh + marker: "{mark} ANSIBLE MANAGED BLOCK Custom SCA Policies" + insertafter: + block: | + {% filter indent(width=2, first=true) %} + + + {% for item in custom_sca_policies.files %} + etc/shared/{{ item.path | basename }} + {% endfor %} + + + {% endfilter %} + when: + - custom_sca_policies_folder.stat.exists + - custom_sca_policies.files | length > 0 + + - name: Add JVM proxy settings to wazuh-indexer + ansible.builtin.blockinfile: + path: /etc/wazuh-indexer/jvm.options + state: present + owner: root + group: wazuh + marker: "# {mark} ANSIBLE MANAGED BLOCK JVM PROXY SETTINGS" + block: | + -Dhttp.proxyHost={{ http_proxy_url | urlsplit('hostname') }} + -Dhttp.proxyPort={{ http_proxy_url | urlsplit('port') }} + -Dhttps.proxyHost={{ http_proxy_url | urlsplit('hostname') }} + -Dhttps.proxyPort={{ http_proxy_url | urlsplit('port') }} + backup: true + when: http_proxy_url is defined + notify: + - Restart wazuh-indexer + + - name: Perform health check against filebeat + ansible.builtin.command: filebeat test output + changed_when: false + retries: 2 + + - name: Correct permissions on alerts manifest + ansible.builtin.file: + path: "/usr/share/filebeat/module/wazuh/alerts/manifest.yml" + mode: "go-w" + + handlers: + - name: Restart wazuh + ansible.builtin.service: + name: wazuh-manager + state: restarted + + - name: Restart wazuh-indexer + ansible.builtin.service: + name: wazuh-indexer + state: restarted diff --git a/etc/kayobe/ansible/wazuh-secrets.yml b/etc/kayobe/ansible/wazuh-secrets.yml new file mode 100644 index 000000000..fbaa3d3e9 --- /dev/null +++ b/etc/kayobe/ansible/wazuh-secrets.yml @@ -0,0 +1,31 @@ +--- +- name: Wazuh secrets + hosts: localhost + gather_facts: false + vars: + wazuh_secrets_path: "{{ kayobe_env_config_path }}/wazuh-secrets.yml" + override_special_characters: '"#$%&()*+,-./:;<=>?@[\]^_{|}~' + tasks: + - name: Install passlib[bcrypt] + ansible.builtin.pip: + name: passlib[bcrypt] + virtualenv: "{{ ansible_playbook_python | dirname | dirname }}" + + - name: Ensure secrets directory exists + ansible.builtin.file: + path: "{{ wazuh_secrets_path | dirname }}" + state: directory + + - name: Template new secrets + no_log: true + ansible.builtin.template: + src: wazuh-secrets.yml.j2 + dest: "{{ wazuh_secrets_path }}" + + - name: In-place encrypt wazuh-secrets + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.file', wazuh_secrets_path) | ansible.builtin.vault(ansible_vault_password) }}" + dest: "{{ wazuh_secrets_path }}" + decrypt: false + vars: + ansible_vault_password: "{{ lookup('ansible.builtin.env', 'KAYOBE_VAULT_PASSWORD') }}" diff --git a/etc/kayobe/ansible/write-github-workflows.yml b/etc/kayobe/ansible/write-github-workflows.yml new file mode 100644 index 000000000..f45319833 --- /dev/null +++ b/etc/kayobe/ansible/write-github-workflows.yml @@ -0,0 +1,7 @@ +--- +- name: Write Kayobe Automation Workflows for GitHub + hosts: github-writer + vars: + github_output_directory: "{{ kayobe_config_path }}/../../.github/workflows" + roles: + - stackhpc.kayobe_workflows.github diff --git a/etc/kayobe/ansible/write-gitlab-pipelines.yml b/etc/kayobe/ansible/write-gitlab-pipelines.yml new file mode 100644 index 000000000..c85e6adcd --- /dev/null +++ b/etc/kayobe/ansible/write-gitlab-pipelines.yml @@ -0,0 +1,7 @@ +--- +- name: Write Kayobe Automation Pipeline for GitLab + hosts: gitlab-writer + vars: + gitlab_output_directory: "{{ kayobe_config_path }}/../../" + roles: + - stackhpc.kayobe_workflows.gitlab diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index e4bb5b179..282336236 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -17,16 +17,28 @@ # * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the # configuration # Default is an empty list. -#apt_config: +apt_config: + # NOTE: Currently the Pulp verbatim publisher does not sync translation + # files, which results in apt update failing. Disable translations until this + # is resolved. + - content: | + Acquire::Languages "none"; + filename: 99no-languages # List of apt keys. Each item is a dict containing the following keys: # * url: URL of key # * filename: Name of a file in which to store the downloaded key. The # extension should be '.asc' for ASCII-armoured keys, or '.gpg' otherwise. # Default is an empty list. -#apt_keys: +apt_keys: + - url: "https://download.docker.com/linux/ubuntu/gpg" + filename: docker.asc + - url: "https://download.ceph.com/keys/release.asc" + filename: ceph.asc # A list of Apt repositories. Each item is a dict with the following keys: +# * name: the .sources filename part. Optional. Default is 'kayobe' and +# the default filename is 'kayobe.sources'. # * types: whitespace-separated list of repository types, e.g. deb or deb-src # (optional, default is 'deb') # * url: URL of the repository @@ -39,23 +51,64 @@ # * architecture: whitespace-separated list of architectures that will be used # (optional, default is unset) # Default is an empty list. -#apt_repositories: +stackhpc_apt_repositories: + - url: "{{ stackhpc_repo_ubuntu_noble_url if ansible_facts.distribution_release == 'noble' else stackhpc_repo_ubuntu_jammy_url }}" + suites: "{{ ansible_facts.distribution_release }} {{ ansible_facts.distribution_release }}-updates {{ ansible_facts.distribution_release }}-backports" + components: main restricted universe multiverse + architecture: amd64 + required: true + - url: "{{ stackhpc_repo_ubuntu_noble_security_url if ansible_facts.distribution_release == 'noble' else stackhpc_repo_ubuntu_jammy_security_url }}" + suites: "{{ ansible_facts.distribution_release }}-security" + components: main restricted universe multiverse + architecture: amd64 + required: true + - url: "{{ stackhpc_repo_docker_ce_ubuntu_noble_url if ansible_facts.distribution_release == 'noble' else stackhpc_repo_docker_ce_ubuntu_jammy_url }}" + suites: "{{ ansible_facts.distribution_release }}" + components: stable + signed_by: docker.asc + architecture: amd64 + required: true + - url: "{{ stackhpc_repo_ubuntu_jammy_cve_2024_6387_url }}" + suites: "pulp" + components: upload + architecture: amd64 + trusted: yes + required: "{{ ansible_facts.distribution_release == 'jammy' }}" + - url: "{{ stackhpc_repo_ceph_reef_debian_url }}" + suites: "{{ ansible_facts.distribution_release }}" + components: main + signed_by: ceph.asc + architecture: amd64 + required: "{{ ansible_facts.distribution_release == 'jammy' }}" + +# Do not replace apt configuration for non-overcloud hosts. This can result in +# errors if apt reconfiguration is performed before local repository mirrors +# are deployed. +apt_repositories: "{{ stackhpc_apt_repositories | selectattr('required') | list if stackhpc_repos_enabled | bool else [] }}" # Whether to disable repositories in /etc/apt/sources.list. This may be used # when replacing the distribution repositories via apt_repositories. # Default is false. -#apt_disable_sources_list: +# Do not disable the default apt configuration for non-overcloud hosts. This +# can result in errors if apt reconfiguration is performed before local +# repository mirrors are deployed. +apt_disable_sources_list: "{{ stackhpc_repos_enabled | bool }}" + +# Apt auth configuration for accessing the package repository mirror. +stackhpc_apt_auth: + - machine: "{{ stackhpc_repo_mirror_url }}" + login: "{{ stackhpc_repo_mirror_username }}" + password: "{{ stackhpc_repo_mirror_password }}" + filename: "kayobe.conf" # List of Apt auth configurations. Each item is a dict with the following keys: # * machine: 'machine' entry in the auth file # * login: 'login' entry in the auth file # * password: 'password' entry in the auth file -# * filename: Name of a file in which to store the auth configuration. The -# extension should be '.conf'. # * filename: Name of a file in /etc/apt/auth.conf.d in which to store # the auth configuration. The extension should be ``.conf``. # Default is an empty list. -#apt_auth: +apt_auth: "{{ stackhpc_apt_auth if stackhpc_repos_enabled | bool and stackhpc_repo_mirror_username is truthy else [] }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index 123c53d23..aa75405b4 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -66,7 +66,7 @@ # UUID of the root filesystem contained within the deployment image. # See below URL for instructions on how to extract it: # https://docs.openstack.org/ironic/latest/admin/raid.html#image-requirements -# Default is none. +# Default is an empty string. #kolla_bifrost_deploy_image_rootfs: # Custom cloud-init user-data passed to deploy of the deployment image. @@ -135,7 +135,7 @@ # Ironic Python Agent (IPA) configuration. # URL of Ironic Python Agent (IPA) kernel image. -#kolla_bifrost_ipa_kernel_upstream_url: +kolla_bifrost_ipa_kernel_upstream_url: "{{ (stackhpc_ipa_image_url + '/ipa.kernel') if stackhpc_ipa_image_bifrost_enabled | bool }}" # URL of checksum of Ironic Python Agent (IPA) kernel image. #kolla_bifrost_ipa_kernel_checksum_url: @@ -144,7 +144,7 @@ #kolla_bifrost_ipa_kernel_checksum_algorithm: # URL of Ironic Python Agent (IPA) ramdisk image. -#kolla_bifrost_ipa_ramdisk_upstream_url: +kolla_bifrost_ipa_ramdisk_upstream_url: "{{ (stackhpc_ipa_image_url + '/ipa.initramfs') if stackhpc_ipa_image_bifrost_enabled | bool }}" # URL of checksum of Ironic Python Agent (IPA) ramdisk image. #kolla_bifrost_ipa_ramdisk_checksum_url: diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index ad83c83dc..5b52d094b 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -2,13 +2,13 @@ ############################################################################### # Cephadm deployment configuration. -# Ceph release name. -cephadm_ceph_release: "reef" - # TODO placeholders stackhpc_docker_registry: "" stackhpc_sync_ceph_images: false +# Ceph release name. +cephadm_ceph_release: "{{ 'squid' if os_release == 'noble' else 'reef' }}" + # Ceph FSID. #cephadm_fsid: @@ -16,10 +16,10 @@ stackhpc_sync_ceph_images: false cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}" # Ceph container image tag. -cephadm_image_tag: "v18.2.7" +cephadm_image_tag: "{{ 'v19.2.3' if cephadm_ceph_release == 'squid' else 'v18.2.7' }}" -# Ceph custom repo workaround for Ubuntu Jammy as there are no official ceph repos for jammy. -cephadm_custom_repos: "{{ ansible_facts['distribution_release'] == 'jammy' }}" +# Ceph custom repo workaround for Ubuntu as there are no official ceph repos for Ubuntu Jammy and Noble. +cephadm_custom_repos: "{{ ansible_facts['distribution'] == 'Ubuntu' }}" # HAProxy container image. cephadm_haproxy_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/haproxy:{{ cephadm_haproxy_image_tag }}" @@ -34,13 +34,13 @@ cephadm_keepalived_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_ima cephadm_keepalived_image_tag: "2.2.4" # Ceph container image registry URL. -# cephadm_registry_url: "{{ stackhpc_docker_registry }}" +cephadm_registry_url: "{{ stackhpc_docker_registry }}" # Ceph container image registry username. -# cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" +cephadm_registry_username: "{{ stackhpc_docker_registry_username }}" # Ceph container image registry password. -# cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" +cephadm_registry_password: "{{ stackhpc_docker_registry_password }}" # Cephadm admin network interface. cephadm_admin_interface: "{{ admin_oc_net_name | net_interface }}" @@ -65,6 +65,7 @@ cephadm_cluster_network: "{{ storage_mgmt_net_name | net_cidr }}" #cephadm_osd_devices: # Ceph OSD specification. +#cephadm_osd_spec: cephadm_osd_spec: | service_type: osd service_id: osd_spec_r760 @@ -190,14 +191,17 @@ cephadm_commands_pre: "{{ cephadm_commands_pre_default + cephadm_commands_pre_ex cephadm_commands_post: "{{ cephadm_commands_post_default + cephadm_commands_post_extra }}" cephadm_commands_pre_default: [] + cephadm_commands_pre_extra: - "config set global ms_cluster_mode secure" - "config set global ms_service_mode secure" - "config set global ms_client_mode secure" + cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | default(False) | bool else [] }}" cephadm_commands_post_extra: - "fs new manila-cephfs cephfs_metadata cephfs_data --force" - "orch apply mds manila-cephfs" + ############################################################################### # Kolla Ceph auto-configuration. @@ -239,3 +243,6 @@ kolla_ceph_manila_required: "{{ kolla_enable_manila | bool }}" # Whether to generate Ceph configuration for Nova. kolla_ceph_nova_required: "{{ kolla_enable_nova | bool }}" + +# A (multiline) string to append to all Ceph configuration files. +#kolla_ceph_conf_append: diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index 69a47b7eb..a5ed68157 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -2,8 +2,9 @@ ############################################################################### # Compute node configuration. -# User with which to access the computes via SSH during bootstrap, in order -# to setup the Kayobe user account. Default is {{ os_distribution }}. +# User with which to access the compute nodes via SSH during bootstrap, in +# order to setup the Kayobe user account. Default is 'cloud-user' if +# os_distribution is set to centos, otherwise 'os_distribution'. #compute_bootstrap_user: compute_bootstrap_user: ileigh @@ -210,6 +211,13 @@ compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" # compute_libvirt_ceph_repo_install is true. Default is 'pacific'. #compute_libvirt_ceph_repo_release: +############################################################################### +# Compute node swap configuration. + +# List of swap devices. Each item is a dict containing a 'device' item. Default +# is an empty list. +# compute_swap: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/containers/pulp/post.yml b/etc/kayobe/containers/pulp/post.yml new file mode 100644 index 000000000..34c25093f --- /dev/null +++ b/etc/kayobe/containers/pulp/post.yml @@ -0,0 +1,48 @@ +--- +- name: Wait for Pulp to become ready + uri: + url: "{{ pulp_url }}/pulp/api/v3/status/" + register: pulp_status + until: pulp_status is success + retries: "{{ pulp_timeout_retries | default(120) }}" + delay: "{{ pulp_timeout_delay | default(3) }}" + +- name: Set the Pulp admin password + become: true + command: >- + docker exec -u root pulp + bash -c + "pulpcore-manager reset-admin-password -p \"{{ pulp_password }}\"" + no_log: true + register: pulp_manager_result + failed_when: + - "'Successfully set password' not in pulp_manager_result.stdout" + +- name: Manage Pulp content checksums + become: true + command: >- + docker exec -u root pulp + bash -c + 'pulpcore-manager handle-artifact-checksums' + when: + - stackhpc_pulp_sync_for_local_container_build | bool + - pulp_settings.changed + +- name: Ensure Pulp stack user exists + ansible.builtin.include_role: + name: stackhpc.pulp.pulp_user + vars: + pulp_users: + - username: "{{ pulp_stack_username }}" + password: "{{ pulp_stack_password }}" + is_staff: false + when: + - pulp_stack_username is defined and pulp_stack_username | length > 0 + - pulp_stack_password is defined and pulp_stack_password | length > 0 + +- name: Login to docker registry + docker_login: + registry_url: "{{ kolla_docker_registry or omit }}" + username: "{{ kolla_docker_registry_username }}" + password: "{{ kolla_docker_registry_password }}" + reauthorize: yes diff --git a/etc/kayobe/containers/pulp/pre.yml b/etc/kayobe/containers/pulp/pre.yml new file mode 100644 index 000000000..5e53a57ce --- /dev/null +++ b/etc/kayobe/containers/pulp/pre.yml @@ -0,0 +1,43 @@ +--- +- name: Ensure /opt/kayobe/containers/pulp exists + file: + path: "/opt/kayobe/containers/pulp" + state: directory + become: true + +- name: Ensure required Docker volumes exist + docker_volume: + name: "{{ item }}" + loop: + - pulp_containers + - pulp_pgsql + - pulp_storage + +- name: Copy modified settings.py + template: + src: "{{ kayobe_config_path }}/containers/pulp/settings.py" + dest: /opt/kayobe/containers/pulp/settings.py + mode: 0644 + become: true + register: pulp_settings + +- name: Configure TLS for local Pulp + when: pulp_enable_tls | bool + become: true + block: + - name: Ensure /opt/kayobe/containers/pulp/certs exists + file: + path: "/opt/kayobe/containers/pulp/certs" + state: directory + + - name: Copy TLS cert and key into container directory + template: + src: "{{ item.src }}" + dest: "/opt/kayobe/containers/pulp/certs/{{ item.dest }}" + mode: 0600 + become: true + loop: + - src: "{{ pulp_cert_path }}" + dest: 'pulp_webserver.crt' + - src: "{{ pulp_key_path }}" + dest: 'pulp_webserver.key' diff --git a/etc/kayobe/containers/pulp/settings.py b/etc/kayobe/containers/pulp/settings.py new file mode 100644 index 000000000..2afa6a08e --- /dev/null +++ b/etc/kayobe/containers/pulp/settings.py @@ -0,0 +1,9 @@ +CONTENT_ORIGIN='{{ pulp_url }}' +ANALYTICS=False +ANSIBLE_API_HOSTNAME='{{ pulp_url }}' +ANSIBLE_CONTENT_HOSTNAME='{{ pulp_url }}/pulp/content' +TOKEN_AUTH_DISABLED=True +{% if stackhpc_pulp_sync_for_local_container_build | bool %} +ALLOWED_CONTENT_CHECKSUMS = ["sha1", "sha224", "sha256", "sha384", "sha512"] +{% endif %} + diff --git a/etc/kayobe/containers/redfish_exporter/post.yml b/etc/kayobe/containers/redfish_exporter/post.yml new file mode 100644 index 000000000..39b3e1e2e --- /dev/null +++ b/etc/kayobe/containers/redfish_exporter/post.yml @@ -0,0 +1,6 @@ +--- +- name: Restart redfish exporter container if config changed + become: true + command: docker kill -s SIGHUP redfish_exporter + when: + - redfish_exporter_config is changed diff --git a/etc/kayobe/containers/redfish_exporter/pre.yml b/etc/kayobe/containers/redfish_exporter/pre.yml new file mode 100644 index 000000000..6bc852925 --- /dev/null +++ b/etc/kayobe/containers/redfish_exporter/pre.yml @@ -0,0 +1,16 @@ +--- + +- name: Ensure /opt/kayobe/containers/redfish_exporter exists + file: + path: "/opt/kayobe/containers/redfish_exporter" + state: directory + mode: 0770 + become: true + +- name: Template redfish exporter configuration file + template: + src: "{{ kayobe_config_path }}/containers/redfish_exporter/redfish_exporter.yml" + dest: /opt/kayobe/containers/redfish_exporter/ + mode: 0660 + become: true + register: redfish_exporter_config diff --git a/etc/kayobe/containers/redfish_exporter/redfish_exporter.yml b/etc/kayobe/containers/redfish_exporter/redfish_exporter.yml new file mode 100644 index 000000000..f7a4d4d8d --- /dev/null +++ b/etc/kayobe/containers/redfish_exporter/redfish_exporter.yml @@ -0,0 +1,5 @@ +--- +hosts: + default: + username: "{{ redfish_exporter_default_username }}" + password: "{{ redfish_exporter_default_password }}" diff --git a/etc/kayobe/containers/squid_proxy/pre.yml b/etc/kayobe/containers/squid_proxy/pre.yml new file mode 100644 index 000000000..39e33a7f9 --- /dev/null +++ b/etc/kayobe/containers/squid_proxy/pre.yml @@ -0,0 +1,14 @@ +--- +- name: Ensure /srv/docker/squid directory exists + file: + path: /srv/docker/squid + state: directory + mode: 0755 + become: true + +- name: Copy modified squid.conf + copy: + src: "{{ kayobe_config_path }}/containers/squid_proxy/squid.conf" + dest: /srv/docker/squid/ + mode: 0644 + become: true diff --git a/etc/kayobe/containers/squid_proxy/squid.conf b/etc/kayobe/containers/squid_proxy/squid.conf new file mode 100644 index 000000000..565fa7e53 --- /dev/null +++ b/etc/kayobe/containers/squid_proxy/squid.conf @@ -0,0 +1,83 @@ +# +# Recommended minimum configuration: +# + +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 10.0.0.0/8 # RFC1918 possible internal network +acl localnet src 172.16.0.0/12 # RFC1918 possible internal network +acl localnet src 192.168.0.0/16 # RFC1918 possible internal network +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines + +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT + +# +# Recommended minimum Access Permission configuration: +# +# Deny requests to certain unsafe ports +http_access deny !Safe_ports + +# Deny CONNECT to other than secure SSL ports +http_access deny CONNECT !SSL_ports + +# Only allow cachemgr access from localhost +http_access allow localhost manager +http_access deny manager + +# We strongly recommend the following be uncommented to protect innocent +# web applications running on the proxy server who think the only +# one who can access services on "localhost" is a local user +#http_access deny to_localhost + +# +# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS +# + +# Example rule allowing access from your local networks. +# Adapt localnet in the ACL section to list your (internal) IP networks +# from where browsing should be allowed +http_access allow localnet +http_access allow localhost + +# And finally deny all other access to this proxy +http_access deny all + +# Squid normally listens to port 3128 +http_port 3128 + +# Uncomment and adjust the following to add a disk cache directory. +cache_dir ufs /var/spool/squid 4096 16 256 +cache_mem 768 MB +maximum_object_size_in_memory 64 MB + +# Set a sensible cap on the number of open file descriptors. +# Rocky Linux 8.6 sets this to ~2^30 which results in alloc failure on start. +max_filedescriptors 10000 + +# Leave coredumps in the first cache dir +coredump_dir /var/spool/squid + +# +# Add any of your own refresh_pattern entries above these. +# +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern . 0 20% 4320 + +# Disable connections over IPv6 +# https://ramesh-sahoo.medium.com/squid-proxy-server-has-stopped-handling-connection-resulting-in-none-503-0-connect-errors-55477316850a +dns_v4_first on diff --git a/etc/kayobe/controllers.yml b/etc/kayobe/controllers.yml index e8679a657..d72ad6ece 100644 --- a/etc/kayobe/controllers.yml +++ b/etc/kayobe/controllers.yml @@ -3,7 +3,8 @@ # Controller node configuration. # User with which to access the controllers via SSH during bootstrap, in order -# to setup the Kayobe user account. Default is {{ os_distribution }}. +# to setup the Kayobe user account. Default is 'cloud-user' if os_distribution +# is set to centos, otherwise 'os_distribution'. #controller_bootstrap_user: controller_bootstrap_user: ileigh @@ -185,6 +186,13 @@ controller_firewalld_default_zone: trusted # - state: enabled controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" +############################################################################### +# Controller node swap configuration. + +# List of swap devices. Each item is a dict containing a 'device' item. Default +# is an empty list. +# controller_swap: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml index da4ea22eb..c337db281 100644 --- a/etc/kayobe/dnf.yml +++ b/etc/kayobe/dnf.yml @@ -41,12 +41,148 @@ # file: myrepo # gpgkey: http://gpgkey # gpgcheck: yes -#dnf_custom_repos: +dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [] }}" + +# A dict of custom repositories that point to the local Pulp server. +# To use these repos, set stackhpc_repos_enabled to true. +# This is done by default for hosts in the overcloud group via a group_vars +# file. +stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_doca if dnf_install_doca | bool else {}) }}" + +# DOCA repositories +dnf_custom_repos_doca: + doca: + baseurl: "{{ stackhpc_repo_rhel9_doca_url }}" + description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL $releasever" + enabled: "{{ dnf_enable_doca | bool | default(false) }}" + priority: -1 + file: doca + gpgcheck: no + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + doca-modules: + baseurl: "{{ stackhpc_repo_rhel9_doca_modules_url }}" + description: "OFED Kernel module repository for DOCA {{ stackhpc_pulp_doca_version }} - RHEL $releasever" + enabled: "{{ dnf_enable_doca_modules | bool | default(false) }}" + priority: -1 + file: doca + gpgcheck: no + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + +# Custom repositories shared between all RHEL 9 derivatives. +dnf_custom_repos_el9: + epel: + baseurl: "{{ stackhpc_repo_epel_9_url }}" + description: "Extra Packages for Enterprise Linux $releasever - $basearch" + enabled: "{{ dnf_enable_epel | bool }}" + file: epel + gpgkey: "{{ dnf_epel_9_gpg_key_url }}" + gpgcheck: yes + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + docker: + baseurl: "{{ stackhpc_repo_centos_stream_9_docker_url }}" + description: "Package repository for installing docker" + enabled: "{{ dnf_enable_docker | bool }}" + file: docker + gpgkey: "{{ dnf_docker_gpg_key_url }}" + gpgcheck: yes + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + +# ELRepo 9 +dnf_custom_repos_elrepo_9: + elrepo: + baseurl: "{{ stackhpc_repo_elrepo_9_url }}" + description: "ELRepo.org Community Enterprise Linux Repository - el9" + enabled: "{{ dnf_enable_elrepo_9 | bool }}" + file: elrepo + gpgkey: https://www.elrepo.org/RPM-GPG-KEY-elrepo.org + gpgcheck: yes + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + +# Rocky 9 specific repositories +dnf_custom_repos_rocky_9: + appstream: + baseurl: "{{ stackhpc_repo_rocky_9_appstream_url }}" + description: "Rocky Linux $releasever - AppStream" + file: rocky + gpgkey: "{{ rocky_9_gpg_key }}" + gpgcheck: yes + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + baseos: + baseurl: "{{ stackhpc_repo_rocky_9_baseos_url }}" + description: "Rocky Linux $releasever - BaseOS" + file: rocky + gpgkey: "{{ rocky_9_gpg_key }}" + gpgcheck: yes + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + crb: + baseurl: "{{ stackhpc_repo_rocky_9_crb_url }}" + description: "Rocky Linux $releasever - CRB" + file: rocky + gpgkey: "{{ rocky_9_gpg_key }}" + gpgcheck: yes + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + extras: + baseurl: "{{ stackhpc_repo_rocky_9_extras_url }}" + description: "Rocky Linux $releasever - Extras" + file: rocky-extras + gpgkey: "{{ rocky_9_gpg_key }}" + gpgcheck: yes + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + security-common: + baseurl: "{{ stackhpc_repo_rocky_9_sig_security_common_url }}" + description: "Rocky Linux $releasever - SIG Security Common" + file: Rocky-SIG-Security-Common + gpgkey: "{{ rocky_9_sig_security_gpg_key }}" + gpgcheck: yes + includepkgs: "openssh*" + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + +# Whether to enable EPEL repositories. This affects RedHat-based systems only. +dnf_enable_epel: "{{ dnf_install_epel | bool }}" + +# Whether to enable the ELRepo repository. This affects RedHat-based, 9.x release systems only. +dnf_enable_elrepo_9: "{{ dnf_install_elrepo_9 | bool }}" + +# Whether to enable DOCA repositories. This affects RedHat-based systems only. +dnf_enable_doca: "{{ dnf_install_doca | bool }}" + +# Whether to enable the DOCA kernel module repository. This affects RedHat-based systems only. +dnf_enable_doca_modules: "{{ dnf_install_doca | bool }}" + +# URL of EPEL GPG keys. +dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9" + +rocky_9_gpg_key: "https://dl.rockylinux.org/pub/rocky/RPM-GPG-KEY-Rocky-9" +rocky_9_sig_security_gpg_key: "https://dl.rockylinux.org/pub/sig/9/security/x86_64/security-common/RPM-GPG-KEY-Rocky-SIG-Security" # Whether to install the epel-release package. This affects RedHat-based # systems only. Default value is 'false'. #dnf_install_epel: +# Whether to create a repo file for ELRepo. This affects RedHat-based +# systems only. +dnf_install_elrepo_9: false + +# Whether to enable docker dnf repo in stackhpc_dnf_repos +dnf_enable_docker: true + +#URL of docker repo GPG key +dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg" + +# Whether to create a repo file for DOCA. This affects RedHat-based +# systems only. +dnf_install_doca: "{{ 'mlnx' in group_names }}" + ############################################################################### # DNF Automatic configuration. diff --git a/etc/kayobe/docker-registry.yml b/etc/kayobe/docker-registry.yml index ea6ca9c84..1bf693890 100644 --- a/etc/kayobe/docker-registry.yml +++ b/etc/kayobe/docker-registry.yml @@ -3,7 +3,7 @@ # Docker registry configuration. # Whether a docker registry is enabled. Default is false. -#docker_registry_enabled: +docker_registry_enabled: false # Dict of environment variables to provide to the docker registry container. # This allows to configure the registry by overriding specific configuration diff --git a/etc/kayobe/docker.yml b/etc/kayobe/docker.yml index ba3f22c9a..26fac1346 100644 --- a/etc/kayobe/docker.yml +++ b/etc/kayobe/docker.yml @@ -22,12 +22,12 @@ #docker_storage_volume_thinpool_meta_size: # URL of docker registry -#docker_registry: +docker_registry: "{{ stackhpc_docker_registry }}" # Whether docker should be configured to use an insecure registry. # Default is false, unless docker_registry_enabled is true and # docker_registry_enable_tls is false. -#docker_registry_insecure: +docker_registry_insecure: "{{ 'https' not in stackhpc_repo_mirror_url }}" # CA of docker registry #docker_registry_ca: @@ -36,7 +36,7 @@ #docker_registry_mirrors: # Enable live-restore on docker daemon -#docker_daemon_live_restore: +docker_daemon_live_restore: true ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh new file mode 100755 index 000000000..9dbbeb990 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +########################################### +# STACKHPC-KAYOBE-CONFIG AUFN ENV VERSION # +########################################### + +# Cheat script for a full deployment. +# This should be used for testing only. + +set -eu + +BASE_PATH=~ +KAYOBE_BRANCH=stackhpc/2024.1 +KAYOBE_CONFIG_BRANCH=stackhpc/2024.1 +KAYOBE_ENVIRONMENT=aufn-ceph + +# Install git and tmux. +if $(which dnf 2>/dev/null >/dev/null); then + sudo dnf -y install git tmux +else + sudo apt update + sudo apt -y install git tmux gcc libffi-dev python3-dev python-is-python3 +fi + +# Disable the firewall. +sudo systemctl is-enabled firewalld && sudo systemctl stop firewalld && sudo systemctl disable firewalld + +# Disable SELinux both immediately and permanently. +if $(which setenforce 2>/dev/null >/dev/null); then + sudo setenforce 0 + sudo sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config +fi + +# Prevent sudo from performing DNS queries. +echo 'Defaults !fqdn' | sudo tee /etc/sudoers.d/no-fqdn + +# Clone repositories +cd $BASE_PATH +mkdir -p src +pushd src +[[ -d kayobe ]] || git clone https://github.com/stackhpc/kayobe.git -b $KAYOBE_BRANCH +[[ -d kayobe-config ]] || git clone https://github.com/stackhpc/stackhpc-kayobe-config kayobe-config -b $KAYOBE_CONFIG_BRANCH +[[ -d kayobe/tenks ]] || (cd kayobe && git clone https://opendev.org/openstack/tenks.git) +popd + +# Create Kayobe virtualenv +mkdir -p venvs +pushd venvs +if [[ ! -d kayobe ]]; then + python3 -m venv kayobe +fi +# NOTE: Virtualenv's activate and deactivate scripts reference an +# unbound variable. +set +u +source kayobe/bin/activate +set -u +pip install -U pip +pip install ../src/kayobe +popd + +# Activate environment +pushd $BASE_PATH/src/kayobe-config +source kayobe-env --environment $KAYOBE_ENVIRONMENT + +# Configure host networking (bridge, routes & firewall) +$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh + +# Bootstrap the Ansible control host. +kayobe control host bootstrap + +# Configure the seed hypervisor host. +kayobe seed hypervisor host configure + +# Provision the seed VM. +kayobe seed vm provision + +# Configure the seed host, and deploy a local registry. +kayobe seed host configure + +# Deploy local pulp server as a container on the seed VM +kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none + +# Deploying the seed restarts networking interface, run configure-local-networking.sh again to re-add routes. +$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh + +# Sync package & container repositories. +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml + +# Re-run full task to set up bifrost_deploy etc. using newly-populated pulp repo +kayobe seed service deploy + +# NOTE: Make sure to use ./tenks, since just ‘tenks’ will install via PyPI. +(export TENKS_CONFIG_PATH=$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/tenks.yml && \ + export KAYOBE_CONFIG_SOURCE_PATH=$BASE_PATH/src/kayobe-config && \ + export KAYOBE_VENV_PATH=$BASE_PATH/venvs/kayobe && \ + cd $BASE_PATH/src/kayobe && \ + ./dev/tenks-deploy-overcloud.sh ./tenks) + +# Inspect and provision the overcloud hardware: +kayobe overcloud inventory discover +kayobe overcloud hardware inspect +kayobe overcloud provision +kayobe overcloud host configure +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml +kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml +kayobe overcloud container image pull +kayobe overcloud service deploy +source $KOLLA_CONFIG_PATH/public-openrc.sh +kayobe overcloud post configure +source $KOLLA_CONFIG_PATH/public-openrc.sh + + +# Use openstack-config-multinode here instead of init-runonce.sh script from standard aufn + +#Deactivate current kayobe venv +set +u +deactivate +set -u +$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-openstack.sh $BASE_PATH + +# Create a test vm +VENV_DIR=$BASE_PATH/venvs/openstack +if [[ ! -d $VENV_DIR ]]; then + python3 -m venv $VENV_DIR +fi +source $VENV_DIR/bin/activate +pip install -U pip +pip install python-openstackclient +source $KOLLA_CONFIG_PATH/public-openrc.sh +echo "Creating openstack key:" +openstack keypair create --public-key ~/.ssh/id_rsa.pub mykey +echo "Creating test vm:" +openstack server create --key-name mykey --flavor m1.tiny --image cirros --network admin-tenant test-vm-1 +echo "Attaching floating IP:" +openstack floating ip create external +openstack server add floating ip test-vm-1 `openstack floating ip list -c ID -f value` +echo -e "Done! \nopenstack server list:" +openstack server list diff --git a/etc/kayobe/environments/aufn-ceph/cephadm.yml b/etc/kayobe/environments/aufn-ceph/cephadm.yml new file mode 100644 index 000000000..2b8457dca --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/cephadm.yml @@ -0,0 +1,63 @@ +--- +############################################################################### +# Cephadm deployment configuration. + +# List of additional cephadm commands to run before deployment +# cephadm_commands: +# - "config set global osd_pool_default_size {{ [3, groups['osds'] | length] | min }}" +# - "config set global osd_pool_default_min_size {{ [3, groups['osds'] | length] | min }}" + +# Ceph OSD specification. +cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: "*" + data_devices: + all: true + +############################################################################### +# Ceph post-deployment configuration. + +# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role +# for format. +cephadm_ec_profiles: [] + +# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. +cephadm_crush_rules: [] + +# List of Ceph pools. See stackhpc.cephadm.pools role for format. +cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: vms + application: rbd + state: present + +# List of Cephx keys. See stackhpc.cephadm.keys role for format. +cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + state: present + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + state: present + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present diff --git a/etc/kayobe/environments/aufn-ceph/configure-local-networking.sh b/etc/kayobe/environments/aufn-ceph/configure-local-networking.sh new file mode 100755 index 000000000..c22bbd518 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/configure-local-networking.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -e +set -o pipefail + +# This should be run on the seed hypervisor. + +# IP addresses on the all-in-one Kayobe cloud network. +# These IP addresses map to those statically configured in +# etc/kayobe/network-allocation.yml and etc/kayobe/networks.yml. +controller_vip=192.168.39.2 +seed_hv_ip=192.168.33.4 + +iface=$(ip route | awk '$1 == "default" {print $5; exit}') + +# Private IP address by which the seed hypervisor is accessible in the cloud +# hosting the VM. +seed_hv_private_ip=$(ip a show dev $iface | awk '$1 == "inet" { gsub(/\/[0-9]*/,"",$2); print $2; exit }') + +# Forward the following ports to the controller. +# 80: Horizon +# 6080: VNC console +forwarded_ports="80 6080" + +# Install iptables. +if $(which dnf >/dev/null 2>&1); then + sudo dnf -y install iptables +else + sudo apt update + sudo apt -y install iptables +fi + +# Configure local networking. +# Add bridges for the Kayobe networks. +if ! sudo ip l show brprov >/dev/null 2>&1; then + sudo ip l add brprov type bridge + sudo ip l set brprov up + sudo ip a add $seed_hv_ip/24 dev brprov +fi + +if ! sudo ip l show brcloud >/dev/null 2>&1; then + sudo ip l add brcloud type bridge + sudo ip l set brcloud up +fi + +# On Rocky Linux, bridges without a port are DOWN, which causes network +# configuration to fail. Add a dummy interface and plug it into the bridge. +for i in mgmt prov cloud; do + if ! sudo ip l show dummy-$i >/dev/null 2>&1; then + sudo ip l add dummy-$i type dummy + fi +done + +# Configure IP routing and NAT to allow the seed VM and overcloud hosts to +# route via this route to the outside world. +sudo iptables -A POSTROUTING -t nat -o $iface -j MASQUERADE +sudo sysctl -w net.ipv4.conf.all.forwarding=1 + +# FIXME: IP MASQUERADE from control plane fails without this on Ubuntu. +if ! $(which dnf >/dev/null 2>&1); then + sudo modprobe br_netfilter + echo 0 | sudo tee /proc/sys/net/bridge/bridge-nf-call-iptables +fi + +# Configure port forwarding from the hypervisor to the Horizon GUI on the +# controller. +sudo iptables -A FORWARD -i $iface -o brprov -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +sudo iptables -A FORWARD -i brprov -o $iface -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +for port in $forwarded_ports; do + # Allow new connections. + sudo iptables -A FORWARD -i $iface -o brcloud -p tcp --syn --dport $port -m conntrack --ctstate NEW -j ACCEPT + # Destination NAT. + sudo iptables -t nat -A PREROUTING -i $iface -p tcp --dport $port -j DNAT --to-destination $controller_vip + # Source NAT. + sudo iptables -t nat -A POSTROUTING -o brcloud -p tcp --dport $port -d $controller_vip -j SNAT --to-source $seed_hv_private_ip +done + +echo +echo "NOTE: The network configuration applied by this script is not" +echo "persistent across reboots." +echo "If you reboot the system, please re-run this script." diff --git a/etc/kayobe/environments/aufn-ceph/configure-openstack.sh b/etc/kayobe/environments/aufn-ceph/configure-openstack.sh new file mode 100755 index 000000000..3bac0a351 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/configure-openstack.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +BASE_PATH=$1 +VENV_DIR=$BASE_PATH/venvs/ansible +cd $BASE_PATH/src/ +[[ -d openstack-config ]] || git clone https://github.com/stackhpc/openstack-config-multinode.git openstack-config +cd openstack-config +if [[ ! -d $VENV_DIR ]]; then + # Using virtualenv causes a strange bug with python3.6 where + # nested virtual env creation leads to envs without pip... + # virtualenv $VENV_DIR + python3 -m venv $VENV_DIR +fi + +# NOTE: Virtualenv's activate and deactivate scripts reference an unbound variable. +set +u +source $VENV_DIR/bin/activate +set -u + +pip install -U pip +pip install -r requirements.txt +ansible-galaxy role install -p ansible/roles -r requirements.yml +ansible-galaxy collection install -p ansible/collections -r requirements.yml + +source $BASE_PATH/src/kayobe-config/etc/kolla/public-openrc.sh + +# Run script to configure openstack cloud +tools/openstack-config diff --git a/etc/kayobe/environments/aufn-ceph/globals.yml b/etc/kayobe/environments/aufn-ceph/globals.yml new file mode 100644 index 000000000..9fe78517c --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/globals.yml @@ -0,0 +1,21 @@ +--- +# Kayobe global configuration. + +############################################################################### +# OS distribution. + +# OS distribution name. Valid options are "rocky", "ubuntu". Default is +# "rocky". +os_distribution: "{{ lookup('pipe', '. /etc/os-release && echo $ID') | trim }}" + +# OS release. Valid options are "9" when os_distribution is "rocky", or +# "jammy" and "noble" when os_distribution is "ubuntu". +os_release: >- + {{ (lookup('pipe', '. /etc/os-release && echo $VERSION_CODENAME') | trim) if os_distribution == 'ubuntu' else + (lookup('pipe', '. /etc/os-release && echo $VERSION_ID') | trim | split('.') | first) if os_distribution == 'rocky' }} + +############################################################################### +# Extra vars. + +# Don't prompt when rebooting hosts. +confirm_reboot: true diff --git a/etc/kayobe/environments/aufn-ceph/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/compute/network-interfaces new file mode 100644 index 000000000..b44b1b048 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/compute/network-interfaces @@ -0,0 +1,19 @@ +--- +############################################################################### +# Network interface definitions for the compute group. + +provision_oc_interface: "{{ 'ens2' if os_distribution == 'ubuntu' else 'eth0' }}" +# Route via the seed-hypervisor to the outside world. +provision_oc_gateway: 192.168.33.4 + +internal_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ internal_vlan }}" + +storage_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ storage_vlan }}" + +tunnel_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ tunnel_vlan }}" + +external_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ external_vlan }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/aufn-ceph/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/controllers/network-interfaces new file mode 100644 index 000000000..d608a62e7 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/controllers/network-interfaces @@ -0,0 +1,27 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +provision_oc_interface: "{{ 'ens2' if os_distribution == 'ubuntu' else 'eth0' }}" +# Route via the seed-hypervisor to the outside world. +provision_oc_gateway: 192.168.33.4 + +mgmt_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}" + +provision_wl_interface: "br{{ 'ens4' if os_distribution == 'ubuntu' else 'eth2' }}" +provision_wl_bridge_ports: + - "{{ 'ens4' if os_distribution == 'ubuntu' else 'eth2' }}" + +internal_interface: "{{ 'ens4' if os_distribution == 'ubuntu' else 'eth2' }}.{{ internal_vlan }}" + +external_interface: "br{{ 'ens4' if os_distribution == 'ubuntu' else 'eth2' }}.{{ external_vlan }}" + +public_interface: "{{ 'ens4' if os_distribution == 'ubuntu' else 'eth2' }}.{{ public_vlan }}" + +storage_interface: "{{ 'ens4' if os_distribution == 'ubuntu' else 'eth2' }}.{{ storage_vlan }}" + +tunnel_interface: "{{ 'ens4' if os_distribution == 'ubuntu' else 'eth2' }}.{{ tunnel_vlan }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed-hypervisor/network-interfaces b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed-hypervisor/network-interfaces new file mode 100644 index 000000000..61d0a836b --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed-hypervisor/network-interfaces @@ -0,0 +1,18 @@ +--- +mgmt_interface: brmgmt +mgmt_bridge_ports: + - dummy-mgmt + +provision_oc_interface: brprov +provision_oc_bridge_ports: + - dummy-prov + +provision_wl_interface: brcloud +provision_wl_bridge_ports: + - dummy-cloud + +internal_interface: "{{ provision_wl_interface }}.{{ internal_vlan }}" + +public_interface: "{{ provision_wl_interface }}.{{ public_vlan }}" + +external_interface: "{{ provision_wl_interface }}.{{ external_vlan }}" diff --git a/etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed/network-interfaces new file mode 100644 index 000000000..ca04864d4 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/seed/network-interfaces @@ -0,0 +1,13 @@ +--- +############################################################################### +# Network interface definitions for the seed group. + +mgmt_interface: "{{ 'ens2' if os_distribution == 'ubuntu' else 'eth0' }}" + +provision_oc_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}" +# Route via the seed-hypervisor to the outside world. +provision_oc_gateway: 192.168.33.4 + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/aufn-ceph/inventory/group_vars/storage/network-interfaces b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/storage/network-interfaces new file mode 100644 index 000000000..c2b0dac84 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/inventory/group_vars/storage/network-interfaces @@ -0,0 +1,17 @@ +--- +############################################################################### +# Network interface definitions for the compute group. + +provision_oc_interface: "{{ 'ens2' if os_distribution == 'ubuntu' else 'eth0' }}" +# Route via the seed-hypervisor to the outside world. +provision_oc_gateway: 192.168.33.4 + +internal_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ internal_vlan }}" + +storage_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ storage_vlan }}" + +storage_mgmt_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ storage_mgmt_vlan }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/aufn-ceph/inventory/groups b/etc/kayobe/environments/aufn-ceph/inventory/groups new file mode 100644 index 000000000..71b723a91 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/inventory/groups @@ -0,0 +1,29 @@ +# Kayobe groups inventory file. This file should generally not be modified. +# It declares the top-level groups and sub-groups. + +############################################################################### +# Ceph groups + +# Empty group with hosts added in overcloud.yml +[storage-ceph] + +[mons:children] +storage-ceph + +[mgrs:children] +storage-ceph + +[osds:children] +storage-ceph + +[rgws:children] +storage-ceph + +[storage:children] +storage-ceph + +############################################################################### +# Monitoring groups + +[monitoring:children] +controllers diff --git a/etc/kayobe/environments/aufn-ceph/inventory/hosts b/etc/kayobe/environments/aufn-ceph/inventory/hosts new file mode 100644 index 000000000..c0d1b51fd --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/inventory/hosts @@ -0,0 +1,27 @@ +# This host acts as the configuration management Ansible control host. This must be +# localhost. +localhost ansible_connection=local + +[seed-hypervisor] +seed-hypervisor + +[seed] +seed + +[controllers] +#controller0 + +[compute:children] +#controllers + +[baremetal-compute] +# Add baremetal compute nodes here if required. + +[mgmt-switches] +# Add management network switches here if required. + +[ctl-switches] +# Add control and provisioning switches here if required. + +[hs-switches] +# Add high speed switches here if required. diff --git a/etc/kayobe/environments/aufn-ceph/ipa.yml b/etc/kayobe/environments/aufn-ceph/ipa.yml new file mode 100644 index 000000000..65616cfe1 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/ipa.yml @@ -0,0 +1,13 @@ +--- +# Ironic Python Agent (IPA) configuration. + +############################################################################### +# Ironic Python Agent (IPA) images configuration. + +# URL of Ironic deployment kernel image to download. +# yamllint disable-line rule:line-length +ipa_kernel_upstream_url: "https://tarballs.openstack.org/ironic-python-agent/tinyipa/files/tinyipa{{ ipa_images_upstream_url_suffix }}.vmlinuz" + +# URL of Ironic deployment ramdisk image to download. +# yamllint disable-line rule:line-length +ipa_ramdisk_upstream_url: "https://tarballs.openstack.org/ironic-python-agent/tinyipa/files/tinyipa{{ ipa_images_upstream_url_suffix }}.gz" diff --git a/etc/kayobe/environments/aufn-ceph/kolla.yml b/etc/kayobe/environments/aufn-ceph/kolla.yml new file mode 100644 index 000000000..4c6d1a57f --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/kolla.yml @@ -0,0 +1,16 @@ +--- +# Kayobe Kolla configuration. + +############################################################################### +# Kolla-ansible inventory configuration. + +# Custom overcloud inventory containing a mapping from services to components. +kolla_overcloud_inventory_custom_services: "{{ lookup('template', kayobe_env_config_path ~ '/kolla/inventory/overcloud-services.j2') }}" + +# Don't give storage nodes to kolla-ansible - we're using Ceph-ansible. +# kolla_overcloud_inventory_storage_groups: [] + +############################################################################### +# Kolla feature flag configuration. +kolla_enable_cinder: true +kolla_enable_neutron_provider_networks: true diff --git a/etc/kayobe/environments/aufn-ceph/kolla/config/bifrost/bifrost.yml b/etc/kayobe/environments/aufn-ceph/kolla/config/bifrost/bifrost.yml new file mode 100644 index 000000000..94b0cd1d9 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/kolla/config/bifrost/bifrost.yml @@ -0,0 +1,15 @@ +# yamllint disable-file +--- +# Don't build an IPA deployment image, instead download upstream images. +create_ipa_image: false +download_ipa: true + +# Use a locally hosted cloud image. +use_cirros: true +{% if os_distribution == 'ubuntu' and os_release == 'jammy' %} +cirros_deploy_image_upstream_url: "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" +{% elif os_distribution == 'ubuntu' and os_release == 'noble' %} +cirros_deploy_image_upstream_url: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" +{% else %} +cirros_deploy_image_upstream_url: "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" +{% endif %} diff --git a/etc/kayobe/environments/aufn-ceph/kolla/config/neutron.conf b/etc/kayobe/environments/aufn-ceph/kolla/config/neutron.conf new file mode 100644 index 000000000..1014e6730 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/kolla/config/neutron.conf @@ -0,0 +1,2 @@ +[DEFAULT] +global_physnet_mtu = {{ tunnel_net_name | net_mtu }} diff --git a/etc/kayobe/environments/aufn-ceph/kolla/config/neutron/ml2_conf.ini b/etc/kayobe/environments/aufn-ceph/kolla/config/neutron/ml2_conf.ini new file mode 100644 index 000000000..7695e792b --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/kolla/config/neutron/ml2_conf.ini @@ -0,0 +1,2 @@ +[ml2] +path_mtu = {{ tunnel_net_name | net_mtu }} diff --git a/etc/kayobe/environments/aufn-ceph/kolla/globals.yml b/etc/kayobe/environments/aufn-ceph/kolla/globals.yml new file mode 100644 index 000000000..872a57b80 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/kolla/globals.yml @@ -0,0 +1,17 @@ +--- +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu + +# Reduce the control plane's memory footprint by limiting the number of worker +# processes to one per-service. +openstack_service_workers: "1" +openstack_service_rpc_workers: "1" + +glance_backend_ceph: "yes" +cinder_backend_ceph: "yes" +nova_backend_ceph: "yes" + +# OpenSearch memory tuning +opensearch_heap_size: 1g diff --git a/etc/kayobe/environments/aufn-ceph/kolla/inventory/overcloud-services.j2 b/etc/kayobe/environments/aufn-ceph/kolla/inventory/overcloud-services.j2 new file mode 100644 index 000000000..f2b89aabb --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/kolla/inventory/overcloud-services.j2 @@ -0,0 +1,466 @@ +# This inventory section provides a mapping of services to components. +# +# Top level groups define the roles of hosts, e.g. controller or compute. +# Components define groups of services, e.g. nova or ironic. +# Services define single containers, e.g. nova-compute or ironic-api. + +# Additional control implemented here. These groups allow you to control which +# services run on which hosts at a per-service level. +# +# Word of caution: Some services are required to run on the same host to +# function appropriately. For example, neutron-metadata-agent must run on the +# same host as the l3-agent and (depending on configuration) the dhcp-agent. + +# Common +[cron:children] +common + +[fluentd:children] +common + +[kolla-logs:children] +common + +[kolla-toolbox:children] +common + +[opensearch:children] +control + +# Opensearch dashboards +[opensearch-dashboards:children] +opensearch + +# Glance +[glance-api:children] +glance + +# Nova +[nova-api:children] +nova + +[nova-conductor:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +[nova-super-conductor:children] +nova + +[nova-novncproxy:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +[nova-scheduler:children] +nova + +[nova-spicehtml5proxy:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +# NOTE: HA for nova-compute services with ironic is still an experimental +# feature. Provide the option to use a single compute host, even when multiple +# controllers are in use. +{% if kolla_nova_compute_ironic_host is not none %} +[nova-compute-ironic] +{{ kolla_nova_compute_ironic_host }} +{% else %} +[nova-compute-ironic:children] +nova +{% endif %} + +[nova-serialproxy:children] +{% if 'cell-control' in kolla_overcloud_inventory_top_level_group_map %} +cell-control +{% else %} +nova +{% endif %} + +# Neutron +[neutron-server:children] +control + +[neutron-dhcp-agent:children] +neutron + +[neutron-l3-agent:children] +neutron + +[neutron-metadata-agent:children] +neutron + +[neutron-ovn-metadata-agent:children] +compute +network + +[neutron-bgp-dragent:children] +neutron + +[neutron-infoblox-ipam-agent:children] +neutron + +[neutron-metering-agent:children] +neutron + +[ironic-neutron-agent:children] +neutron + +[neutron-ovn-agent:children] +compute +network + +# Cinder +[cinder-api:children] +cinder + +[cinder-backup:children] +cinder + +[cinder-scheduler:children] +cinder + +[cinder-volume:children] +cinder + +# Cloudkitty +[cloudkitty-api:children] +cloudkitty + +[cloudkitty-processor:children] +cloudkitty + +# iSCSI +[iscsid:children] +compute +storage +ironic + +[tgtd:children] +storage + +# Manila +[manila-api:children] +manila + +[manila-scheduler:children] +manila + +[manila-share:children] +network + +[manila-data:children] +manila + +# Swift +[swift-proxy-server:children] +swift + +[swift-account-server:children] +storage + +[swift-container-server:children] +storage + +[swift-object-server:children] +storage + +# Barbican +[barbican-api:children] +barbican + +[barbican-keystone-listener:children] +barbican + +[barbican-worker:children] +barbican + +# Heat +[heat-api:children] +heat + +[heat-api-cfn:children] +heat + +[heat-engine:children] +heat + +# Ironic +[ironic-api:children] +ironic + +[ironic-conductor:children] +ironic + +{% if kolla_ironic_inspector_host %} +[ironic-inspector] +{{ kolla_ironic_inspector_host }} +{% else %} +[ironic-inspector:children] +ironic +{% endif %} + +[ironic-tftp:children] +ironic + +[ironic-http:children] +ironic + +# Magnum +[magnum-api:children] +magnum + +[magnum-conductor:children] +magnum + +# Mistral +[mistral-api:children] +mistral + +[mistral-executor:children] +mistral + +[mistral-engine:children] +mistral + +[mistral-event-engine:children] +mistral + +# Ceilometer +[ceilometer-central:children] +ceilometer + +[ceilometer-notification:children] +ceilometer + +[ceilometer-compute:children] +compute + +[ceilometer-ipmi:children] +compute + +# Aodh +[aodh-api:children] +aodh + +[aodh-evaluator:children] +aodh + +[aodh-listener:children] +aodh + +[aodh-notifier:children] +aodh + +# Cyborg +[cyborg-api:children] +cyborg + +[cyborg-agent:children] +compute + +[cyborg-conductor:children] +cyborg + +# Gnocchi +[gnocchi-api:children] +gnocchi + +[gnocchi-statsd:children] +gnocchi + +[gnocchi-metricd:children] +gnocchi + +# Trove +[trove-api:children] +trove + +[trove-conductor:children] +trove + +[trove-taskmanager:children] +trove + +# Multipathd +[multipathd:children] +compute +storage + +# Watcher +[watcher-api:children] +watcher + +[watcher-engine:children] +watcher + +[watcher-applier:children] +watcher + +# Octavia +[octavia-api:children] +octavia + +[octavia-driver-agent:children] +octavia + +[octavia-health-manager:children] +octavia + +[octavia-housekeeping:children] +octavia + +[octavia-worker:children] +octavia + +# Designate +[designate-api:children] +designate + +[designate-central:children] +designate + +[designate-producer:children] +designate + +[designate-mdns:children] +network + +[designate-worker:children] +designate + +[designate-sink:children] +designate + +[designate-backend-bind9:children] +designate + +# Placement +[placement-api:children] +placement + +# Zun +[zun-api:children] +zun + +[zun-wsproxy:children] +zun + +[zun-compute:children] +compute + +[zun-cni-daemon:children] +compute + +# Skyline +[skyline-apiserver:children] +skyline + +[skyline-console:children] +skyline + +# Tacker +[tacker-server:children] +tacker + +[tacker-conductor:children] +tacker + +# Blazar +[blazar-api:children] +blazar + +[blazar-manager:children] +blazar + +# Prometheus +[prometheus-node-exporter:children] +monitoring +control +compute +network +storage + +[prometheus-mysqld-exporter:children] +mariadb + +[prometheus-memcached-exporter:children] +memcached + +[prometheus-cadvisor:children] +monitoring +control +compute +network +storage + +[prometheus-alertmanager:children] +monitoring + +[prometheus-openstack-exporter:children] +monitoring + +[prometheus-elasticsearch-exporter:children] +opensearch + +[prometheus-blackbox-exporter:children] +monitoring + +[prometheus-libvirt-exporter:children] +compute + +[prometheus-msteams:children] +prometheus-alertmanager + +[masakari-api:children] +control + +[masakari-engine:children] +control + +[masakari-hostmonitor:children] +control + +[masakari-instancemonitor:children] +compute + +[ovn-controller:children] +ovn-controller-compute +ovn-controller-network + +[ovn-controller-compute:children] +compute + +[ovn-controller-network:children] +network + +[ovn-database:children] +control + +[ovn-northd:children] +ovn-database + +[ovn-nb-db:children] +ovn-database + +[ovn-sb-db:children] +ovn-database + +[venus-api:children] +venus + +[venus-manager:children] +venus + +[letsencrypt-webserver:children] +letsencrypt + +[letsencrypt-lego:children] +letsencrypt diff --git a/etc/kayobe/environments/aufn-ceph/kolla/kolla-build.conf b/etc/kayobe/environments/aufn-ceph/kolla/kolla-build.conf new file mode 100644 index 000000000..0a665e1bd --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/kolla/kolla-build.conf @@ -0,0 +1,4 @@ +[DEFAULT] +# This is necessary for network connectivity of kolla-build, when Docker +# default iptables rules are disabled. +network_mode = host diff --git a/etc/kayobe/environments/aufn-ceph/network-allocation.yml b/etc/kayobe/environments/aufn-ceph/network-allocation.yml new file mode 100644 index 000000000..ba8a1241c --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/network-allocation.yml @@ -0,0 +1,13 @@ +--- +provision_oc_ips: + compute0: 192.168.33.6 + compute1: 192.168.33.7 + compute2: 192.168.33.8 + controller0: 192.168.33.3 + controller1: 192.168.33.9 + controller2: 192.168.33.10 + seed: 192.168.33.5 + seed-hypervisor: 192.168.33.4 + storage0: 192.168.33.11 + storage1: 192.168.33.12 + storage2: 192.168.33.13 diff --git a/etc/kayobe/environments/aufn-ceph/networks.yml b/etc/kayobe/environments/aufn-ceph/networks.yml new file mode 100644 index 000000000..89a30331c --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/networks.yml @@ -0,0 +1,117 @@ +--- +# Kayobe network configuration. + +############################################################################### +# Network role to network mappings. + +# Name of the network used by the seed to manage the bare metal overcloud +# hosts via their out-of-band management controllers. +oob_oc_net_name: mgmt + +# Name of the network used by the seed to provision the bare metal overcloud +# hosts. +provision_oc_net_name: provision_oc + +# Name of the network used by the overcloud hosts to manage the bare metal +# compute hosts via their out-of-band management controllers. +oob_wl_net_name: mgmt + +# Name of the network used by the overcloud hosts to provision the bare metal +# workload hosts. +provision_wl_net_name: provision_wl + +# Name of the network used to expose the internal OpenStack API endpoints. +internal_net_name: internal + +# List of names of networks used to provide external network access via +# Neutron. +# Deprecated name: external_net_name +# If external_net_name is defined, external_net_names will default to a list +# containing one item, external_net_name. +external_net_names: + - external + +# Name of the network used to expose the public OpenStack API endpoints. +public_net_name: public + +# Name of the network used by Neutron to carry tenant overlay network traffic. +tunnel_net_name: tunnel + +# Name of the network used to carry storage data traffic. +storage_net_name: storage + +# Name of the network used to carry storage management traffic. +storage_mgmt_net_name: storage_mgmt + +# Name of the network used to perform hardware introspection on the bare metal +# workload hosts. +inspection_net_name: provision_wl + +# Name of the network used to perform cleaning on the bare metal workload +# hosts +cleaning_net_name: provision_wl + +############################################################################### +# Network definitions. + +mgmt_cidr: 192.168.35.0/24 +mgmt_mtu: 1442 +# Native VLAN +mgmt_physical_network: mgmt + +provision_oc_cidr: 192.168.33.0/24 +provision_oc_mtu: 1442 +provision_oc_inspection_allocation_pool_start: 192.168.33.128 +provision_oc_inspection_allocation_pool_end: 192.168.33.254 +# Native VLAN +provision_oc_physical_network: provision + +provision_wl_cidr: 192.168.36.0/24 +provision_wl_mtu: 1442 +provision_wl_inspection_allocation_pool_start: 192.168.36.128 +provision_wl_inspection_allocation_pool_end: 192.168.36.254 +provision_wl_neutron_allocation_pool_start: 192.168.36.2 +provision_wl_neutron_allocation_pool_end: 192.168.36.127 +# Native VLAN +provision_wl_physical_network: cloud + +internal_cidr: 192.168.37.0/24 +internal_mtu: 1442 +internal_allocation_pool_start: 192.168.37.3 +internal_allocation_pool_end: 192.168.37.254 +internal_vip_address: 192.168.37.2 +internal_vlan: 101 +internal_physical_network: cloud + +external_cidr: 192.168.38.0/24 +external_mtu: 1392 +external_vlan: 102 +external_physical_network: cloud + +public_cidr: 192.168.39.0/24 +public_mtu: 1442 +public_allocation_pool_start: 192.168.39.3 +public_allocation_pool_end: 192.168.39.254 +public_vip_address: 192.168.39.2 +public_vlan: 103 +public_physical_network: cloud + +tunnel_cidr: 192.168.40.0/24 +tunnel_mtu: 1442 +tunnel_vlan: 104 +tunnel_physical_network: cloud + +storage_cidr: 192.168.41.0/24 +storage_mtu: 1442 +storage_vlan: 105 +storage_physical_network: cloud + +storage_mgmt_cidr: 192.168.42.0/24 +storage_mgmt_mtu: 1442 +storage_mgmt_vlan: 106 +storage_mgmt_physical_network: cloud + +############################################################################### +# MichaelRigart interfaces configuration. + +interfaces_pause_time: 5 diff --git a/etc/kayobe/environments/aufn-ceph/neutron.yml b/etc/kayobe/environments/aufn-ceph/neutron.yml new file mode 100644 index 000000000..258c0a15f --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/neutron.yml @@ -0,0 +1,13 @@ +--- +kolla_neutron_ml2_network_vlan_ranges: + - physical_network: "physnet1" + +kolla_neutron_ml2_type_drivers: + - flat + - vlan + - geneve + +kolla_neutron_ml2_tenant_network_types: + - flat + - vlan + - geneve diff --git a/etc/kayobe/environments/aufn-ceph/overcloud.yml b/etc/kayobe/environments/aufn-ceph/overcloud.yml new file mode 100644 index 000000000..1530c401d --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/overcloud.yml @@ -0,0 +1,21 @@ +--- +############################################################################### +# Overcloud configuration. + +# Dict mapping overcloud Ansible group names to lists of hosts in the group. +# As a special case, the group 'ignore' can be used to specify hosts that +# should not be added to the inventory. +#overcloud_group_hosts_map: +overcloud_group_hosts_map: + controllers: + - controller0 + - controller1 + - controller2 + compute: + - compute0 + - compute1 + - compute2 + storage-ceph: + - storage0 + - storage1 + - storage2 diff --git a/etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml b/etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml new file mode 100644 index 000000000..2f288f030 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/seed-hypervisor.yml @@ -0,0 +1,14 @@ +--- + +############################################################################### +# Seed hypervisor network interface configuration. + +# List of extra networks to which seed hypervisor nodes are attached. +seed_hypervisor_extra_network_interfaces: + - "{{ provision_wl_net_name }}" + - "{{ internal_net_name }}" + - "{{ public_net_name }}" + - "{{ external_net_names[0] }}" + +# Workaround change to cloud-user default login name on Rocky Linux +seed_hypervisor_bootstrap_user: "{{ lookup('env', 'USER') }}" diff --git a/etc/kayobe/environments/aufn-ceph/seed-vm.yml b/etc/kayobe/environments/aufn-ceph/seed-vm.yml new file mode 100644 index 000000000..ddf82f9de --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/seed-vm.yml @@ -0,0 +1,9 @@ +--- +############################################################################### +# Seed node VM configuration. + +# Memory in MB. +seed_vm_memory_mb: "{{ 4 * 1024 }}" + +# Number of vCPUs. +seed_vm_vcpus: 1 diff --git a/etc/kayobe/environments/aufn-ceph/stackhpc.yml b/etc/kayobe/environments/aufn-ceph/stackhpc.yml new file mode 100644 index 000000000..0f31b2851 --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/stackhpc.yml @@ -0,0 +1,15 @@ +--- +############################################################################### +# Kolla configuration. +# +# Docker namespace to use for Kolla images. Default is 'kolla'. +kolla_docker_namespace: stackhpc-dev + +############################################################################### +# StackHPC configuration. + +# Base URL of the StackHPC Test Pulp service. +stackhpc_release_pulp_url: "http://pulp-server.internal.sms-cloud:8080" + +pulp_username: admin +pulp_password: 9e4bfa04-9d9d-493d-9473-ba92e4361dae diff --git a/etc/kayobe/environments/aufn-ceph/storage.yml b/etc/kayobe/environments/aufn-ceph/storage.yml new file mode 100644 index 000000000..4f69e973a --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/storage.yml @@ -0,0 +1,10 @@ +--- +############################################################################### +# Storage node LVM configuration. + +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. +# storage_lvm_groups: + +# Avoid undefined var which would result in 'LVM physical disks have not been configured' error +storage_lvm_groups: [] diff --git a/etc/kayobe/environments/aufn-ceph/tenks.yml b/etc/kayobe/environments/aufn-ceph/tenks.yml new file mode 100644 index 000000000..36d482b2c --- /dev/null +++ b/etc/kayobe/environments/aufn-ceph/tenks.yml @@ -0,0 +1,95 @@ +--- +# This file holds the config given to Tenks when running `tenks-deploy.sh`. It +# assumes the existence of the bridges `brmgmt` and `brcloud`. + +node_types: + controller: + memory_mb: 8192 + vcpus: 4 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 20GiB + physical_networks: + - provision-net + - mgmt-net + - cloud-net + console_log_enabled: true + storage: + memory_mb: 8192 + vcpus: 4 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 15GiB + # Ceph volume + - capacity: 20GiB + physical_networks: + - provision-net + - cloud-net + console_log_enabled: true + compute: + memory_mb: 8192 + vcpus: 4 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 15GiB + physical_networks: + - provision-net + - cloud-net + console_log_enabled: true + baremetal: + memory_mb: 4096 + vcpus: 1 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 5GiB + physical_networks: + - cloud-net + console_log_enabled: true + +specs: + - type: controller + count: 3 + node_name_prefix: controller + ironic_config: + resource_class: test-rc + network_interface: noop + - type: storage + count: 3 + node_name_prefix: storage + ironic_config: + resource_class: test-rc + network_interface: noop + - type: compute + count: 2 + node_name_prefix: compute + ironic_config: + resource_class: test-rc + network_interface: noop + - type: baremetal + count: 0 + node_name_prefix: baremetal + +ipmi_address: 192.168.33.4 +ipmi_port_range_end: 6250 + +nova_flavors: [] + +physnet_mappings: + mgmt-net: brmgmt + provision-net: brprov + cloud-net: brcloud + +bridge_type: linuxbridge + +# No placement service. +wait_for_placement: false + +# NOTE(priteau): Disable libvirt_vm_trust_guest_rx_filters, which when enabled +# triggers the following errors when booting baremetal instances with Tenks on +# Libvirt 9: Cannot set interface flags on 'macvtap1': Value too large for +# defined data type +libvirt_vm_trust_guest_rx_filters: false diff --git a/etc/kayobe/environments/ci-aio/automated-setup.sh b/etc/kayobe/environments/ci-aio/automated-setup.sh new file mode 100644 index 000000000..cbb1e814f --- /dev/null +++ b/etc/kayobe/environments/ci-aio/automated-setup.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +set -eux + +BASE_PATH=~ +KAYOBE_BRANCH=stackhpc/2024.1 +KAYOBE_CONFIG_BRANCH=stackhpc/2024.1 +KAYOBE_AIO_LVM=true +KAYOBE_CONFIG_EDIT_PAUSE=false +AIO_RUN_TEMPEST=false + +if [[ ! -f $BASE_PATH/vault-pw ]]; then + echo "Vault password file not found at $BASE_PATH/vault-pw" + exit 1 +fi + +if sudo vgdisplay | grep -q lvm2; then + sudo pvresize $(sudo pvs --noheadings | head -n 1 | awk '{print $1}') + sudo lvextend -L 4G /dev/rootvg/lv_home -r || true + sudo lvextend -L 4G /dev/rootvg/lv_tmp -r || true +elif $KAYOBE_AIO_LVM; then + echo "This environment is only designed for LVM images. If possible, switch to an LVM image. + To ignore this warning, set KAYOBE_AIO_LVM to false in this script." + exit 1 +fi + +if type dnf; then + sudo dnf -y install git +else + sudo apt update + sudo apt -y install gcc git libffi-dev python3-dev python-is-python3 python3-venv +fi + +cd $BASE_PATH +mkdir -p src +pushd src +[[ -d kayobe ]] || git clone https://github.com/stackhpc/kayobe.git -b $KAYOBE_BRANCH +[[ -d kayobe-config ]] || git clone https://github.com/stackhpc/stackhpc-kayobe-config kayobe-config -b $KAYOBE_CONFIG_BRANCH +popd + +if $KAYOBE_CONFIG_EDIT_PAUSE; then + echo "Deployment is paused, edit configuration in another terminal" + echo "Press enter to continue" + read -s +fi + +if ! sudo vgdisplay | grep -q lvm2; then + rm $BASE_PATH/src/kayobe-config/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml + sed -i -e '/controller_lvm_groups/,+2d' $BASE_PATH/src/kayobe-config/etc/kayobe/environments/ci-aio/controllers.yml +fi + +mkdir -p venvs +pushd venvs +if [[ ! -d kayobe ]]; then + python3 -m venv kayobe +fi +# NOTE: Virtualenv's activate and deactivate scripts reference an +# unbound variable. +set +u +source kayobe/bin/activate +set -u +pip install -U pip +pip install -r ../src/kayobe-config/requirements.txt +popd + +if ! ip l show breth1 >/dev/null 2>&1; then + sudo ip l add breth1 type bridge +fi +sudo ip l set breth1 up +if ! ip a show breth1 | grep 192.168.33.3/24; then + sudo ip a add 192.168.33.3/24 dev breth1 +fi +if ! ip l show dummy1 >/dev/null 2>&1; then + sudo ip l add dummy1 type dummy +fi +sudo ip l set dummy1 up +sudo ip l set dummy1 master breth1 + +export KAYOBE_VAULT_PASSWORD=$(cat $BASE_PATH/vault-pw) +pushd $BASE_PATH/src/kayobe-config +source kayobe-env --environment ci-aio + +kayobe control host bootstrap + +kayobe playbook run etc/kayobe/ansible/growroot.yml etc/kayobe/ansible/purge-command-not-found.yml + +kayobe overcloud host configure + +kayobe overcloud service deploy + +if $AIO_RUN_TEMPEST; then + pushd $BASE_PATH/src/kayobe-config + git submodule init + git submodule update + sudo DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=rockylinux:9 --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest --network host . + export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(cat ~/.ssh/id_rsa) + mkdir -p tempest-artifacts + sudo -E docker run --name kayobe-automation --detach -it --rm --network host \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + sleep 300 + sudo docker logs -f tempest +else + export KAYOBE_CONFIG_SOURCE_PATH=$BASE_PATH/src/kayobe-config + export KAYOBE_VENV_PATH=$BASE_PATH/venvs/kayobe + pushd $BASE_PATH/src/kayobe + ./dev/overcloud-test-vm.sh +fi diff --git a/etc/kayobe/environments/ci-aio/compute.yml b/etc/kayobe/environments/ci-aio/compute.yml new file mode 100644 index 000000000..e5bd95d6d --- /dev/null +++ b/etc/kayobe/environments/ci-aio/compute.yml @@ -0,0 +1,3 @@ +--- + +compute_libvirt_sasl_password: "{{ lookup('password', kayobe_env_config_path ~ '/secrets/libvirt_sasl_password') }}" diff --git a/etc/kayobe/environments/ci-aio/controllers.yml b/etc/kayobe/environments/ci-aio/controllers.yml new file mode 100644 index 000000000..12fe3afcb --- /dev/null +++ b/etc/kayobe/environments/ci-aio/controllers.yml @@ -0,0 +1,23 @@ +--- +############################################################################### +# Controller node configuration. + +# User with which to access the controllers via SSH during bootstrap, in order +# to setup the Kayobe user account. Default is {{ os_distribution }}. +controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" + +controller_extra_network_interfaces: + - ethernet + +# Controller lvm configuration. See intentory/group_vars/controllers/lvm.yml +# for the exact configuration. +controller_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +# Controller firewalld configuration. See inventory/group_vars/all/firewall for +# the exact configuration. +controller_firewalld_zones: "{{ stackhpc_firewalld_zones }}" +controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" +# FIXME(wszumski): Firewall disabled in OVS temporarily until someone has a change to fix it. +controller_firewalld_enabled: "{{ kolla_enable_ovn | bool }}" +controller_firewalld_default_zone: "drop" diff --git a/etc/kayobe/environments/ci-aio/globals.yml b/etc/kayobe/environments/ci-aio/globals.yml new file mode 100644 index 000000000..12d3fc767 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/globals.yml @@ -0,0 +1,66 @@ +--- +# Kayobe global configuration. + +############################################################################### +# Local path configuration (Ansible control host). + +# Path to Kayobe configuration directory on Ansible control host, with an +# environment path appended if kayobe_environment is set. +#kayobe_config_path: + +# Name of Kayobe environment to use. Default is $KAYOBE_ENVIRONMENT, or an +# empty string if $KAYOBE_ENVIRONMENT is not set. Can also be set via the +# --environment argument when invoking kayobe. +#kayobe_environment: + +# Path to Kayobe configuration directory on Ansible control host with an +# environment path appended if kayobe_environment is set. +#kayobe_env_config_path: + +############################################################################### +# Remote path configuration (seed, seed-hypervisor and overcloud hosts). + +# Base path for kayobe state on remote hosts. +#base_path: + +# Path in which to store configuration on remote hosts. +#config_path: + +# Path in which to cache downloaded images on remote hosts. +#image_cache_path: + +# Path on which to checkout source code repositories on remote hosts. +#source_checkout_path: + +# Path on which to create python virtualenvs on remote hosts. +#virtualenv_path: + +############################################################################### +# User configuration. + +# User with which to access remote hosts. This user will be created if it does +# not exist. +#kayobe_ansible_user: + +############################################################################### +# OS distribution. + +# OS distribution name. Valid options are "rocky", "ubuntu". Default is +# "rocky". +os_distribution: "{{ lookup('pipe', '. /etc/os-release && echo $ID') | trim }}" + +# OS release. Valid options are "9" when os_distribution is "rocky", or +# "jammy" and "noble" when os_distribution is "ubuntu". +os_release: >- + {{ (lookup('pipe', '. /etc/os-release && echo $VERSION_CODENAME') | trim) if os_distribution == 'ubuntu' else + (lookup('pipe', '. /etc/os-release && echo $VERSION_ID') | trim | split('.') | first) if os_distribution == 'rocky' }} + +############################################################################### +# Extra vars. + +# Don't prompt when rebooting hosts. +confirm_reboot: true + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml b/etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml new file mode 100644 index 000000000..f0fa29978 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/hooks/overcloud-host-configure/post.d/10-debug-firewall.yml @@ -0,0 +1,9 @@ +--- + +- hosts: overcloud + gather_facts: false + tasks: + - name: Configure firewalld to log blocked traffic + command: firewall-cmd --set-log-denied=all + become: true + when: firewalld_enabled | bool diff --git a/etc/kayobe/environments/ci-aio/inventory/group_vars/cis-hardening/cis b/etc/kayobe/environments/ci-aio/inventory/group_vars/cis-hardening/cis new file mode 100644 index 000000000..252c97bf7 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/inventory/group_vars/cis-hardening/cis @@ -0,0 +1,28 @@ +--- +############################################################################## +# Rocky 9 CIS Hardening Configuration + +# NOTE: Using DEFAULT crypto policy in CI. FIPS breaks ed25519 SSH keys, and +# FUTURE breaks wazuh agent repo metadata download. +rhel9cis_crypto_policy: DEFAULT + +# Disable shell timeout for inactivity which can be disruptive to +# development work. +rhel9cis_rule_5_4_3_2: false + +############################################################################## +# Ubuntu Jammy CIS Hardening Configuration + +# Disable shell timeout for inactivity which can be disruptive to +# development work. +ubtu22cis_rule_5_4_3_2: false + +############################################################################## +# Ubuntu Noble CIS Hardening Configuration +# TODO: Test CIS rules for Ubuntu Noble + +# Disable shell timeout for inactivity which can be disruptive to +# development work. +ubtu24cis_rule_5_4_3_2: false + +############################################################################## diff --git a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml new file mode 100644 index 000000000..b204f1e6a --- /dev/null +++ b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 500m + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 2g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 2g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 5g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 2g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 2g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 250m + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 2g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 95%FREE diff --git a/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces new file mode 100644 index 000000000..85f318f42 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/inventory/group_vars/controllers/network-interfaces @@ -0,0 +1,17 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +# Ethernet interface is the `primary` or `physical` interface associated +# with the instance that the AIO deployment runs inside of. It is the interface used +# to reach the instance. +ethernet_interface: "{{ ansible_facts['default_ipv4']['interface'] }}" + +# Controller interface on all-in-one network. +aio_interface: breth1 +# Use dummy1 if it exists, otherwise the bridge will have no ports. +aio_bridge_ports: "{{ ['dummy1'] if ('dummy1' in hostvars[inventory_hostname].ansible_facts) or (os_distribution == 'rocky' and os_release == '9') else [] }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-aio/inventory/groups b/etc/kayobe/environments/ci-aio/inventory/groups new file mode 100644 index 000000000..39b72601a --- /dev/null +++ b/etc/kayobe/environments/ci-aio/inventory/groups @@ -0,0 +1,8 @@ +[controllers] + +[container-image-builders:children] +# Build container images on the all-in-one controller. +controllers + +[fix-hostname:children] +controllers diff --git a/etc/kayobe/environments/ci-aio/inventory/hosts b/etc/kayobe/environments/ci-aio/inventory/hosts new file mode 100644 index 000000000..a16582738 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/inventory/hosts @@ -0,0 +1,9 @@ +# A single all-in-one controller/compute/monitoring host. +[controllers] +controller0 + +[compute:children] +controllers + +[monitoring:children] +controllers diff --git a/etc/kayobe/environments/ci-aio/inventory/kayobe-automation b/etc/kayobe/environments/ci-aio/inventory/kayobe-automation new file mode 100644 index 000000000..5ab47462a --- /dev/null +++ b/etc/kayobe/environments/ci-aio/inventory/kayobe-automation @@ -0,0 +1,2 @@ +[tempest_runner] +controller0 diff --git a/etc/kayobe/environments/ci-aio/kolla/config/ironic.conf b/etc/kayobe/environments/ci-aio/kolla/config/ironic.conf new file mode 100644 index 000000000..45f848825 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/kolla/config/ironic.conf @@ -0,0 +1,2 @@ +[DEFAULT] +minimum_required_memory = 256 diff --git a/etc/kayobe/environments/ci-aio/kolla/globals.yml b/etc/kayobe/environments/ci-aio/kolla/globals.yml new file mode 100644 index 000000000..a54a5468f --- /dev/null +++ b/etc/kayobe/environments/ci-aio/kolla/globals.yml @@ -0,0 +1,19 @@ +--- +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu + +# Reduce the control plane's memory footprint by limiting the number of worker +# processes to one per-service. +openstack_service_workers: "1" +openstack_service_rpc_workers: "1" + +# OpenSearch memory tuning +opensearch_heap_size: 200m + +# Increase Grafana timeout +grafana_start_first_node_retries: 20 + +# Open up ports in firewalld for services on the public API network. +enable_external_api_firewalld: "{{ controller_firewalld_enabled | bool }}" diff --git a/etc/kayobe/environments/ci-aio/network-allocation.yml b/etc/kayobe/environments/ci-aio/network-allocation.yml new file mode 100644 index 000000000..fa7158222 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/network-allocation.yml @@ -0,0 +1,3 @@ +--- +aio_ips: + controller0: 192.168.33.3 diff --git a/etc/kayobe/environments/ci-aio/networks.yml b/etc/kayobe/environments/ci-aio/networks.yml new file mode 100644 index 000000000..4bf4e96cd --- /dev/null +++ b/etc/kayobe/environments/ci-aio/networks.yml @@ -0,0 +1,126 @@ +--- +# Kayobe network configuration. + +############################################################################### +# Network role to network mappings. + +# Map all networks to the all-in-one network. + +# Name of the network used for admin access to the overcloud +#admin_oc_net_name: +admin_oc_net_name: aio + +# Name of the network used by the seed to manage the bare metal overcloud +# hosts via their out-of-band management controllers. +#oob_oc_net_name: +oob_oc_net_name: aio + +# Name of the network used by the seed to provision the bare metal overcloud +# hosts. +#provision_oc_net_name: +provision_oc_net_name: aio + +# Name of the network used by the overcloud hosts to manage the bare metal +# compute hosts via their out-of-band management controllers. +#oob_wl_net_name: +oob_wl_net_name: aio + +# Name of the network used by the overcloud hosts to provision the bare metal +# workload hosts. +#provision_wl_net_name: +provision_wl_net_name: aio + +# Name of the network used to expose the internal OpenStack API endpoints. +#internal_net_name: +internal_net_name: aio + +# List of names of networks used to provide external network access via +# Neutron. +# Deprecated name: external_net_name +# If external_net_name is defined, external_net_names will default to a list +# containing one item, external_net_name. +#external_net_names: +external_net_names: + - aio + +# Name of the network used to expose the public OpenStack API endpoints. +#public_net_name: +public_net_name: aio + +# Name of the network used by Neutron to carry tenant overlay network traffic. +#tunnel_net_name: +tunnel_net_name: aio + +# Name of the network used to carry storage data traffic. +#storage_net_name: +storage_net_name: aio + +# Name of the network used to carry storage management traffic. +#storage_mgmt_net_name: +storage_mgmt_net_name: aio + +# Name of the network used to carry swift storage data traffic. +#swift_storage_net_name: +swift_storage_net_name: aio + +# Name of the network used to carry swift storage replication traffic. +#swift_storage_replication_net_name: +swift_storage_replication_net_name: aio + +# Name of the network used to perform hardware introspection on the bare metal +# workload hosts. +#inspection_net_name: +inspection_net_name: aio + +# Name of the network used to perform cleaning on the bare metal workload +# hosts +#cleaning_net_name: +cleaning_net_name: aio + +############################################################################### +# Network definitions. + +# This network is required to be defined within `ci-aio` environment to ensure that +# the network interface files are created appropriately and to provide easy inclusion +# within the firewall configuration. +ethernet_bootproto: dhcp +ethernet_zone: trusted + +# All-in-one network. +aio_cidr: 192.168.33.0/24 +aio_allocation_pool_start: 192.168.33.3 +aio_allocation_pool_end: 192.168.33.30 +aio_neutron_allocation_pool_start: 192.168.33.31 +aio_neutron_allocation_pool_end: 192.168.33.127 +aio_inspection_allocation_pool_start: 192.168.33.128 +aio_inspection_allocation_pool_end: 192.168.33.254 +aio_vip_address: 192.168.33.2 +aio_zone: aio + +############################################################################### +# Network virtual patch link configuration. + +# Suffix for Open vSwitch bridge names. +#network_bridge_suffix_ovs: + +# Prefix for virtual patch interface names. +#network_patch_prefix: + +# Suffix for virtual patch link interface names when connected towards the +# physical interface. +#network_patch_suffix_phy: + +# Suffix for virtual patch link interface names when connected towards the +# OVS bridge. +#network_patch_suffix_ovs: + +############################################################################### +# Network routing table configuration. + +# List of IP routing tables. Each item should be a dict containing 'id' and +# 'name' items. These tables will be added to /etc/iproute2/rt_tables. +#network_route_tables: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-aio/stackhpc-ci.yml b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml new file mode 100644 index 000000000..9edf4f237 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/stackhpc-ci.yml @@ -0,0 +1,76 @@ +--- +############################################################################### +# Kolla configuration. + +# Docker namespace to use for Kolla images. Default is 'kolla'. +kolla_docker_namespace: stackhpc-dev + +############################################################################### +# StackHPC configuration. + +# Host and port of a package repository mirror. +# Build and deploy the development Pulp service repositories. +# Use Ark's package repositories to install packages. +stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" +stackhpc_repo_mirror_username: "skc-ci-aio-caracal" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 61356434323736346666336432643266653237343035366333386332633338393136303238346163 + 3464316466336439643762653632323736663132313934370a356135356361383938613261343830 + 65613534353136306630316336366132643932633731376564316331373135316265663234323463 + 6165323733663065620a386537613930303461343234633331306132323666626531643839353532 + 33613231376165313934313233366630326662353839366132303932636164333136 + +# Build against released Pulp repository versions. +stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_26_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_26_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_27_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_27_version }}" +stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}" +stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}" +stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}" +stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: "" +stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" +stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}" +stackhpc_repo_ubuntu_noble_version: "{{ stackhpc_pulp_repo_ubuntu_noble_version }}" +stackhpc_repo_ubuntu_noble_security_version: "{{ stackhpc_pulp_repo_ubuntu_noble_security_version }}" +stackhpc_repo_docker_ce_ubuntu_noble_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_noble_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" +stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}" +stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version }}" +stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}" +stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version }}" +stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}" +stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}" +stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}" +stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}" +stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}" +stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}" +stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}" +## Use derived vars from etc/kayobe/pulp.yml to switch between +## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version +stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}" +stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstream_version }}" +stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}" +stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}" +stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}" +stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version }}" +stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}" + +# Rocky-and-CI-specific Pulp urls +stackhpc_include_os_minor_version_in_repo_url: true + +# Host and port of container registry. +# Push built images to the development Pulp service registry. +stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" +stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}" +stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}" + +# Override Pulp credentials to allow querying container image tags in the +# check-tags.yml custom playbook. +pulp_url: "{{ stackhpc_repo_mirror_url }}" +pulp_username: "{{ stackhpc_repo_mirror_username }}" +pulp_password: "{{ stackhpc_repo_mirror_password }}" + +# Ensure Blackbox monitoring configuration is generated correctly +seed_pulp_container_enabled: false diff --git a/etc/kayobe/environments/ci-aio/stackhpc.yml b/etc/kayobe/environments/ci-aio/stackhpc.yml new file mode 100644 index 000000000..a2c7858bb --- /dev/null +++ b/etc/kayobe/environments/ci-aio/stackhpc.yml @@ -0,0 +1,3 @@ +--- + +stackhpc_enable_cis_benchmark_hardening_hook: true diff --git a/etc/kayobe/environments/ci-aio/tempest.yml b/etc/kayobe/environments/ci-aio/tempest.yml new file mode 100644 index 000000000..766776417 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/tempest.yml @@ -0,0 +1,3 @@ +--- +# Show Rally output on failure. +rally_no_sensitive_log: false diff --git a/etc/kayobe/environments/ci-aio/time.yml b/etc/kayobe/environments/ci-aio/time.yml new file mode 100644 index 000000000..6c1bef686 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/time.yml @@ -0,0 +1,3 @@ +--- +# Force system clock synchronisation +ntp_force_sync: True diff --git a/etc/kayobe/environments/ci-builder/globals.yml b/etc/kayobe/environments/ci-builder/globals.yml new file mode 100644 index 000000000..f33fd5e05 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/globals.yml @@ -0,0 +1,15 @@ +--- +# Kayobe global configuration. + +############################################################################### +# OS distribution. + +# OS distribution name. Valid options are "rocky", "ubuntu". Default is +# "rocky". +os_distribution: "{{ lookup('pipe', '. /etc/os-release && echo $ID') | trim }}" + +############################################################################### +# Extra vars. + +# Don't prompt when rebooting hosts. +confirm_reboot: true diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 000000000..6e84a7104 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-container-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 000000000..6e84a7104 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-deployment-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml new file mode 120000 index 000000000..6e84a7104 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/hooks/overcloud-host-image-build/pre.d/10-pulp-auth-proxy.yml @@ -0,0 +1 @@ +../../../../../ansible/pulp-auth-proxy.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces new file mode 100644 index 000000000..a583b7214 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces @@ -0,0 +1,13 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +# Controller interface on all-in-one network. +aio_interface: breth1 +# Use dummy1 if it exists, otherwise the bridge will have no ports. +aio_bridge_ports: "{{ ['dummy1'] if 'dummy1' in hostvars[inventory_hostname].ansible_facts else [] }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes + diff --git a/etc/kayobe/environments/ci-builder/inventory/groups b/etc/kayobe/environments/ci-builder/inventory/groups new file mode 100644 index 000000000..155f353e2 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/inventory/groups @@ -0,0 +1,14 @@ +[seed] + +[container-image-builders:children] +seed + +[network] + +[docker:children] +# Hosts in this group will have Docker installed. +seed + +[ntp:children] +# Kayobe will configure Chrony on members of this group. +seed diff --git a/etc/kayobe/environments/ci-builder/inventory/hosts b/etc/kayobe/environments/ci-builder/inventory/hosts new file mode 100644 index 000000000..a272c6b7f --- /dev/null +++ b/etc/kayobe/environments/ci-builder/inventory/hosts @@ -0,0 +1,8 @@ +# A 'seed' host used for building images. + +# This is neccesary to run `overcloud deployment image build` +[controllers] +builder + +[seed] +builder diff --git a/etc/kayobe/environments/ci-builder/kolla/.keep b/etc/kayobe/environments/ci-builder/kolla/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/etc/kayobe/environments/ci-builder/network-allocation.yml b/etc/kayobe/environments/ci-builder/network-allocation.yml new file mode 100644 index 000000000..75ed29143 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/network-allocation.yml @@ -0,0 +1,3 @@ +--- +aio_ips: + builder: 192.168.33.3 diff --git a/etc/kayobe/environments/ci-builder/networks.yml b/etc/kayobe/environments/ci-builder/networks.yml new file mode 100644 index 000000000..a72481bc6 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/networks.yml @@ -0,0 +1,114 @@ +--- +# Kayobe network configuration. + +############################################################################### +# Network role to network mappings. + +# Map all networks to the all-in-one network. + +# Name of the network used for admin access to the overcloud +#admin_oc_net_name: +admin_oc_net_name: aio + +# Name of the network used by the seed to manage the bare metal overcloud +# hosts via their out-of-band management controllers. +#oob_oc_net_name: +oob_oc_net_name: aio + +# Name of the network used by the seed to provision the bare metal overcloud +# hosts. +#provision_oc_net_name: +provision_oc_net_name: aio + +# Name of the network used by the overcloud hosts to manage the bare metal +# compute hosts via their out-of-band management controllers. +#oob_wl_net_name: +oob_wl_net_name: aio + +# Name of the network used by the overcloud hosts to provision the bare metal +# workload hosts. +#provision_wl_net_name: +provision_wl_net_name: aio + +# Name of the network used to expose the internal OpenStack API endpoints. +#internal_net_name: +internal_net_name: aio + +# List of names of networks used to provide external network access via +# Neutron. +# Deprecated name: external_net_name +# If external_net_name is defined, external_net_names will default to a list +# containing one item, external_net_name. +#external_net_names: +external_net_names: + - aio + +# Name of the network used to expose the public OpenStack API endpoints. +#public_net_name: +public_net_name: aio + +# Name of the network used by Neutron to carry tenant overlay network traffic. +#tunnel_net_name: +tunnel_net_name: aio + +# Name of the network used to carry storage data traffic. +#storage_net_name: +storage_net_name: aio + +# Name of the network used to carry storage management traffic. +#storage_mgmt_net_name: +storage_mgmt_net_name: aio + +# Name of the network used to carry swift storage data traffic. +#swift_storage_net_name: +swift_storage_net_name: aio + +# Name of the network used to carry swift storage replication traffic. +#swift_storage_replication_net_name: +swift_storage_replication_net_name: aio + +# Name of the network used to perform hardware introspection on the bare metal +# workload hosts. +#inspection_net_name: +inspection_net_name: aio + +# Name of the network used to perform cleaning on the bare metal workload +# hosts +#cleaning_net_name: +cleaning_net_name: aio + +############################################################################### +# Network definitions. + +# All-in-one network. +aio_cidr: 192.168.33.0/24 +aio_allocation_pool_start: 192.168.33.3 +aio_allocation_pool_end: 192.168.33.30 + +############################################################################### +# Network virtual patch link configuration. + +# Suffix for Open vSwitch bridge names. +#network_bridge_suffix_ovs: + +# Prefix for virtual patch interface names. +#network_patch_prefix: + +# Suffix for virtual patch link interface names when connected towards the +# physical interface. +#network_patch_suffix_phy: + +# Suffix for virtual patch link interface names when connected towards the +# OVS bridge. +#network_patch_suffix_ovs: + +############################################################################### +# Network routing table configuration. + +# List of IP routing tables. Each item should be a dict containing 'id' and +# 'name' items. These tables will be added to /etc/iproute2/rt_tables. +#network_route_tables: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-builder/overcloud-dib.yml b/etc/kayobe/environments/ci-builder/overcloud-dib.yml new file mode 100644 index 000000000..6a8f3b0f1 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/overcloud-dib.yml @@ -0,0 +1,20 @@ +--- +# Overcloud host disk image configuration. + +############################################################################### +# Diskimage-builder configuration for overcloud host disk images. + +# Whether to build host disk images with DIB directly instead of through +# Bifrost. Setting it to true disables Bifrost image build and allows images to +# be built with the `kayobe overcloud host image build` command. Default value +# is False. This will change in a future release. +overcloud_dib_build_host_images: true + +# List of overcloud host disk images to build. Each element is a dict defining +# an image in a format accepted by the stackhpc.os-images role. Default is to +# build an image named "deployment_image" configured with the overcloud_dib_* +# variables defined below: {"name": "deployment_image", "elements": "{{ +# overcloud_dib_elements }}", "env": "{{ overcloud_dib_env_vars }}", +# "packages": "{{ overcloud_dib_packages }}"}. +overcloud_dib_host_images: + - "{{ stackhpc_overcloud_dib_host_image }}" diff --git a/etc/kayobe/environments/ci-builder/seed.yml b/etc/kayobe/environments/ci-builder/seed.yml new file mode 100644 index 000000000..5054fe36e --- /dev/null +++ b/etc/kayobe/environments/ci-builder/seed.yml @@ -0,0 +1,13 @@ +--- +############################################################################### +# Seed node configuration. + +# User with which to access the seed via SSH during bootstrap, in order +# to setup the Kayobe user account. Default is {{ os_distribution }}. +seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" + +# Uncomment this if using an LVM-based image. +# Seed lvm configuration. See inventory/group_vars/seed/lvm.yml +# for the exact configuration. +# seed_lvm_groups: +# - "{{ stackhpc_lvm_group_rootvg }}" diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml new file mode 100644 index 000000000..6fa5cdab1 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -0,0 +1,118 @@ +--- +############################################################################### +# Kolla configuration. + +# Docker namespace to use for Kolla images. Default is 'kolla'. +kolla_docker_namespace: stackhpc-dev + +# Kolla feature flag configuration. +kolla_enable_barbican: true +kolla_enable_blazar: true +kolla_enable_caso: true +kolla_enable_central_logging: true +kolla_enable_cinder: true +kolla_enable_cloudkitty: true +kolla_enable_designate: true +kolla_enable_etcd: true +kolla_enable_grafana: true +kolla_enable_heat: true +kolla_enable_influxdb: true +kolla_enable_ironic: true +kolla_enable_iscsid: true +kolla_enable_letsencrypt: true +kolla_enable_magnum: true +kolla_enable_manila: true +kolla_enable_mariabackup: true +kolla_enable_neutron_bgp_dragent: true +kolla_enable_neutron_mlnx: true +kolla_enable_neutron_sriov: true +kolla_enable_octavia: true +kolla_enable_opensearch: true +kolla_enable_prometheus: true +kolla_enable_redis: true +kolla_build_neutron_ovs: true + +############################################################################### +# StackHPC configuration. + +# Host and port of a package repository mirror. +# Build against the development Pulp service repositories. +# Use Ark's package repositories to install packages. +stackhpc_repo_mirror_auth_proxy_enabled: true +stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}" +stackhpc_repo_mirror_username: "skc-ci-builder-reader-caracal" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 64663534323334386635306339313431616237633661326530396436396339346339373838636433 + 6531316638333138633536623062303432356366353962370a333238653830626264373732643432 + 34313763343830353136383131303636653235646337663830383733633832363735336163633034 + 3732636136633364660a623961613934643761663437323132656361666262376439616461376637 + 65343938306332353163346233666135646261313131363638313131323239376235 + +# Build against released Pulp repository versions. +stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_26_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_26_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_27_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_27_version }}" +stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}" +stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}" +stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}" +stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: "" +stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" +stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}" +stackhpc_repo_ubuntu_noble_version: "{{ stackhpc_pulp_repo_ubuntu_noble_version }}" +stackhpc_repo_ubuntu_noble_security_version: "{{ stackhpc_pulp_repo_ubuntu_noble_security_version }}" +stackhpc_repo_docker_ce_ubuntu_noble_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_noble_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" +stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}" +stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version }}" +stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}" +stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version }}" +stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}" +stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}" +stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}" +stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}" +stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}" +stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}" +stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}" +## Use derived vars from etc/kayobe/pulp.yml to switch between +## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version +stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}" +stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstream_version }}" +stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}" +stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}" +stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}" +stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version }}" +stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}" + +# Rocky-and-CI-specific Pulp urls +stackhpc_include_os_minor_version_in_repo_url: true + +# Host and port of container registry. +# Push built images to the Ark registry. +stackhpc_docker_registry: "{{ stackhpc_release_pulp_url | regex_replace('^https?://', '') }}" + +# Username and password of container registry. +stackhpc_docker_registry_username: "release-train-ci" +stackhpc_docker_registry_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 62653437333237376336336264366133326434326333326332346561313837306462396163343962 + 3565316433336436366630393865383030623732313765330a333962633233613965303636333939 + 61323663383139303332636266643766343961306234316532663533616132303335366139636137 + 6263313063363636360a386464363933363263613365373533643036333730616532636264633338 + 39383762313638303835666235663864653938306339663235356538666261653638 + +# Username and password of the overcloud host image repository. +stackhpc_release_pulp_username: "{{ stackhpc_docker_registry_username }}" +stackhpc_release_pulp_password: "{{ stackhpc_docker_registry_password }}" + +# Build during IPA builder workflow +ipa_build_images: true +ipa_build_dib_env_extra: + DISTRO_NAME: "{{ ipa_ci_builder_distribution | default('ubuntu') }}" + DIB_RELEASE: "{{ ipa_ci_builder_release | default('jammy') }}" + +# Ensure Ark repos are disabled during CI runs, this is due to +# builder being a member of the 'overcloud' group for IPA builds. +stackhpc_repos_enabled: false +enable_docker_repo: true diff --git a/etc/kayobe/environments/ci-doca-builder/.kayobe-environment b/etc/kayobe/environments/ci-doca-builder/.kayobe-environment new file mode 100644 index 000000000..0cedd00c8 --- /dev/null +++ b/etc/kayobe/environments/ci-doca-builder/.kayobe-environment @@ -0,0 +1,4 @@ +--- + +dependencies: + - ci-builder diff --git a/etc/kayobe/environments/ci-doca-builder/controllers.yml b/etc/kayobe/environments/ci-doca-builder/controllers.yml new file mode 100644 index 000000000..f0dfd79a3 --- /dev/null +++ b/etc/kayobe/environments/ci-doca-builder/controllers.yml @@ -0,0 +1,15 @@ +--- +############################################################################### +# Controller node configuration. + +# User with which to access the controllers via SSH during bootstrap, in order +# to setup the Kayobe user account. Default is {{ os_distribution }}. +controller_bootstrap_user: cloud-user + +############################################################################### +# Controller node LVM configuration. + +# List of controller volume groups. See mrlesmithjr.manage_lvm role for +# format. +controller_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" diff --git a/etc/kayobe/environments/ci-doca-builder/globals.yml b/etc/kayobe/environments/ci-doca-builder/globals.yml new file mode 100644 index 000000000..f08f5e2e4 --- /dev/null +++ b/etc/kayobe/environments/ci-doca-builder/globals.yml @@ -0,0 +1,6 @@ +--- +############################################################################### +# Extra vars. + +# Don't prompt when rebooting hosts. +confirm_reboot: true diff --git a/etc/kayobe/environments/ci-doca-builder/inventory/group_vars/ofed-builder/lvm.yml b/etc/kayobe/environments/ci-doca-builder/inventory/group_vars/ofed-builder/lvm.yml new file mode 100644 index 000000000..ab553dfd3 --- /dev/null +++ b/etc/kayobe/environments/ci-doca-builder/inventory/group_vars/ofed-builder/lvm.yml @@ -0,0 +1,35 @@ +--- + +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 500m + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 2g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 2g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 3g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 2g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 2g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 250m + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 2g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 95%FREE diff --git a/etc/kayobe/environments/ci-doca-builder/inventory/hosts b/etc/kayobe/environments/ci-doca-builder/inventory/hosts new file mode 100644 index 000000000..c0b6fd7cb --- /dev/null +++ b/etc/kayobe/environments/ci-doca-builder/inventory/hosts @@ -0,0 +1,3 @@ +[ofed-builder:children] +seed +controllers diff --git a/etc/kayobe/environments/ci-doca-builder/seed.yml b/etc/kayobe/environments/ci-doca-builder/seed.yml new file mode 100644 index 000000000..ecba7d050 --- /dev/null +++ b/etc/kayobe/environments/ci-doca-builder/seed.yml @@ -0,0 +1,15 @@ +--- +############################################################################### +# Seed node configuration. + +# User with which to access the seed via SSH during bootstrap, in order +# to setup the Kayobe user account. Default is {{ os_distribution }}. +seed_bootstrap_user: cloud-user + +############################################################################### +# Seed node LVM configuration. + +# Seed lvm configuration. See inventory/group_vars/seed/lvm.yml +# for the exact configuration. +seed_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" diff --git a/etc/kayobe/environments/ci-doca-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-doca-builder/stackhpc-ci.yml new file mode 100644 index 000000000..e3cc8dd3e --- /dev/null +++ b/etc/kayobe/environments/ci-doca-builder/stackhpc-ci.yml @@ -0,0 +1,12 @@ +--- +############################################################################### +# StackHPC configuration. + +# Enable StackHPC Ark repositories for DOCA builds +stackhpc_repos_enabled: true +enable_docker_repo: false +dnf_install_doca: true +dnf_enable_doca_modules: false + +# Disable Pulp auth proxy (deployed too late for DOCA DNF repository setup). +stackhpc_repo_mirror_auth_proxy_enabled: false diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml new file mode 100644 index 000000000..7885a5735 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -0,0 +1,77 @@ +--- +############################################################################### +# Cephadm deployment configuration. + +# Ceph OSD specification. +cephadm_osd_spec: + service_type: osd + service_id: osd_spec_default + placement: + host_pattern: "*" + data_devices: + all: true + +############################################################################### +# Ceph post-deployment configuration. + +# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role +# for format. +cephadm_ec_profiles: [] + +# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format. +cephadm_crush_rules: [] + +# List of Ceph pools. See stackhpc.cephadm.pools role for format. +cephadm_pools: + - name: backups + application: rbd + state: present + - name: images + application: rbd + state: present + - name: volumes + application: rbd + state: present + - name: vms + application: rbd + state: present + - name: cephfs_data + application: cephfs + state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}" + - name: cephfs_metadata + application: cephfs + state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}" + +# List of Cephx keys. See stackhpc.cephadm.keys role for format. +cephadm_keys: + - name: client.cinder + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images" + mgr: "profile rbd pool=volumes, profile rbd pool=vms" + state: present + - name: client.cinder-backup + caps: + mon: "profile rbd" + osd: "profile rbd pool=volumes, profile rbd pool=backups" + mgr: "profile rbd pool=volumes, profile rbd pool=backups" + state: present + - name: client.glance + caps: + mon: "profile rbd" + osd: "profile rbd pool=images" + mgr: "profile rbd pool=images" + state: present + - name: client.manila + caps: + mon: "allow r" + mgr: "allow rw" + state: "{{ 'present' if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else 'absent' }}" + +# List of Cephadm commands to run. See stackhpc.cephadm.commands role for format. +cephadm_commands_pre: [] + +cephadm_commands_post: "{{ cephadm_commands_manila_cephfs_native if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else [] }}" +cephadm_commands_manila_cephfs_native: + - "fs new manila-cephfs cephfs_metadata cephfs_data" + - "orch apply mds manila-cephfs" diff --git a/etc/kayobe/environments/ci-multinode/compute.yml b/etc/kayobe/environments/ci-multinode/compute.yml new file mode 100644 index 000000000..a00207b41 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/compute.yml @@ -0,0 +1,29 @@ +--- +compute_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. +compute_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +############################################################################### +# Compute node firewalld configuration. + +# Whether to install and enable firewalld. +compute_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +compute_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +compute_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +compute_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml new file mode 100644 index 000000000..173bcc371 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/controllers.yml @@ -0,0 +1,30 @@ +--- +controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. +controller_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + + +############################################################################### +# Controller node firewalld configuration. + +# Whether to install and enable firewalld. +controller_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +controller_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +controller_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +controller_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml new file mode 100644 index 000000000..21f217354 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/globals.yml @@ -0,0 +1,74 @@ +--- +# Kayobe global configuration. + +############################################################################### +# Local path configuration (Ansible control host). + +# Path to Kayobe configuration directory on Ansible control host, with an +# environment path appended if kayobe_environment is set. +#kayobe_config_path: + +# Name of Kayobe environment to use. Default is $KAYOBE_ENVIRONMENT, or an +# empty string if $KAYOBE_ENVIRONMENT is not set. Can also be set via the +# --environment argument when invoking kayobe. +#kayobe_environment: + +# Path to Kayobe configuration directory on Ansible control host with an +# environment path appended if kayobe_environment is set. +#kayobe_env_config_path: + +############################################################################### +# Remote path configuration (seed, seed-hypervisor and overcloud hosts). + +# Base path for kayobe state on remote hosts. +#base_path: + +# Path in which to store configuration on remote hosts. +#config_path: + +# Path in which to cache downloaded images on remote hosts. +#image_cache_path: + +# Path on which to checkout source code repositories on remote hosts. +#source_checkout_path: + +# Path on which to create python virtualenvs on remote hosts. +#virtualenv_path: + +############################################################################### +# User configuration. + +# User with which to access remote hosts. This user will be created if it does +# not exist. +#kayobe_ansible_user: + +############################################################################### +# OS distribution. + +# OS distribution name. Valid options are "rocky", "ubuntu". Default is +# "rocky". +os_distribution: "{{ lookup('pipe', '. /etc/os-release && echo $ID') | trim }}" + +# OS release. Valid options are "9" when os_distribution is "rocky", or +# "jammy" and "noble" when os_distribution is "ubuntu". +os_release: >- + {{ (lookup('pipe', '. /etc/os-release && echo $VERSION_CODENAME') | trim) if os_distribution == 'ubuntu' else + (lookup('pipe', '. /etc/os-release && echo $VERSION_ID') | trim | split('.') | first) if os_distribution == 'rocky' }} +############################################################################### +# Hashicorp vault, Barbican, and TLS configuration + +stackhpc_write_barbican_role_id_to_file: true +stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id" + +# Enable rebooting to update SELinux state +selinux_do_reboot: true + +############################################################################### +# Extra vars. + +# Don't prompt when rebooting hosts. +confirm_reboot: true + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/growroot.yml b/etc/kayobe/environments/ci-multinode/growroot.yml new file mode 100644 index 000000000..66322676f --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/growroot.yml @@ -0,0 +1,2 @@ +--- +growroot_group: "seed:overcloud:infra-vms" diff --git a/etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/pre.d/10-fix-hostname.yml b/etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/pre.d/10-fix-hostname.yml new file mode 120000 index 000000000..829fbfd77 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/pre.d/10-fix-hostname.yml @@ -0,0 +1 @@ +../../../../../ansible/fix-hostname.yml \ No newline at end of file diff --git a/etc/kayobe/environments/ci-multinode/infra-vms.yml b/etc/kayobe/environments/ci-multinode/infra-vms.yml new file mode 100644 index 000000000..33bb0b91e --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/infra-vms.yml @@ -0,0 +1,29 @@ +--- +infra_vm_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. +infra_vm_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +############################################################################### +# Infrastructure VM node firewalld configuration + +# Whether to install and enable firewalld. +infra_vm_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +infra_vm_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +infra_vm_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +infra_vm_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml new file mode 100644 index 000000000..ca1f9b475 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/firewall.yml @@ -0,0 +1,7 @@ +--- + +stackhpc_firewalld_rules_extra: + - port: "{{ vxlan_dstport }}/udp" + network: "{{ admin_oc_net_name }}" + zone: "{{ admin_oc_net_name | net_zone }}" + state: enabled diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml new file mode 100644 index 000000000..e26394f26 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/main.yml @@ -0,0 +1,2 @@ +--- +cephadm_host_labels: [] diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml new file mode 100644 index 000000000..fe92c42a0 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml @@ -0,0 +1,18 @@ +--- +############################################################################### +# VXLAN configuration. + +# These variables are used by stackhpc.vxlan to construct a VXLAN interface. +# The physical device that the VXLAN is plugged into. +vxlan_phys_dev: "{{ admin_oc_interface }}" + +# Destination port used by the VXLAN interface. Note: OpenStack uses 4789. +vxlan_dstport: 4790 + +# Virtual network identifier used by the VLXAN. This must be unique between multinode environments. +vxlan_vni: + +# This is a dictionary of interfaces to be created by the stackhpc.vxlan role. +vxlan_interfaces: + - device: "vxlan{{ vxlan_vni }}" + group: "{{ '239.0.0.0/8' | ansible.utils.next_nth_usable(vxlan_vni) }}" diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/cis-hardening/cis b/etc/kayobe/environments/ci-multinode/inventory/group_vars/cis-hardening/cis new file mode 100644 index 000000000..252c97bf7 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/cis-hardening/cis @@ -0,0 +1,28 @@ +--- +############################################################################## +# Rocky 9 CIS Hardening Configuration + +# NOTE: Using DEFAULT crypto policy in CI. FIPS breaks ed25519 SSH keys, and +# FUTURE breaks wazuh agent repo metadata download. +rhel9cis_crypto_policy: DEFAULT + +# Disable shell timeout for inactivity which can be disruptive to +# development work. +rhel9cis_rule_5_4_3_2: false + +############################################################################## +# Ubuntu Jammy CIS Hardening Configuration + +# Disable shell timeout for inactivity which can be disruptive to +# development work. +ubtu22cis_rule_5_4_3_2: false + +############################################################################## +# Ubuntu Noble CIS Hardening Configuration +# TODO: Test CIS rules for Ubuntu Noble + +# Disable shell timeout for inactivity which can be disruptive to +# development work. +ubtu24cis_rule_5_4_3_2: false + +############################################################################## diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml new file mode 100644 index 000000000..aef63ee30 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 5g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 5g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 2g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 1g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 10g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 2g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 5g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 95%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces new file mode 100644 index 000000000..96c6f15c3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces @@ -0,0 +1,15 @@ +--- +############################################################################### +# Network interface definitions for the compute group. + +admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" + +internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" + +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + +tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml new file mode 100644 index 000000000..aef63ee30 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 5g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 5g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 2g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 1g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 10g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 2g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 5g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 95%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces new file mode 100644 index 000000000..7df8bec28 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces @@ -0,0 +1,21 @@ +--- +############################################################################### +# Network interface definitions for the controller group. + +admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" + +internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" + +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + +tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}" + +public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}" + +public_routes: + - cidr: "{{ external_cidr }}" + gateway: "{{ public_net_name | net_ip( groups['seed'][0] ) }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml new file mode 100644 index 000000000..a46fa96d3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 10g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 5g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 10g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 5g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 5g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 75%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces new file mode 100644 index 000000000..e604b02ec --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/seed/network-interfaces @@ -0,0 +1,19 @@ +--- +############################################################################### +# Network interface definitions for the seed group. + +admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" + +provision_oc_interface: "{{ vxlan_interfaces[0].device}}.{{ provision_oc_vlan }}" + +external_interface: "{{ vxlan_interfaces[0].device }}.{{ external_vlan }}" + +public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}" + +# The storage interface is required for routing manila traffic between VMs and +# the Storage nodes. +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm.yml new file mode 100644 index 000000000..4d14cfb32 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 5g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 2g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 2g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 1g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 2g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 1g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 2g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 95%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces new file mode 100644 index 000000000..6594897c4 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/storage/network-interfaces @@ -0,0 +1,15 @@ +--- +############################################################################### +# Network interface definitions for the storage group. + +admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" + +internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" + +storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" + +storage_mgmt_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_mgmt_vlan }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/lvm.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/lvm.yml new file mode 100644 index 000000000..a46fa96d3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/lvm.yml @@ -0,0 +1,34 @@ +--- +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: + - "{{ stackhpc_lvm_lv_docker }}" + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 1g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 10g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 5g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 10g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 5g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 5g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 75%FREE diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/network-interfaces new file mode 100644 index 000000000..080c11bdc --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/network-interfaces @@ -0,0 +1,9 @@ +--- +############################################################################### +# Network interface definitions for the wazuh-manager group. + +admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/wazuh-manager b/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/wazuh-manager new file mode 100644 index 000000000..4ab556dd9 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/wazuh-manager/wazuh-manager @@ -0,0 +1,4 @@ +--- +# Not using provision_oc, use admin_oc instead +wazuh_manager_ip: "{{ admin_oc_net_name | net_ip }}" +indexer_network_host: "{{ admin_oc_net_name | net_ip }}" diff --git a/etc/kayobe/environments/ci-multinode/inventory/groups b/etc/kayobe/environments/ci-multinode/inventory/groups new file mode 100644 index 000000000..8bc65f541 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/groups @@ -0,0 +1,8 @@ +[controllers] + +[container-image-builders:children] +# Build container images on the all-in-one controller. +controllers + +[fix-hostname:children] +overcloud diff --git a/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation new file mode 100644 index 000000000..7add0ce97 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/inventory/kayobe-automation @@ -0,0 +1,2 @@ +[tempest_runner:children] +seed diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml new file mode 100644 index 000000000..809c71142 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -0,0 +1,25 @@ +--- +kolla_enable_cinder: true +kolla_enable_cinder_backup: true +kolla_enable_neutron_provider_networks: true +kolla_enable_octavia: true +kolla_enable_magnum: true +kolla_enable_designate: true +kolla_enable_redis: true +kolla_enable_barbican: true + +kolla_enable_tls_external: true +kolla_enable_tls_internal: true + +kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}" +kolla_admin_openrc_cacert: "{{ kolla_public_openrc_cacert }}" + +# The multinode environment supports Manila but it is not enabled by default. +# kolla_enable_manila: true +# kolla_enable_manila_backend_cephfs_native: true + +kolla_internal_vip_address: "192.168.37.2" +kolla_internal_fqdn: "internal.infra.mos.{{ root_domain | default('example.com') }}" + +kolla_external_vip_address: "192.168.39.2" +kolla_external_fqdn: "public.infra.mos.{{ root_domain | default('example.com') }}" diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/barbican.conf b/etc/kayobe/environments/ci-multinode/kolla/config/barbican.conf new file mode 100644 index 000000000..25a49e2b7 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/barbican.conf @@ -0,0 +1,15 @@ +# Example barbican config for vault integration +[secretstore] +namespace=barbican.secretstore.plugin +enable_multiple_secret_stores=false +enabled_secretstore_plugins=vault_plugin + +[vault_plugin] +vault_url = https://{{ kolla_internal_fqdn }}:8200 +use_ssl = True +{% raw %} +ssl_ca_crt_file = {{ openstack_cacert }} +{% endraw %} +approle_role_id = {{ secrets_barbican_approle_role_id }} +approle_secret_id = {{ secrets_barbican_approle_secret_id }} +kv_mountpoint = barbican diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/haproxy/services.d/vault.cfg b/etc/kayobe/environments/ci-multinode/kolla/config/haproxy/services.d/vault.cfg new file mode 100644 index 000000000..e125ecd59 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/haproxy/services.d/vault.cfg @@ -0,0 +1,21 @@ +{% raw %} +frontend vault_front + mode tcp + option tcplog + bind {{ kolla_internal_vip_address }}:8200 + default_backend vault_back + +backend vault_back + mode tcp + option httpchk GET /v1/sys/health + # https://www.vaultproject.io/api-docs/system/health + # 200: initialized, unsealed, and active + # 429: standby + http-check expect rstatus (200|429) + +{% for host in groups['control'] %} +{% set host_name = hostvars[host].ansible_facts.hostname %} +{% set host_ip = 'api' | kolla_address(host) %} + server {{ host_name }} {{ host_ip }}:8200 check check-ssl verify none inter 2000 rise 2 fall 5 +{% endfor %} +{% endraw %} diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf b/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf new file mode 100644 index 000000000..45f848825 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/ironic.conf @@ -0,0 +1,2 @@ +[DEFAULT] +minimum_required_memory = 256 diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf b/etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf new file mode 100644 index 000000000..71b1682cb --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/keepalived/keepalived.conf @@ -0,0 +1,48 @@ +{% raw %} +vrrp_script check_alive { + script "/check_alive.sh" + interval 2 + fall 2 + rise 10 +} + +vrrp_instance kolla_internal_vip_{{ keepalived_virtual_router_id }} { + state BACKUP + nopreempt + interface {{ api_interface }} + virtual_router_id {{ keepalived_virtual_router_id }} + priority {{ groups['loadbalancer'].index(inventory_hostname) + 1 }} + advert_int 3 # Increase advert interval from 1 to 3 as there was a lot of jumping amongst controllers on the internal network +{% if keepalived_traffic_mode == 'unicast' %} + unicast_src_ip {{ api_interface_address }} +{% if groups['loadbalancer'] | length > 1 %} + unicast_peer { +{% for host in groups['loadbalancer'] %} +{% set ip_addr = 'api' | kolla_address(host) %} +{% if ip_addr != api_interface_address %} + {{ ip_addr }} +{% endif %} +{% endfor %} + } +{% endif %} +{% endif %} + virtual_ipaddress { + {{ kolla_internal_vip_address }} dev {{ api_interface }} +{% if haproxy_enable_external_vip | bool %} + {{ kolla_external_vip_address }} dev {{ kolla_external_vip_interface }} +{% endif %} + } +{% if haproxy_enable_external_vip | bool and api_interface != kolla_external_vip_interface %} + track_interface { + {{ kolla_external_vip_interface }} + } +{% endif %} + authentication { + auth_type PASS + auth_pass {{ keepalived_password }} + } + track_script { + check_alive + } +} +{% endraw %} diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/magnum.conf b/etc/kayobe/environments/ci-multinode/kolla/config/magnum.conf new file mode 100644 index 000000000..3a3c45373 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/magnum.conf @@ -0,0 +1,2 @@ +[trust] +cluster_user_trust = true diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf b/etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf new file mode 100644 index 000000000..265f67b26 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/neutron.conf @@ -0,0 +1,3 @@ +[DEFAULT] +global_physnet_mtu = {{ tunnel_net_name | net_mtu }} + diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini b/etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini new file mode 100644 index 000000000..ff31f09a5 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/neutron/ml2_conf.ini @@ -0,0 +1,3 @@ +[ml2] +path_mtu = {{ tunnel_net_name | net_mtu }} + diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml new file mode 100644 index 000000000..2fe551d8d --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -0,0 +1,69 @@ +# yamllint disable-file +--- +############################################################################ +# TLS + +# Internal TLS configuration +# Copy the self-signed CA into the kolla containers +kolla_copy_ca_into_containers: "yes" +# Use the following trust store within the container +openstack_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution == 'rocky' else '/etc/ssl/certs/ca-certificates.crt' }}" + +# Enable backend TLS +kolla_enable_tls_backend: "yes" + +# Enable RabbitMQ TLS +rabbitmq_enable_tls: "yes" + +############################################################################ +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu + +# Reduce the control plane's memory footprint by limiting the number of worker +# processes to two per-service when running in a VM. +openstack_service_workers: "{% raw %}{{ [ansible_facts.processor_vcpus, 2 if ansible_facts.virtualization_role == 'guest' else 5] | min }}{% endraw %}" +openstack_service_rpc_workers: "{% raw %}{{ [ansible_facts.processor_vcpus, 2 if ansible_facts.virtualization_role == 'guest' else 3] | min }}{% endraw %}" + +# Glance Ceph configuration +glance_backend_ceph: "yes" + +# Cinder Ceph configuration +cinder_backend_ceph: "yes" + +# Nova Ceph configuration +nova_backend_ceph: "yes" + +# Because this is a vxlan interface, Kayobe can't generate the config for it. +# Therefore we can't use this interface as the external network +# (external_net_names). Instead, make external_net_names an empty +# list, and add the config that would be generated from it here. +neutron_bridge_name: "{{ vxlan_interfaces[0].device }}-ovs" +neutron_external_interface: "{{ vxlan_interfaces[0].device }}" + +neutron_dns_domain: "{{ root_domain }}." + +# OpenSearch memory tuning +opensearch_heap_size: 1g + +# Octavia load balancer configuration +octavia_auto_configure: "no" +octavia_provider_drivers: "ovn:OVN provider" +octavia_provider_agents: "ovn" + +# Manila CephFS configuration +manila_cephfs_filesystem_name: manila-cephfs + +# Designate configuration +designate_ns_record: + - "ns1.mos.{{ root_domain | default('example.com') }}" + - "ns2.mos.{{ root_domain | default('example.com') }}" + - "ns3.mos.{{ root_domain | default('example.com') }}" +designate_backend: "bind9" +designate_recursion: "yes" +designate_forwarders_addresses: "1.1.1.1; 8.8.8.8" + +# Open up ports in firewalld for services on the public API network. +enable_external_api_firewalld: true +external_api_firewalld_zone: "{{ public_net_name | net_zone }}" diff --git a/etc/kayobe/environments/ci-multinode/monitoring.yml b/etc/kayobe/environments/ci-multinode/monitoring.yml new file mode 100644 index 000000000..7057bb8fe --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/monitoring.yml @@ -0,0 +1,23 @@ +--- +############################################################################### +# Monitoring node firewalld configuration. + +# Whether to install and enable firewalld. +monitoring_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +monitoring_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +monitoring_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +monitoring_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml new file mode 100644 index 000000000..59d3760b9 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -0,0 +1,160 @@ +--- +# Kayobe network configuration. + +############################################################################### +# Network role to network mappings. + +# Map all networks to the all-in-one network. + +# Name of the network used for admin access to the overcloud +#admin_oc_net_name: +admin_oc_net_name: admin_oc + +# Name of the network used by the seed to manage the bare metal overcloud +# hosts via their out-of-band management controllers. +#oob_oc_net_name: +oob_oc_net_name: + +# Name of the network used by the seed to provision the bare metal overcloud +# hosts. +#provision_oc_net_name: +provision_oc_net_name: provision_oc + +# Name of the network used by the overcloud hosts to manage the bare metal +# compute hosts via their out-of-band management controllers. +#oob_wl_net_name: + +# Name of the network used by the overcloud hosts to provision the bare metal +# workload hosts. +#provision_wl_net_name: + +# Name of the network used to expose the internal OpenStack API endpoints. +#internal_net_name: +internal_net_name: internal + +# List of names of networks used to provide external network access via +# Neutron. +# Deprecated name: external_net_name +# If external_net_name is defined, external_net_names will default to a list +# containing one item, external_net_name. +#external_net_names: +external_net_names: [] + +# Name of the network used to expose the public OpenStack API endpoints. +public_net_name: public + +# Name of the network used by Neutron to carry tenant overlay network traffic. +tunnel_net_name: tunnel + +# Name of the network used to carry storage data traffic. +#storage_net_name: +storage_net_name: storage + +# Name of the network used to carry storage management traffic. +#storage_mgmt_net_name: +storage_mgmt_net_name: storage_mgmt + +# Name of the network used to carry swift storage data traffic. +#swift_storage_net_name: + +# Name of the network used to carry swift storage replication traffic. +#swift_storage_replication_net_name: + +# Name of the network used to perform hardware introspection on the bare metal +# workload hosts. +#inspection_net_name: + +# Name of the network used to perform cleaning on the bare metal workload +# hosts +#cleaning_net_name: + +############################################################################### +# Network definitions. + +# Admin overcloud network +admin_oc_zone: "admin_oc" + +# Internal network +internal_cidr: 192.168.37.0/24 +internal_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +internal_allocation_pool_start: 192.168.37.3 +internal_allocation_pool_end: 192.168.37.254 +internal_vlan: 101 +internal_zone: "trusted" + +# External network +external_cidr: 192.168.38.0/24 +external_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +external_allocation_pool_start: 192.168.38.3 +external_allocation_pool_end: 192.168.38.128 +external_vlan: 102 +external_zone: "external" + +# Public network +public_cidr: 192.168.39.0/24 +public_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +public_allocation_pool_start: 192.168.39.3 +public_allocation_pool_end: 192.168.39.254 +public_vlan: 103 +public_zone: "public" + +# Tunnel network +tunnel_cidr: 192.168.40.0/24 +tunnel_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +tunnel_allocation_pool_start: 192.168.40.3 +tunnel_allocation_pool_end: 192.168.40.254 +tunnel_vlan: 104 +tunnel_zone: "tunnel" + +# Storage network +storage_cidr: 192.168.41.0/24 +storage_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +storage_allocation_pool_start: 192.168.41.3 +storage_allocation_pool_end: 192.168.41.254 +storage_vlan: 105 +storage_zone: "storage" + +# Storage management network +# NOTE: Skipping the .42 subnet to avoid a collision with a popular number. +storage_mgmt_cidr: 192.168.43.0/24 +storage_mgmt_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +storage_mgmt_allocation_pool_start: 192.168.43.3 +storage_mgmt_allocation_pool_end: 192.168.43.254 +storage_mgmt_vlan: 106 +storage_mgmt_zone: "storage_mgmt" + +# Provision overcloud network +provision_oc_cidr: 192.168.33.0/24 +provision_oc_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +provision_oc_allocation_pool_start: 192.168.33.128 +provision_oc_allocation_pool_end: 192.168.33.254 +provision_oc_vlan: 107 +provision_oc_zone: "provision_oc" + +############################################################################### +# Network virtual patch link configuration. + +# Suffix for Open vSwitch bridge names. +#network_bridge_suffix_ovs: + +# Prefix for virtual patch interface names. +#network_patch_prefix: + +# Suffix for virtual patch link interface names when connected towards the +# physical interface. +#network_patch_suffix_phy: + +# Suffix for virtual patch link interface names when connected towards the +# OVS bridge. +#network_patch_suffix_ovs: + +############################################################################### +# Network routing table configuration. + +# List of IP routing tables. Each item should be a dict containing 'id' and +# 'name' items. These tables will be added to /etc/iproute2/rt_tables. +#network_route_tables: + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/environments/ci-multinode/neutron.yml b/etc/kayobe/environments/ci-multinode/neutron.yml new file mode 100644 index 000000000..44f50d00c --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/neutron.yml @@ -0,0 +1,12 @@ +--- +kolla_neutron_ml2_network_vlan_ranges: + - physical_network: "physnet1" + +kolla_neutron_ml2_type_drivers: + - flat + - vlan + - "{{ 'geneve' if kolla_enable_ovn | bool else 'vxlan' }}" + +kolla_neutron_ml2_tenant_network_types: + - vlan + - "{{ 'geneve' if kolla_enable_ovn | bool else 'vxlan' }}" diff --git a/etc/kayobe/environments/ci-multinode/secrets.yml b/etc/kayobe/environments/ci-multinode/secrets.yml new file mode 100644 index 000000000..4d30eaa55 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/secrets.yml @@ -0,0 +1,10 @@ +$ANSIBLE_VAULT;1.1;AES256 +36343166376362356138343762623637323234613132353766303631393136613038346161663036 +3062613031653937656163323439316563353634336162620a656239623532316639373638336561 +39353236313266316139363761353135616334623838366431626534656531323765383736623739 +3639653362663539340a643434663832363638633465326566353939373832316361626433393362 +38313765396434353136666662316434393232386531306436353062393533303235643638353466 +39386432666364663937616134353431393631323164383634613763313837616665643464613561 +63623665646536366534633264343566653964616366363530396535336665613437396438396630 +64356333643834313335643633626638333733663836393333393138643635616161616361616431 +6331 diff --git a/etc/kayobe/environments/ci-multinode/seed-hypervisor.yml b/etc/kayobe/environments/ci-multinode/seed-hypervisor.yml new file mode 100644 index 000000000..a88fd27ca --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/seed-hypervisor.yml @@ -0,0 +1,23 @@ +--- +############################################################################### +# Seed hypervisor node firewalld configuration. + +# Whether to install and enable firewalld. +seed_hypervisor_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +seed_hypervisor_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +seed_hypervisor_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +seed_hypervisor_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/seed.yml b/etc/kayobe/environments/ci-multinode/seed.yml new file mode 100644 index 000000000..c5f99a30a --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/seed.yml @@ -0,0 +1,52 @@ +--- +seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" +seed_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +seed_extra_network_interfaces: "{{ seed_extra_network_interfaces_external + (seed_extra_network_interfaces_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else []) }}" + +# Seed has been provided an external interface +# for tempest tests and SSH access to machines. +seed_extra_network_interfaces_external: + - "external" + - "public" + +# Seed requires access to the storage network for manila-cephfs. +seed_extra_network_interfaces_manila: + - "storage" + +# Enable IP routing and source NAT on the seed, allowing it to be used as the +# external subnet gateway and provide internet access for VMs in the deployment. +seed_enable_snat: true + +snat_rules_default: + - interface: "{{ ansible_facts.default_ipv4.interface }}" + source_ip: "{{ ansible_facts.default_ipv4.address }}" +snat_rules_manila: + - interface: "{{ storage_interface }}" + source_ip: "{{ ansible_facts[storage_interface].ipv4.address | default }}" +# Only add the storage snat rule if we are using manila-cephfs. +snat_rules: "{{ snat_rules_default + snat_rules_manila if (kolla_enable_manila | bool and kolla_enable_manila_backend_cephfs_native | bool) else snat_rules_default }}" + +############################################################################### +# Seed node firewalld configuration. + +# Whether to install and enable firewalld. +seed_firewalld_enabled: "{{ kolla_enable_ovn | bool }}" + +# A list of zones to create. Each item is a dict containing a 'zone' item. +seed_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +seed_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +seed_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml new file mode 100644 index 000000000..0247fdb3f --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -0,0 +1,67 @@ +--- +############################################################################### +# Kolla configuration. + +# Docker namespace to use for Kolla images. Default is 'kolla'. +kolla_docker_namespace: stackhpc-dev + +############################################################################### +# StackHPC configuration. + +# Host and port of a package repository mirror. +# Build and deploy the development Pulp service repositories. +# Use Ark's package repositories to install packages. +stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" +stackhpc_repo_mirror_username: "skc-ci-multinode" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 37623033316339326163333132316637653566366662646133633038623137306435323935326161 + 3363666433323562316537373539386133363838633461650a333838386436346261323466616337 + 61636262366161323534633066376232393733316632623030383735346335363437636464663061 + 3138623135353862640a373032663662653930333932636534343232643737343234636130613031 + 33306539316466373763656137653232313732646533353333626665313936613736 + +# Build and deploy released Pulp repository versions. +stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_26_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_26_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_27_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_27_version }}" +stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}" +stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}" +stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}" +stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: "" +stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" +stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}" +stackhpc_repo_ubuntu_noble_version: "{{ stackhpc_pulp_repo_ubuntu_noble_version }}" +stackhpc_repo_ubuntu_noble_security_version: "{{ stackhpc_pulp_repo_ubuntu_noble_security_version }}" +stackhpc_repo_docker_ce_ubuntu_noble_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_noble_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" +stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}" +stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version }}" +stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}" +stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version }}" +stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}" +stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}" +stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}" +stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}" +stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}" +stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}" +stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}" +## Use derived vars from etc/kayobe/pulp.yml to switch between +## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version +stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}" +stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstream_version }}" +stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}" +stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}" +stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}" +stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version }}" + +# Rocky-and-CI-specific Pulp urls +stackhpc_include_os_minor_version_in_repo_url: true + +# Host and port of container registry. +# Push built images to the development Pulp service registry. +stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" + +stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}" +stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}" diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml b/etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml new file mode 100644 index 000000000..d9693381e --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/stackhpc-monitoring.yml @@ -0,0 +1,3 @@ +--- +# Path to a CA certificate file to trust in the OpenStack Capacity exporter. +stackhpc_os_capacity_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/{{ stackhpc_ca_secret_store }}.crt" diff --git a/etc/kayobe/environments/ci-multinode/stackhpc.yml b/etc/kayobe/environments/ci-multinode/stackhpc.yml new file mode 100644 index 000000000..a2c7858bb --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/stackhpc.yml @@ -0,0 +1,3 @@ +--- + +stackhpc_enable_cis_benchmark_hardening_hook: true diff --git a/etc/kayobe/environments/ci-multinode/storage.yml b/etc/kayobe/environments/ci-multinode/storage.yml new file mode 100644 index 000000000..bcf8d3800 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/storage.yml @@ -0,0 +1,29 @@ +--- +storage_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cloud-user' }}" +# List of storage volume groups. See mrlesmithjr.manage-lvm role for +# format. +storage_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" + +############################################################################### +# Storage node firewalld configuration. + +# Whether to install and enable firewalld. +storage_firewalld_enabled: true + +# A list of zones to create. Each item is a dict containing a 'zone' item. +storage_firewalld_zones: "{{ stackhpc_firewalld_zones }}" + +# A firewalld zone to set as the default. Default is unset, in which case +# the default zone will not be changed. +# Predefined zones are listed here: +# https://firewalld.org/documentation/zone/predefined-zones.html +storage_firewalld_default_zone: trusted + +# A list of firewall rules to apply. Each item is a dict containing +# arguments to pass to the firewalld module. Arguments are omitted if not +# provided, with the following exceptions: +# - offline: true +# - permanent: true +# - state: enabled +storage_firewalld_rules: "{{ stackhpc_firewalld_rules }}" diff --git a/etc/kayobe/environments/ci-multinode/tempest.yml b/etc/kayobe/environments/ci-multinode/tempest.yml new file mode 100644 index 000000000..327fbef34 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/tempest.yml @@ -0,0 +1,6 @@ +--- +# Show Rally output on failure. +rally_no_sensitive_log: false + +# Add the Vault CA certificate to the rally container when running tempest. +tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/{{ stackhpc_ca_secret_store }}.crt" diff --git a/etc/kayobe/globals.yml b/etc/kayobe/globals.yml index b71640251..40a5a689a 100644 --- a/etc/kayobe/globals.yml +++ b/etc/kayobe/globals.yml @@ -50,8 +50,8 @@ kayobe_ansible_user: "stack" os_distribution: "rocky" # OS release. Valid options are "9-stream" when os_distribution is "centos", or -# "9" when os_distribution is "rocky", or "jammy" when os_distribution is -# "ubuntu". +# "9" when os_distribution is "rocky", or "jammy" and "noble" (with "jammy" +# being the default) when os_distribution is "ubuntu". #os_release: os_release: 9 diff --git a/etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/control-host-bootstrap/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/control-host-upgrade/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/infra-vm-host-configure/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/infra-vm-service-deploy/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml new file mode 100644 index 000000000..9c132efbc --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml @@ -0,0 +1,4 @@ +--- + +- import_playbook: ../../../ansible/cis.yml + when: stackhpc_enable_cis_benchmark_hardening_hook | bool diff --git a/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-configure/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-host-configure/pre.d/pci-passthrough.yml b/etc/kayobe/hooks/overcloud-host-configure/pre.d/pci-passthrough.yml new file mode 120000 index 000000000..ffdf55f6a --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-configure/pre.d/pci-passthrough.yml @@ -0,0 +1 @@ +../../../ansible/pci-passthrough.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-host-package-update/post.d/10-fix-grub-rl9.yml b/etc/kayobe/hooks/overcloud-host-package-update/post.d/10-fix-grub-rl9.yml new file mode 120000 index 000000000..775762abf --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-package-update/post.d/10-fix-grub-rl9.yml @@ -0,0 +1 @@ +../../../ansible/fix-grub-rl9.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-upgrade/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/.gitkeep b/etc/kayobe/hooks/overcloud-service-deploy/post.d/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml new file mode 120000 index 000000000..0cc70aace --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml @@ -0,0 +1 @@ +../../../ansible/deploy-os-capacity-exporter.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml new file mode 120000 index 000000000..3d939329a --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-radosgw-usage-exporter.yml @@ -0,0 +1 @@ +../../../ansible/deploy-radosgw-usage-exporter.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-deploy/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-reconfigure/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-upgrade/post.d/deploy-os-capacity-exporter.yml b/etc/kayobe/hooks/overcloud-service-upgrade/post.d/deploy-os-capacity-exporter.yml new file mode 120000 index 000000000..0cc70aace --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-upgrade/post.d/deploy-os-capacity-exporter.yml @@ -0,0 +1 @@ +../../../ansible/deploy-os-capacity-exporter.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-upgrade/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/seed-host-configure/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/seed-hypervisor-host-configure/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml new file mode 120000 index 000000000..bec8ed4c6 --- /dev/null +++ b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kayobe-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kayobe-version.yml \ No newline at end of file diff --git a/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-ansible-version.yml b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-ansible-version.yml new file mode 120000 index 000000000..0f7651f53 --- /dev/null +++ b/etc/kayobe/hooks/seed-service-deploy/pre.d/check-kolla-ansible-version.yml @@ -0,0 +1 @@ +../../../ansible/check-kolla-ansible-version.yml \ No newline at end of file diff --git a/etc/kayobe/infra-vms.yml b/etc/kayobe/infra-vms.yml index e48004103..0256a0ecc 100644 --- a/etc/kayobe/infra-vms.yml +++ b/etc/kayobe/infra-vms.yml @@ -31,13 +31,16 @@ # Base image for the infra VM root volume. Default is # "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" +# when os_distribution is "ubuntu" and distribution_major_version is '22', +# "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" +# when os_distribution is "ubuntu" and distribution_major_version is not '22', # when os_distribution is "ubuntu", # "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" # when os_distribution is "rocky" and seed_vm_boot_firmware is "efi", # "https://dl.rockylinux.org/vault/rocky/9.3/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" # when os_distribution is "rocky" and seed_vm_boot_firmware is not "efi" # (default is "bios"), or -# "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20221206.0.x86_64.qcow2" +# "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" # otherwise. #infra_vm_root_image: @@ -70,7 +73,8 @@ # Infrastructure VM node configuration. # User with which to access the infrastructure vm via SSH during bootstrap, in -# order to setup the Kayobe user account. +# order to setup the Kayobe user account. Default is 'cloud-user' if +# os_distribution is set to centos, otherwise 'os_distribution'. #infra_vm_bootstrap_user: ############################################################################### @@ -178,6 +182,13 @@ # - state: enabled #infra_vm_firewalld_rules: +############################################################################### +# Infrastructure VM node swap configuration. + +# List of swap devices. Each item is a dict containing a 'device' item. Default +# is an empty list. +# infra_vm_swap: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/all/growroot b/etc/kayobe/inventory/group_vars/all/growroot new file mode 100644 index 000000000..280cee665 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/growroot @@ -0,0 +1,3 @@ +--- +# Name of the LVM VG containing the root PV for ansible/growroot.yml +growroot_vg: "rootvg" diff --git a/etc/kayobe/inventory/group_vars/all/openbao.yml b/etc/kayobe/inventory/group_vars/all/openbao.yml new file mode 100644 index 000000000..1c3c0b39a --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/openbao.yml @@ -0,0 +1,82 @@ +############################################################################### +# Openbao deployment configuration. + +# Registry information for seed. +seed_openbao_registry_url: "{{ stackhpc_docker_registry if stackhpc_sync_openbao_images | bool else '' }}" +seed_openbao_registry_username: "{{ stackhpc_docker_registry_username if stackhpc_sync_openbao_images | bool else '' }}" +seed_openbao_registry_password: "{{ stackhpc_docker_registry_password if stackhpc_sync_openbao_images | bool else '' }}" + +# Seed OpenBao container image. +seed_openbao_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_openbao_images | bool else '' }}openbao/openbao" + +# Seed OpenBao container image tag. +seed_openbao_docker_tag: "2.2.1" + +# Seed OpenBao PKI Role name +seed_openbao_pki_role_name: "ServerCert" + +# Seed OpenBao PKI Roles definition +seed_openbao_pki_roles: + - name: "{{ seed_openbao_pki_role_name }}" + config: + max_ttl: 730d + ttl: 730d + allow_any_name: true + allow_ip_sans: true + require_cn: false + server_flag: true + key_type: rsa + key_bits: 4096 + country: ["UK"] + locality: ["Bristol"] + organization: ["StackHPC"] + ou: ["OpenStack"] + +# Registry information for overcloud. +overcloud_openbao_registry_url: "{{ stackhpc_docker_registry if stackhpc_sync_openbao_images | bool else '' }}" +overcloud_openbao_registry_username: "{{ stackhpc_docker_registry_username if stackhpc_sync_openbao_images | bool else '' }}" +overcloud_openbao_registry_password: "{{ stackhpc_docker_registry_password if stackhpc_sync_openbao_images | bool else '' }}" + +# Overcloud OpenBao container image. +overcloud_openbao_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_openbao_images | bool else '' }}openbao/openbao" + +# Overcloud OpenBao container image tag. +overcloud_openbao_docker_tag: "2.2.1" + +# Overcloud OpenBao PKI Default Role name +overcloud_openbao_pki_default_role_name: "ServerCert" + +# Overcloud OpenBao PKI Internal TLS Role name +overcloud_openbao_pki_internal_tls_role_name: "{{ overcloud_openbao_pki_default_role_name }}" + +# Overcloud OpenBao PKI Backend TLS Role name +overcloud_openbao_pki_backend_tls_role_name: "{{ overcloud_openbao_pki_default_role_name }}" + +# Overcloud OpenBao PKI External TLS Role name (for testing only) +overcloud_openbao_pki_external_tls_role_name: "{{ overcloud_openbao_pki_default_role_name }}" + +# Overcloud OpenBao PKI Roles definition +overcloud_openbao_pki_roles: + - name: "{{ overcloud_openbao_pki_default_role_name }}" + config: + max_ttl: 730d + ttl: 730d + allow_any_name: true + allow_ip_sans: true + require_cn: false + server_flag: true + key_type: rsa + key_bits: 4096 + country: ["UK"] + locality: ["Bristol"] + organization: ["StackHPC"] + ou: ["OpenStack"] + +seed_openbao_pki_certificate_subject: + - common_name: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}" + role: "{{ seed_openbao_pki_role_name }}" + extra_params: + ip_sans: "{% for host in groups['controllers'] %}{{ internal_net_name | net_ip(host) }}{% if not loop.last %},{% endif %}{% endfor %},{{ kolla_internal_vip_address }}" + +# Enable OpenBao UI +openbao_enable_ui: true diff --git a/etc/kayobe/inventory/group_vars/all/selinux b/etc/kayobe/inventory/group_vars/all/selinux new file mode 100644 index 000000000..80d084d7e --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/selinux @@ -0,0 +1,5 @@ +--- +# Target SELinux state +# NOTE(MaxN) In StackHPC cloud tests we're checking the host's SELinux state matches the targeted state +# but we can't access what was defined upstream so we redefine here - this must follow any upstream change. +selinux_state: permissive diff --git a/etc/kayobe/inventory/group_vars/all/stackhpc b/etc/kayobe/inventory/group_vars/all/stackhpc new file mode 100644 index 000000000..427bb0201 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/stackhpc @@ -0,0 +1,6 @@ +--- +############################################################################### +# Feature flags + +# Whether or not to run CIS benchmark hardening playbooks. Default is false. +stackhpc_enable_cis_benchmark_hardening_hook: false diff --git a/etc/kayobe/inventory/group_vars/all/stackhpc-lvm b/etc/kayobe/inventory/group_vars/all/stackhpc-lvm new file mode 100644 index 000000000..8d710ba0d --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/stackhpc-lvm @@ -0,0 +1,150 @@ +--- +############################################################################### +# StackHPC LVM Volume Group (VG) configuration. + +# StackHPC rootvg VG. +# This VG contains a set of LVs that conform to the CIS partition benchmarks. +# This layout is compatible with the overcloud DIB host image configuration in +# stackhpc-overcloud-dib.yml. This may be used as an item in one of the +# *_lvm_groups variables. +# Example: +# controller_lvm_groups: +# - "{{ stackhpc_lvm_group_rootvg }}" +stackhpc_lvm_group_rootvg: + vgname: rootvg + disks: "{{ stackhpc_lvm_group_rootvg_disks }}" + create: true + lvnames: "{{ stackhpc_lvm_group_rootvg_lvs_default + stackhpc_lvm_group_rootvg_lvs_extra }}" + +# List of disks to include in the rootvg VG. +stackhpc_lvm_group_rootvg_disks: + - /dev/disk/by-partlabel/root + +# List of default LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_default: + - "{{ stackhpc_lvm_lv_swap }}" + - "{{ stackhpc_lvm_lv_root }}" + - "{{ stackhpc_lvm_lv_tmp }}" + - "{{ stackhpc_lvm_lv_var }}" + - "{{ stackhpc_lvm_lv_var_tmp }}" + - "{{ stackhpc_lvm_lv_log }}" + - "{{ stackhpc_lvm_lv_audit }}" + - "{{ stackhpc_lvm_lv_home }}" + +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: [] + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 16g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 50g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 2g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 20g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 10g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 10g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 100%FREE + +# StackHPC LVM lv_swap LV. +stackhpc_lvm_lv_swap: + lvname: lv_swap + size: "{{ stackhpc_lvm_lv_swap_size }}" + create: true + mount: false + +# StackHPC LVM lv_root LV. +stackhpc_lvm_lv_root: + lvname: lv_root + size: "{{ stackhpc_lvm_lv_root_size }}" + create: true + filesystem: ext4 + mount: true + mntp: / + +# StackHPC LVM lv_tmp LV. +stackhpc_lvm_lv_tmp: + lvname: lv_tmp + size: "{{ stackhpc_lvm_lv_tmp_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /tmp + +# StackHPC LVM lv_var LV. +stackhpc_lvm_lv_var: + lvname: lv_var + size: "{{ stackhpc_lvm_lv_var_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var + +# StackHPC LVM lv_var_tmp LV. +stackhpc_lvm_lv_var_tmp: + lvname: lv_var_tmp + size: "{{ stackhpc_lvm_lv_var_tmp_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/tmp + +# StackHPC LVM lv_log LV. +stackhpc_lvm_lv_log: + lvname: lv_log + size: "{{ stackhpc_lvm_lv_log_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/log + +# StackHPC LVM lv_audit LV. +stackhpc_lvm_lv_audit: + lvname: lv_audit + size: "{{ stackhpc_lvm_lv_audit_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/log/audit + +# StackHPC LVM lv_home LV. +stackhpc_lvm_lv_home: + lvname: lv_home + size: "{{ stackhpc_lvm_lv_home_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /home + +# StackHPC LVM lv_docker LV. +# NOTE: This is not included in the rootvg by default, but may be added via +# stackhpc_lvm_group_rootvg_lvs_extra. +# Example: +# stackhpc_lvm_group_rootvg_lvs_extra: +# - "{{ stackhpc_lvm_lv_docker }}" +# Alternatively, this may reside in a separate VG. +stackhpc_lvm_lv_docker: + lvname: lv_docker + size: "{{ stackhpc_lvm_lv_docker_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/lib/docker diff --git a/etc/kayobe/inventory/group_vars/all/stackhpc-repos b/etc/kayobe/inventory/group_vars/all/stackhpc-repos new file mode 100644 index 000000000..6273f2b4a --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/stackhpc-repos @@ -0,0 +1,6 @@ +--- +# Use upstream package repos by default to avoid situations where the +# seed-hypervisor tries to use a local pulp repo on the seed VM before the seed +# vm has been provisioned +# This behaviour is overriden for Overcloud hosts. +stackhpc_repos_enabled: false diff --git a/etc/kayobe/inventory/group_vars/all/vault b/etc/kayobe/inventory/group_vars/all/vault index e3da5a39f..25a8453bc 100644 --- a/etc/kayobe/inventory/group_vars/all/vault +++ b/etc/kayobe/inventory/group_vars/all/vault @@ -83,3 +83,9 @@ overcloud_vault_pki_roles: locality: ["Bristol"] organization: ["StackHPC"] ou: ["OpenStack"] + +seed_vault_pki_certificate_subject: + - common_name: "{% if kolla_internal_fqdn != kolla_internal_vip_address %}{{ kolla_internal_fqdn }}{% else %}overcloud{% endif %}" + role: "{{ seed_vault_pki_role_name }}" + extra_params: + ip_sans: "{% for host in groups['controllers'] %}{{ internal_net_name | net_ip(host) }}{% if not loop.last %},{% endif %}{% endfor %},{{ kolla_internal_vip_address }}" diff --git a/etc/kayobe/inventory/group_vars/all/wazuh b/etc/kayobe/inventory/group_vars/all/wazuh new file mode 100644 index 000000000..b78eb8c47 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/wazuh @@ -0,0 +1,3 @@ +--- +# Ansible custom SCA policies directory +local_custom_sca_policies_path: "{{ kayobe_env_config_path }}/wazuh/custom_sca_policies" diff --git a/etc/kayobe/inventory/group_vars/cis-hardening/cis b/etc/kayobe/inventory/group_vars/cis-hardening/cis new file mode 100644 index 000000000..731be1cd9 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/cis-hardening/cis @@ -0,0 +1,313 @@ +--- +############################################################################## +# Common CIS Hardening Configuration + +# Enable collecting auditd logs +update_audit_template: true + +############################################################################## +# Rocky 9 CIS Hardening Configuration + +# Allow IP forwarding +rhel9cis_is_router: true + +# Skip configuration of chrony +rhel9cis_rule_2_1_1: false +rhel9cis_rule_2_1_2: false + +# Skip configuration of the firewall +rhel9cis_firewall: None +rhel9cis_rule_3_4_1_2: false + +# Don't configure selinux +rhel9cis_selinux_disable: true + +# NOTE: FUTURE breaks wazuh agent repo metadata download +rhel9cis_crypto_policy: FIPS + +# Skip package updates +rhel9cis_rule_1_9: false + +# Disable requirement for password when using sudo +rhel9cis_rule_5_3_4: false + +# Disable check for root password being set, we should be locking root passwords instead. +# Please double-check yourself with: sudo passwd -S root +rhel9cis_rule_5_6_6: false + +# Stop the CIS benchmark scanning all files on every filesystem since this +# takes a long time. Related to the changing permissions block below. This +# would normally warn you about violations, but we can use Wazuh to continually +# monitor this. +rhel9cis_rule_6_1_9: false +rhel9cis_rule_6_1_10: false +rhel9cis_rule_6_1_11: false +rhel9cis_rule_6_1_12: false +rhel9cis_rule_6_1_13: false +rhel9cis_rule_6_1_14: false +rhel9cis_rule_6_1_15: false + +# The following rules change permissions on all files on every mounted +# filesystem. We do not want to change /var/lib/docker permissions. +rhel9cis_no_world_write_adjust: false + +# Prevent hardening from recursivley changing permissions on log files +rhel9cis_rule_4_2_3: false + +# Configure log rotation to prevent audit logs from filling the disk +rhel9cis_auditd: + space_left_action: syslog + action_mail_acct: root + admin_space_left_action: syslog + max_log_file: 10 + max_log_file_action: rotate + +# Max size of audit logs (MB) +rhel9cis_max_log_file_size: 1024 + +# Disable setting of boatloader password. This requires setting the variable +# `rhel9cis_bootloader_password_hash` +rhel9cis_set_boot_pass: false + +# NOTICE: rule disabled otherwise rule will prevent access to accounts +# as it will expire passwords older than one year. +rhel9cis_rule_5_6_1_1: false + +############################################################################## +# Ubuntu Jammy CIS Hardening Configuration + +# Ubuntu 22 CIS configuration +# Disable changing routing rules +ubtu22cis_is_router: true + +# Set Chrony as the time sync tool +ubtu22cis_time_sync_tool: "chrony" + +# Disable CIS from configuring the firewall +ubtu22cis_firewall_package: "none" + +# Stop CIS from installing Network Manager +ubtu22cis_install_network_manager: false + +# Set syslog service to journald +ubtu22cis_syslog_service: journald + +# Squashfs is compiled into the kernel +ubtu22cis_rule_1_1_1_2: false + +# This updates the system. Let's do this explicitly. +ubtu22cis_rule_1_9: false + +# Do not change Chrony Time servers +ubtu22cis_rule_2_1_2_1: false + +# Disable CIS from touching sudoers +ubtu22cis_rule_5_3_4: false + +# Add stack and kolla to allowed ssh users +ubtu22cis_sshd: + log_level: "INFO" + max_auth_tries: 4 + ciphers: + - chacha20-poly1305@openssh.com + - aes256-gcm@openssh.com + - aes128-gcm@openssh.com + - aes256-ctr + - aes192-ctr + - aes128-ctr + macs: + - hmac-sha2-512-etm@openssh.com + - hmac-sha2-256-etm@openssh.com + - hmac-sha2-512 + - hmac-sha2-256 + kex_algorithms: + - curve25519-sha256 + - curve25519-sha256@libssh.org + - diffie-hellman-group14-sha256 + - diffie-hellman-group16-sha512 + - diffie-hellman-group18-sha512 + - ecdh-sha2-nistp521 + - ecdh-sha2-nistp384 + - ecdh-sha2-nistp256 + - diffie-hellman-group-exchange-sha256 + client_alive_interval: 300 + client_alive_count_max: 3 + login_grace_time: 60 + max_sessions: 8 + allow_users: "kolla stack ubuntu" + allow_groups: "kolla stack ubuntu" + deny_users: "" + deny_groups: "" + +# Stop the CIS benchmark scanning all files on every filesystem since this +# takes a long time. Related to the changing permissions block below. This +# would normally warn you about violations, but we can use Wazuh to continually +# monitor this. +ubtu22cis_rule_6_1_9: false +ubtu22cis_rule_6_1_10: false +ubtu22cis_rule_6_1_11: false +ubtu22cis_rule_6_1_12: false +ubtu22cis_rule_6_1_13: false + +# The following rules change permissions on all files on every mounted +# filesystem. We do not want to change /var/lib/docker permissions. +ubtu22cis_no_group_adjust: false +ubtu22cis_no_owner_adjust: false +ubtu22cis_no_world_write_adjust: false +ubtu22cis_suid_adjust: false + +# Prevent hardening from recursivley changing permissions on log files +ubtu22cis_rule_4_2_3: false + +# Configure log rotation to prevent audit logs from filling the disk +ubtu22cis_auditd: + action_mail_acct: root + space_left_action: syslog + admin_space_left_action: syslog + max_log_file_action: rotate + +# Max size of audit logs (MB) +ubtu22cis_max_log_file_size: 1024 + +# Disable grub bootloader password. Requires overriding +# ubtu22cis_bootloader_password_hash +ubtu22cis_rule_1_4_1: false +ubtu22cis_rule_1_4_3: false + +# Disable: Ensure minimum days between password changes is configured +ubtu22cis_rule_5_5_1_1: false + +# Disable: Ensure password expiration is 365 days or less +ubtu22cis_rule_5_5_1_2: false + +# Disable: Ensure inactive password lock is 30 days or less +ubtu22cis_rule_5_5_1_4: false + +# Disable: Ensure all users last password change date is in the past +ubtu22cis_rule_5_5_1_5: false + +# The way this is disabled currently breaks kolla's IPV6 check, see: +# https://bugs.launchpad.net/kolla-ansible/+bug/2071443 +# Also matches RHEL hardening behavior. +ubtu22cis_ipv6_required: true + +############################################################################## +# Ubuntu Noble CIS Hardening Configuration +# FIXME: These settings are untested, they are just carried over from Jammy + +# Ubuntu 24 CIS configuration +# Disable changing routing rules +ubtu24cis_is_router: true + +# Set Chrony as the time sync tool +ubtu24cis_time_sync_tool: "chrony" + +# Disable CIS from configuring the firewall +ubtu24cis_firewall_package: "none" + +# Stop CIS from installing Network Manager +ubtu24cis_install_network_manager: false + +# Set syslog service to journald +ubtu24cis_syslog_service: journald + +# Squashfs is compiled into the kernel +ubtu24cis_rule_1_1_1_2: false + +# This updates the system. Let's do this explicitly. +ubtu24cis_rule_1_9: false + +# Do not change Chrony Time servers +ubtu24cis_rule_2_1_2_1: false + +# Disable CIS from touching sudoers +ubtu24cis_rule_5_3_4: false + +# Add stack and kolla to allowed ssh users +ubtu24cis_sshd: + log_level: "INFO" + max_auth_tries: 4 + ciphers: + - chacha20-poly1305@openssh.com + - aes256-gcm@openssh.com + - aes128-gcm@openssh.com + - aes256-ctr + - aes192-ctr + - aes128-ctr + macs: + - hmac-sha2-512-etm@openssh.com + - hmac-sha2-256-etm@openssh.com + - hmac-sha2-512 + - hmac-sha2-256 + kex_algorithms: + - curve25519-sha256 + - curve25519-sha256@libssh.org + - diffie-hellman-group14-sha256 + - diffie-hellman-group16-sha512 + - diffie-hellman-group18-sha512 + - ecdh-sha2-nistp521 + - ecdh-sha2-nistp384 + - ecdh-sha2-nistp256 + - diffie-hellman-group-exchange-sha256 + client_alive_interval: 300 + client_alive_count_max: 3 + login_grace_time: 60 + max_sessions: 8 + allow_users: "kolla stack ubuntu" + allow_groups: "kolla stack ubuntu" + deny_users: "" + deny_groups: "" + +# Stop the CIS benchmark scanning all files on every filesystem since this +# takes a long time. Related to the changing permissions block below. This +# would normally warn you about violations, but we can use Wazuh to continually +# monitor this. +ubtu24cis_rule_6_1_9: false +ubtu24cis_rule_6_1_10: false +ubtu24cis_rule_6_1_11: false +ubtu24cis_rule_6_1_12: false +ubtu24cis_rule_6_1_13: false + +# The following rules change permissions on all files on every mounted +# filesystem. We do not want to change /var/lib/docker permissions. +ubtu24cis_no_group_adjust: false +ubtu24cis_no_owner_adjust: false +ubtu24cis_no_world_write_adjust: false +ubtu24cis_suid_adjust: false + +# Prevent hardening from recursivley changing permissions on log files +ubtu24cis_rule_4_2_3: false + +# Configure log rotation to prevent audit logs from filling the disk +ubtu24cis_auditd: + action_mail_acct: root + space_left_action: syslog + admin_space_left_action: syslog + max_log_file_action: rotate + +# Max size of audit logs (MB) +ubtu24cis_max_log_file_size: 1024 + +# Disable grub bootloader password. Requires overriding +# ubtu24cis_bootloader_password_hash +ubtu24cis_rule_1_4_1: false +ubtu24cis_rule_1_4_3: false + +# Disable: Ensure minimum days between password changes is configured +ubtu24cis_rule_5_5_1_1: false + +# Disable: Ensure password expiration is 365 days or less +ubtu24cis_rule_5_5_1_2: false + +# Disable: Ensure inactive password lock is 30 days or less +ubtu24cis_rule_5_5_1_4: false + +# Disable: Ensure all users last password change date is in the past +ubtu24cis_rule_5_5_1_5: false + +# The way this is disabled currently breaks kolla's IPV6 check, see: +# https://bugs.launchpad.net/kolla-ansible/+bug/2071443 +# Also matches RHEL hardening behavior. +ubtu24cis_ipv6_required: true + +############################################################################## diff --git a/etc/kayobe/inventory/group_vars/controllers/cis b/etc/kayobe/inventory/group_vars/controllers/cis new file mode 100644 index 000000000..354a5353f --- /dev/null +++ b/etc/kayobe/inventory/group_vars/controllers/cis @@ -0,0 +1,4 @@ +--- + +rhel8cis_is_router: true + diff --git a/etc/kayobe/inventory/group_vars/github-runners/runners.yml b/etc/kayobe/inventory/group_vars/github-runners/runners.yml new file mode 100644 index 000000000..1d2d5235e --- /dev/null +++ b/etc/kayobe/inventory/group_vars/github-runners/runners.yml @@ -0,0 +1,38 @@ +--- +runner_user: "{{ kayobe_ansible_user }}" +github_account: "{{ undef(hint='Name of customer's GitHub account') }}" +github_repo: "{{ undef(hint='Name of customer's kayobe config repository') }}" +access_token: "{{ secrets_github_access_token }}" + +base_runner_dir: /opt/actions-runner + +default_runner_labels: + - kayobe + - openstack + - "{{ kayobe_environment }}" + +# Dictionary of GitHub runners to be deployed to each runner host. +# Each dict item can be provided with optional attributes +# * labels - provide a list of labels for a specific runner +# overriding the contents of `default_runner_labels` +# * state - either `started`` or `absent`. By default it will +# be started if however the runner needs to be removed +# then setting it to `absent` will unregister the runner with +# GitHub and remove it from the system. +# Example +# github_runners: +# runner_01: {} +# runner_02: +# labels: ['foo', 'bar', 'baz'] +# runner_03: +# state: absent +github_runners: + runner_01: {} + runner_02: {} + runner_03: {} + +docker_users: + - "{{ runner_user }}" + +pip_install_packages: + - name: docker diff --git a/etc/kayobe/inventory/group_vars/github-writer/writer.yml b/etc/kayobe/inventory/group_vars/github-writer/writer.yml new file mode 100644 index 000000000..4c536087f --- /dev/null +++ b/etc/kayobe/inventory/group_vars/github-writer/writer.yml @@ -0,0 +1,8 @@ +--- +# Configuration of GitHub workflows generated with stackhpc.kayobe_workflows.github should go here. +# See documentation for more information +# https://github.com/stackhpc/ansible-collection-kayobe-workflows/blob/main/roles/github/README.md + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml b/etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml new file mode 100644 index 000000000..346acef27 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/gitlab-runners/runners.yml @@ -0,0 +1,9 @@ +--- +# Configuration of GitLab runners using riemers.gitlab-runner should go here. +# See documentation for more information +# https://github.com/riemers/ansible-gitlab-runner +# https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-2024.1/configuration/ci-cd.html + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/gitlab-writer/writer.yml b/etc/kayobe/inventory/group_vars/gitlab-writer/writer.yml new file mode 100644 index 000000000..be2ce1c7e --- /dev/null +++ b/etc/kayobe/inventory/group_vars/gitlab-writer/writer.yml @@ -0,0 +1,14 @@ +--- +# Configuration of GitLab pipelines generated with stackhpc.kayobe_workflows.gitlab should go here. +# See documentation for more information +# https://github.com/stackhpc/ansible-collection-kayobe-workflows/blob/main/roles/gitlab/README.md + +gitlab_output_directory: $KAYOBE_CONFIG_PATH/../../.gitlab/ + +gitlab_registry: "{{ pulp_url | regex_replace('^https?://|^http?://', '') }}" + +gitlab_openstack_release: "{{ openstack_release }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/overcloud/stackhpc-repos b/etc/kayobe/inventory/group_vars/overcloud/stackhpc-repos new file mode 100644 index 000000000..9f45a416b --- /dev/null +++ b/etc/kayobe/inventory/group_vars/overcloud/stackhpc-repos @@ -0,0 +1,7 @@ +--- +# Only use local pulp mirrors for overcloud hosts +# to avoid situations where the seed-hypervisor +# tries to use a local pulp repo on the seed VM +# before the seed vm has been provisioned +stackhpc_repos_enabled: false +enable_docker_repo: false diff --git a/etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent b/etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent new file mode 100644 index 000000000..e464b3385 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/wazuh-agent/wazuh-agent @@ -0,0 +1,26 @@ +--- +# Wazuh-Agent role configuration +# Reference: https://documentation.wazuh.com/4.3/deploying-with-ansible/reference.html#wazuh-agent +# Defaults: https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-wazuh-agent/defaults/main.yml + +# Wazuh-Manager IP address +# Convenience var not used by wazuh-agent role +wazuh_manager_address: "{{ admin_oc_net_name | net_ip(groups['wazuh-manager'][0]) }}" + +# Wazuh-Manager API config +wazuh_managers: + - address: "{{ wazuh_manager_address }}" + port: 1514 + protocol: tcp + api_port: 55000 + +# Wazuh-Agent authd config +wazuh_agent_authd: + registration_address: "{{ wazuh_manager_address }}" + enable: true + port: 1515 + ssl_agent_ca: null + ssl_auto_negotiate: 'no' + +# Wazuh-Agent authd password +authd_pass: "{{ secrets_wazuh.authd_pass }}" diff --git a/etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager b/etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager new file mode 100644 index 000000000..26ddb2a5c --- /dev/null +++ b/etc/kayobe/inventory/group_vars/wazuh-manager/wazuh-manager @@ -0,0 +1,147 @@ +--- +# Wazuh deployment details and documetation: +# https://documentation.wazuh.com/current/deployment-options/deploying-with-ansible/index.html +# +# Defaults that are modified by this file: +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-filebeat-oss/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-wazuh-manager/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/wazuh-dashboard/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/wazuh-indexer/defaults/main.yml +# https://github.com/wazuh/wazuh-ansible/blob/4.3/roles/wazuh/ansible-wazuh-manager/defaults/main.yml + +# Wazuh IP accessible from nodes running agents +# Change network as needed +wazuh_manager_ip: "{{ provision_oc_net_name | net_ip }}" + +# Single node setup +single_node: true +minimum_master_nodes: 1 +indexer_node_master: true + +# Ansible control host certificate directory +local_certs_path: "{{ kayobe_env_config_path }}/wazuh" + +# Indexer variables +indexer_node_name: "{{ inventory_hostname }}" + +# IP to which indexer will bind on host. +indexer_network_host: "{{ provision_oc_net_name | net_ip }}" + +# Even in a single node setup this must be defined. If not defaults to 127.0.0.1 +indexer_cluster_nodes: + - "{{ indexer_network_host }}" + +instances: + node1: + name: "{{ groups['wazuh-manager'][0] }}" + ip: "{{ wazuh_manager_ip }}" + role: indexer + +# Filebeat variables +filebeat_node_name: "{{ indexer_node_name }}" + +filebeat_output_indexer_hosts: + - "{{ indexer_network_host }}" + +# Dashboard variables +dashboard_node_name: "{{ indexer_node_name }}" + +dashboard_server_host: "{{ indexer_network_host }}" + +dashboard_server_name: "{{ indexer_node_name }}" + +indexer_admin_password: "{{ secrets_wazuh.opendistro_admin_password }}" + +dashboard_user: kibanaserver + +# TODO refactor creation and usage of secrets_wazuh +dashboard_password: "{{ secrets_wazuh.opendistro_kibana_password }}" + +# Indexer filebeat user +indexer_security_user: admin + +# Indexer filebeat password +indexer_security_password: "{{ secrets_wazuh.opendistro_admin_password }}" + +# Used to override the default Wazuh api user +wazuh_api_users: "{{ secrets_wazuh.wazuh_api_users }}" + +# Dashboard to Wazuh api credentials +wazuh_api_credentials: + - id: "default" + url: "https://{{ indexer_network_host }}" + port: 55000 + username: "wazuh" + password: "{{ secrets_wazuh.wazuh_api_users | selectattr('username', 'match', 'wazuh') | map(attribute='password') | first }}" + +# Perform hash_behaviour=merge at role runtime +wazuh_manager_config_overlay: true + +# JSON formatted alerts to /var/ossec/logs/alerts/alerts.json +wazuh_manager_json_output: 'yes' + +# Plain text alerts to /var/ossec/logs/alerts/alerts.log +wazuh_manager_alerts_log: 'yes' + +# Store events that do not trip rules +wazuh_manager_logall: 'no' + +# Email alert notifications +wazuh_manager_email_notification: 'no' + +# Log format of /var/ossec/logs/ossec.{log|json} +wazuh_manager_log_format: 'plain' + +authd_pass: "{{ secrets_wazuh.authd_pass }}" + +# Wazuh-Manager listener settings +wazuh_manager_connection: + - type: 'secure' + port: '1514' + protocol: 'tcp' + queue_size: 131072 + +# Wazuh-Manager authd settings +# TODO, enable SSL/TLS +wazuh_manager_authd: + enable: true + port: 1515 + use_source_ip: 'no' + force: + enabled: 'no' + key_mismatch: 'no' + disconnected_time: '0' + after_registration_time: '0' + purge: 'no' + use_password: 'yes' + ciphers: 'HIGH:!ADH:!EXP:!MD5:!RC4:!3DES:!CAMELLIA:@STRENGTH' + ssl_agent_ca: null + ssl_verify_host: 'no' + ssl_manager_cert: null + ssl_manager_key: null + ssl_auto_negotiate: 'no' + +# Wazuh-Manager Vulnerability Detector settings +wazuh_manager_vulnerability_detector: + enabled: 'yes' + interval: '5m' + ignore_time: '6h' + run_on_start: 'yes' + providers: + - enabled: 'yes' + os: + - 'focal' + - 'jammy' + - 'noble' + update_interval: '1h' + name: '"canonical"' + - enabled: 'yes' + os: + - '8' + - '9' + update_interval: '1h' + name: '"redhat"' + - enabled: 'yes' + update_from_year: '2018' + update_interval: '1h' + name: '"nvd"' diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index 275ea9bce..a608cd043 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -24,6 +24,26 @@ seed-hypervisor [infra-vms] # Empty group to provide declaration of infra-vms group. +[wazuh-manager] +# Empty group to provide declaration of wazuh-manager group. + +[wazuh-agent:children] +seed +seed-hypervisor +overcloud + +[github-runners] +# Empty group to provide declaration of github-runner group. + +[github-writer] +localhost + +[gitlab-runners] +# Empty group to provide declaration of gitlab-runner group. + +[gitlab-writer] +localhost + ############################################################################### # Overcloud groups. @@ -62,6 +82,9 @@ compute-vgpu [iommu:children] vgpu +[mlnx] +# Empty group to provide declaration of mlnx group. + ############################################################################### # Service groups. @@ -73,6 +96,7 @@ network monitoring storage compute +gitlab-runners [docker-registry:children] # Hosts in this group will have a Docker Registry deployed. This group should @@ -85,6 +109,7 @@ seed seed seed-hypervisor overcloud +infra-vms ############################################################################### # Baremetal compute node groups. @@ -123,3 +148,15 @@ rgws [mgrs] [osds] [rgws] + +############################################################################### +# Feature control groups + +[cis-hardening:children] +overcloud + +############################################################################### +# Monitoring groups. + +[redfish_exporter_targets:children] +overcloud diff --git a/etc/kayobe/ipa.yml b/etc/kayobe/ipa.yml index 0138c6c44..30a1057e6 100644 --- a/etc/kayobe/ipa.yml +++ b/etc/kayobe/ipa.yml @@ -30,7 +30,9 @@ # List of additional Diskimage Builder (DIB) elements to use when building IPA # images. Default is none. -#ipa_build_dib_elements_extra: +ipa_build_dib_elements_extra: + - extra-hardware + - mellanox # List of Diskimage Builder (DIB) elements to use when building IPA images. # Default is combination of ipa_build_dib_elements_default and @@ -85,7 +87,7 @@ #ipa_images_kernel_name: # URL of Ironic deployment kernel image to download. -#ipa_kernel_upstream_url: +ipa_kernel_upstream_url: "{{ (stackhpc_ipa_image_url + '/ipa.kernel') if stackhpc_ipa_image_overcloud_enabled | bool }}" # URL of checksum of Ironic deployment kernel image. #ipa_kernel_checksum_url: @@ -97,7 +99,7 @@ #ipa_images_ramdisk_name: # URL of Ironic deployment ramdisk image to download. -#ipa_ramdisk_upstream_url: +ipa_ramdisk_upstream_url: "{{ (stackhpc_ipa_image_url + '/ipa.initramfs') if stackhpc_ipa_image_overcloud_enabled | bool }}" # URL of checksum of Ironic deployment ramdisk image. #ipa_ramdisk_checksum_url: @@ -105,6 +107,13 @@ # Algorithm of checksum of Ironic deployment ramdisk image. #ipa_ramdisk_checksum_algorithm: +# IPA download parameters +image_download_url_username: "{{ stackhpc_release_pulp_username }}" +image_download_url_password: "{{ stackhpc_release_pulp_password }}" +image_download_force_basic_auth: true +image_download_unredirected_headers: + - Authorization + ############################################################################### # Ironic Python Agent (IPA) deployment configuration. @@ -117,7 +126,7 @@ #ipa_collectors_default: # List of additional inspection collectors to run. -#ipa_collectors_extra: +ipa_collectors_extra: "{{ ['dmi-decode', 'extra-hardware', 'numa-topology'] if ipa_build_images or stackhpc_ipa_image_overcloud_enabled else [] }}" # List of inspection collectors to run. #ipa_collectors: @@ -135,7 +144,11 @@ #ipa_kernel_options_default: # List of additional kernel parameters for Ironic python agent. -#ipa_kernel_options_extra: +ipa_kernel_options_extra: + # Useful until NTP is configured by default + - ipa-insecure=1 + # Avoid disk benchmark failures on some NVMe drives + - nvme_core.multipath=N # List of kernel parameters for Ironic python agent. #ipa_kernel_options: diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml new file mode 100644 index 000000000..d84feac3e --- /dev/null +++ b/etc/kayobe/kolla-image-tags.yml @@ -0,0 +1,89 @@ +--- +# Dict of Kolla image tags to deploy for each service. +# Each key is the tag variable prefix name, and the value is another dict, +# where the key is the OS distro and the value is the tag to deploy. +kolla_image_tags: + openstack: + rocky-9: 2024.1-rocky-9-20260223T075405 + ubuntu-jammy: 2024.1-ubuntu-jammy-20241218T141809 + ubuntu-noble: 2024.1-ubuntu-noble-20250404T150323 + barbican: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + blazar: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + bifrost: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250325T141125 + # not buildable anymore, see https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a + rocky-9: 2024.1-rocky-9-20250716T041633 + cinder: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + cloudkitty: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + designate: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + heat: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + glance: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250903T130503 + ubuntu-noble: 2024.1-ubuntu-noble-20250903T130503 + horizon: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250227T091118 + ironic: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + ironic_dnsmasq: + ubuntu-jammy: 2024.1-ubuntu-jammy-20241218T141809 + ubuntu-noble: 2024.1-ubuntu-noble-20250404T150323 + ironic_prometheus_exporter: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250124T081816 + iscsid: + ubuntu-jammy: 2024.1-ubuntu-jammy-20251103T065408 + ubuntu-noble: 2024.1-ubuntu-noble-20251103T065408 + keystone: + rocky-9: 2024.1-rocky-9-20260527T155618 + ubuntu-jammy: 2024.1-ubuntu-jammy-20260527T155618 + ubuntu-noble: 2024.1-ubuntu-noble-20260527T155618 + kolla_toolbox: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147 + ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147 + magnum: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250922T105039 + ubuntu-noble: 2024.1-ubuntu-noble-20250922T105039 + manila: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + neutron: + ubuntu-jammy: 2024.1-ubuntu-jammy-20260128T065158 + ubuntu-noble: 2024.1-ubuntu-noble-20260128T065158 + neutron_bgp_dragent: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147 + ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147 + nova: + rocky-9: 2024.1-rocky-9-20260226T161729 + ubuntu-jammy: 2024.1-ubuntu-jammy-20260220T123443 + ubuntu-noble: 2024.1-ubuntu-noble-20260220T123443 + octavia: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250627T102805 + ubuntu-noble: 2024.1-ubuntu-noble-20250627T102805 + openvswitch: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250529T081147 + ubuntu-noble: 2024.1-ubuntu-noble-20250529T081147 + ovn: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250925T130625 + ubuntu-noble: 2024.1-ubuntu-noble-20250925T130625 + prometheus: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250219T145255 + prometheus_alertmanager: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250422T103147 + prometheus_libvirt_exporter: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250819T151035 + ubuntu-noble: 2024.1-ubuntu-noble-20250819T151035 + rabbitmq: + ubuntu-jammy: 2024.1-ubuntu-jammy-20250722T083943 + ubuntu-noble: 2024.1-ubuntu-noble-20250722T083943 diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 33f446ef0..3b7ea8bd0 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -11,11 +11,11 @@ #kolla_source_path: # URL of Kolla source code repository if type is 'source'. -#kolla_source_url: +kolla_source_url: "{{ stackhpc_kolla_source_url }}" # Version (branch, tag, etc.) of Kolla source code repository if type is # 'source'. Default is {{ openstack_branch }}. -#kolla_source_version: +kolla_source_version: "{{ stackhpc_kolla_source_version }}" # Path to virtualenv in which to install kolla. #kolla_venv: @@ -23,13 +23,14 @@ # Path in which to generate kolla configuration. #kolla_build_config_path: +# View debug logs, like source downloads, during image build. +kolla_build_extra_config: | + [DEFAULT] + debug = True + ############################################################################### # Kolla-ansible installation. -# Type of Kolla-ansible control installation. One of 'binary' or 'source'. -# Default is 'source'. -#kolla_ansible_ctl_install_type: - # Path to directory for kolla-ansible source code checkout. # Default is $KOLLA_SOURCE_PATH, or $PWD/src/kolla-ansible if # $KOLLA_SOURCE_PATH is not set. @@ -37,11 +38,11 @@ # URL of Kolla Ansible source code repository if type is 'source'. Default is # https://opendev.org/openstack/kolla-ansible. -#kolla_ansible_source_url: +kolla_ansible_source_url: "{{ stackhpc_kolla_ansible_source_url }}" # Version (branch, tag, etc.) of Kolla Ansible source code repository if type # is 'source'. Default is {{ openstack_branch }}. -#kolla_ansible_source_version: +kolla_ansible_source_version: "{{ stackhpc_kolla_ansible_source_version }}" # Path to virtualenv in which to install kolla-ansible. Default is # $KOLLA_VENV_PATH or $PWD/venvs/kolla-ansible if $KOLLA_VENV_PATH is not set. @@ -66,6 +67,11 @@ ############################################################################### # Kolla configuration. +# Kolla base container image architecture. Options are "x86_64", "aarch64". +# Default is "{{ ansible_facts.architecture }}" +# NOTE(bbezak): evaluating this var locally for non facts gathering playbooks +kolla_base_arch: "{{ 'aarch64' if lookup('pipe','uname -m') in ['aarch64','arm64'] else 'x86_64' }}" + # Kolla base container image distribution. Options are "centos", "debian", # "rocky", "ubuntu". Default is {{ os_distribution }}. #kolla_base_distro: @@ -78,29 +84,31 @@ # Default is kolla_base_distro_version_default_map[kolla_base_distro]. #kolla_base_distro_version: +kolla_base_distro_and_version: "{{ kolla_base_distro }}-{{ kolla_base_distro_version }}" + # URL of docker registry to use for Kolla images. Default is not set, in which # case Quay.io will be used. #kolla_docker_registry: -# Docker namespace to use for Kolla images. Default is 'kolla'. -#kolla_docker_namespace: +# Docker namespace to use for Kolla images. Default is 'openstack.kolla'. +kolla_docker_namespace: stackhpc # Username to use to access a docker registry. Default is not set, in which # case the registry will be used without authentication. -#kolla_docker_registry_username: +kolla_docker_registry_username: "{{ stackhpc_docker_registry_username }}" # Password to use to access a docker registry. Default is not set, in which # case the registry will be used without authentication. -#kolla_docker_registry_password: +kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -#kolla_openstack_release: +# kolla_openstack_release: # Docker tag applied to built container images. Default is {{ # kolla_openstack_release }}-{{ kolla_base_distro }}-{{ # kolla_base_distro_version }}. -#kolla_tag: +kolla_tag: "{{ openstack_release }}-{{ kolla_base_distro }}-{{ kolla_base_distro_version }}-{{ ansible_facts.date_time.iso8601_basic_short }}" # Dict mapping names of sources to their definitions. # See kolla.common.config for details. @@ -110,13 +118,356 @@ # type: git # location: https://github.com/openstack/ironic # reference: master -#kolla_sources: +# NOTE(upgrade): These sources should be checked with each release. StackHPC +# branches are only required when we have custom backports. For a new release, +# we may have caught up with upstream. +kolla_sources: + bifrost-base-additions-stackhpc-inspector-plugins: + # Install our custom inspector plugins. + type: git + location: https://github.com/stackhpc/stackhpc-inspector-plugins.git + reference: 1.3.0 + bifrost-base: + type: git + location: https://github.com/stackhpc/bifrost.git + reference: stackhpc/{{ openstack_release }} + cloudkitty-base: + type: git + location: https://github.com/stackhpc/cloudkitty.git + reference: stackhpc/{{ openstack_release }} + glance-base: + type: git + location: https://github.com/stackhpc/glance.git + reference: stackhpc/{{ openstack_release }} + horizon-plugin-cloudkitty-dashboard: + type: git + location: https://github.com/stackhpc/cloudkitty-dashboard.git + reference: stackhpc/{{ openstack_release }} + ironic-inspector-additions-stackhpc-inspector-plugins: + # Install our custom inspector plugins. + type: git + location: https://github.com/stackhpc/stackhpc-inspector-plugins.git + reference: 1.3.0 + magnum-base: + type: git + location: https://github.com/stackhpc/magnum.git + reference: stackhpc/{{ openstack_release }} + neutron-base-plugin-networking-generic-switch: + type: git + location: https://github.com/stackhpc/networking-generic-switch.git + reference: stackhpc/{{ openstack_release }} + neutron-server-plugin-networking-mlnx: + type: git + location: https://github.com/stackhpc/networking-mlnx.git + reference: stackhpc/{{ openstack_release }} + nova-base: + type: git + location: https://github.com/stackhpc/nova.git + reference: stackhpc/{{ openstack_release }} + nova-compute-plugin-networking-mlnx: + type: git + location: https://github.com/stackhpc/networking-mlnx.git + reference: stackhpc/{{ openstack_release }} + octavia-base: + type: git + location: https://github.com/stackhpc/octavia.git + reference: stackhpc/{{ openstack_release }} + blazar-base: + type: git + location: https://github.com/stackhpc/blazar.git + reference: stackhpc/master ############################################################################### # Kolla image build configuration. +# List of repositories for Rocky Linux 9 +stackhpc_rocky_9_repos: + - url: "{{ stackhpc_repo_rocky_9_baseos_url }}" + file: "rocky.repo" + tag: "baseos" + - url: "{{ stackhpc_repo_rocky_9_appstream_url }}" + file: "rocky.repo" + tag: "appstream" + - url: "{{ stackhpc_repo_rocky_9_extras_url }}" + file: "rocky-extras.repo" + tag: "extras" + +# List of additional repositories for Rocky Linux 9 +stackhpc_rocky_9_additional_repos: + - url: "{{ stackhpc_repo_centos_stream_9_nfv_openvswitch_url }}" + file: "CentOS-NFV-OpenvSwitch.repo" + tag: "centos-nfv-openvswitch" + - url: "{{ stackhpc_repo_centos_stream_9_openstack_caracal_url }}" + file: "CentOS-OpenStack-caracal.repo" + tag: "centos-openstack-caracal" + - url: "{{ stackhpc_repo_centos_stream_9_opstools_url }}" + file: "CentOS-OpsTools.repo" + tag: "centos-opstools" + - url: "{{ stackhpc_repo_rocky_9_crb_url }}" + file: "rocky.repo" + tag: "crb" + - url: "{{ stackhpc_repo_rocky_9_highavailability_url }}" + file: "rocky-addons.repo" + tag: "highavailability" + +# List of third-party repositories for Rocky Linux 9 +stackhpc_rocky_9_third_party_repos: + - url: "{{ stackhpc_repo_grafana_url }}" + file: "grafana.repo" + tag: "grafana" + - url: "{{ stackhpc_repo_rhel_9_treasuredata_5_url }}" + file: "td.repo" + tag: "treasuredata" + - url: "{{ stackhpc_repo_rhel_9_mariadb_10_11_url }}" + file: "mariadb.repo" + tag: "mariadb" + - url: "{{ stackhpc_repo_rhel_9_influxdb_url }}" + file: "influxdb.repo" + tag: "influxdb" + - url: "{{ stackhpc_repo_centos_stream_9_storage_ceph_reef_url }}" + file: "CentOS-Ceph-Reef.repo" + tag: "centos-ceph-reef" + - url: "{{ stackhpc_repo_rhel9_rabbitmq_erlang_url }}" + file: "rabbitmq_rabbitmq-erlang.repo" + tag: "rabbitmq_rabbitmq-erlang" + - url: "{{ stackhpc_repo_rhel9_rabbitmq_erlang_26_url }}" + file: "copr-rabbitmq-erlang-26.repo" + tag: "copr-rabbitmq-erlang-26" + - url: "{{ stackhpc_repo_rhel9_rabbitmq_erlang_27_url }}" + file: "copr-rabbitmq-erlang-27.repo" + tag: "copr-rabbitmq-erlang-27" + - url: "{{ stackhpc_repo_rhel9_rabbitmq_server_url }}" + file: "rabbitmq_rabbitmq-server.repo" + tag: "rabbitmq_rabbitmq-server" + - url: "{{ stackhpc_repo_opensearch_2_x_url }}" + file: "opensearch.repo" + tag: "opensearch-2.x" + - url: "{{ stackhpc_repo_opensearch_dashboards_2_x_url }}" + file: "opensearch.repo" + tag: "opensearch-dashboards-2.x" + +# List of EPEL 9 +stackhpc_epel_9_repos: + - url: "{{ stackhpc_repo_epel_9_url }}" + file: "epel.repo" + tag: "epel" + +base_centos_repo_overrides_post_yum_list: "{{ stackhpc_rocky_9_repos + stackhpc_epel_9_repos + stackhpc_rocky_9_additional_repos + stackhpc_rocky_9_third_party_repos }}" +stackhpc_yum_repos: "{{ stackhpc_rocky_9_repos }}" + +# Apt sources.list entry prefix. +# If using an authenticating Pulp proxy we need to trust the repository because +# the certificate provided by the upstream repo will not match the proxy's IP. +stackhpc_ubuntu_repo_prefix: "deb {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}[trusted=yes] {% endif %}" + +# List of base repositories for Ubuntu Jammy. +stackhpc_ubuntu_jammy_base_repos: + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_url }} jammy main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_url }} jammy-updates main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_url }} jammy-backports main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_jammy_security_url }} jammy-security main universe" + +# List of UCA repositories for Ubuntu Jammy. +stackhpc_ubuntu_jammy_uca_repos: + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_cloud_archive_url }} jammy-updates/{{ openstack_release_codename }} main" + +# List of repositories for Ubuntu Jammy. +stackhpc_ubuntu_jammy_repos: "{{ stackhpc_ubuntu_jammy_base_repos + stackhpc_ubuntu_jammy_uca_repos }}" + +# List of base repositories for Ubuntu Noble. +stackhpc_ubuntu_noble_base_repos: + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_noble_url }} noble main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_noble_url }} noble-updates main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_noble_url }} noble-backports main universe" + - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_noble_security_url }} noble-security main universe" + +# List of UCA repositories for Ubuntu Noble. +stackhpc_ubuntu_noble_uca_repos: [] + +# List of repositories for Ubuntu Noble. +stackhpc_ubuntu_noble_repos: "{{ stackhpc_ubuntu_noble_base_repos + stackhpc_ubuntu_noble_uca_repos }}" + +# Whether to revert to the upstream mirrors in built Kolla container images. +stackhpc_kolla_clean_up_repo_mirrors: true + # Dict mapping Jinja2 block names in kolla's Docker images to their contents. -#kolla_build_blocks: +kolla_build_blocks: + base_header: | + {% if kolla_base_distro == 'rocky' %} + RUN \ + {% if stackhpc_kolla_clean_up_repo_mirrors | bool %} + mkdir -p /etc/yum.repos.d.backup && \ + tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \ + {% endif %} + {% for repo in stackhpc_yum_repos %} + sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \ + {% endif %} + {% endfor %} + {% else %} + RUN \ + rm /etc/apt/sources.list && \ + {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %} + {# We lack the ca-certificates package at this stage, so don't verify the CA initially #} + echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \ + {% endif %} + {% if kolla_base_distro_version == 'jammy' %} + {% for repo in stackhpc_ubuntu_jammy_base_repos %} + echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ + {% endif %} + {% endfor %} + {% else %} + {% for repo in stackhpc_ubuntu_noble_repos %} + echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ + {% endif %} + {% endfor %} + {% endif %} + {% endif %} + base_centos_repo_overrides_post_yum: | + {# fixme #} + {% if kolla_base_distro == 'rocky' %} + && \ + {% if stackhpc_kolla_clean_up_repo_mirrors | bool %} + tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \ + tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \ + {% endif %} + {% for repo in base_centos_repo_overrides_post_yum_list %} + if [ -f /etc/yum.repos.d/{{ repo.file }} ]; then \ + sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[ \t]*\(https.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \ + -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}; \ + fi{% if not loop.last %} &&{% endif %} \ + {% endfor %} + {% endif %} + # With the UCA keyring installed we can now add all repos. + base_ubuntu_package_sources_list: | + {% if stackhpc_kolla_clean_up_repo_mirrors | bool %} + {% if kolla_base_distro_version == 'jammy' %} + COPY sources.list.ubuntu.jammy /etc/apt/sources.list.backup + {% else %} + COPY sources.list.ubuntu.noble /etc/apt/sources.list.backup + {% endif %} + + {% endif %} + RUN \ + rm /etc/apt/sources.list && \ + rm -f /etc/apt/apt.conf.d/90no-verify-peer && \ + {% if kolla_base_distro_version == 'jammy' %} + {% for repo in stackhpc_ubuntu_jammy_base_repos %} + echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ + {% endif %} + {% endfor %} + {% else %} + {% for repo in stackhpc_ubuntu_noble_repos %} + echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ + {% endif %} + {% endfor %} + {% endif %} + footer: | + {% if stackhpc_kolla_clean_up_repo_mirrors | bool %} + {% if kolla_base_distro == 'rocky' %} + RUN \ + if [ -f /etc/yum.repos.d.backup/repos.tar.gz ]; then \ + tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \ + rm -rf /etc/yum.repos.d.backup/; \ + fi && \ + if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \ + echo "Found repository mirror in Yum repositories"; \ + exit 1; \ + fi + {% else %} + RUN \ + if [ -f /etc/apt/sources.list.backup ]; then \ + mv /etc/apt/sources.list.backup /etc/apt/sources.list; \ + fi && \ + if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \ + echo "Found repository mirror in APT repositories"; \ + exit 1; \ + fi + {% endif %} + {% endif %} + kolla_toolbox_header: | + ENV UPPER_CONSTRAINTS_FILE=https://raw.githubusercontent.com/stackhpc/requirements/stackhpc/{{ openstack_release }}/upper-constraints.txt + bifrost_base_header: | + ADD additions-archive / + ENV TOX_CONSTRAINTS_FILE=/requirements/upper-constraints.txt + grafana_plugins_install: | + RUN grafana-cli plugins install vonage-status-panel \ + && grafana-cli plugins install grafana-opensearch-datasource + ironic_inspector_header: | + ADD additions-archive / + +# Dict mapping image customization variable names to their values. +# Each variable takes the form: +# __ +# Hyphens in the image name must be replaced with underscores. The +# customization is most commonly packages. The operation should be one of +# override, append or remove. The value should be a list. +kolla_build_customizations_common: + bifrost_base_pip_packages_append: + - /additions/* + glance_base_pip_packages_override: + - "/glance" + - "glance_store[cinder,vmware,swift,s3]@git+https://github.com/stackhpc/glance_store@stackhpc/4.7.1.1" + ironic_inspector_pip_packages_append: + - /additions/* + magnum_base_pip_packages_override: + - /magnum[osprofiler] + - magnum-capi-helm==1.3.0 + neutron_server_packages_append: + - python3-libvirt + - python3-ethtool + nova_compute_packages_append: + - python3-libvirt + - python3-ethtool + neutron_mlnx_agent_pip_packages_override: + - networking-mlnx@git+https://github.com/stackhpc/networking-mlnx.git@stackhpc/{{ openstack_release }} + +kolla_build_customizations_rocky: + kolla_toolbox_packages_remove: + - openvswitch + kolla_toolbox_packages_append: + - openvswitch3.3 + manila_base_packages_remove: + - openvswitch + manila_base_packages_append: + - openvswitch3.3 + neutron_base_packages_remove: + - openvswitch + - python3-openvswitch + neutron_base_packages_append: + - openvswitch3.3 + - python3-openvswitch3.3 + nova_base_packages_remove: + - openvswitch + - python3-openvswitch + nova_base_packages_append: + - openvswitch3.3 + - python3-openvswitch3.3 + octavia_base_packages_remove: + - python3-openvswitch + openvswitch_base_packages_remove: + - openvswitch + - python3-openvswitch + openvswitch_base_packages_append: + - openvswitch3.3 + - python3-openvswitch3.3 + ovn_base_packages_override: + - ovn24.03 + ovn_controller_packages_override: + - ovn24.03-host + ovn_nb_db_server_packages_override: + - ovn24.03-central + ovn_northd_packages_override: + - ovn24.03-central + ovn_sb_db_server_packages_override: + - ovn24.03-central + +kolla_build_customizations_ubuntu: {} # Dict mapping image customization variable names to their values. # Each variable takes the form: @@ -124,7 +475,10 @@ # Hyphens in the image name must be replaced with underscores. The # customization is most commonly packages. The operation should be one of # override, append or remove. The value should be a list. -#kolla_build_customizations: +kolla_build_customizations: "{{ kolla_build_customizations_common | combine(kolla_build_customizations_rocky if kolla_base_distro == 'rocky' else kolla_build_customizations_ubuntu) }}" + +# Dict mapping Kolla Dockerfile ARG names to their values. +kolla_build_args: {} ############################################################################### # Kolla-ansible inventory configuration. @@ -186,6 +540,24 @@ kolla_overcloud_inventory_custom_services: "{{ lookup('template', kayobe_config_ # * groups: A list of kayobe ansible groups to map to this kolla-ansible group. # * vars: A dict mapping variable names to values for hosts in this # kolla-ansible group. +# NOTE(Alex-Welsh): If you want to extend the map rather than replace it, you +# must include the Kayobe defaults in the mapping. +# Standard Kayobe defaults: +# compute: +# groups: +# - "compute" +# control: +# groups: +# - "controllers" +# monitoring: +# groups: +# - "controllers" +# network: +# groups: +# - "controllers" +# storage: +# groups: +# - "controllers" #kolla_overcloud_inventory_top_level_group_map: # List of names of top level kolla-ansible groups. Any of these groups which @@ -200,7 +572,11 @@ kolla_overcloud_inventory_custom_services: "{{ lookup('template', kayobe_config_ # List of names of additional host variables to pass through from kayobe hosts # to kolla-ansible hosts, if set. See also # kolla_overcloud_inventory_pass_through_host_vars_map. -#kolla_overcloud_inventory_pass_through_host_vars_extra: +kolla_overcloud_inventory_pass_through_host_vars_extra: + - stackhpc_gpu_data + - gpu_group_map + - stackhpc_radosgw_usage_exporter_frontend_port + - stackhpc_radosgw_usage_exporter_backend_port # List of names of host variables to pass through from kayobe hosts to # kolla-ansible hosts, if set. See also @@ -287,7 +663,7 @@ kolla_overcloud_inventory_pass_through_host_vars_extra: #kolla_enable_ceilometer_ipmi: #kolla_enable_ceilometer_prometheus_pushgateway: #kolla_enable_cells: -kolla_enable_central_logging: yes +kolla_enable_central_logging: true #kolla_enable_ceph_rgw: #kolla_enable_ceph_rgw_loadbalancer: kolla_enable_cinder: yes @@ -317,12 +693,12 @@ kolla_enable_cinder: yes #kolla_enable_glance_image_cache: #kolla_enable_gnocchi: #kolla_enable_gnocchi_statsd: -kolla_enable_grafana: yes +kolla_enable_grafana: true #kolla_enable_grafana_external: #kolla_enable_hacluster: #kolla_enable_haproxy: #kolla_enable_haproxy_memcached: -#kolla_enable_heat: +kolla_enable_heat: false #kolla_enable_heat_horizon_policy_file: #kolla_enable_horizon: #kolla_enable_horizon_blazar: @@ -344,7 +720,7 @@ kolla_enable_grafana: yes #kolla_enable_influxdb: #kolla_enable_ironic: #kolla_enable_ironic_neutron_agent: -#kolla_enable_ironic_prometheus_exporter: +kolla_enable_ironic_prometheus_exporter: false #kolla_enable_iscsid: #kolla_enable_keepalived: #kolla_enable_keystone: @@ -360,7 +736,7 @@ kolla_enable_manila_backend_cephfs_native: yes #kolla_enable_manila_backend_generic: #kolla_enable_manila_backend_glusterfs_nfs: #kolla_enable_manila_backend_hnas: -#kolla_enable_mariabackup: +kolla_enable_mariabackup: True #kolla_enable_mariadb: #kolla_enable_masakari: #kolla_enable_masakari_hostmonitor: @@ -402,17 +778,16 @@ kolla_enable_octavia: yes #kolla_enable_openstack_core: #kolla_enable_openvswitch: #kolla_enable_osprofiler: -#kolla_enable_outward_rabbitmq: -kolla_enable_ovn: yes +kolla_enable_ovn: true +#kolla_enable_ovn_sb_db_relay: #kolla_enable_ovs_dpdk: #kolla_enable_placement: -kolla_enable_prometheus: yes +kolla_enable_prometheus: true #kolla_enable_prometheus_alertmanager: #kolla_enable_prometheus_alertmanager_external: #kolla_enable_prometheus_blackbox_exporter: #kolla_enable_prometheus_cadvisor: #kolla_enable_prometheus_ceph_mgr_exporter: -#kolla_enable_prometheus_elasticsearch_exporter: #kolla_enable_prometheus_etcd_integration: #kolla_enable_prometheus_fluentd_integration: #kolla_enable_prometheus_haproxy_exporter: diff --git a/etc/kayobe/kolla/config/bifrost/bifrost.yml b/etc/kayobe/kolla/config/bifrost/bifrost.yml new file mode 100644 index 000000000..6310d4931 --- /dev/null +++ b/etc/kayobe/kolla/config/bifrost/bifrost.yml @@ -0,0 +1,19 @@ +# yamllint disable-file +--- +# Use prebuilt release train images from Ark. +{% if stackhpc_download_overcloud_host_images | bool %} +use_cirros: true +cirros_deploy_image_upstream_url: "{{ stackhpc_overcloud_host_image_url }}" +{% endif %} + +# Disable debug logging to avoid generating large log files +ironic_debug: false + +# Use prebuilt release train IPA images from Ark. +{% if stackhpc_ipa_image_bifrost_enabled | bool %} +ipa_download_url_username: "{{ stackhpc_release_pulp_username }}" +ipa_download_url_password: "{{ stackhpc_release_pulp_password }}" +ipa_download_force_basic_auth: true +ipa_download_unredirected_headers: + - Authorization +{% endif %} diff --git a/etc/kayobe/kolla/config/fluentd/output/00-local.conf b/etc/kayobe/kolla/config/fluentd/output/00-local.conf new file mode 100644 index 000000000..a00daa5c9 --- /dev/null +++ b/etc/kayobe/kolla/config/fluentd/output/00-local.conf @@ -0,0 +1,85 @@ +{% raw %} +{% for item in syslog_facilities | selectattr('enabled') %} + + @type copy + + @type file + path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }} + append true + # Disable timestamp in filename for logs + + path /var/log/kolla/{{ item.logdir }}/{{ item.logfile }}.*.buffer + + + output_tag {{ item.output_tag | default(false) | lower }} + output_time {{ item.output_time | default(false) | lower }} + + +{% if log_direct_to_elasticsearch %} + + @type elasticsearch + host {{ elasticsearch_address }} + port {{ elasticsearch_port | default('9200') }} + scheme {{ fluentd_elasticsearch_scheme }} +{% if fluentd_elasticsearch_path != '' %} + path {{ fluentd_elasticsearch_path }} +{% endif %} +{% if fluentd_elasticsearch_scheme == 'https' %} + ssl_version {{ fluentd_elasticsearch_ssl_version }} + ssl_verify {{ fluentd_elasticsearch_ssl_verify }} +{% if fluentd_elasticsearch_cacert | length > 0 %} + ca_file {{ fluentd_elasticsearch_cacert }} +{% endif %} +{% endif %} +{% if fluentd_elasticsearch_user != '' and fluentd_elasticsearch_password != ''%} + user {{ fluentd_elasticsearch_user }} + password {{ fluentd_elasticsearch_password }} +{% endif %} + logstash_format true + logstash_prefix {{ opensearch_log_index_prefix }} + reconnect_on_error true + request_timeout {{ fluentd_elasticsearch_request_timeout }} + suppress_type_name true + + @type file + path /var/lib/fluentd/data/elasticsearch.buffer/{{ item.facility }}.* + flush_interval 15s + + +{% elif log_direct_to_opensearch %} + + @type opensearch + host {{ opensearch_address }} + port {{ opensearch_port }} + scheme {{ fluentd_opensearch_scheme }} +{% if fluentd_opensearch_path != '' %} + path {{ fluentd_opensearch_path }} +{% endif %} +{% if fluentd_opensearch_scheme == 'https' %} + ssl_version {{ fluentd_opensearch_ssl_version }} + ssl_verify {{ fluentd_opensearch_ssl_verify }} +{% if fluentd_opensearch_cacert | length > 0 %} + ca_file {{ fluentd_opensearch_cacert }} +{% endif %} +{% endif %} +{% if fluentd_opensearch_user != '' and fluentd_opensearch_password != ''%} + user {{ fluentd_opensearch_user }} + password {{ fluentd_opensearch_password }} +{% endif %} + logstash_format true + logstash_prefix {{ opensearch_log_index_prefix }} + reconnect_on_error true + request_timeout {{ fluentd_opensearch_request_timeout }} + suppress_type_name true + bulk_message_request_threshold 20M + + @type file + path /var/lib/fluentd/data/opensearch.buffer/{{ item.facility }}.* + flush_interval 15s + chunk_limit_size 8M + + +{% endif %} + +{% endfor %} +{% endraw %} diff --git a/etc/kayobe/kolla/config/fluentd/output/03-opensearch.conf b/etc/kayobe/kolla/config/fluentd/output/03-opensearch.conf new file mode 100644 index 000000000..6b873e3f9 --- /dev/null +++ b/etc/kayobe/kolla/config/fluentd/output/03-opensearch.conf @@ -0,0 +1,51 @@ +{% raw %} +{% if enable_caso | bool and inventory_hostname in groups['caso'] %} + + @type copy + + @type opensearch + host {{ opensearch_address }} + port {{ opensearch_port }} + logstash_format true + logstash_prefix apel + flush_interval 15s + + +{% endif %} + + + @type copy + + @type opensearch + host {{ opensearch_address }} + port {{ opensearch_port }} + scheme {{ fluentd_opensearch_scheme }} +{% if fluentd_opensearch_path != '' %} + path {{ fluentd_opensearch_path }} +{% endif %} +{% if fluentd_opensearch_scheme == 'https' %} + ssl_version {{ fluentd_opensearch_ssl_version }} + ssl_verify {{ fluentd_opensearch_ssl_verify }} +{% if fluentd_opensearch_cacert | length > 0 %} + ca_file {{ fluentd_opensearch_cacert }} +{% endif %} +{% endif %} +{% if fluentd_opensearch_user != '' and fluentd_opensearch_password != ''%} + user {{ fluentd_opensearch_user }} + password {{ fluentd_opensearch_password }} +{% endif %} + logstash_format true + logstash_prefix {{ opensearch_log_index_prefix }} + reconnect_on_error true + request_timeout {{ fluentd_opensearch_request_timeout }} + suppress_type_name true + bulk_message_request_threshold 20M + + @type file + path /var/lib/fluentd/data/opensearch.buffer/openstack.* + flush_interval 15s + chunk_limit_size 8M + + + +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana.ini b/etc/kayobe/kolla/config/grafana.ini new file mode 100644 index 000000000..ee8bc0301 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana.ini @@ -0,0 +1,2 @@ +[unified_alerting] +enabled = true diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json new file mode 100644 index 000000000..e539584ff --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_mds.json @@ -0,0 +1,354 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "MDS Performance", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Reads(-) / Writes (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Reads/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(irate(ceph_objecter_op_r{ceph_daemon=~\"($mds_servers).*\"}[10m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Read Ops", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(irate(ceph_objecter_op_w{ceph_daemon=~\"($mds_servers).*\"}[10m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write Ops", + "refId": "B" + } + ], + "title": "MDS Workload - $mds_servers", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Client Requests", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "irate(ceph_mds_server_handle_client_request{ceph_daemon=~\"($mds_servers).*\"}[30m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ceph_daemon}}", + "refId": "A" + } + ], + "title": "Client Request Load - $mds_servers", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "MDS Server", + "multi": false, + "name": "mds_servers", + "options": [], + "query": "label_values(ceph_mds_inodes, ceph_daemon)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Ceph MDS", + "uid": "tbO9LAiZz", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json new file mode 100644 index 000000000..b03cab8cd --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_osds.json @@ -0,0 +1,1471 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 17069, + "links": [], + "panels": [ + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "@95%ile" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#e0752d", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AVG read", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MAX read", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "quantile(0.95,\n (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "@95%ile", + "refId": "C" + } + ], + "title": "OSD Read Latencies", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "This table shows the osd's that are delivering the 10 highest read latencies within the cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ceph_daemon" + }, + "properties": [ + { + "id": "displayName", + "value": "OSD ID" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Latency (ms)" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OSD ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 15, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "topk(10,\n (sort(\n (irate(ceph_osd_op_r_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_r_latency_count[5m]) * 1000)\n ))\n)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Highest READ Latencies", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "@95%ile write" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#e0752d", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg (rate(ceph_osd_op_w_latency_sum[10m]) / on (ceph_daemon) rate(ceph_osd_op_w_latency_count[10m]) * 1000)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AVG write", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max (irate(ceph_osd_op_w_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[5m]) * 1000)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "MAX write", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "quantile(0.95,\n (irate(ceph_osd_op_w_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[5m]) * 1000)\n)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "@95%ile write", + "refId": "C" + } + ], + "title": "OSD Write Latencies", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "This table shows the osd's that are delivering the 10 highest write latencies within the cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ceph_daemon" + }, + "properties": [ + { + "id": "displayName", + "value": "OSD ID" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Latency (ms)" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OSD ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 97 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 16, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk(10,\n (sort(\n (irate(ceph_osd_op_w_latency_sum[5m]) / on (ceph_daemon) irate(ceph_osd_op_w_latency_count[5m]) * 1000)\n ))\n)\n\n", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Highest WRITE Latencies", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "device_class" + }, + "properties": [ + { + "id": "custom.width", + "value": 134 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "device_class" + }, + "properties": [ + { + "id": "displayName", + "value": "Device Class" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count by (device_class) (ceph_osd_metadata)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{device_class}}", + "refId": "A" + } + ], + "title": "OSD Types Summary", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "bluestore" + }, + "properties": [ + { + "id": "custom.width", + "value": 123 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 4, + "y": 8 + }, + "hideTimeOverride": true, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_bluefs_wal_total_bytes)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "bluestore", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_metadata) - count(ceph_bluefs_wal_total_bytes)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "filestore", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "absent(ceph_bluefs_wal_total_bytes)*count(ceph_osd_metadata)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "filestore", + "refId": "C", + "step": 240 + } + ], + "title": "OSD Objectstore Types", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "The pie chart shows the various OSD sizes used within the cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 8, + "y": 8 + }, + "hideTimeOverride": true, + "id": 8, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes < 1099511627776)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<1 TB", + "refId": "A", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 1099511627776 < 2199023255552)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<2 TB", + "refId": "B", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 2199023255552 < 3298534883328)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<3TB", + "refId": "C", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 3298534883328 < 4398046511104)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<4TB", + "refId": "D", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 4398046511104 < 6597069766656)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<6TB", + "refId": "E", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 6597069766656 < 8796093022208)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<8TB", + "refId": "F", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 8796093022208 < 10995116277760)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<10TB", + "refId": "G", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 10995116277760 < 13194139533312)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "<12TB", + "refId": "H", + "step": 2 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_stat_bytes >= 13194139533312)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "12TB+", + "refId": "I", + "step": 2 + } + ], + "title": "OSD Size Summary", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Each bar indicates the number of OSD's that have a PG count in a specific range as shown on the x axis.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "ceph_osd_numpg", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "legendFormat": "PGs per OSD", + "refId": "A" + } + ], + "title": "Distribution of PGs per OSD", + "transformations": [ + { + "id": "histogram", + "options": { + "bucketSize": 1, + "combine": true, + "fields": {} + } + } + ], + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of OSDs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 84, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "displayName": "${__field.labels.ceph_daemon}", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 21, + "options": { + "combine": true, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "right", + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "ceph_osd_stat_bytes_used / ceph_osd_stat_bytes", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Per OSD Space Used", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of BluestoreDB's", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds", + "seriesBy": "last" + }, + "custom": { + "fillOpacity": 84, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "displayName": "${__field.labels.ceph_daemon}", + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 24, + "options": { + "combine": true, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "right", + "showLegend": false + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "ceph_bluefs_db_used_bytes / ceph_bluefs_db_total_bytes ", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "BluestoreDB Space Used", + "type": "histogram" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 20, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "R/W Profile", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Show the read/write workload profile overtime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "round(sum(irate(ceph_pool_rd[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "round(sum(irate(ceph_pool_wr[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Writes", + "refId": "B" + } + ], + "title": "Read/Write Profile", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Ceph OSD Overview", + "uid": "lo02I1Aiz", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json new file mode 100644 index 000000000..15db4aac2 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_overview.json @@ -0,0 +1,3341 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Ceph Cluster overview.\r\n", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 2842, + "graphTooltip": 0, + "id": 17071, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 37, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "CLUSTER STATE", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "HEALTHY" + }, + "1": { + "text": "WARNING" + }, + "2": { + "text": "ERROR" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 21, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_health_status", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A", + "step": 300 + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 2 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 3 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 14, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(ceph_mon_quorum_status)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Monitors In Quorum", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 22, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_pool_bytes_used)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Pools", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0.025 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 0.1 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 33, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_cluster_total_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Cluster Capacity", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0.025 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 0.1 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 34, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_cluster_total_used_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Used Capacity", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 30 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 23, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "((ceph_cluster_total_bytes-ceph_cluster_total_used_bytes)/ceph_cluster_total_bytes)*100", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Available Capacity", + "type": "gauge" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 38, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "OSD STATE", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 6 + }, + "id": 26, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(ceph_osd_in)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "OSDs IN", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 40, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 6 + }, + "id": 27, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_up) - count(ceph_osd_in)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "OSDs OUT", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 4, + "y": 6 + }, + "id": 28, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(ceph_osd_up)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "OSDs UP", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 40, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 6, + "y": 6 + }, + "id": 29, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(ceph_osd_up == 0) OR vector(0)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "OSDs DOWN", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 250 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 300 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 30, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(ceph_osd_numpg)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Average PGs per OSD", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 50 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 31, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(ceph_osd_apply_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Average OSD Apply Latency", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 50 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 32, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(ceph_osd_commit_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Average OSD Commit Latency", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 24, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(ceph_monitor_latency_seconds)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 300 + } + ], + "title": "Average Monitor Latency", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 39, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "CLUSTER", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Capacity" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#447EBC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_avail" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_space" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Capacity" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 3 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 1, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_cluster_total_bytes-ceph_cluster_total_used_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Available", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_cluster_total_used_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "B", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_cluster_total_bytes", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Total Capacity", + "refId": "C", + "step": 300 + } + ], + "title": "Capacity", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total Capacity" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_avail" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_space" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 3, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(rate(ceph_osd_op_w[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(rate(ceph_osd_op_r[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B", + "step": 300 + } + ], + "title": "IOPS", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 7, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(irate(ceph_osd_op_w_in_bytes[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(irate(ceph_osd_op_r_out_bytes[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B", + "step": 300 + } + ], + "title": "Throughput", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 40, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "LATENCY", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 16, + "x": 0, + "y": 19 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(ceph_osd_apply_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "apply", + "metric": "ceph_osd_perf_apply_latency_seconds", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(ceph_osd_commit_latency_ms)", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "commit", + "metric": "ceph_osd_perf_commit_latency_seconds", + "refId": "B", + "step": 4 + } + ], + "title": "OSD Apply + Commit Latency", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "avg(ceph_monitor_latency_seconds)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "latency", + "metric": "ceph_monitor_latency_seconds", + "refId": "A", + "step": 10 + } + ], + "title": "Monitor Latency (Currently not available with Ceph MGR)", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 41, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "OBJECTS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "ceph_pool_objects * on(pool_id) group_left(instance,name) ceph_pool_metadata", + "format": "time_series", + "instant": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "title": "Objects in the Cluster", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Total.*$/" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_active", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "M" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_clean", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Clean", + "refId": "U" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_peering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peering", + "refId": "I" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_degraded", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Degraded", + "refId": "B", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_stale", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Stale", + "refId": "C", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_unclean_pgs", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Unclean", + "refId": "D", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_undersized", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Undersized", + "refId": "E", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_incomplete", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Incomplete", + "refId": "G" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_forced_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Backfill", + "refId": "H" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_inconsistent", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inconsistent", + "refId": "F" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_forced_recovery", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Recovery", + "refId": "J" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_creating", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Creating", + "refId": "K" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_wait_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Wait Backfill", + "refId": "L" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_deep", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Deep", + "refId": "N" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_scrubbing", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Scrubbing", + "refId": "O" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_recovering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Recovering", + "refId": "P" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_repair", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Repair", + "refId": "Q" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_down", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Down", + "refId": "R" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_peered", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peered", + "refId": "S" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill", + "refId": "T" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_remapped", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Remapped", + "refId": "V" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_backfill_toofull", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill Toofull", + "refId": "W" + } + ], + "title": "PGs", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Total.*$/" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_active", + "format": "time_series", + "hide": true, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "M" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_clean", + "format": "time_series", + "hide": true, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Clean", + "refId": "U" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_peering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peering", + "refId": "I" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_degraded", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Degraded", + "refId": "B", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_stale", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Stale", + "refId": "C", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_unclean_pgs", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Unclean", + "refId": "D", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_undersized", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Undersized", + "refId": "E", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_incomplete", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Incomplete", + "refId": "G" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_forced_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Backfill", + "refId": "H" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_inconsistent", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inconsistent", + "refId": "F" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_forced_recovery", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Forced Recovery", + "refId": "J" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_creating", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Creating", + "refId": "K" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_wait_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Wait Backfill", + "refId": "L" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_deep", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Deep", + "refId": "N" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_scrubbing", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Scrubbing", + "refId": "O" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_recovering", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Recovering", + "refId": "P" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_repair", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Repair", + "refId": "Q" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_down", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Down", + "refId": "R" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_peered", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Peered", + "refId": "S" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_backfill", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill", + "refId": "T" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_remapped", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Remapped", + "refId": "V" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_backfill_toofull", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backfill Toofull", + "refId": "W" + } + ], + "title": "Bad PGs", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^Total.*$/" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_degraded", + "format": "time_series", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Degraded", + "refId": "F", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_stale", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Stale", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "ceph_pg_undersized", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Undersized", + "refId": "B", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "H" + } + ], + "title": "Stuck PGs", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 42, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "RECOVERY", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(irate(ceph_osd_recovery_ops[$interval]))", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "OPS", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "title": "Operations", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 10, + "auto_min": "1m", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Ceph Cluster Overview", + "uid": "oVDYSH1Wz", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json new file mode 100644 index 000000000..2823507a8 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/ceph/ceph_pools.json @@ -0,0 +1,962 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 17063, + "links": [], + "panels": [ + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Read (-) / Write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.* read/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk($topk,rate(ceph_pool_rd[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{name}} - read", + "refId": "F" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk($topk,rate(ceph_pool_wr[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} - write", + "refId": "A" + } + ], + "title": "Top $topk Client IOPS by Pool", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Read (-) / Writes (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.* read/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk($topk,rate(ceph_pool_rd_bytes[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} - read", + "refId": "A", + "textEditor": true + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk($topk,rate(ceph_pool_wr_bytes[5m]) + on(pool_id) group_left(instance,name) ceph_pool_metadata)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}} - write", + "refId": "B" + } + ], + "title": "Top $topk Client Throughput by Pool", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "displayName", + "value": "Pool Name" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pool_id" + }, + "properties": [ + { + "id": "displayName", + "value": "Pool ID" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "IOPS (R+W)" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 83 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 339 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk($topk,((irate(ceph_pool_rd[5m]) + irate(ceph_pool_wr[5m])) + on(pool_id) group_left(instance,name) ceph_pool_metadata))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "refId": "A", + "textEditor": true + } + ], + "title": "Top $topk Pools by Client IOPS", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "displayName", + "value": "Pool Name" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pool_id" + }, + "properties": [ + { + "id": "displayName", + "value": "Pool ID" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "displayName", + "value": "Throughput" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 226 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "topk($topk,(irate(ceph_pool_rd_bytes[5m]) + irate(ceph_pool_wr_bytes[5m])) + on(pool_id) group_left(instance,name) ceph_pool_metadata) ", + "format": "table", + "instant": true, + "intervalFactor": 2, + "refId": "A", + "textEditor": true + } + ], + "title": "Top $topk Pools by Throughput", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "displayName", + "value": "Pool Name" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "pool_id" + }, + "properties": [ + { + "id": "displayName", + "value": "Pool ID" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Capacity Used" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 85 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 201 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pool ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "topk($topk,((ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail)) * on(pool_id) group_left(name) ceph_pool_metadata))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "D" + } + ], + "title": "Top $topk Pools By Capacity Used", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "15", + "value": "15" + }, + "hide": 0, + "label": "Top K", + "name": "topk", + "options": [ + { + "selected": true, + "text": "15", + "value": "15" + } + ], + "query": "15", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "15s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Ceph Pools Overview", + "uid": "z99hzWtmk", + "version": 3, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json new file mode 100644 index 000000000..115ec73b8 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/blackbox.json @@ -0,0 +1,1162 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Prometheus Blackbox Exporter Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7587, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 138, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "title": "Global Probe Duration", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 15, + "panels": [], + "repeat": "target", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "$target status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "DOWN" + }, + "1": { + "text": "UP" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 2, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_success{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 4, + "y": 9 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ phase }}", + "refId": "B" + } + ], + "title": "HTTP Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 9 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "seconds", + "refId": "A" + } + ], + "title": "Probe Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "text": "N/A" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 201 + }, + { + "color": "#d44a3a", + "value": 399 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 20, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_status_code{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Status Code", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 27, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_version{instance=~\"$target\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "HTTP Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_ssl{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "0": { + "text": "NO" + }, + "1": { + "text": "YES" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1209600 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 10, + "x": 4, + "y": 15 + }, + "id": 19, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_ssl_earliest_cert_expiry{instance=~\"$target\"} - time()", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "SSL Expiry", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 14, + "y": 15 + }, + "id": 23, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(probe_duration_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average Probe Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 24, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(probe_dns_lookup_time_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Average DNS Lookup", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "blackbox", + "prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "target", + "options": [], + "query": { + "query": "label_values(probe_success, instance)", + "refId": "Prometheus-target-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Blackbox Exporter", + "uid": "xtkCtBkiz", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json new file mode 100644 index 000000000..a4c2a0621 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/cadvisor.json @@ -0,0 +1,1506 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A simple overview of the most important Docker host and container metrics. (cAdvisor/Prometheus)", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 893, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 43, + "panels": [], + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 31, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "count(rate(container_last_seen{name=~\".+\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + }, + { + "expr": "", + "refId": "B" + } + ], + "title": "Global Container Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 38, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "count(container_last_seen{job=\"$job\", instance=\"$node\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 1800 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Selected Container Count", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 52, + "panels": [], + "title": "Host Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "avg by (instance) (irate(container_cpu_usage_seconds_total{name=~\".+\",job=\"$job\", instance=\"$node\"}[$interval])) * 100", + "hide": false, + "interval": "", + "intervalFactor": 5, + "legendFormat": "{{ name }}", + "metric": "", + "refId": "C", + "step": 600 + } + ], + "title": "Per Host CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(container_memory_usage_bytes{name=~\".+\",job=\"$job\", instance=\"$node\"}) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 5, + "legendFormat": "{{ instance }}", + "metric": "", + "refId": "C", + "step": 600 + } + ], + "title": "Per Host Memory Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 50, + "panels": [], + "title": "Network summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "rate(container_network_receive_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "rx - {{interface}}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "- rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "tx - {{interface}}", + "refId": "B" + } + ], + "title": "Network Traffic ", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 47, + "panels": [], + "title": "CPU and Memory per Container", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\".+\",job=\"$job\", instance=\"$node\"}[$interval])) by (name) * 100", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "F", + "step": 240 + } + ], + "title": "CPU Usage per Container", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(container_memory_usage_bytes{name=~\".+\",job=\"$job\", instance=\"$node\"}) by (name)", + "hide": false, + "legendFormat": "{{ name }}", + "refId": "C" + } + ], + "title": "Memory Usage per Container", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 45, + "panels": [], + "title": "Network details", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "rate(container_network_receive_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{interface}} {{instance}}", + "refId": "A" + } + ], + "title": "Receive Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{interface}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "rate(container_network_transmit_bytes_total{id=\"/\",job=\"$job\", instance=\"$node\"}[$interval])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{interface}} {{instance}}", + "refId": "A" + } + ], + "title": "Transmit Network Traffic", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(container_cpu_load_average_10s, name)", + "hide": 0, + "includeAll": true, + "label": "Container Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "query": "label_values(container_cpu_load_average_10s, name)", + "refId": "Prometheus-name-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 50, + "auto_min": "50s", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "2m", + "value": "2m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "7m", + "value": "7m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "cadvisor", + "value": "cadvisor" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(container_cpu_usage_seconds_total, job)", + "hide": 0, + "includeAll": false, + "label": "job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "kef1p-phycon0001", + "value": "kef1p-phycon0001" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(container_cpu_usage_seconds_total{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Node", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(container_cpu_usage_seconds_total{job=\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Control Plane Containers (CAdvisor)", + "uid": "DV5zI13Wi", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json new file mode 100644 index 000000000..0c4fcad46 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/database.json @@ -0,0 +1,1128 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Basic Mysql dashboard for the prometheus exporter ", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 6239, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [], + "title": "Global status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#bf1b00", + "value": null + }, + { + "color": "#508642", + "value": 1 + }, + { + "color": "#ef843c", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 11, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "min(mysql_up)", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "expr": "mysql_galera_status_info", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "title": "Galera status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 25200 + }, + { + "color": "#508642", + "value": 32400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 15, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "min(mysql_global_status_uptime)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "mysql_global_status_max_used_connections", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "current {{ instance }}", + "refId": "A" + }, + { + "expr": "min(mysql_global_variables_max_connections)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Max allowed", + "refId": "B" + } + ], + "title": "Mysql Connections", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 19, + "panels": [], + "title": "I/O", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "write" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "irate(mysql_global_status_innodb_data_reads[10m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "reads {{ instance }}", + "refId": "A" + }, + { + "expr": "irate(mysql_global_status_innodb_data_written[10m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{ instance }}", + "refId": "B" + }, + { + "expr": "", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "write {{ instance }}", + "refId": "C" + } + ], + "title": "mysql disk reads vs writes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sent/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "irate(mysql_global_status_bytes_received[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "received {{ instance }}", + "refId": "A" + }, + { + "expr": "irate(mysql_global_status_bytes_sent[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "sent {{ instance }}", + "refId": "B" + } + ], + "title": "mysql network received vs sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "sum(irate(mysql_global_status_commands_total[5m])) by (command)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ command }}", + "refId": "A" + } + ], + "title": "Query rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 15, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "mysql_global_status_threads_running ", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "title": "Running Threads", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 21, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of connections that were aborted because the client died without closing the connection properly. See Section B.5.2.10, “Communication Errors and Aborted Connections”.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "irate(mysql_global_status_aborted_clients[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "title": "Aborted clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of failed attempts to connect to the MySQL server. See Section B.5.2.10, “Communication Errors and Aborted Connections”.\n\nFor additional connection-related information, check the Connection_errors_xxx status variables and the host_cache table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "expr": "irate(mysql_global_status_aborted_connects[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "title": "mysql aborted Connects", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [ + "Databases", + "backgroundservices" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(mysql_version_info, innodb_version)", + "hide": 0, + "includeAll": false, + "label": "MariaDB Version", + "multi": false, + "name": "release", + "options": [], + "query": { + "query": "label_values(mysql_version_info, innodb_version)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Database", + "uid": "6-kPlS7il", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json new file mode 100644 index 000000000..5cc166184 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/fluentd.json @@ -0,0 +1,3422 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "fluentd 1.x deployed in kubernetes and scraped by prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13042, + "graphTooltip": 1, + "id": 17116, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 19, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "If you see errors then you probbaly have serious issues with log processing, see https://docs.fluentd.org/buffer#handling-unrecoverable-errors\n\nRetries are normal but should occur only from time to time, otherwise check for network errors or destination is too slow and requires additional tuning per given provider.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Error.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(fluentd_output_status_retry_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Retry rate", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(fluentd_output_status_num_errors[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Error rate", + "refId": "C" + } + ], + "title": "Fluentd output error/retry rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Input and output total rates", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "input", + "refId": "A" + }, + { + "expr": "sum(rate(fluentd_output_status_write_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "output", + "refId": "B" + } + ], + "title": "Input / output rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This should not reach 0 otherwise logs are blocked from processing or even dropped", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "min(fluentd_output_status_buffer_available_space_ratio)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "lowest across all hosts", + "refId": "A" + } + ], + "title": "fluentd output status buffer available space ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "total flush time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_flush_time_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "time", + "refId": "A" + }, + { + "expr": "sum(rate(fluentd_output_status_slow_flush_count[5m]))", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "count", + "refId": "B" + } + ], + "title": "fluentd output status flush time count rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Current total size of stage and queue buffers.\nfluentd_output_status_buffer_total_bytes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_total_bytes) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }}", + "refId": "A" + } + ], + "title": "Current total size of stage and queue buffers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_queue_length)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "A" + } + ], + "title": "Fluentd output buffer queue", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 41, + "panels": [], + "title": "Buffer Stage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_buffer_stage_length", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_stage_length) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Current length of stage buffers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_buffer_stage_byte_size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_stage_byte_size) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Current total size of stage buffers", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 43, + "panels": [], + "title": "Buffer Queue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "max_over_time(fluentd_output_status_buffer_queue_length[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Maximum buffer length in last 5min", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "max_over_time(fluentd_output_status_buffer_total_bytes[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Maximum buffer bytes in last 5min", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_buffer_queue_length", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_queue_length) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Current length of queue buffers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_queue_byte_size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_queue_byte_size) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Current total size of queue buffers", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 46, + "panels": [], + "title": "Buffer Space", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_buffer_available_space_ratio", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_available_space_ratio) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Ratio of available space in buffer", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 53, + "panels": [], + "title": "Buffer Retries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_retry_count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_retry_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current retry counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_emit_records", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_emit_records[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current emit records", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_emit_count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_emit_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current emit counts rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_rollback_count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_rollback_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current rollback counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_write_count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_write_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current write counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_slow_flush_count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_slow_flush_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current slow flush counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_retry_wait", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_retry_wait[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current retry wait", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_flush_time_count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_flush_time_count[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Total flush time", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 57, + "panels": [], + "title": "Buffer Errors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_num_errors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(rate(fluentd_output_status_num_errors[5m])) by (type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current number of errors rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_buffer_newest_timekey", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_newest_timekey) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Newest timekey in buffer", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 55, + "panels": [], + "title": "Buffer timekeys", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "fluentd_output_status_buffer_newest_timekey - fluentd_output_status_buffer_oldest_timekey", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type}}", + "refId": "A" + } + ], + "title": "Timekey diff", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_output_status_buffer_oldest_timekey", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "sum(fluentd_output_status_buffer_oldest_timekey) by (pod, type)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{pod}} {{ type }}", + "refId": "A" + } + ], + "title": "Oldest timekey in buffer", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 17, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.6", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(fluentd_input_status_num_records_total{cluster=\"$cluster\", instance=\"$instance\"}[5m])) ", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "A" + } + ], + "title": "Input entries rate (total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.6", + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (hostname)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "title": "Input entries rate per hostname", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.6", + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (namespace)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Input entries rate per namespace", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.6", + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Input entries rate per instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "fluentd_input_status_num_records_total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 6, + "y": 90 + }, + "id": 49, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.6", + "targets": [ + { + "expr": "sum(rate(fluentd_input_status_num_records_total[5m])) by (tag)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{tag}}", + "refId": "A" + } + ], + "title": "Input entries rate per tag", + "type": "timeseries" + } + ], + "title": "Input details - Not implemented", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "fluentd", + "logging" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "kolla_logging", + "value": "kolla_logging" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(cluster)", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Fluentd 1.x - Update", + "uid": "bNn5LUtizs2", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json new file mode 100644 index 000000000..eed9b1e3a --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json @@ -0,0 +1,905 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 18, + "title": "Summary Flavor Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4.8, + "x": 0, + "y": 1 + }, + "id": 20, + "maxPerRow": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.5", + "repeat": "flavors", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "round(avg_over_time(openstack_free_capacity_by_flavor_total{flavor_name=~\"$flavors\"}[30m]), 1)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$flavors", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 13, + "panels": [], + "title": "Summary Hypervisor Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"MEMORY_MB\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"})", + "hide": false, + "legendFormat": "Free Memory", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"})", + "hide": false, + "legendFormat": "Allocated Memory", + "range": true, + "refId": "B" + } + ], + "title": "Total Memory", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"VCPU\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"VCPU\"})", + "hide": false, + "legendFormat": "Free VCPU", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"PCPU\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"PCPU\"})", + "hide": false, + "legendFormat": "Free PCPU", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"VCPU\"})", + "hide": false, + "legendFormat": "Allocated VCPU", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"PCPU\"})", + "hide": false, + "legendFormat": "Allocated PCPU", + "range": true, + "refId": "D" + } + ], + "title": "Total CPU", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decgbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"DISK_GB\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"DISK_GB\"})", + "hide": false, + "legendFormat": "Free Local Disk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"DISK_GB\"})", + "hide": false, + "legendFormat": "Allocated Local Disk", + "range": true, + "refId": "D" + } + ], + "title": "Total Local Disk", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 11, + "panels": [], + "title": "Project Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 5, + "interval": "10m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(openstack_project_usage{placement_resource=\"MEMORY_MB\"}[30m])", + "legendFormat": "{{project_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Used by Project", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 16, + "interval": "10m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(openstack_project_usage{placement_resource=\"VCPU\"}[30m])", + "legendFormat": "VCPU {{project_name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(openstack_project_usage{placement_resource=\"PCPU\"}[30m])", + "hide": false, + "legendFormat": "PCPU {{project_name}}", + "range": true, + "refId": "B" + } + ], + "title": "CPUs Used by Project", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 15, + "panels": [], + "title": "Per Hypervisor Free Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 28, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 6, + "interval": "10m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(openstack_free_capacity_hypervisor_by_flavor{flavor_name=~\"$flavors\"}[30m])", + "format": "time_series", + "instant": false, + "legendFormat": "{{flavor_name}} on {{hypervisor}}", + "range": true, + "refId": "Available Capacity on Hypervisors" + } + ], + "title": "Available Capacity for $flavors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 21, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 4, + "interval": "10m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(openstack_hypervisor_placement_allocatable_capacity{resource=\"MEMORY_MB\"}[30m]) - on(hypervisor) avg_over_time(openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"}[30m])", + "legendFormat": "{{hypervisor}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Free by Hypervisor", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(openstack_free_capacity_by_flavor_total, flavor_name)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "flavors", + "options": [], + "query": { + "query": "label_values(openstack_free_capacity_by_flavor_total, flavor_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cloud Metrics", + "uid": "g__ksD67z", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json new file mode 100644 index 000000000..d29cfda66 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json @@ -0,0 +1,1165 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Project Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-yellow" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 18, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(openstack_project_usage{project_id=~\"${project_id}\"}) by (placement_resource)", + "format": "time_series", + "hide": false, + "legendFormat": "{{flavor_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Max usage", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 5, + "interval": "10m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(avg_over_time(openstack_project_usage{project_id=~\"${project_id}\"}[30m:])) by (placement_resource)", + "hide": false, + "legendFormat": "{{placement_resource}}", + "range": true, + "refId": "A" + } + ], + "title": "Resources per Project", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 19, + "interval": "10m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg_over_time(openstack_project_quota{project_id=~\"${project_id}\"}[30m:])", + "hide": false, + "legendFormat": "{{project_name}}:{{quota_resource}}", + "range": true, + "refId": "B" + } + ], + "title": "Project quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(count(libvirt_domain_openstack_info{project_id=~\"${project_id}\"}) by (domain,flavor_name)) by (flavor_name)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Servers by flavor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 20, + "interval": "30m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(sum(irate(libvirt_domain_vcpu_time_seconds_total{}[5m]) / ignoring(instance,vcpu) group_left(domain) libvirt_domain_info_virtual_cpus{}) by (domain) * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"}) by (project_name)", + "hide": false, + "legendFormat": "{{instance_name}}", + "range": true, + "refId": "B" + } + ], + "title": "CPU utilization per project", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(libvirt_domain_memory_stats_used_percent * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"}) by (project_name)", + "hide": false, + "legendFormat": "{{instance_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory utilization per project", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(rate(libvirt_domain_block_stats_read_time_seconds_total[5m]) * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"}) by (project_name)", + "hide": false, + "legendFormat": "read: {{project_name}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(rate(libvirt_domain_block_stats_write_time_seconds_total[5m]) * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"} * -1) by (project_name)", + "hide": false, + "legendFormat": " write: {{project_name}}", + "range": true, + "refId": "C" + } + ], + "title": "Disk utilization per project", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 15, + "panels": [], + "title": "Per Instance Utilization", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 30 + }, + "id": 23, + "interval": "30m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(libvirt_domain_vcpu_time_seconds_total{}[5m]) / ignoring(instance,vcpu) group_left(domain) libvirt_domain_info_virtual_cpus{}) by (domain) * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"}", + "hide": false, + "legendFormat": "{{instance_name}}", + "range": true, + "refId": "B" + } + ], + "title": "CPU utilization per instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 30 + }, + "id": 24, + "interval": "30m", + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "libvirt_domain_memory_stats_used_percent * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"}", + "hide": false, + "legendFormat": "{{instance_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory utilization per instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 30 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.1+security-01", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(libvirt_domain_block_stats_read_time_seconds_total[5m]) * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"}", + "hide": false, + "legendFormat": "{{instance_name}} : read {{target_device}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(libvirt_domain_block_stats_write_time_seconds_total[5m]) * on(domain) group_left(instance_name,project_name,project_id) libvirt_domain_openstack_info{project_id=~\"${project_id}\"} * -1", + "hide": false, + "legendFormat": "{{instance_name}} : write {{target_device}}", + "range": true, + "refId": "C" + } + ], + "title": "Disk utilization per instance", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [ + "capacity", + "azimuth" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "The prometheus datasource used for queries.", + "includeAll": false, + "label": "datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "openstack_project_usage", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project_id", + "options": [], + "query": { + "query": "openstack_project_usage", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/project_name=\"(?[^\"]+)|project_id=\"(?[^\"]+)/g", + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "OpenStack Project Metrics", + "uid": "mXiuBDe7z", + "version": 1 +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json new file mode 100644 index 000000000..236cbb748 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/haproxy.json @@ -0,0 +1,5445 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "HAProxy backend servers", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 2428, + "graphTooltip": 1, + "id": 27, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 86, + "panels": [], + "title": "Basic Backend status", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Backends Down", + "color": "#890F02" + }, + { + "alias": "Backends Down", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(haproxy_backend_status{state=\"UP\", proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"} == 1)", + "hide": false, + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backends Up", + "range": true, + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(haproxy_backend_status{state=\"DOWN\", proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"} == 1)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backends Down", + "range": true, + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Backends Up / Down", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 87, + "panels": [], + "title": "Basic General Info", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Back.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*1.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*2.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*3.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*4.*/", + "color": "#CCA300" + }, + { + "alias": "/.*5.*/", + "color": "#890F02" + }, + { + "alias": "/.*other.*/", + "color": "#806EB7" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_frontend_http_responses_total{proxy=~\"$frontend\",code=~\"$code\",instance=~\"$host:?[0-9]*\"}[5m])) by (code)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Frontend {{ code }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_http_responses_total{proxy=~\"$backend\",code=~\"$code\",instance=~\"$host:?[0-9]*\"}[5m])) by (code)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backend {{ code }}", + "metric": "", + "range": true, + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total responses by HTTP code", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 75, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_frontend_bytes_in_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])*8) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN Front", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_frontend_bytes_out_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])*8) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT Front", + "range": true, + "refId": "B", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_bytes_in_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])*8) by (instance)", + "intervalFactor": 2, + "legendFormat": "IN Back", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_bytes_out_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])*8) by (instance)", + "intervalFactor": 2, + "legendFormat": "OUT Back", + "range": true, + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 79, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Back.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_frontend_connections_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Front", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_connections_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Back", + "metric": "", + "range": true, + "refId": "B", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_connection_errors_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Back errors", + "metric": "", + "range": true, + "refId": "C", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total number of connections", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Response.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Backend.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_frontend_http_requests_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Requests", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_response_errors_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Response errors", + "range": true, + "refId": "B", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_frontend_request_errors_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Requests errors", + "metric": "", + "range": true, + "refId": "C", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_redispatch_warnings_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backend redispatch", + "range": true, + "refId": "D", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_backend_retry_warnings_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backend retry", + "range": true, + "refId": "E", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_frontend_requests_denied_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Request denied", + "range": true, + "refId": "F", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(haproxy_backend_current_queue{backend=~\"$backend\",proxy=~\"$host:?[0-9]*\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Backend Queued", + "range": true, + "refId": "G", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total requests / responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Back.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(haproxy_frontend_current_sessions{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Frontend current sessions", + "metric": "", + "range": true, + "refId": "B", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(haproxy_frontend_current_session_rate{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Frontend current session rate", + "metric": "", + "range": true, + "refId": "C", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(haproxy_backend_current_sessions{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backend current sessions", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(haproxy_backend_current_session_rate{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}) by (instance)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Backend current session rate", + "metric": "", + "range": true, + "refId": "D", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Sessions", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 88, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_bytes_in_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN-{{ proxy }}", + "metric": "haproxy_backend_", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_bytes_out_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT-{{ proxy }}", + "range": true, + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_frontend_bytes_in_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN-{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_frontend_bytes_out_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT-{{ proxy }}", + "range": true, + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 95, + "panels": [], + "title": "By Server Throughput", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*OUT.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_bytes_in_total{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "IN-{{ proxy }} / {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_bytes_out_total{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])*8", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "OUT-{{ proxy }} / {{ server }}", + "range": true, + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current total of incoming / outgoing bytes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 89, + "panels": [], + "title": "Connections", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_backend_current_sessions{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "B", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_connection_errors_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{ proxy }} Error", + "range": true, + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total number of connections OK / Error", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 64 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_frontend_connections_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "haproxy_backe", + "range": true, + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Total number of connections", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 90, + "panels": [], + "title": "Queue", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_max_queue{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Maximum observed number of queued requests not assigned to any server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 76 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "haproxy_backend_current_queue{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "haproxy_backend_current_queue", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current number of queued requests not assigned to any server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 91, + "panels": [], + "title": "Requests / Responses", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 87 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_redispatch_warnings_total{proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Redispatch {{ backend }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_retry_warnings_total{proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Retry {{ backend }}", + "range": true, + "refId": "B", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_response_errors_total{proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Error {{ backend }}", + "range": true, + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total of redispatch / error / retry warnings", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 87 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Denied.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_frontend_http_requests_total{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_frontend_request_errors_total{proxy=~\"$frontend\", instance=~\"$host:?[0-9]*\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{ proxy }} Error", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_frontend_requests_denied_total{proxy=~\"$frontend\", instance=~\"$host:?[0-9]*\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{ proxy }} Denied", + "range": true, + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Total HTTP requests OK / Error / Denied", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 99 + }, + "id": 92, + "panels": [], + "title": "Responses by HTTP code", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 100 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_backend_http_responses_total{proxy=~\"$backend\", code=~\"$code\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ code }} {{ backend }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total of HTTP responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 100 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_frontend_http_responses_total{proxy=~\"$frontend\", code=~\"$code\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ code }} {{ frontend }} ", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Total of HTTP responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 111 + }, + "id": 93, + "panels": [], + "title": "Sessions", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 112 + }, + "hiddenSeries": false, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_backend_current_sessions{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 112 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_frontend_current_sessions{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "haproxy_backend_current_queue", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Current number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 122 + }, + "hiddenSeries": false, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(haproxy_backend_sessions_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}[$interval])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "haproxy_backend_current_queue", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current number of sessions rate per second over last elapsed second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 122 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(haproxy_frontend_current_sessions{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}[$interval])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Current number of sessions rate per second over last elapsed second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "fieldMinMax": false, + "unitScale": true + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 132 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1847", + "alias": "/.*limit.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_backend_max_sessions{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} ", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_backend_limit_sessions{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "intervalFactor": 2, + "legendFormat": "{{ proxy }} limit", + "range": true, + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Maximum observed number of active sessions and limit", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 132 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:1809", + "alias": "/.*limit.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_frontend_max_sessions{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_frontend_limit_sessions{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}", + "intervalFactor": 2, + "legendFormat": "{{ proxy }} limit", + "range": true, + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Maximum observed number of active sessions and limit", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1793", + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:1794", + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 142 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_backend_max_session_rate{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Maximum number of sessions rate per second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 142 + }, + "hiddenSeries": false, + "id": 69, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 3, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*limit.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_frontend_max_session_rate{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_frontend_limit_session_rate{proxy=~\"$frontend\",instance=~\"$host:?[0-9]*\"}", + "intervalFactor": 2, + "legendFormat": "{{ proxy}} limit", + "range": true, + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Front - Maximum number of sessions rate per second and limit", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 152 + }, + "id": 94, + "panels": [], + "title": "Health and Weight", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 0, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 153 + }, + "hiddenSeries": false, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:4007" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_backend_status{state=\"UP\", proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": " {{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Current health status of the backend", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 153 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_backend_weight{proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Back - Total weight of the servers in the backend / Active servers backend", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 161 + }, + "id": 96, + "panels": [], + "title": "By Server Connections", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 162 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "irate(haproxy_server_sessions_total[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "haproxy_backe", + "range": true, + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total number of connections", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 173 + }, + "id": 97, + "panels": [], + "title": "By Server Queue", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 174 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_max_queue{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy}} / {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Maximum observed number of queued requests assigned to this server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 174 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_current_queue{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current number of queued requests assigned to this server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 184 + }, + "id": 98, + "panels": [], + "title": "By Server Requests / Responses", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 185 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(haproxy_server_http_responses_total{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])) by (server)", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of HTTP responses OK", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 185 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Error.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_retry_warnings_total{proxy=~\"$backend\", instance=~\"$host:?[0-9]*\"}[5m])", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Retry {{ proxy }} {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_server_redispatch_warnings_total{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Redispatch {{ proxy }} {{ server }}", + "range": true, + "refId": "B", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_response_errors_total{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])", + "hide": false, + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "Error {{ proxy }} {{ server }}", + "range": true, + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of retry / error / redispatch warnings", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 197 + }, + "id": 99, + "panels": [], + "title": "By Server Responses by HTTP code", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 198 + }, + "height": "400px", + "hiddenSeries": false, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_http_responses_total{proxy=~\"$backend\",server=~\"$server\",code=~\"$code\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ backend }} {{ server }} {{ code }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of HTTP responses", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 208 + }, + "id": 100, + "panels": [], + "title": "By Server Sessions", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 209 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_server_current_sessions{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 209 + }, + "hiddenSeries": false, + "id": 67, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_server_max_sessions{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Maximum observed number of active sessions", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 218 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_server_max_session_rate{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Maximum observed number of sessions rate per second", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 227 + }, + "id": 101, + "panels": [], + "title": "By Server Health Weight", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 228 + }, + "hiddenSeries": false, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_server_downtime_seconds_total{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total downtime in seconds", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 228 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "increase(haproxy_server_check_failures_total{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total number of failed health checks", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 0, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 240 + }, + "hiddenSeries": false, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(haproxy_server_connection_errors_total{proxy=~\"$backend\",server=~\"$server\",instance=~\"$host:?[0-9]*\"}[5m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Total of connection errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unitScale": true + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 240 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "maxPerRow": 2, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.3.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "haproxy_server_weight{proxy=~\"$backend\",instance=~\"$host:?[0-9]*\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ proxy }} / {{ server }}", + "metric": "haproxy_backend_current_queue", + "range": true, + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Server - Current weight of the server", + "tooltip": { + "msResolution": true, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "influxdb", + "uid": "GOTaYLf7z" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 252 + }, + "id": 102, + "panels": [], + "title": "Scrape info", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "haproxy", + "servers" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "${datasource}" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(haproxy_backend_status,instance)", + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": false, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(haproxy_backend_status,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/([^:]+).*/", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(haproxy_backend_bytes_in_total{instance=~\"$host:?[0-9]*\"},proxy)", + "hide": 0, + "includeAll": true, + "label": "Backend", + "multi": true, + "name": "backend", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(haproxy_backend_bytes_in_total{instance=~\"$host:?[0-9]*\"},proxy)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(haproxy_frontend_bytes_in_total{instance=~\"$host:?[0-9]*\"},proxy)", + "hide": 0, + "includeAll": true, + "label": "Frontend", + "multi": true, + "name": "frontend", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(haproxy_frontend_bytes_in_total{instance=~\"$host:?[0-9]*\"},proxy)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(haproxy_server_bytes_in_total{instance=~\"$host:?[0-9]*\", proxy=~\"$backend\"},server)", + "hide": 0, + "includeAll": true, + "label": "Server", + "multi": true, + "name": "server", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(haproxy_server_bytes_in_total{instance=~\"$host:?[0-9]*\", proxy=~\"$backend\"},server)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(haproxy_server_http_responses_total{instance=~\"$host:?[0-9]*\", proxy=~\"$backend\", server=~\"$server\"},code)", + "hide": 0, + "includeAll": true, + "label": "HTTP Code", + "multi": true, + "name": "code", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(haproxy_server_http_responses_total{instance=~\"$host:?[0-9]*\", proxy=~\"$backend\", server=~\"$server\"},code)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "30s,1m,5m,1h,6h,1d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "HAProxy", + "uid": "3eATHHJWl", + "version": 2, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json new file mode 100644 index 000000000..b30550222 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/hardware_overview.json @@ -0,0 +1,1549 @@ +{% raw %} +{ + "__inputs": [ + { + "name": "datasource", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "title": "All Disks Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of healthy drives", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"} > 0 )", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Healthy Drives", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of healthy drives", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"} < 1) ", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Unhealthy Drives", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of healthy drives", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(smartmon_device_smart_healthy{instance=~\"$node\"})", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Drives", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #Health" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "custom.width" + }, + { + "id": "displayName", + "value": "Health" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bad" + }, + "1": { + "color": "green", + "index": 0, + "text": "Ok" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "type 1" + }, + "properties": [ + { + "id": "custom.width", + "value": 153 + }, + { + "id": "displayName", + "value": "Type" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "serial_number" + }, + "properties": [ + { + "id": "custom.width", + "value": 208 + }, + { + "id": "displayName", + "value": "Serial Number" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time 1" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "disk 1" + }, + "properties": [ + { + "id": "custom.width", + "value": 146 + }, + { + "id": "displayName", + "value": "Disk" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance 1" + }, + "properties": [ + { + "id": "custom.width", + "value": 203 + }, + { + "id": "displayName", + "value": "Hostname" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hostname" + }, + "properties": [ + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "group 1" + }, + "properties": [ + { + "id": "displayName", + "value": "Group" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "smart_id" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".* 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #Temp" + }, + "properties": [ + { + "id": "displayName", + "value": "Temperature" + }, + { + "id": "unit", + "value": "celsius" + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 20, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "$$hashKey": "object:40", + "aggregation": "Last", + "alias": "Healthy", + "crit": 0, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 0, + "displayAliasType": "Warning / Critical", + "displayType": "Regular", + "displayValueWithAlias": "Never", + "editorMode": "code", + "exemplar": false, + "expr": "smartmon_device_smart_healthy{instance=~\"$node\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "Health", + "units": "none", + "valueHandler": "Number Threshold", + "warn": 0 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "smartmon_temperature{instance=~\"$node\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "Temp" + } + ], + "title": "SMART Info", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "serial_number" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "job": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 3, + "Time 2": 10, + "Value #Health": 8, + "Value #Temp": 9, + "__name__ 1": 4, + "__name__ 2": 11, + "disk 1": 5, + "disk 2": 12, + "group 1": 0, + "group 2": 13, + "instance 1": 1, + "instance 2": 14, + "job 1": 6, + "job 2": 15, + "serial_number": 7, + "smart_id": 16, + "type 1": 2, + "type 2": 17 + }, + "renameByName": { + "disk": "Disk", + "instance": "Instance", + "job": "", + "type": "Type" + } + } + } + ], + "transparent": true, + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Temperature (°C)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 9, + "x": 0, + "y": 18 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(smartmon_temperature{instance=~\"$node\"}[1h])", + "instant": false, + "interval": "", + "legendFormat": "{{instance}} - {{disk}} - {{serial_number}}", + "range": true, + "refId": "A" + } + ], + "title": "All Disk Temperatures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 11, + "panels": [], + "title": "NVMe Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unique_device" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #Health" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "custom.width" + }, + { + "id": "displayName", + "value": "Health" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "Ok" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "red", + "index": 1, + "text": "Bad" + }, + "to": 1000000000000000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".* 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #Temp" + }, + "properties": [ + { + "id": "displayName", + "value": "Temperature" + }, + { + "id": "unit", + "value": "celsius" + }, + { + "id": "noValue", + "value": "-" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".* 1" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".* 3" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #TBW" + }, + "properties": [ + { + "id": "unit", + "value": "deckbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #TBR" + }, + "properties": [ + { + "id": "unit", + "value": "deckbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #Capacity" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Device" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/uesjf83hh/nvme-monitoring?var-serial_number=${__data.fields[\"Serial Number\"]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 20, + "x": 0, + "y": 32 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "TBW" + } + ] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "$$hashKey": "object:40", + "aggregation": "Last", + "alias": "Healthy", + "crit": 0, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "decimals": 0, + "displayAliasType": "Warning / Critical", + "displayType": "Regular", + "displayValueWithAlias": "Never", + "editorMode": "code", + "exemplar": false, + "expr": "label_join(nvme_critical_warning_total{instance=~\"$node\"},\"unique_device\", \"-\", \"instance\", \"device\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "Health", + "units": "none", + "valueHandler": "Number Threshold", + "warn": 0 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(nvme_temperature_celsius{instance=~\"$node\"},\"unique_device\", \"-\", \"instance\", \"device\")", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "Temp" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(nvme_data_units_written_total{instance=~\"$node\"},\"unique_device\", \"-\", \"instance\", \"device\") * 512", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "TBW" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(nvme_data_units_read_total{instance=~\"$node\"},\"unique_device\", \"-\", \"instance\", \"device\") * 512", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "TBR" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(nvme_physical_size_bytes{instance=~\"$node\"},\"unique_device\", \"-\", \"instance\", \"device\")", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "Capacity" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(delta(nvme_data_units_written_total{instance=~\"$node\"}[24h])*512000,\"unique_device\", \"-\", \"instance\", \"device\")/label_join(nvme_physical_size_bytes{instance=~\"$node\"},\"unique_device\", \"-\", \"instance\", \"device\")", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "", + "range": false, + "refId": "DWPD" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(nvme_rated_dwpd{instance=~\"$node\"},\"unique_device\", \"-\", \"instance\", \"device\")", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "Rated DWPD" + } + ], + "title": "SMART Info", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "unique_device", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Value #Health": false, + "__name__ 1": true, + "__name__ 2": true, + "__name__ 3": true, + "__name__ 4": true, + "device 1": false, + "device 2": true, + "device 3": true, + "device 4": true, + "device 5": true, + "device 6": true, + "device 7": true, + "instance 1": false, + "instance 2": true, + "instance 3": true, + "instance 4": true, + "instance 5": true, + "instance 6": true, + "instance 7": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true, + "job 5": true, + "job 6": true, + "job 7": true, + "model 2": true, + "model 3": true, + "model 4": true, + "model 5": true, + "model 6": true, + "model 7": true, + "original_device 1": true, + "original_device 2": true, + "original_device 3": true, + "original_device 4": true, + "original_device 5": true, + "original_device 6": true, + "original_device 7": true, + "serial_number 2": true, + "serial_number 3": true, + "serial_number 4": true, + "serial_number 5": true, + "serial_number 6": true, + "serial_number 7": true, + "unique_device": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 11, + "Time 2": 15, + "Time 3": 23, + "Time 4": 27, + "Time 5": 32, + "Time 6": 38, + "Time 7": 53, + "Value #Capacity": 6, + "Value #DWPD": 8, + "Value #Health": 2, + "Value #Rated DWPD": 7, + "Value #TBR": 5, + "Value #TBW": 4, + "Value #Temp": 3, + "__name__ 1": 12, + "__name__ 2": 16, + "__name__ 3": 37, + "__name__ 4": 54, + "device 1": 1, + "device 2": 21, + "device 3": 24, + "device 4": 28, + "device 5": 33, + "device 6": 39, + "device 7": 55, + "instance 1": 0, + "instance 2": 17, + "instance 3": 14, + "instance 4": 29, + "instance 5": 34, + "instance 6": 40, + "instance 7": 56, + "job 1": 13, + "job 2": 18, + "job 3": 25, + "job 4": 30, + "job 5": 35, + "job 6": 41, + "job 7": 57, + "model 1": 9, + "model 2": 43, + "model 3": 45, + "model 4": 47, + "model 5": 49, + "model 6": 51, + "model 7": 58, + "original_device 1": 20, + "original_device 2": 22, + "original_device 3": 26, + "original_device 4": 31, + "original_device 5": 36, + "original_device 6": 42, + "original_device 7": 59, + "serial_number 1": 10, + "serial_number 2": 44, + "serial_number 3": 46, + "serial_number 4": 48, + "serial_number 5": 50, + "serial_number 6": 52, + "serial_number 7": 60, + "unique_device": 19 + }, + "renameByName": { + "Time 1": "", + "Value #Capacity": "Disk Size", + "Value #DWPD": "DWPD", + "Value #Rated DWPD": "Rated DWPD", + "Value #TBR": "TBR", + "Value #TBW": "TBW", + "__name__ 1": "", + "device 1": "Device", + "instance 1": "Hostname", + "model 1": "Model Name", + "serial_number 1": "Serial Number" + } + } + } + ], + "transparent": true, + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The data written to the disk in the last 24h period divided by the physical capacity of the disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 10, + "x": 0, + "y": 42 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "delta(nvme_data_units_written_total{instance=~\"$node\"}[24h])*512000 / nvme_physical_size_bytes{instance=~\"$node\"}", + "legendFormat": "{{instance}} - {{device}}", + "range": true, + "refId": "A" + } + ], + "title": "DWPD", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Temperature (°C)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 9, + "x": 10, + "y": 42 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(nvme_temperature_celsius{instance=~\"$node\"}[1h]) ", + "instant": false, + "interval": "", + "legendFormat": "{{instance}} - {{device}}", + "range": true, + "refId": "A" + } + ], + "title": "NVMe Temperatures", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + }, + { + "current": {}, + "includeAll": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_cpu_seconds_total{job=\"node\"}, instance)", + "includeAll": true, + "label": "Host:", + "name": "node", + "options": [], + "query": { + "query": "label_values(node_cpu_seconds_total{job=\"node\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Hardware Overview", + "uid": "TCN51Y25P", + "version": 10, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json new file mode 100644 index 000000000..c113ed482 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/libvirt.json @@ -0,0 +1,2237 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 18, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "CPU", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_info_cpu_time_seconds_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}", + "refId": "A" + } + ], + "title": "Total CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(libvirt_domain_info_cpu_time_seconds_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])/libvirt_domain_info_virtual_cpus", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ uuid }} {{ instance }}/{{ domain }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Per-cpu average usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "libvirt_domain_info_memory_usage_bytes{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}", + "refId": "A" + } + ], + "title": "RAM usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "libvirt_domain_info_memory_usage_bytes{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"} / libvirt_domain_info_maximum_memory_bytes", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}", + "refId": "A" + } + ], + "title": "RAM used vs allocated", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 19, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Block devices", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_block_stats_read_requests_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{ uuid }} ({{ instance }}/{{ domain }}:{{ target_device }})", + "refId": "A" + } + ], + "title": "Read requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "wps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_block_stats_write_requests_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{uuid}} ({{ instance }}/{{ domain }}:{{ target_device }})", + "refId": "A" + } + ], + "title": "Write requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_block_stats_read_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{uuid}} {{ instance }}/{{ domain }}:{{ target_device }}", + "refId": "A" + } + ], + "title": "Read bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_block_stats_write_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "legendFormat": "{{ target_device }} {{uuid}} {{ instance }}/{{ domain }}:{{ target_device }}", + "refId": "A" + } + ], + "title": "Write bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 20, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Networking", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_interface_stats_receive_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{ target_device }} {{ uuid }} {{ instance }}/{{ domain }}:{{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "title": "Packets received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "wps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_interface_stats_transmit_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Packets transmitted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "asc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_interface_stats_receive_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Receive bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_interface_stats_transmit_bytes_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Transmit bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "asc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_interface_stats_receive_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Receive packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_interface_stats_transmit_packets_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m]) ", + "legendFormat": "{{uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Transmit bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 52 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "libvirt_domain_interface_stats_receive_errors_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{ uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "title": "Receive errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 52 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(libvirt_domain_interface_stats_receive_drops_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}[5m])", + "instant": false, + "legendFormat": "{{uuid}} on {{ instance }} {{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "title": "Receive drops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 52 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "libvirt_domain_interface_stats_transmit_errors_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{uuid}} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "refId": "B" + } + ], + "title": "Transmit errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 52 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "libvirt_domain_interface_stats_transmit_drops_total{domain=~\"$domain\",instance=~\"$instance\",job=\"libvirt_exporter\"}", + "legendFormat": "{{ uuid }} {{ instance }}/{{ domain }}: {{ target_device }} ({{ source_bridge }})", + "refId": "A" + } + ], + "title": "Transmit drops", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(libvirt_up{job=\"libvirt_exporter\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Hypervisor", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(libvirt_up{job=\"libvirt_exporter\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(libvirt_domain_info_virtual_cpus{job=\"libvirt_exporter\"}, domain)", + "hide": 0, + "includeAll": true, + "label": "Domain", + "multi": true, + "name": "domain", + "options": [], + "query": "label_values(libvirt_domain_info_virtual_cpus{job=\"libvirt_exporter\"}, domain)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(libvirt_domain_interface_stats_receive_bytes_total{job=\"libvirt_exporter\"}, source_bridge)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "source_bridge", + "options": [], + "query": "label_values(libvirt_domain_interface_stats_receive_bytes_total{job=\"libvirt_exporter\"}, source_bridge)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Libvirt (KVM)", + "uid": "gh5hj2qWZ", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/loadbalancer.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/loadbalancer.json new file mode 100644 index 000000000..395271cce --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/loadbalancer.json @@ -0,0 +1,1003 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 14, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Load Balancer Network IP" + }, + "properties": [ + { + "id": "custom.width", + "value": 178 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Load Balancer Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 170 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Operating Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "ERROR": { + "color": "red", + "index": 1 + }, + "ONLINE": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Provisioning Status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "ACTIVE": { + "color": "green", + "index": 0 + }, + "ERROR": { + "color": "red", + "index": 1 + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Amphora Status" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "ALLOCATED": { + "color": "green", + "index": 0 + }, + "ERROR": { + "color": "red", + "index": 2 + }, + "READY": { + "color": "green", + "index": 1 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "openstack_loadbalancer_amphora_status * on (loadbalancer_id) group_left(operating_status, provisioning_status, name) label_replace(openstack_loadbalancer_loadbalancer_status, \"loadbalancer_id\", \"$1\", \"id\", \"(.+)\")", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Amphorae", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "compute_id": true, + "group": true, + "instance": true, + "job": true + }, + "indexByName": { + "Time": 9, + "Value": 14, + "compute_id": 10, + "group": 11, + "ha_ip": 4, + "id": 5, + "instance": 12, + "job": 13, + "lb_network_ip": 8, + "loadbalancer_id": 0, + "name": 1, + "operating_status": 2, + "provisioning_status": 3, + "role": 6, + "status": 7 + }, + "renameByName": { + "Time": "", + "compute_id": "", + "ha_ip": "VIP/HA IP", + "id": "Amphora ID", + "lb_network_ip": "Load Balancer Network IP", + "loadbalancer_id": "Load Balancer ID", + "name": "Load Balancer Name", + "operating_status": "Operating Status", + "provisioning_status": "Provisioning Status", + "role": "Role", + "status": "Amphora Status" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 18, + "panels": [], + "title": "Service Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 0, + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "openstack_loadbalancer_up", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Load Balancer Service Status", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 16, + "panels": [], + "title": "Load Balancers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "openstack_loadbalancer_total_loadbalancers", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Load Balancers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 15 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(openstack_loadbalancer_loadbalancer_status == 0)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Online Load Balancers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 15 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(openstack_loadbalancer_loadbalancer_status == 2)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Offline Load Balancers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 15 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(openstack_loadbalancer_loadbalancer_status == 3)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Errored Load Balancers", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 14, + "panels": [], + "title": "Amphorae", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 21 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "openstack_loadbalancer_total_amphorae", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Amphorae", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 21 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(openstack_loadbalancer_amphora_status == 2)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Ready Amphorae", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 21 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(openstack_loadbalancer_amphora_status == 1)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Allocated Amphorae", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 21 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(openstack_loadbalancer_amphora_status == 6)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Errored Amphorae", + "type": "stat" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "", + "title": "Load Balancers", + "uid": "h-UybNo4z", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json new file mode 100644 index 000000000..9a4a776d1 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/memcached.json @@ -0,0 +1,935 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard for memcached servers monitored by Prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 37, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (memcached_commands_total{instance=~\"$node\", status=\"hit\"}) / sum (memcached_commands_total{instance=~\"$node\"}) ", + "hide": false, + "intervalFactor": 2, + "legendFormat": "hit ratio", + "refId": "A", + "step": 2, + "target": "" + } + ], + "title": "% Hit ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (memcached_current_connections{instance=~\"$node\"}) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(memcached_commands_total{instance=~\"$node\", command=\"set\"}) / sum(memcached_commands_total{instance=~\"$node\", command=\"get\"})", + "intervalFactor": 2, + "legendFormat": "ratio {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "title": "Set / Get ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (memcached_commands_total{instance=~\"$node\"}) by (command)", + "intervalFactor": 2, + "legendFormat": "{{command}}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "title": "Commands", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(memcached_items_evicted_total{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "evicts {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(memcached_items_reclaimed_total{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "reclaims {{ instance }}", + "refId": "B", + "step": 2 + } + ], + "title": "evicts / reclaims", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(memcached_read_bytes_total{instance=~\"$node\"}[10m])", + "intervalFactor": 2, + "legendFormat": "read from network {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(memcached_written_bytes_total{instance=~\"$node\"}[10m])", + "intervalFactor": 2, + "legendFormat": "written to network {{ instance }}", + "refId": "B", + "step": 5, + "target": "" + } + ], + "title": "Read / written bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "memcached_current_bytes{instance=~\"$node\"} / memcached_limit_bytes{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "% used memory used {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "title": "Total memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "memcached_current_items{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "items {{ instance }}", + "refId": "A", + "step": 2, + "target": "" + } + ], + "title": "Items in cache", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [ + "prometheus", + "memcached" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "node", + "options": [], + "query": "label_values(memcached_up, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Memcached", + "uid": "DjolHN1Zk", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json new file mode 100644 index 000000000..f9c90981c --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_full.json @@ -0,0 +1,22607 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 0, + "id": 69, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 261, + "panels": [], + "title": "Quick CPU / Mem / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 20, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "(((count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))) - avg(sum by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[5m])))) * 100) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 900 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Busy state of all CPU cores together (5 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 155, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "avg(node_load5{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "Sys Load (5m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Busy state of all CPU cores together (15 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 19, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "avg(node_load15{instance=\"$node\",job=\"$job\"}) / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)) * 100", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "Sys Load (15m avg)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "hideTimeOverride": false, + "id": 16, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "((node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "A", + "step": 900 + }, + { + "expr": "100 - ((node_memory_MemAvailable_bytes{instance=\"$node\",job=\"$job\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "B", + "step": 900 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 21, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} )) * 100", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 154, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "Root FS Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 14, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 900 + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "hideTimeOverride": true, + "id": 15, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 1800 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total RootFS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 18, + "y": 3 + }, + "id": 23, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "RootFS Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 20, + "y": 3 + }, + "id": 75, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total SWAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 2, + "x": 22, + "y": 3 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 263, + "panels": [], + "title": "Basic CPU / Mem / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#1F78C1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "guest" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "irq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "nice" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "softirq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "steal" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "system" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{job=\"$job\",instance=\"$node\", mode!='idle'}[5m])) * 100", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ mode }}", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=~\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ mode }}", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "CPU Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SWAP Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avaliable" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEDAF7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RAM Total", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RAM Used", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Cache + Buffer", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Free", + "refId": "D", + "step": 240 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SWAP Used", + "refId": "E", + "step": 240 + } + ], + "title": "Memory Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "recv {{device}}", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trans {{device}} ", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Basic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Disk space used of all filesystems mounted", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 152, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used Basic", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 265, + "panels": [], + "title": "CPU / Memory / Net / Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "percentage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "guest" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "irq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "nice" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "softirq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "steal" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "system" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "user" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='user',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "User - Normal processes executing in user mode", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='nice',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Nice - Niced processes executing in user mode", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='iowait',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Iowait - Waiting for I/O to complete", + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='irq',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Irq - Servicing interrupts", + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='softirq',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Softirq - Servicing softirqs", + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='steal',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='guest',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", + "refId": "I", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=\"$node\",job=\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Idle - Waiting for something to happen", + "refId": "D", + "step": 240 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap - Swap memory usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused - Free memory unassigned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Apps - Memory used by user-space applications", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "D", + "step": 240 + }, + { + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Cache - Parked file data (file content) cache", + "refId": "E", + "step": 240 + }, + { + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "refId": "F", + "step": 240 + }, + { + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Unused - Free memory unassigned", + "refId": "G", + "step": 240 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Swap - Swap space used", + "refId": "H", + "step": 240 + }, + { + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "I", + "step": 240 + } + ], + "title": "Memory Stack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "bits out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 84, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[5m])*8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 156, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 229, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 480 + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*read*./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Successfully read bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Successfully written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "I/O Usage Read / Write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "io time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "hidden" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 127, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"$diskdevices\"} [5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Time spent doing I/Os", + "refId": "A", + "step": 240 + } + ], + "title": "I/O Usage Times", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 266, + "panels": [], + "title": "Memory Meminfo", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 136, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Active / Inactive", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 135, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Commited", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 86 + }, + "id": 191, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "refId": "C", + "step": 4 + }, + { + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "refId": "D", + "step": 4 + } + ], + "title": "Memory Active / Inactive Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 86 + }, + "id": 130, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writeback - Memory which is actively being written back to disk", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "refId": "C", + "step": 4 + } + ], + "title": "Memory Writeback and Dirty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 138, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Mapped - Used memory in mapped pages files which have been mmaped, such as libraries", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "refId": "C", + "step": 4 + }, + { + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "ShmemPmdMapped - Ammount of shared (shmem/tmpfs) memory backed by huge pages", + "refId": "D", + "step": 4 + } + ], + "title": "Memory Shared and Mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 96 + }, + "id": 131, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 106 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocChunk - Largest contigious block of vmalloc area which is free", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "refId": "C", + "step": 4 + } + ], + "title": "Memory Vmalloc", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 106 + }, + "id": 159, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bounce - Memory used for block device bounce buffers", + "refId": "A", + "step": 4 + } + ], + "title": "Memory Bounce", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive *./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 116 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "AnonPages - Memory in user pages not backed by files", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Anonymous", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 116 + }, + "id": 160, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Kernel / CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "pages", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 126 + }, + "id": 140, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "refId": "C", + "step": 4 + } + ], + "title": "Memory HugePages Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 126 + }, + "id": 71, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages - Total size of the pool of huge pages", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Hugepagesize - Huge Page size", + "refId": "B", + "step": 4 + } + ], + "title": "Memory HugePages Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 136 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DirectMap1G - Amount of pages mapped as this size", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DirectMap2M - Amount of pages mapped as this size", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DirectMap4K - Amount of pages mapped as this size", + "refId": "C", + "step": 4 + } + ], + "title": "Memory DirectMap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 136 + }, + "id": 137, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Unevictable and MLocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 146 + }, + "id": 132, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet commited to the storage", + "refId": "A", + "step": 4 + } + ], + "title": "Memory NFS", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 156 + }, + "id": 267, + "panels": [], + "title": "Memory Vmstat", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 157 + }, + "id": 176, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pagesin - Page in operations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pagesout - Page out operations", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Pages In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "pages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 157 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pswpin - Pages swapped in", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pswpout - Pages swapped out", + "refId": "B", + "step": 4 + } + ], + "title": "Memory Pages Swap In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "faults", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault operations" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 167 + }, + "id": 175, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgfault - Page major and minor fault operations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgmajfault - Major page fault operations", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[5m]) - irate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgminfault - Minor page fault operations", + "refId": "C", + "step": 4 + } + ], + "title": "Memory Page Faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 167 + }, + "id": 307, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "max_over_time(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval:]) - (min_over_time(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval:]))", + "format": "time_series", + "interval": "30s", + "intervalFactor": 2, + "legendFormat": "oom killer invocations ", + "refId": "A", + "step": 4 + } + ], + "title": "OOM Killer", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 177 + }, + "id": 293, + "panels": [], + "title": "System Timesync", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 178 + }, + "id": 260, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Estimated error in seconds", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Time offset in between local system and reference clock", + "refId": "B", + "step": 240 + }, + { + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "title": "Time Syncronized Drift", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 178 + }, + "id": 291, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Phase-locked loop time adjust", + "refId": "A", + "step": 240 + } + ], + "title": "Time PLL Adjust", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 188 + }, + "id": 168, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Local clock frequency adjustment", + "refId": "B", + "step": 240 + } + ], + "title": "Time Syncronized Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 188 + }, + "id": 294, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Seconds between clock ticks", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "International Atomic Time (TAI) offset", + "refId": "B", + "step": 240 + } + ], + "title": "Time Misc", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 198 + }, + "id": 312, + "panels": [], + "title": "System Processes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 199 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Processes blocked waiting for I/O to complete", + "refId": "A", + "step": 240 + }, + { + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Processes in runnable state", + "refId": "B", + "step": 240 + } + ], + "title": "Processes Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 199 + }, + "id": 315, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ state }}", + "refId": "A", + "step": 240 + } + ], + "title": "Processes State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "forks / sec", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 209 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_forks_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Processes forks second", + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 209 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "process_resident_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "B", + "step": 240 + }, + { + "expr": "irate(process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "C", + "step": 240 + }, + { + "expr": "irate(process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum amount of virtual memory available in bytes", + "refId": "D", + "step": 240 + } + ], + "title": "Processes Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PIDs limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 219 + }, + "id": 313, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Number of PIDs", + "refId": "A", + "step": 240 + }, + { + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PIDs limit", + "refId": "B", + "step": 240 + } + ], + "title": "PIDs Number and Limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 219 + }, + "id": 305, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{ cpu }} - seconds spent running a process", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "refId": "B", + "step": 240 + } + ], + "title": "Process schdeule stats Running / Waiting", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Threads limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 229 + }, + "id": 314, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Allocated threads", + "refId": "A", + "step": 240 + }, + { + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Threads limit", + "refId": "B", + "step": 240 + } + ], + "title": "Threads Number and Limit", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 239 + }, + "id": 269, + "panels": [], + "title": "System Misc", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 240 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Context switches", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_intr_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Interrupts", + "refId": "B", + "step": 240 + } + ], + "title": "Context Switches / Interrupts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 240 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 1m", + "refId": "A", + "step": 480 + }, + { + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 5m", + "refId": "B", + "step": 480 + }, + { + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 15m", + "refId": "C", + "step": 480 + } + ], + "title": "System Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 250 + }, + "id": 259, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ type }} - {{ info }}", + "refId": "A", + "step": 240 + } + ], + "title": "Interrupts Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 250 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{ cpu }}", + "refId": "A", + "step": 240 + } + ], + "title": "Schedule timeslices executed by each cpu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 260 + }, + "id": 151, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Entropy available to random number generators", + "refId": "A", + "step": 240 + } + ], + "title": "Entropy", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 260 + }, + "id": 308, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Time spent", + "refId": "A", + "step": 240 + } + ], + "title": "CPU time spent in user and system contexts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 270 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptors", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 280 + }, + "id": 304, + "panels": [], + "title": "Hardware Misc", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "temperature", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 281 + }, + "id": 158, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} temp", + "refId": "A", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical", + "refId": "C", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "title": "Hardware temperature monitor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 281 + }, + "id": 300, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Current {{ name }} in {{ type }}", + "refId": "A", + "step": 240 + }, + { + "expr": "node_cooling_device_max_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Max {{ name }} in {{ type }}", + "refId": "B", + "step": 240 + } + ], + "title": "Throttle cooling device", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 291 + }, + "id": 302, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ power_supply }} online", + "refId": "A", + "step": 240 + } + ], + "title": "Power supply", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 301 + }, + "id": 296, + "panels": [], + "title": "Systemd", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 302 + }, + "id": 297, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} Connections", + "refId": "A", + "step": 240 + } + ], + "title": "Systemd Sockets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deactivating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFCB7D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C8F2C2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 302 + }, + "id": 298, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Activating", + "refId": "A", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "refId": "B", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Deactivating", + "refId": "C", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Failed", + "refId": "D", + "step": 240 + }, + { + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "refId": "E", + "step": 240 + } + ], + "title": "Systemd Units State", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 312 + }, + "id": 270, + "panels": [], + "title": "Storage Disk", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 313 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 4 + } + ], + "title": "Disk IOps Completed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 313 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 4 + } + ], + "title": "Disk R/W Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "time. read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 323 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "intervalFactor": 4, + "legendFormat": "{{device}} - Read time", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Write time", + "refId": "B", + "step": 4 + } + ], + "title": "Disk R/W Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 323 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO time weighted", + "refId": "A", + "step": 8 + } + ], + "title": "Disk IOs Weighted", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "I/Os", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 333 + }, + "id": 133, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Read merged", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Write merged", + "refId": "B", + "step": 4 + } + ], + "title": "Disk R/W Merged", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 333 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO time", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - discard time", + "refId": "B", + "step": 8 + } + ], + "title": "Time Spent Doing I/Os", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "I/Os", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 343 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_io_now{instance=\"$node\",job=\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO now", + "refId": "A", + "step": 8 + } + ], + "title": "Disk IOs Current in Progress", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 343 + }, + "id": 301, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Discards completed", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}} - Discards merged", + "refId": "B", + "step": 4 + } + ], + "title": "Disk IOps Discards completed / merged", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 353 + }, + "id": 271, + "panels": [], + "title": "Storage Filesystem", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 354 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 4 + }, + { + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Free", + "refId": "B", + "step": 2 + }, + { + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Size", + "refId": "C", + "step": 2 + } + ], + "title": "Filesystem space available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 354 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Free file nodes", + "refId": "A", + "step": 4 + } + ], + "title": "File Nodes Free", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "files", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 364 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Max open files", + "refId": "A", + "step": 8 + }, + { + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open files", + "refId": "B", + "step": 4 + } + ], + "title": "File Descriptor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "file Nodes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 364 + }, + "id": 219, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - File nodes total", + "refId": "A", + "step": 4 + } + ], + "title": "File Nodes Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "/ ReadOnly" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 374 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - ReadOnly", + "refId": "A", + "step": 4 + }, + { + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Device error", + "refId": "B", + "step": 4 + } + ], + "title": "Filesystem in ReadOnly / Error", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 384 + }, + "id": 272, + "panels": [], + "title": "Network Traffic", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 385 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 4 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 385 + }, + "id": 142, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive errors", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Rransmit errors", + "refId": "B", + "step": 4 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 395 + }, + "id": 143, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive drop", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit drop", + "refId": "B", + "step": 4 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 395 + }, + "id": 141, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive compressed", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit compressed", + "refId": "B", + "step": 4 + } + ], + "title": "Network Traffic Compressed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 405 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive multicast", + "refId": "A", + "step": 4 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 405 + }, + "id": 144, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive fifo", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit fifo", + "refId": "B", + "step": 4 + } + ], + "title": "Network Traffic Fifo", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 415 + }, + "id": 145, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive frame", + "refId": "A", + "step": 4 + } + ], + "title": "Network Traffic Frame", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 415 + }, + "id": 231, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Statistic transmit_carrier", + "refId": "A", + "step": 4 + } + ], + "title": "Network Traffic Carrier", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 425 + }, + "id": 232, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Network Traffic Collisions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "entries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "NF conntrack limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 425 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NF conntrack entries", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NF conntrack limit", + "refId": "B", + "step": 4 + } + ], + "title": "NF Contrack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 435 + }, + "id": 230, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "ARP Entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 435 + }, + "id": 288, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 10 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Link MTU", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [ + { + "options": { + "1250000": { + "index": 0, + "text": "10MB" + }, + "12500000": { + "index": 1, + "text": "100MB" + }, + "125000000": { + "index": 2, + "text": "1G" + }, + "1250000000": { + "index": 3, + "text": "10G" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 445 + }, + "id": 280, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (device)(node_network_speed_bytes{instance=\"$node\",job=\"$job\", device!~\"tap.*\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Link Speed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 445 + }, + "id": 289, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 10 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "node_network_transmit_queue_length{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ device }}", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": " Interface transmit queue length", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packetes drop (-) / process (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 455 + }, + "id": 290, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{cpu}} - Processed", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{cpu}} - Dropped", + "refId": "B", + "step": 4 + } + ], + "title": "Softnet Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 455 + }, + "id": 310, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "CPU {{cpu}} - Squeezed", + "refId": "A", + "step": 4 + } + ], + "title": "Softnet Out of Quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 465 + }, + "id": 309, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{interface}} - Operational state UP", + "refId": "A", + "step": 4 + }, + { + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link state", + "refId": "B" + } + ], + "title": "Network Operational Status", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 475 + }, + "id": 273, + "panels": [], + "title": "Network Sockstat", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 476 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_alloc - Allocated sockets", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_mem - Used memory for tcp", + "refId": "C", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_orphan - Orphan sockets", + "refId": "D", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCP_tw - Sockets waiting close", + "refId": "E", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 476 + }, + "id": 124, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "UDP_mem - Used memory for udp", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "sockets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 486 + }, + "id": 126, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Sockets_used - Sockets currently in use", + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 486 + }, + "id": 220, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "mem_bytes - TCP sockets in that state", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "mem_bytes - UDP sockets in that state", + "refId": "B", + "step": 240 + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 496 + }, + "id": 125, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "FRAG_memory - Used memory for frag", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat FRAG / RAW", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 506 + }, + "id": 274, + "panels": [], + "title": "Network Netstat", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "octects out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 507 + }, + "id": 221, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InOctets - Received octets", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutOctets - Sent octets", + "refId": "B", + "step": 4 + } + ], + "title": "Netstat IP In / Out Octets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 507 + }, + "id": 81, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Forwarding - IP forwarding", + "refId": "A", + "step": 4 + } + ], + "title": "Netstat IP Forwarding", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 517 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "B", + "step": 4 + } + ], + "title": "ICMP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 517 + }, + "id": 50, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "refId": "A", + "step": 4 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 527 + }, + "id": 55, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InDatagrams - Datagrams received", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "OutDatagrams - Datagrams sent", + "refId": "B", + "step": 4 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 527 + }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", + "refId": "C" + }, + { + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "RcvbufErrors - UDP buffer errors received", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SndbufErrors - UDP buffer errors send", + "refId": "E", + "step": 4 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 537 + }, + "id": 299, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "refId": "B", + "step": 4 + } + ], + "title": "TCP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 537 + }, + "id": 104, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "refId": "D" + }, + { + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[5m])", + "interval": "", + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "refId": "E" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*MaxConn *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 547 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "refId": "A", + "step": 4 + }, + { + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dinamic is \"-1\")", + "refId": "B", + "step": 4 + } + ], + "title": "TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "counter out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Sent.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 547 + }, + "id": 91, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "refId": "C", + "step": 4 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 557 + }, + "id": 82, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "refId": "B", + "step": 4 + } + ], + "title": "TCP Direct Transition", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 567 + }, + "id": 279, + "panels": [], + "title": "Node Exporter", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 568 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape duration", + "refId": "A", + "step": 4 + } + ], + "title": "Node Exporter Scrape Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*error.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 568 + }, + "id": 157, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape success", + "refId": "A", + "step": 4 + }, + { + "expr": "node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "refId": "B", + "step": 4 + } + ], + "title": "Node Exporter Scrape", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "node", + "value": "node" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_cpu_seconds_total, job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(node_cpu_seconds_total, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "192.168.33.3:9100", + "value": "192.168.33.3:9100" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_cpu_seconds_total{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_cpu_seconds_total{job=\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "[a-z]+|nvme[0-9]+n[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "diskdevices", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Full", + "uid": "node-exporter-full", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json new file mode 100644 index 000000000..9351c41ce --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/node_exporter_side_by_side.json @@ -0,0 +1,2043 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard to view multiple servers", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 405, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "editable": true, + "error": false, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "maxPerRow": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "html" + }, + "pluginVersion": "11.0.0", + "repeat": "node", + "style": {}, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "$node", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 20, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "repeat": "node", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(node_cpu_seconds_total{instance=~\"$node\", mode=\"system\"}) or count(node_cpu{instance=~\"$node\", mode=\"system\"})", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400, + "target": "" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "%", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "rgba(216, 200, 27, 0.27)", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "repeat": "node", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"system\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"system\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "system", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"user\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"user\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "user", + "refId": "B", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"nice\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"nice\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "nice", + "refId": "C", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"iowait\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"iowait\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "iowait", + "refId": "E", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"steal\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"steal\",instance=~'$node'}[5m]))", + "intervalFactor": 2, + "legendFormat": "steal", + "refId": "H", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"idle\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "idle", + "refId": "D", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"irq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"irq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "irq", + "refId": "F", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"softirq\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"softirq\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "softirq", + "refId": "G", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(irate(node_cpu_seconds_total{mode=\"guest\",instance=~'$node'}[5m])) or sum(irate(node_cpu{mode=\"guest\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "guest", + "refId": "I", + "step": 1200 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E5A8E2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 50 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 50 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "repeat": "node", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "( node_memory_MemTotal_bytes{instance=~'$node'} - node_memory_MemFree_bytes{instance=~'$node'} - node_memory_Buffers_bytes{instance=~'$node'} - node_memory_Cached_bytes{instance=~'$node'} - node_memory_SwapCached_bytes{instance=~'$node'} - node_memory_Slab_bytes{instance=~'$node'} - node_memory_PageTables_bytes{instance=~'$node'} - node_memory_VmallocUsed_bytes{instance=~'$node'} ) or ( node_memory_MemTotal{instance=~'$node'} - node_memory_MemFree{instance=~'$node'} - node_memory_Buffers{instance=~'$node'} - node_memory_Cached{instance=~'$node'} - node_memory_SwapCached{instance=~'$node'} - node_memory_Slab{instance=~'$node'} - node_memory_PageTables{instance=~'$node'} - node_memory_VmallocUsed{instance=~'$node'} )", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Apps", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Buffers_bytes{instance=~'$node'} or node_memory_Buffers{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Cached_bytes{instance=~'$node'} or node_memory_Cached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_MemFree_bytes{instance=~'$node'} or node_memory_MemFree{instance=~'$node'}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "E", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Slab_bytes{instance=~'$node'} or node_memory_Slab{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Slab", + "refId": "F", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_SwapCached_bytes{instance=~'$node'} or node_memory_SwapCached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SwapCached", + "refId": "G", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_PageTables_bytes{instance=~'$node'} or node_memory_PageTables{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PageTables", + "refId": "H", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_VmallocUsed_bytes{instance=~'$node'} or node_memory_VmallocUsed{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "VmallocUsed", + "metric": "", + "refId": "I", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'}) or (node_memory_SwapTotal{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Swap", + "metric": "", + "refId": "C", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Committed_AS_bytes{instance=~'$node'} or node_memory_Committed_AS{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Committed", + "metric": "", + "refId": "J", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Mapped_bytes{instance=~'$node'} or node_memory_Mapped{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Mapped", + "refId": "K", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Active_bytes{instance=~'$node'} or node_memory_Active{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "metric": "", + "refId": "L", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_memory_Inactive_bytes{instance=~'$node'} or node_memory_Inactive{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "metric": "", + "refId": "M", + "step": 1200 + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "repeat": "node", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_load1{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "repeat": "node", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "100.0 - 100 * (node_filesystem_avail_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size_bytes{instance=~'$node',device !~'tmpfs',device!~'by-uuid'}) or 100.0 - 100 * (node_filesystem_avail{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{instance=~'$node',device !~'tmpfs',device!~'by-uuid'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "title": "Disk Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "repeat": "node", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node\"}[5m])*100 or irate(node_disk_io_time_ms{instance=~\"$node\"}[5m])/10", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "title": "Disk Utilization per Device", + "type": "timeseries" + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_reads_completed_total{instance=~'$node'}[5m]) or irate(node_disk_reads_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_writes_completed_total{instance=~'$node'}[5m]) or irate(node_disk_writes_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk IOs per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "IO/second read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "/.*_read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_read_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_read{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "refId": "B", + "step": 2400 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_disk_written_bytes_total{instance=~'$node'}[5m]) or irate(node_disk_sectors_written{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Throughput per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes/second read (-) / write (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_disk_io_now{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Queue Length", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_context_switches_total{instance=~\"$node\"}[5m]) or irate(node_context_switches{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "context switches", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Context Switches", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_receive_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_receive_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_in", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=~'$node'}[5m])*8 or irate(node_network_transmit_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_out", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(node_network_receive_drop_total{instance=~'$node'}[5m])", + "hide": false, + "interval": "", + "legendFormat": "dropped receive {{ device }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(node_network_transmit_total{instance=~'$node'}[5m]) * -1", + "hide": false, + "interval": "", + "legendFormat": "dropped transmit {{ device }}", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Network drops", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 83 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "intervalFactor": 2, + "legendFormat": "established", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Netstat", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 90 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "Udp_NoPorts", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InDatagrams", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InErrors", + "refId": "B", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Udp_OutDatagrams", + "refId": "C", + "step": 1200 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_NoPorts", + "refId": "D", + "step": 1200 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "UDP Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 97 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "maxPerRow": 6, + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "node_nf_conntrack_entries_limit{instance=~\"$node\"} - node_nf_conntrack_entries{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "free", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Conntrack", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [ + "prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allFormat": "glob", + "current": { + "selected": true, + "text": [ + "192.168.33.3:9100" + ], + "value": [ + "192.168.33.3:9100" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "", + "multi": true, + "multiFormat": "regex values", + "name": "node", + "options": [], + "query": "label_values(node_exporter_build_info, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Server Metrics Side by Side", + "uid": "snCzSH1Zk", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/nvme.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/nvme.json new file mode 100644 index 000000000..1669b02a0 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/nvme.json @@ -0,0 +1,1217 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 17197, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nvme_data_units_written_total{serial_number=~\"$serial_number\"}", + "instant": true, + "legendFormat": "{{instance}} - {{device}} - {{serial_number}}", + "refId": "A" + } + ], + "title": "Device & Serial Number", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 26, + "panels": [], + "title": "Device Information", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_physical_size_bytes{serial_number=\"$serial_number\"}", + "legendFormat": "Physical Size", + "refId": "A" + } + ], + "title": "Physical Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 65 + }, + { + "color": "red", + "value": 75 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 6, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_temperature_celsius{serial_number=\"$serial_number\"}", + "legendFormat": "Temperature", + "refId": "A" + } + ], + "title": "Temperature", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 4 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_rated_dwpd{serial_number=\"$serial_number\"}", + "legendFormat": "Rated DWPD", + "refId": "A" + } + ], + "title": "Rated DWPD", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 50000 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 4 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_power_on_hours_total{serial_number=\"$serial_number\"}", + "legendFormat": "Power Hours", + "refId": "A" + } + ], + "title": "Power-On Hours", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 4 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_power_cycles_total{serial_number=\"$serial_number\"}", + "legendFormat": "Power Cycles", + "refId": "A" + } + ], + "title": "Power Cycles", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 10, + "panels": [], + "title": "Health Indicators", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 12 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_critical_warning_total{serial_number=\"$serial_number\"}", + "legendFormat": "Critical Warnings", + "refId": "A" + } + ], + "title": "Critical Warnings", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 12 + }, + "id": 5, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_percentage_used_ratio{serial_number=\"$serial_number\"} * 100", + "legendFormat": "Percentage Used", + "refId": "A" + } + ], + "title": "Percentage Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 12 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_num_err_log_entries_total{serial_number=\"$serial_number\"}", + "legendFormat": "Error Log Entries", + "refId": "A" + } + ], + "title": "Error Log Entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 12 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_media_errors_total{serial_number=\"$serial_number\"}", + "legendFormat": "Media Errors", + "refId": "A" + } + ], + "title": "Media Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 20 + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 12 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_available_spare_ratio{serial_number=\"$serial_number\"} * 100", + "legendFormat": "Available Spare", + "refId": "A" + } + ], + "title": "Available Spare Ratio", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 12 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "nvme_available_spare_threshold_ratio{serial_number=\"$serial_number\"} * 100", + "legendFormat": "Spare Threshold", + "refId": "A" + } + ], + "title": "Spare Threshold Ratio", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 11, + "panels": [], + "title": "Performance Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nvme_data_units_read_total{serial_number=\"$serial_number\"}[5m])*512000", + "legendFormat": "Data Read", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(nvme_data_units_written_total{serial_number=\"$serial_number\"}[5m])*512000", + "legendFormat": "Data Written", + "range": true, + "refId": "B" + } + ], + "title": "Disk I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 20 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "nvme_data_units_written_total{serial_number=\"$serial_number\"} * 512000", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "All Time TBW", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 20 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "nvme_data_units_read_total{serial_number=\"$serial_number\"} * 512000", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "All Time TBR", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "irate(nvme_controller_busy_time_seconds{serial_number=\"$serial_number\"}[5m])", + "legendFormat": "Controller Busy Time", + "range": true, + "refId": "A" + } + ], + "title": "Controller Busy Time", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "", + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": "Z2M0A13LTCD8", + "value": "Z2M0A13LTCD8" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(nvme_data_units_read_total,serial_number)", + "includeAll": false, + "label": "Serial Number", + "name": "serial_number", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(nvme_data_units_read_total,serial_number)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "NVMe Monitoring", + "uid": "uesjf83hh", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/opensearch.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/opensearch.json new file mode 100644 index 000000000..211e70aee --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/opensearch.json @@ -0,0 +1,5801 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "OpenSearch cluster stats", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 2322, + "graphTooltip": 1, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "OS" + ], + "targetBlank": true, + "title": "OS", + "type": "dashboards" + }, + { + "asDropdown": true, + "icon": "external link", + "keepTime": true, + "tags": [ + "MySQL" + ], + "targetBlank": true, + "title": "MySQL", + "type": "dashboards" + }, + { + "asDropdown": true, + "icon": "external link", + "keepTime": true, + "tags": [ + "MongoDB" + ], + "targetBlank": true, + "title": "MongoDB", + "type": "dashboards" + }, + { + "asDropdown": true, + "icon": "external link", + "keepTime": true, + "tags": [ + "App" + ], + "targetBlank": true, + "title": "App", + "type": "dashboards" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 90, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "KPI", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "1": { + "text": "Red" + }, + "3": { + "text": "Yellow" + }, + "5": { + "text": "Green" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 2 + }, + { + "color": "#299c46", + "value": 4 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 53, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_status{cluster=\"$cluster\",color=\"red\"}==1 or (elasticsearch_cluster_health_status{cluster=\"$cluster\",color=\"green\"}==1)+4 or (elasticsearch_cluster_health_status{cluster=\"$cluster\",color=\"yellow\"}==1)+2)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cluster health", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "N/A": { + "color": "#299c46", + "text": "0" + }, + "no value": { + "color": "#299c46", + "text": "0" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "#299c46", + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#d44a3a", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 4, + "y": 1 + }, + "id": 81, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^__name__$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(elasticsearch_breakers_tripped{cluster=\"$cluster\",name=~\"$name\"}>0)", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Tripped for breakers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 1 + }, + "id": 51, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (elasticsearch_process_cpu_percent{cluster=\"$cluster\",name=~\"$name\"} ) / count (elasticsearch_process_cpu_percent{cluster=\"$cluster\",name=~\"$name\"} )", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "title": "CPU usage Avg.", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 1 + }, + "id": 50, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (elasticsearch_jvm_memory_used_bytes{cluster=\"$cluster\",name=~\"$name\"}) / sum (elasticsearch_jvm_memory_max_bytes{cluster=\"$cluster\",name=~\"$name\"}) * 100", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "title": "JVM memory used Avg.", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of nodes in the cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 14, + "y": 1 + }, + "id": 10, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_number_of_nodes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of data nodes in the cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 16, + "y": 1 + }, + "id": 9, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_number_of_data_nodes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "title": "Data nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Cluster level changes which have not yet been executed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 18, + "y": 1 + }, + "hideTimeOverride": true, + "id": 16, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_number_of_pending_tasks{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "title": "Pending tasks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 89, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum (elasticsearch_process_open_files_count{cluster=\"$cluster\",name=~\"$name\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "title": "Open file descriptors per cluster", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 91, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Shards", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of primary shards in your cluster. This is an aggregate total across all indices.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 11, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "repeat": "shard_type", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_active_primary_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Active primary shards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Aggregate total of all shards across all indices, which includes replica shards", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 39, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_active_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Active shards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Count of shards that are being freshly created", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 40, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_initializing_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Initializing shards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of shards that are currently moving from one node to another node.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 5 + }, + "id": 41, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_relocating_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Relocating shards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Shards delayed to reduce reallocation overhead", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 5 + }, + "id": 42, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_delayed_unassigned_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Delayed shards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The number of shards that exist in the cluster state, but cannot be found in the cluster itself", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 82, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(1, elasticsearch_cluster_health_unassigned_shards{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Unassigned shards", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 92, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "JVM Garbage Collection", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GCs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_jvm_gc_collection_seconds_count{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{gc}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "title": "GC count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_jvm_gc_collection_seconds_sum{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{gc}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "title": "GC time", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 93, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Translog", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 77, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_translog_operations{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Total translog operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 78, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_translog_size_in_bytes{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Total translog size in bytes", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 94, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Breakers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 79, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_breakers_tripped{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{breaker}}", + "refId": "A" + } + ], + "title": "Tripped for breakers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 80, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_breakers_estimated_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{breaker}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_breakers_limit_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: limit for {{breaker}}", + "refId": "B" + } + ], + "title": "Estimated size in bytes of breaker", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 95, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "CPU and Memory", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "CPU usage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_os_load1{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "load1: {{name}}", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_os_load5{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "load5: {{name}}", + "metric": "", + "refId": "B", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_os_load15{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "load15: {{name}}", + "metric": "", + "refId": "C", + "step": 20 + } + ], + "title": "Load average", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "CPU usage", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 88, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_process_cpu_percent{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Memory", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_jvm_memory_used_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} used: {{area}}", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} max: {{area}}", + "refId": "C", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_jvm_memory_pool_peak_used_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} peak used pool: {{pool}}", + "refId": "D", + "step": 20 + } + ], + "title": "JVM memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Memory", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 54, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_jvm_memory_committed_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} committed: {{area}}", + "refId": "B", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} max: {{area}}", + "refId": "C", + "step": 20 + } + ], + "title": "JVM memory committed", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 96, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Disk and Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Disk Usage %", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "rgba(216, 200, 27, 0.27)", + "value": 0.8 + }, + { + "color": "rgba(234, 112, 112, 0.22)", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "1-(elasticsearch_filesystem_data_available_bytes{cluster=\"$cluster\",name=~\"$name\"}/elasticsearch_filesystem_data_size_bytes{cluster=\"$cluster\",name=~\"$name\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{path}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Disk usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes/sec", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sent" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_transport_tx_size_bytes_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: sent ", + "refId": "D", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "-irate(elasticsearch_transport_rx_size_bytes_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: received", + "refId": "C", + "step": 20 + } + ], + "title": "Network usage", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 97, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Documents", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_indices_docs{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Documents count on node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "index calls/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_indexing_index_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Documents indexed rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Count of deleted documents on this node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Documents/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 90 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_docs_deleted{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Documents deleted rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Documents/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 90 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_merges_docs_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Documents merged rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 90 + }, + "id": 52, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_merges_total_size_bytes_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Documents merged bytes", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 98, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Times", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 102 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_search_query_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval]) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "title": "Query time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 102 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "title": "Indexing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 113 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_merges_total_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "title": "Merging time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 113 + }, + "id": 87, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_store_throttle_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "title": "Throttle time for index store", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 124 + }, + "id": 99, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Total Operations stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Operations/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 125 + }, + "id": 48, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_search_query_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: query", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_search_fetch_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: fetch", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_merges_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: merges", + "refId": "D", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_refresh_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: refresh", + "refId": "E", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_flush_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: flush", + "refId": "F", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_get_exists_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_exists", + "refId": "G", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_get_missing_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_missing", + "refId": "H", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_get_tota{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get", + "refId": "I", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_indexing_delete_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing_delete", + "refId": "J", + "step": 10 + } + ], + "title": "Total Operations rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Time", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 136 + }, + "id": 49, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_search_query_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: query", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_search_fetch_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: fetch", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_merges_total_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: merges", + "refId": "D", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_refresh_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: refresh", + "refId": "E", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_flush_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: flush", + "refId": "F", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_get_exists_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_exists", + "refId": "G", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_get_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_time", + "refId": "H", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_get_missing_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get_missing", + "refId": "I", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_indexing_delete_time_seconds_total{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: indexing_delete", + "refId": "J", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_indices_get_time_seconds{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: get", + "refId": "K", + "step": 10 + } + ], + "title": "Total Operations time", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 147 + }, + "id": 100, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Thread Pool", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 20, + "w": 6, + "x": 0, + "y": 148 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_thread_pool_rejected_count{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "title": "Thread Pool operations rejected", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 20, + "w": 6, + "x": 6, + "y": 148 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_thread_pool_active_count{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "title": "Thread Pool operations queued", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 20, + "w": 6, + "x": 12, + "y": 148 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_thread_pool_active_count{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "title": "Thread Pool threads active", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 20, + "w": 6, + "x": 18, + "y": 148 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(elasticsearch_thread_pool_completed_count{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}: {{ type }}", + "refId": "A", + "step": 20 + } + ], + "title": "Thread Pool operations completed", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 168 + }, + "id": 101, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Caches", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Memory", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 169 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_indices_fielddata_memory_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Field data memory size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Evictions/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 169 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_fielddata_evictions{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Field data evictions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Size", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 180 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_indices_query_cache_memory_size_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Query cache size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Evictions/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 180 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_query_cache_evictions{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Query cache evictions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Evictions/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 180 + }, + "id": 84, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(elasticsearch_indices_filter_cache_evictions{cluster=\"$cluster\",name=~\"$name\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 20 + } + ], + "title": "Evictions from filter cache", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 191 + }, + "id": 102, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Segments", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 192 + }, + "id": 85, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_indices_segments_count{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Count of index segments", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 199 + }, + "id": 86, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "elasticsearch_indices_segments_memory_bytes{cluster=\"$cluster\",name=~\"$name\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Current memory size of segments in bytes", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [ + "App", + "opensearch" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "kolla_logging", + "value": "kolla_logging" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Сluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(elasticsearch_indices_docs,cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Node name", + "multi": true, + "name": "name", + "options": [], + "query": "label_values(elasticsearch_indices_docs{cluster=\"$cluster\", name!=\"\"},name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "192.168.33.3:9108", + "value": "192.168.33.3:9108" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Source of metrics", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(elasticsearch_indices_docs{cluster=\"$cluster\", name!=\"\"},instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "OpenSearch", + "uid": "LNua-zOMC", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json new file mode 100644 index 000000000..1299713e9 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus.json @@ -0,0 +1,4035 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": true, + "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}[10m])) by (instance)", + "hide": false, + "iconColor": "rgb(0, 96, 19)", + "limit": 100, + "name": "reloads", + "showIn": 0, + "step": "5m", + "type": "alert" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "enable": false, + "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "down", + "showIn": 0, + "step": "5m", + "type": "alert" + } + ] + }, + "description": "Overview of metrics from Prometheus 2.0. \r\nUseful for using prometheus to monitor your prometheus.\r\nRevisions welcome!", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 3662, + "graphTooltip": 0, + "id": 29, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 34, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "at a glance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Percentage of uptime during the most recent $interval period. Change the period with the 'interval' dropdown above.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 3, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 90 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 99 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(avg_over_time(up{instance=~\"$instance\",job=~\"$job\"}[$interval]) * 100)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "title": "Uptime [$interval]", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Servers which are DOWN RIGHT NOW! \nFIX THEM!!", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/__name__|job|Value/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": " " + }, + { + "id": "unit", + "value": "string" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "hideTimeOverride": true, + "id": 25, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "up{instance=~\"$instance\",job=~\"$job\"} < 1", + "format": "table", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "timeFrom": "1s", + "title": "Currently Down", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of time series in prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1000000 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 2000000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 12, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "B", + "step": 40 + } + ], + "title": "Total Series", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 14, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "B", + "step": 40 + } + ], + "title": "Memory Chunks", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 35, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "quick numbers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The total number of rule group evaluations missed due to slow rule group evaluation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 16, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(prometheus_evaluator_iterations_missed_total{job=~\"$job\",instance=~\"$instance\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "Missed Iterations [$interval]", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The total number of rule group evaluations skipped due to throttled metric storage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(sum_over_time(prometheus_evaluator_iterations_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "Skipped Iterations [$interval]", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of scrapes that hit the sample limit and were rejected.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 19, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "Tardy Scrapes [$interval]", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of times the database failed to reload block data from disk.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 9 + }, + "id": 13, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(sum_over_time(prometheus_tsdb_reloads_failures_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "Reload Failures [$interval]", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Sum of all skipped scrapes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 20, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) + \nirate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) + \nirate(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) + \nirate(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}[$interval]) ", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40 + } + ], + "title": "Skipped Scrapes [$interval]", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 36, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "errors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "All non-zero failures and errors", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Errors", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(net_conntrack_dialer_conn_failed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Failed Connections", + "refId": "A", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_evaluator_iterations_missed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Missed Iterations", + "refId": "B", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_evaluator_iterations_skipped_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Skipped Iterations", + "refId": "C", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_rule_evaluation_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Evaluation", + "refId": "D", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_azure_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Azure Refresh", + "refId": "E", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_consul_rpc_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consul RPC", + "refId": "F", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_dns_lookup_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DNS Lookup", + "refId": "G", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_ec2_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "EC2 Refresh", + "refId": "H", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_gce_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "GCE Refresh", + "refId": "I", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_marathon_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Marathon Refresh", + "refId": "J", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_openstack_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Openstack Refresh", + "refId": "K", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_sd_triton_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Triton Refresh", + "refId": "L", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sample Limit", + "refId": "M", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Duplicate Timestamp", + "refId": "N", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_bounds_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Timestamp Out of Bounds", + "refId": "O", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_order_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sample Out of Order", + "refId": "P", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_treecache_zookeeper_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Zookeeper", + "refId": "Q", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_tsdb_compactions_failed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TSDB Compactions", + "refId": "R", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_tsdb_head_series_not_found{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Series Not Found", + "refId": "S", + "step": 2 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(prometheus_tsdb_reloads_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reload", + "refId": "T", + "step": 2 + } + ], + "title": "Failures and Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 37, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "up", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Up", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "up{instance=~\"$instance\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "title": "Upness (stacked)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Chunks", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "title": "Storage Memory Chunks", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 38, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "series", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Series", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "title": "Series Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Series Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "removed" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum( increase(prometheus_tsdb_head_series_created_total{instance=~\"$instance\"}[5m]) )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "created", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum( increase(prometheus_tsdb_head_series_removed_total{instance=~\"$instance\"}[5m]) )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "removed", + "refId": "B", + "step": 4 + } + ], + "title": "Series Created / Removed", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 39, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "appended samples", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rate of total number of appended samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Samples / Second", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "title": "Appended Samples per Second", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 40, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "sync", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of syncs that were executed on a scrape pool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Syncs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_target_scrape_pool_sync_total{job=~\"$job\",instance=~\"$instance\"}) by (scrape_job)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "refId": "B", + "step": 4 + } + ], + "title": "Scrape Sync Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Actual interval to sync the scrape pool.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Milliseconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[$interval])) by (scrape_job) * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "refId": "A", + "step": 4 + } + ], + "title": "Target Sync", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 41, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "scrapes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "scrape_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "title": "Scrape Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of rejected scrapes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Scrapes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "duplicate timestamp", + "refId": "B", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "out of bounds", + "refId": "C", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}) ", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "out of order", + "refId": "D", + "step": 4 + } + ], + "title": "Rejected Scrapes", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 42, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "durations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The duration of rule group evaluations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Milliseconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 65 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "1000 * rate(prometheus_evaluator_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$interval]) / rate(prometheus_evaluator_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "E", + "step": 4 + } + ], + "title": "Average Rule Evaluation Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Microseconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 65 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(http_request_duration_microseconds_count{job=~\"$job\",instance=~\"$instance\"}[$interval])) by (handler) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "refId": "A", + "step": 4 + } + ], + "title": "HTTP Request Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(prometheus_engine_query_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"}) by (slice)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "refId": "A", + "step": 4 + } + ], + "title": "Prometheus Engine Query Duration Seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Rule-group evaluations \n - total\n - missed due to slow rule group evaluation\n - skipped due to throttled metric storage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "iterations", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(prometheus_evaluator_iterations_total{job=~\"$job\", instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "refId": "B", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(prometheus_evaluator_iterations_missed_total{job=~\"$job\", instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Missed", + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(prometheus_evaluator_iterations_skipped_total{job=~\"$job\", instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Skipped", + "refId": "C", + "step": 4 + } + ], + "title": "Rule Evaluator Iterations", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 43, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "notifications", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Notifications", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(prometheus_notifications_sent_total[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "title": "Notifications Sent", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 87 + }, + "id": 44, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "config", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Minutes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(time() - prometheus_config_last_reload_success_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) / 60", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "title": "Minutes Since Successful Config Reload", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Success", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 88 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 4 + } + ], + "title": "Successful Config Reload", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 45, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "garbage collection", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "GC invocation durations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(go_gc_duration_seconds_sum{instance=~\"$instance\",job=~\"$job\"}[2m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "title": "GC Rate / 2m", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 46, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Broken, ignore", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This is probably wrong! Please help.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "allocated" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_alloc_bytes_total{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "alloc_bytes_total", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "allocated", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_buck_hash_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "profiling bucket hash table", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_gc_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "GC metadata", + "refId": "D", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap in-use", + "refId": "E", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_heap_idle_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap idle", + "refId": "F", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap in use", + "refId": "G", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap released", + "refId": "H", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_heap_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap system", + "refId": "I", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_mcache_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mcache in use", + "refId": "J", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_mcache_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mcache sys", + "refId": "K", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_mspan_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mspan in use", + "refId": "L", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_mspan_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mspan sys", + "refId": "M", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_next_gc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap next gc", + "refId": "N", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_other_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "other sys", + "refId": "O", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "stack in use", + "refId": "P", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_stack_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "stack sys", + "refId": "Q", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "sys", + "refId": "R", + "step": 10 + } + ], + "title": "Go Memory Usage (FIXME)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 104 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_target_interval_length_seconds{instance=~\"$instance\", job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{interval}}", + "refId": "A", + "step": 20 + } + ], + "title": "Scrape Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Scrapes", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 104 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m])) by (interval)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{interval}}", + "refId": "A", + "step": 20 + } + ], + "title": "Target Scrapes / 5m", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": "query_result(prometheus_tsdb_head_samples_appended_total)", + "refresh": 2, + "regex": "/.*job=\"([^\"]+)/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": "query_result(up{job=~\"$job\"})", + "refresh": 2, + "regex": "/.*instance=\"([^\"]+).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "1h", + "value": "1h" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "2d", + "value": "2d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + }, + { + "selected": false, + "text": "90d", + "value": "90d" + }, + { + "selected": false, + "text": "180d", + "value": "180d" + } + ], + "query": "1h, 3h, 6h, 12h, 1d, 2d, 7d, 30d, 90d, 180d", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus 2.0 Overview", + "uid": "b7DPIZ6Zz", + "version": 2, + "weekStart": "" +} +{% endraw %} \ No newline at end of file diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json new file mode 100644 index 000000000..d956bb727 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_alertmanager.json @@ -0,0 +1,4313 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": { + "uid": "$datasource" + }, + "enable": false, + "expr": "changes(process_start_time_seconds{ instance=~\"$instance\"}[2m]) > 0", + "hide": false, + "iconColor": "#bf1b00", + "name": "Restarts", + "showIn": 0, + "step": "1m", + "tagKeys": "instance", + "titleFormat": "Restart" + } + ] + }, + "description": "Dashboard showing Prometheus Alertmanager metrics for observing status of the cluster and possible debugging.", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 9578, + "graphTooltip": 1, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Docs", + "tooltip": "Official documentation of Alertmanager", + "type": "link", + "url": "https://prometheus.io/docs/alerting/alertmanager/" + }, + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "GitHub", + "tooltip": "Alertmanager sources on GitHub", + "type": "link", + "url": "https://github.com/prometheus/alertmanager" + }, + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Twitter", + "tooltip": "Twitter account with prometheus related info", + "type": "link", + "url": "https://twitter.com/PrometheusIO" + }, + { + "icon": "question", + "tags": [], + "targetBlank": true, + "title": "Mailing list", + "tooltip": "Prometheus users mailing list", + "type": "link", + "url": "https://groups.google.com/forum/#!forum/prometheus-users" + }, + { + "icon": "question", + "tags": [], + "targetBlank": true, + "title": "IRC", + "tooltip": "Join IRC using Riot", + "type": "link", + "url": "https://riot.im/app/#/room/#prometheus:matrix.org" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 36, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "General info", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#f4d598", + "mode": "fixed" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 4, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "count(alertmanager_build_info{instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Number of instances", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Table containing list of Alertmanager instances showing its version, uptime, last reload time and if it was successful.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 50 + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "Instance" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version" + }, + "properties": [ + { + "id": "displayName", + "value": "Version" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Uptime" + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Last reload" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Last reload successful" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "branch" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "goversion" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "revision" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 9, + "x": 3, + "y": 1 + }, + "id": 26, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "time() - (alertmanager_build_info{instance=~\"$instance\"} * on (instance, cluster) group_left process_start_time_seconds{instance=~\"$instance\", job=\"alertmanager\"})", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "time() - alertmanager_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_config_last_reload_successful{instance=~\"$instance\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "C" + } + ], + "title": "Instance versions and uptime", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Number of peers in the Alertmanager cluster.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#e5ac0e", + "mode": "fixed" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 207, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max(alertmanager_cluster_members{instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cluster size", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Current number of active alerts.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#bf1b00", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 2, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max(alertmanager_alerts{state=\"active\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Number of active alerts", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Current number of suppressed alerts.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#f9e2d2", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 3, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max(alertmanager_alerts{state=\"suppressed\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Number of suppressed alerts", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Current number of active silences.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#f9e2d2", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 121, + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max(alertmanager_silences{state=\"active\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Number of active silences", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 113, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Notifications", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Number of sent notifications to distinct integrations such as PagerDuty, Slack and so on. On negative axis are displayed failed notifications.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 118, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(alertmanager_notifications_total{instance=~\"$instance\"}[$__interval])) by (integration)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ integration}}", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(alertmanager_notifications_failed_total{instance=~\"$instance\"}[$__interval])) by (integration)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Failed {{ integration }}", + "refId": "A" + } + ], + "title": "Notifications sent from $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Duration of notification sends in 0.99 and 0.9 quantiles per integration.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/0.99.*/" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/0.5 .*/" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 115, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(histogram_quantile(0.9,rate(alertmanager_notification_latency_seconds_bucket{instance=~\"$instance\"}[$__interval]))) by (integration)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "0.9q {{ integration }}", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(histogram_quantile(0.99,rate(alertmanager_notification_latency_seconds_bucket{instance=~\"$instance\"}[$__interval]))) by (integration)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "0.99q {{ integration }}", + "refId": "A" + } + ], + "title": "Notification durations per integration on $instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 18, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Alerts", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Number of alerts by state such as `active`, `suppressed` etc.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(alertmanager_alerts{instance=~\"$instance\"}) by (state)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Active alerts in $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Number of received alerts from Prometheus by status `firing` on positive axis and `resolved` on negative axis.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "resolved" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(alertmanager_alerts_received_total{instance=~\"$instance\"}[$__interval])) by (status)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ status }}", + "refId": "A" + } + ], + "title": "Received alerts by status for $instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 34, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Cluster members", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Shows cluster score representing cluster health. From Hashicorps official documentation: \n> This metric describes a node's perception of its own health based on how well it is meeting the soft real-time requirements of the protocol. This metric ranges from 0 to 8, where 0 indicates \"totally healthy\".\n\nFor more info see https://www.consul.io/docs/agent/telemetry.html#cluster-health", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_cluster_health_score{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster health score", + "refId": "A" + } + ], + "title": "Clusterhealth score for $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Shows gossip cluster members count in time and failing peers in case of any in red color.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_cluster_members{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of cluster members", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_cluster_failed_peers{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Number of failed peers", + "refId": "B" + } + ], + "title": "Cluster members count on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "On positive axis shows number of peers that joined the cluster and on negative axis number of peers that left the cluster.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Cluster left peers" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_cluster_peers_left_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_cluster_peers_joined_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster left peers", + "refId": "B" + } + ], + "title": "Cluster peers left/joined on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "On positive axis is number of attempts to reconnect the cluster. On negative axis if number of failed attempts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed reconnections" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_cluster_reconnections_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Successful reconnections", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_cluster_reconnections_failed_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed reconnections", + "refId": "B" + } + ], + "title": "Cluster reconnections on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "On positive axis is number of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/received.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(alertmanager_cluster_messages_sent_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(alertmanager_cluster_messages_received_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "title": "Cluster messages count on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "On positive axis is size of sent cluster messages by type `update` or `full_state` and on negative axis the same for received messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/received.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(alertmanager_cluster_messages_sent_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "sent {{msg_type}}", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(increase(alertmanager_cluster_messages_received_size_total{instance=~\"$instance\"}[$__interval])) by (msg_type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "received {{msg_type}}", + "refId": "B" + } + ], + "title": "Cluster messages size on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "On positive axis is number of queued cluster messages and on negative axis number of pruned messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pruned messaged" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_cluster_messages_pruned_total{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pruned messaged", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_cluster_messages_queued{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Queued messages", + "refId": "B" + } + ], + "title": "Cluster messages queue on $instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 284, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Gossip messages", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/dropped/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/failed/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 314, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_oversized_gossip_message_sent_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_oversized_gossip_message_dropped_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "dropped {{key}}", + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_oversized_gossip_message_failure_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "failed {{key}}", + "refId": "C" + } + ], + "title": "Count of oversized gossip messages on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 307, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "histogram_quantile(1,rate(alertmanager_oversize_gossip_message_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{key}}", + "refId": "A" + } + ], + "title": "Duration of oversized gossip messages on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 303, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_silences_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "silences", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_nflog_gossip_messages_propagated_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "nf_log", + "refId": "B" + } + ], + "title": "Number of propagated gossip messages on $instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 87 + }, + "id": 84, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Nflog", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 94, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_nflog_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query count", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_nflog_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query errors", + "refId": "B" + } + ], + "title": "Nf log queries count for $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 94 + }, + "id": 106, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "histogram_quantile(1,rate(alertmanager_nflog_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log query duration", + "refId": "A" + } + ], + "title": "Nf log query duration for $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 100 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_nflog_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "title": "Nf log snapshot size for $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 106 + }, + "id": 101, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval]) / rate(alertmanager_nflog_snapshot_duration_seconds_sum{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Nf log snapshot size", + "refId": "A" + } + ], + "title": "Nf log snapshot duration for $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Cluster left peers" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 112 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_nflog_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster joined peers", + "refId": "A" + } + ], + "title": "Nf log Go GC time for $instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 118 + }, + "id": 123, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Silences", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 119 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_silences{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Silences count by state on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Silecnces query fails" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 125 + }, + "id": 134, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_silences_queries_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query count", + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(alertmanager_silences_query_errors_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Silecnces query fails", + "refId": "B" + } + ], + "title": "Silences query count on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 131 + }, + "id": 138, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "histogram_quantile(1,rate(alertmanager_silences_query_duration_seconds_bucket{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces query duration", + "refId": "A" + } + ], + "title": "Silences query duration on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 137 + }, + "id": 149, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_silences_snapshot_size_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot size", + "refId": "A" + } + ], + "title": "Silences snapshot size on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 143 + }, + "id": 143, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_silences_snapshot_duration_seconds{instance=~\"$instance\", quantile=\"0.99\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces snapshot duration", + "refId": "A" + } + ], + "title": "Silences snapshot duration on $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nf log query errors" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 149 + }, + "id": 131, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "alertmanager_silences_gc_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Silecnces GC duration", + "refId": "A" + } + ], + "title": "Silences GC duraton on $instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 155 + }, + "id": 173, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Limit .*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 156 + }, + "id": 175, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(process_cpu_seconds_total{instance=~\"$instance\"}[$__interval])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max(kube_pod_container_resource_limits_cpu_cores{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{pod}}", + "refId": "B" + } + ], + "title": "CPU usage/s for $instance", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Limit .*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 163 + }, + "id": 177, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.2", + "repeat": "instance", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "process_resident_memory_bytes{instance=~\"$instance\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "E" + }, + { + "datasource": { + "uid": "$datasource" + }, + "expr": "max(kube_pod_container_resource_limits_memory_bytes{pod=~\"$instance\"}) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Limit {{ pod }}", + "refId": "A" + } + ], + "title": "Memory usage for $instance", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "alertmanager", + "prometheus", + "alerting" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result(alertmanager_build_info)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": "query_result(alertmanager_build_info)", + "refresh": 2, + "regex": "/.*instance=\"([^\"]+)\".*/", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Alertmanager", + "uid": "eea-9_siZ", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json new file mode 100644 index 000000000..09fb38fa0 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/prometheus_benchmark.json @@ -0,0 +1,5356 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics useful for benchmarking and loadtesting Prometheus itself. Designed primarily for Prometheus 2.17.x.", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 12054, + "graphTooltip": 1, + "id": 28, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 49, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Basics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_build_info{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{version}} - {{revision}}", + "refId": "A" + } + ], + "title": "Prometheus Version", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() - process_start_time_seconds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Age", + "refId": "A" + } + ], + "title": "Uptime", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 107, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() - prometheus_config_last_reload_success_timestamp_seconds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Age", + "refId": "A" + } + ], + "title": "Last Successful Config Reload", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 46, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Ingestion", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_head_series{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time series", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "title": "Head Time series", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_head_active_appenders{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head Appenders", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 10 + } + ], + "title": "Head Active Appenders", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "samples/s", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "title": "Samples Appended/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_head_chunks{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunks", + "metric": "prometheus_local_storage_memory_chunks", + "refId": "A", + "step": 10 + } + ], + "title": "Head Chunks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Created", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Removed", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "title": "Head Chunks/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_isolation_high_watermark{job=\"prometheus\",instance=\"$Prometheus\"} - prometheus_tsdb_isolation_low_watermark{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Difference", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 10 + } + ], + "title": "Isolation Watermarks", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 52, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Compaction", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 35, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_head_min_time{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Min", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() * 1000", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Now", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_head_max_time{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Max", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "title": "Head Time Range", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "GC Time/s", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "title": "Head GC Time/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_blocks_loaded{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks Loaded", + "metric": "prometheus_local_storage_indexing_batch_sizes_sum", + "refId": "A", + "step": 10 + } + ], + "title": "Blocks Loaded", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_reloads_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reloads", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "title": "TSDB Reloads/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Fsync Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_tsdb_wal_trunacte_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Truncate Latency", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + } + ], + "title": "WAL Fsync&Truncate Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Corruptions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Reload Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Head Series Not Found", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compaction Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "D", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Retention Cutoff Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "E", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_checkpoint_creations_failed_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Checkpoint Creation Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "F", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_checkpoint_deletions_failed_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "WAL Checkpoint Deletion Failures", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "G", + "step": 10 + } + ], + "title": "TSDB Problems/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 38 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_compactions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Compactions", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "title": "Compactions/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 38 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "title": "Compaction Time/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 38 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_time_retentions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time Cutoffs", + "metric": "last", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_size_retentions_total{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Size Cutoffs", + "metric": "last", + "refId": "B", + "step": 10 + } + ], + "title": "Retention Cutoffs/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurationms" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 45 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunk Time Range", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "title": "First Compaction, Avg Chunk Time Range", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 45 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes/Sample", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "title": "First Compaction, Avg Bytes/Sample", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 45 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",instance=\"$Prometheus\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",instance=\"$Prometheus\"}[10m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chunk Samples", + "metric": "prometheus_local_storage_series_chunks_persisted_count", + "refId": "A", + "step": 10 + } + ], + "title": "First Compaction, Avg Chunk Samples", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 55, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Resource Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_resident_memory_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "RSS", + "metric": "process_resident_memory_bytes", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "Target heap size", + "metric": "go_memstats_alloc_bytes", + "refId": "D", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "go_memstats_next_gc_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "Next GC", + "metric": "go_memstats_next_gc_bytes", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "go_memstats_alloc_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "intervalFactor": 2, + "legendFormat": "Allocated", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Allocated Bytes/s", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 10 + } + ], + "title": "Allocations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "irate(process_cpu_seconds_total{job=\"prometheus\",instance=\"$Prometheus\"}[1m])", + "intervalFactor": 2, + "legendFormat": "Irate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(process_cpu_seconds_total{job=\"prometheus\",instance=\"$Prometheus\"}[5m])", + "intervalFactor": 2, + "legendFormat": "5m rate", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "B", + "step": 10 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 60 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_symbol_table_size_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Used", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "title": "Symbol Tables Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 60 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_tsdb_storage_blocks_bytes_total{job=\"prometheus\",instance=\"$Prometheus\"} or prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disk Used", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 10 + } + ], + "title": "Block Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 60 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_max_fds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "process_open_fds{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open", + "refId": "B" + } + ], + "title": "File Descriptors", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 91, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Service Discovery", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 68 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_sd_discovered_targets{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}-{{config}}", + "refId": "A" + } + ], + "title": "Discovered Targets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 68 + }, + "id": 96, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_sd_updates_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Sent Updates/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 68 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_sd_received_updates_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Received Updates/s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 99, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Scraping", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 76 + }, + "id": 105, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_target_interval_length_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{interval}}", + "refId": "A" + } + ], + "title": "Scrape Interval", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 76 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_target_scrapes_exceeded_sample_limit_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Exceeded Sample Limit", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Duplicate Timestamp", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_target_scrapes_sample_out_of_bounds_total{job=\"prometheus\",instance=\"$Prometheus\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out Of Bounds ", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_target_scrapes_sample_out_of_order_total{job=\"prometheus\",instance=\"$Prometheus\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out of Order", + "refId": "E" + } + ], + "title": "Scrape Problems/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 76 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_target_metadata_cache_bytes{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{scrape_job}}", + "refId": "A" + } + ], + "title": "Metadata Cache Size", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 63, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Query Engine", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time spent in each mode, per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 84 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Query engine timings/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 84 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }} missed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }} failed", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "C", + "step": 10 + } + ], + "title": "Rule group evaulation problems/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 84 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Rule evaluation duration", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Evaluation time of rule groups/s", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 77, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "Notification", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 92 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_notifications_sent_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Notification Sent/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 92 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_notifications_errors_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_notifications_sent_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Notification Error Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 92 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_notifications_latency_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_notifications_latency_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertmanager}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Notification Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 99 + }, + "id": 85, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_notifications_alertmanagers_discovered{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Alertmanagers", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Alertmanagers Discovered", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 99 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_notifications_dropped_total{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Dropped", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Notifications Dropped/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 99 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_notifications_queue_length{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Pending", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_notifications_queue_capacity{job=\"prometheus\",instance=\"$Prometheus\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Max", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "title": "Notification Queue", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 106 + }, + "id": 58, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "HTTP Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 107 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "HTTP requests/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 107 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "HTTP request latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 107 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",instance=\"$Prometheus\"}[2m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Time spent in HTTP requests/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",instance=\"$Prometheus\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "B", + "step": 10 + } + ], + "title": "Alerting: group rules evaluation time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "prometheus_rule_group_rules{job=\"prometheus\",instance=\"$Prometheus\",rule_group=~\"$RuleGroup\"}\n", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ rule_group }}", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 10 + } + ], + "title": "Alerting: number of rules in group", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "192.168.33.3:9091", + "value": "192.168.33.3:9091" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "Prometheus", + "options": [], + "query": { + "query": "query_result(up{job=\"prometheus\"} == 1)", + "refId": "Prometheus-Prometheus-Variable-Query" + }, + "refresh": 2, + "regex": ".*instance=\"([^\"]+)\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "", + "hide": 2, + "includeAll": true, + "multi": false, + "name": "RuleGroup", + "options": [], + "query": { + "query": "prometheus_rule_group_last_duration_seconds{job=\"prometheus\",instance=\"$Prometheus\"}", + "refId": "Prometheus-RuleGroup-Variable-Query" + }, + "refresh": 2, + "regex": ".*rule_group=\"(.*?)\".*", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Prometheus Benchmark", + "uid": "L0HBvojWz", + "version": 8, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json new file mode 100644 index 000000000..010adfa2b --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/rabbitmq.json @@ -0,0 +1,5895 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A new RabbitMQ Management Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 10991, + "graphTooltip": 1, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Monitoring with Prometheus & Grafana", + "tooltip": "", + "type": "link", + "url": "https://www.rabbitmq.com/prometheus.html" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 10000 + }, + { + "color": "#C4162A", + "value": 100000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "id": 64, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Ready messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 62, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(irate(rabbitmq_channel_messages_published_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Incoming messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 66, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) - sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Publishers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 37, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "sum(rabbitmq_connections * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 40, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "sum(rabbitmq_queues * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Queues", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#37872D", + "value": null + }, + { + "color": "#1F60C4", + "value": 100 + }, + { + "color": "#C4162A", + "value": 500 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 65, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Unacknowledged messages", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": -1 + }, + { + "color": "#37872D", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 63, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_ack_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_ack_total[5m]) * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Outgoing messages / s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 41, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Consumers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#C4162A", + "value": null + }, + { + "color": "#1F60C4", + "value": 0 + }, + { + "color": "#37872D", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 38, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Channels", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#1F60C4", + "value": null + }, + { + "color": "#37872D", + "value": 3 + }, + { + "color": "#C4162A", + "value": 8 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 67, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "sum(rabbitmq_build_info * on(instance) group_left(rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 4, + "panels": [], + "title": "NODES", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "erlang_version" + }, + "properties": [ + { + "id": "displayName", + "value": "Erlang/OTP" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": null + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_version" + }, + "properties": [ + { + "id": "displayName", + "value": "RabbitMQ" + }, + { + "id": "unit", + "value": "none" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": null + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "Host" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_node" + }, + "properties": [ + { + "id": "displayName", + "value": "Node name" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": null + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "rabbitmq_node" + }, + "properties": [ + { + "id": "displayName", + "value": "Cluster" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "prometheus_client_version" + }, + "properties": [ + { + "id": "displayName", + "value": "prometheus.erl" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "prometheus_plugin_version" + }, + "properties": [ + { + "id": "displayName", + "value": "rabbitmq_prometheus" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 69, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "expr": "rabbitmq_build_info * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0 + }, + { + "color": "transparent", + "value": 536870912 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+0/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+1/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+2/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+3/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+4/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+5/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+6/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+7/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+8/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+9/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "(rabbitmq_resident_memory_limit_bytes * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) -\n(rabbitmq_process_resident_memory_bytes * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Memory available before publishers blocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1073741824 + }, + { + "color": "transparent", + "value": 5368709120 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+0/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+1/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+2/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+3/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+4/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+5/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+6/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+7/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+8/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+9/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 11 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "rabbitmq_disk_space_available_bytes * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "Disk space available before publishers blocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "transparent", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+0/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+1/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+2/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+3/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+4/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+5/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+6/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+7/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+8/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+9/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 11 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "(rabbitmq_process_max_fds * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_fds * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "File descriptors available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "transparent", + "value": 1000 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+0/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+1/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+2/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+3/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A352CC", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+4/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+5/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#96D98D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+6/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFEE52", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+7/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#8AB8FF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+8/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CA95E5", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/^rabbit@[\\w.-]+9/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFB357", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "asc" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "expr": "(rabbitmq_process_max_tcp_sockets * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_tcp_sockets * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "title": "TCP sockets available", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 27, + "panels": [], + "title": "QUEUED MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages ready to be delivered to consumers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages pending consumer acknowledgement", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 11, + "panels": [], + "title": "INCOMING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_published_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages published / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_confirmed_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages confirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queue_messages_published_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages routed to queues / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unconfirmed[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages unconfirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_dropped_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Unroutable messages dropped / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 36 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_returned_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Unroutable messages returned to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 29, + "panels": [], + "title": "OUTGOING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n (rate(rabbitmq_channel_messages_delivered_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) +\n (rate(rabbitmq_channel_messages_delivered_ack_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"})\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages delivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 20, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 100, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Messages redelivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_ack_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages delivered with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages delivered auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_acked_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages acknowledged / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 52 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Polling operations with auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_empty_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Polling operations that yield no result / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_ack_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Polling operations with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 53, + "panels": [], + "title": "QUEUES", + "type": "row" + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_queues * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total queues", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 63 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_declared_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Queues declared / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 63 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_created_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Queues created / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 63 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_deleted_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Queues deleted / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 51, + "panels": [], + "title": "CHANNELS", + "type": "row" + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_channels * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total channels", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 69 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_opened_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Channels opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 69 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_closed_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Channels closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 46, + "panels": [], + "title": "CONNECTIONS", + "type": "row" + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_connections * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total connections", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 75 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_opened_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Connections opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "autoMigrateFrom": "graph", + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 75 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.5.3", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_closed_total[5m]) * on(instance) group_left(rabbitmq_node, rabbitmq_node) rabbitmq_identity_info{rabbitmq_node=\"$rabbitmq_node\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeRegions": [], + "title": "Connections closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "timeseries", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "rabbitmq-prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rabbitmq_identity_info, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info, namespace)", + "refId": "Prometheus-namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "rabbit@kef1p-phycon0001", + "value": "rabbit@kef1p-phycon0001" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(rabbitmq_identity_info, rabbitmq_node)", + "hide": 0, + "includeAll": false, + "label": "RabbitMQ Cluster", + "multi": false, + "name": "rabbitmq_node", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info, rabbitmq_node)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "15s", + "30s", + "1m", + "5m", + "10m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RabbitMQ-Overview", + "uid": "Hz7D2_LGp", + "version": 2, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json new file mode 100644 index 000000000..6ba67ad27 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/redfish.json @@ -0,0 +1,3588 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 12403, + "graphTooltip": 0, + "id": 91, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 31, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(redfish_system_power_state{group=\"$group\", job!=\"redfish-exporter-collectlog\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + } + ], + "title": "Redfish Up", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 54, + "interval": "30m", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(redfish_system_power_state{group=\"$group\", job!=\"redfish-exporter-collectlog\"} == 1)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Powered On", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 55, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(redfish_system_power_state{group=\"$group\", job!=\"redfish-exporter-collectlog\"} != 1)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Powered Off", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 56, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(redfish_chassis_health{group=\"$group\", job!=\"redfish-exporter-collectlog\" } != 1)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Unhealthy Nodes", + "type": "stat" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "1": { + "color": "dark-green", + "index": 1, + "text": "On" + }, + "2": { + "color": "dark-red", + "index": 0, + "text": "Off" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 2 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": "BMC" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "https://${__data.fields.instance}" + } + ] + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Power state" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "#73BF69", + "value": null + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "env" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "resource" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "system_id" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "server" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "https://${__data.fields.instance}" + } + ] + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "hostname" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Power state" + }, + "properties": [ + { + "id": "custom.width", + "value": 197 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 59, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Power state" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sort_desc(redfish_system_power_state{group=\"$group\", job!=\"redfish-exporter-collectlog\"})", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Power states", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "1": { + "index": 1, + "text": "Healthy" + }, + "2": { + "index": 2, + "text": "Unknown" + }, + "3": { + "index": 0, + "text": "Unhealthy" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/__name__|chassis_id|Time|env|job|resource|instance/" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "unit", + "value": "" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "#EAB839", + "value": 2 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 3 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "server" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "https://${__data.fields.instance}" + } + ] + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 61, + "interval": "", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Status" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sort(redfish_chassis_health{group=\"$group\", job!=\"redfish-exporter-collectlog\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Chassis status", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "1": { + "index": 1, + "text": "Healthy" + }, + "2": { + "index": 2, + "text": "Warning" + }, + "3": { + "index": 0, + "text": "Critical" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/__name__|chassis_id|Time|env|job|resource|instance/" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.hidden", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "unit", + "value": "" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "#EAB839", + "value": 2 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 3 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "server" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "https://${__data.fields.instance}" + } + ] + }, + { + "id": "custom.align" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 62, + "interval": "", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Status" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(last_over_time({__name__=~\"redfish_chassis_log_entry_severity_state|redfish_manager_log_entry_severity_state\", group=\"$group\"}[1h])) by (group, server, log_service_id, instance)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Log severity summary", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": { + "Time": 0, + "Value": 4, + "group": 1, + "instance": 5, + "log_service_id": 3, + "server": 2 + }, + "renameByName": { + "Value": "Status", + "log_service_id": "Log Service", + "server": "Server" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(redfish_chassis_power_average_consumed_watts{group=\"$group\", job!=\"redfish-exporter-collectlog\"}) by (env)", + "interval": "", + "legendFormat": "{{ env }}", + "range": true, + "refId": "A" + } + ], + "title": "Power consumption", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 42, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(redfish_system_power_state{group=\"$group\", job!=\"redfish-exporter-collectlog\"} == 1) by (env)", + "hide": false, + "interval": "", + "legendFormat": "Powered up {{ env }}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(redfish_system_power_state{group=\"$group\", job!=\"redfish-exporter-collectlog\"} == 2) by (env) * -1", + "hide": true, + "interval": "", + "legendFormat": "Powered down {{ env }}", + "range": true, + "refId": "B" + } + ], + "title": "Powered status", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 16, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 7, + "x": 0, + "y": 23 + }, + "id": 39, + "interval": "5m", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(redfish_chassis_temperature_celsius{sensor=~\".*Ambient.*Temp|.*Inlet.*\", group=\"$group\", job!=\"redfish-exporter-collectlog\"}) by (env)", + "hide": false, + "interval": "", + "legendFormat": "max inlet {{ env }}", + "range": true, + "refId": "A" + } + ], + "title": "Max Inlet Temp", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 20, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 7, + "y": 23 + }, + "id": 40, + "interval": "5m", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(redfish_chassis_temperature_celsius{sensor=~\".*CPU.*1.*Temp\", group=\"$group\", job!=\"redfish-exporter-collectlog\"}) by (env)", + "hide": false, + "interval": "", + "legendFormat": "{{ env }}", + "range": true, + "refId": "A" + } + ], + "title": "Max CPU1 Temp", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 20, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 9, + "x": 15, + "y": 23 + }, + "id": 41, + "interval": "5m", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(redfish_chassis_temperature_celsius{sensor=~\"CPU2 Temp|.*CPU.2.*Temp\", group=\"$group\", job!=\"redfish-exporter-collectlog\"}) by (env)", + "hide": false, + "interval": "", + "legendFormat": "{{ env }}", + "range": true, + "refId": "A" + } + ], + "title": "Max CPU2 Temp", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 34 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 46, + "interval": "1m", + "legend": { + "show": false + }, + "options": { + "calculate": true, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_temperature_celsius{sensor=~\".*Ambient.*Temp|.*Inlet.*\", group=\"$group\", job!=\"redfish-exporter-collectlog\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Max Inlet Temp", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "short", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 34 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 48, + "interval": "1m", + "legend": { + "show": false + }, + "options": { + "calculate": true, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "95", + "min": "25", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_temperature_celsius{sensor=~\".*CPU.*1.*Temp\", group=\"$group\", job!=\"redfish-exporter-collectlog\"} != 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ env }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU1 Temp", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "short", + "logBase": 1, + "max": "95", + "min": "25", + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 34 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 50, + "interval": "1m", + "legend": { + "show": false + }, + "options": { + "calculate": true, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "95", + "min": "25", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_temperature_celsius{sensor=~\".*CPU.*2.*Temp\", group=\"$group\", job!=\"redfish-exporter-collectlog\"} != 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ env }}", + "range": true, + "refId": "A" + } + ], + "title": "CPU2 Temp", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "short", + "logBase": 1, + "max": "95", + "min": "25", + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 34 + }, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 51, + "interval": "1m", + "legend": { + "show": false + }, + "options": { + "calculate": true, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "short" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(redfish_chassis_fan_rpm_percentage{group=\"$group\", job!=\"redfish-exporter-collectlog\"}) by (server) > 0", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A" + } + ], + "title": "Max server fan speed", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "short", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 47, + "interval": "5m", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_temperature_celsius{sensor=~\".*Ambient.*Temp\", job!=\"redfish-exporter-collectlog\"} or redfish_chassis_temperature_celsius{sensor_id=~\".*InletTemp\", job!=\"redfish-exporter-collectlog\"} or redfish_chassis_temperature_celsius{sensor=~\".*Inlet.*\", job!=\"redfish-exporter-collectlog\"}", + "hide": false, + "interval": "", + "legendFormat": "{{ env }} {{ server }}", + "range": true, + "refId": "A" + } + ], + "title": "Max Inlet Temp", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 49, + "interval": "1m", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_temperature_celsius{sensor=~\".*CPU.*1.*Temp\", group=\"$group\", job!=\"redfish-exporter-collectlog\"} != 0", + "hide": false, + "interval": "", + "legendFormat": "{{ env }} {{ server }}", + "range": true, + "refId": "A" + } + ], + "title": "Max CPU1 Temp", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 29, + "panels": [], + "repeat": "server", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "refId": "A" + } + ], + "title": "$server", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 500, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 55 + }, + "id": 19, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_power_average_consumed_watts{server=\"$server\", job!=\"redfish-exporter-collectlog\"}", + "hide": false, + "interval": "", + "legendFormat": "{{power_voltage}}", + "range": true, + "refId": "A" + } + ], + "title": "Power consumption", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "displayName": "Disks", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 8, + "y": 55 + }, + "id": 24, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(redfish_system_storage_drive_state{server=~\"$server\", job!=\"redfish-exporter-collectlog\"} != 1) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Disks with errors", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "displayName": "Controllers", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 10, + "y": 55 + }, + "id": 25, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(redfish_system_pcie_device_health_state{server=~\"$server\", job!=\"redfish-exporter-collectlog\"} != 1) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "PCI-E with errors", + "type": "gauge" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "displayName": "Sensor", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 12, + "y": 55 + }, + "id": 26, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(redfish_chassis_temperature_sensor_state{server=~\"$server\", job!=\"redfish-exporter-collectlog\"} > 2) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sensors with errors", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "displayName": "Power Supply", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 14, + "y": 55 + }, + "id": 27, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(redfish_chassis_power_powersupply_health{server=~\"$server\", job!=\"redfish-exporter-collectlog\"} > 1) or vector(0)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "PS with errors", + "type": "gauge" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "max": 80, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "degree" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Temperature" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + } + ] + }, + "gridPos": { + "h": 18, + "w": 8, + "x": 16, + "y": 55 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Temperature" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (sensor) (sum_over_time(redfish_chassis_temperature_celsius{server=\"$server\", job!=\"redfish-exporter-collectlog\"}[15m])) / sum by (sensor) (count_over_time(redfish_chassis_temperature_celsius{server=\"$server\", job!=\"redfish-exporter-collectlog\"}[15m])) > 0", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{sensor}}", + "range": false, + "refId": "A" + } + ], + "title": "Temperatures", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Temperature", + "sensor": "Sensor" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "1": { + "text": "Up" + } + }, + "type": "value" + }, + { + "options": { + "2": { + "text": "Down" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a" + }, + { + "color": "#299c46", + "value": 0 + }, + { + "color": "#299c46", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 59 + }, + "id": 6, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_system_power_state{server=\"$server\", job!=\"redfish-exporter-collectlog\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Power Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "1": { + "text": "OK" + }, + "2": { + "text": "WARNING" + }, + "3": { + "text": "CRITICAL" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "#299c46", + "value": 1 + }, + { + "color": "#d44a3a", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 3, + "y": 59 + }, + "id": 66, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "/^Value$/", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "redfish_chassis_health{server=~\"$server\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{chassis_id}}", + "range": false, + "refId": "A" + } + ], + "title": "General Health", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 80, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 50 + }, + { + "color": "red", + "value": 70 + } + ] + }, + "unit": "degree" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 59 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "redfish_chassis_temperature_celsius{server=~\"$server\", job!=\"redfish-exporter-collectlog\"}", + "interval": "", + "legendFormat": "{{sensor}}", + "refId": "A" + } + ], + "title": "Temperatures", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-background" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46" + }, + { + "color": "#299c46", + "value": 1 + }, + { + "color": "#d44a3a", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "color": "dark-green", + "index": 0, + "text": "OK" + }, + "2": { + "color": "dark-yellow", + "index": 1, + "text": "WARNING" + }, + "3": { + "color": "dark-red", + "index": 2, + "text": "CRITICAL" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 63 + }, + "id": 7, + "maxDataPoints": 100, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_health{server=\"$server\", job!=\"redfish-exporter-collectlog\"}", + "format": "table", + "instant": true, + "interval": "30m", + "legendFormat": "Chassis {{chassis_id}}", + "refId": "A" + } + ], + "title": "Health by Chassis", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "env": true, + "group": true, + "instance": true, + "job": true, + "resource": true, + "server": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 66 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "redfish_chassis_power_average_consumed_watts{server=~\"$server\", job!=\"redfish-exporter-collectlog\"}", + "interval": "", + "legendFormat": "{{power_voltage}}", + "range": true, + "refId": "A" + } + ], + "title": "Power consumption", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "production", + "value": "production" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(redfish_exporter_collector_duration_seconds,env)", + "hide": 0, + "includeAll": false, + "label": "env", + "multi": false, + "name": "env", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(redfish_exporter_collector_duration_seconds,env)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "compute", + "value": "compute" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(redfish_exporter_collector_duration_seconds{env=\"$env\"},group)", + "hide": 0, + "includeAll": false, + "label": "group", + "multi": false, + "name": "group", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(redfish_exporter_collector_duration_seconds{env=\"$env\"},group)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "svn1-dr07-u10", + "value": "svn1-dr07-u10" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(redfish_exporter_collector_duration_seconds{group=\"$group\", env=\"$env\"},server)", + "hide": 0, + "includeAll": false, + "label": "server", + "multi": false, + "name": "server", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(redfish_exporter_collector_duration_seconds{group=\"$group\", env=\"$env\"},server)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now-5m" + }, + "timepicker": { + "nowDelay": "1m", + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Redfish exporter", + "uid": "redfish", + "version": 3, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json b/etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json new file mode 100644 index 000000000..48dad8f42 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/grafana_home_dashboard.json @@ -0,0 +1,3016 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": false, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "OpenStack cloud overall status dashboard", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 9701, + "graphTooltip": 0, + "id": 53, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 47, + "panels": [], + "title": "Active alerts", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 40, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "folder": "", + "groupBy": [], + "groupMode": "default", + "maxItems": 200, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true + }, + "viewMode": "list" + }, + "title": "Prometheus Alertmanager", + "type": "alertlist" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 36, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Service status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0.1 + }, + { + "color": "#d44a3a", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 18 + }, + "id": 14, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(openstack_nova_agent_state{adminState=\"enabled\"} or openstack_nova_agent_state{adminState=\"disabled\", disabledReason=~\"^AUTO:.*\"})-sum(openstack_nova_agent_state{adminState=\"enabled\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Nova agents down", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.hidden", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.align" + }, + { + "id": "custom.width", + "value": 263 + }, + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 3, + "y": 18 + }, + "id": 16, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openstack_nova_agent_state", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openstack_nova_service_state", + "format": "table", + "hide": true, + "instant": true, + "refId": "B" + } + ], + "title": "Nova agent status", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 22 + }, + "id": 26, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_agent_state{adminState=\"enabled\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Nova agents up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0.1 + }, + { + "color": "#d44a3a", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 26 + }, + "id": 22, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "count(openstack_neutron_agent_state{adminState=\"up\"})-sum(openstack_neutron_agent_state{adminState=\"up\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Neutron agents down", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm" + }, + { + "id": "custom.align" + }, + { + "id": "custom.width", + "value": 202 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "#890f02", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 1 + } + ] + } + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "adminState" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 332 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "service" + }, + "properties": [ + { + "id": "custom.width", + "value": 222 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 3, + "y": 26 + }, + "id": 25, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openstack_neutron_agent_state", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Neutron agent status", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 30 + }, + "id": 28, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_neutron_agent_state{adminState=\"up\"})", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Neutron agents up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "#d44a3a" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 34 + }, + "id": 18, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(count(openstack_cinder_agent_state{adminState=\"enabled\"})-sum(openstack_cinder_agent_state{adminState=\"enabled\"}))", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cinder agents down", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "#890f02", + "value": 0 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 1 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 19, + "x": 3, + "y": 34 + }, + "id": 20, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openstack_cinder_agent_state", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cinder agent status", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#d44a3a", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 0 + }, + { + "color": "#299c46", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 38 + }, + "id": 27, + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_cinder_agent_state{adminState=\"enabled\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cinder agent up", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 12, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Resource Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Memory in TiB", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_memory_used_bytes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "memory in use", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_memory_available_bytes)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "memory available", + "refId": "B" + } + ], + "title": "Overall memory usage (TiB)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "cores", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_vcpus_used)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "used vcpu cores", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_vcpus_available)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total vcpu cores", + "refId": "B" + } + ], + "title": "Overall CPU cores usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_local_storage_used_bytes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "local storage used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_local_storage_available_bytes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "local storage available", + "refId": "B" + } + ], + "title": "Local Storage (TB)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_neutron_floating_ips) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "floating ips", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_neutron_networks)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "networks", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_neutron_security_groups)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "security groups", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_neutron_subnets)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "subnets", + "refId": "D" + } + ], + "title": "Neutron Stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min", + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "projects", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_identity_users)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "users", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_identity_groups)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "groups", + "refId": "C" + } + ], + "title": "Keystone stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 4, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_running_vms)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "running vms", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(openstack_nova_total_vms)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total vms", + "refId": "B" + } + ], + "title": "Virtual Machines", + "transparent": true, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_cinder_volumes)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "volumes", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_cinder_snapshots)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "snapshots", + "refId": "B" + } + ], + "title": "Cinder volumes/snapshots", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "avg(openstack_glance_images)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "images", + "refId": "A" + } + ], + "title": "Glance Images", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 500 + }, + { + "id": "displayName", + "value": "Instance" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "max", + "value": 0.5 + }, + { + "id": "displayName", + "value": "Connect Time" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.4 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "decimals", + "value": 2 + }, + { + "id": "displayName", + "value": "Certificate expires in" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red" + }, + { + "color": "semi-dark-yellow", + "value": 604800 + }, + { + "color": "semi-dark-green", + "value": 2592000 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.width", + "value": 220 + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "unit", + "value": "dtdurations" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "displayName", + "value": "HTTP Response" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 301 + }, + { + "color": "red", + "value": 400 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Transfer Time" + }, + { + "id": "max", + "value": 0.5 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.125 + }, + { + "color": "red", + "value": 0.3 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #E" + }, + "properties": [ + { + "id": "displayName", + "value": "TLS Time" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "max", + "value": 1 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.5 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #F" + }, + "properties": [ + { + "id": "displayName", + "value": "Processing Time" + }, + { + "id": "max", + "value": 0.5 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.25 + }, + { + "color": "red", + "value": 0.4 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #G" + }, + "properties": [ + { + "id": "displayName", + "value": "Resolve Time" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "max", + "value": 0.01 + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 0.005 + }, + { + "color": "red", + "value": 0.009 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 341 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "module" + }, + "properties": [ + { + "id": "custom.width", + "value": 109 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "service" + }, + "properties": [ + { + "id": "custom.width", + "value": 102 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 38, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Certificate expires in" + } + ] + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_ssl_earliest_cert_expiry-time()", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_status_code", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_duration_seconds{phase=\"resolve\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_duration_seconds{phase=\"connect\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_duration_seconds{phase=\"tls\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_duration_seconds{phase=\"processing\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "probe_http_duration_seconds{phase=\"transfer\"}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + } + ], + "title": "OpenStack endpoint monitoring", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "instance" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Value #D": false, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "__name__ 3": true, + "__name__ 4": true, + "__name__ 5": true, + "__name__ 6": true, + "instance": false, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true, + "job 5": true, + "job 6": true, + "job 7": true, + "module 1": false, + "module 2": true, + "module 3": true, + "module 4": true, + "module 5": true, + "module 6": true, + "module 7": true, + "phase": true, + "phase 1": true, + "phase 2": true, + "phase 3": true, + "phase 4": true, + "phase 5": true, + "service 2": true, + "service 3": true, + "service 4": true, + "service 5": true, + "service 6": true, + "service 7": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 39, + "tags": [ + "openstack", + "openstack-exporter" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "auto": true, + "auto_count": 10, + "auto_min": "1m", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Overview Dashboard", + "uid": "YZCsB1QmZ", + "version": 9, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/provisioning.yaml b/etc/kayobe/kolla/config/grafana/provisioning.yaml new file mode 100644 index 000000000..07dfa91dd --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/provisioning.yaml @@ -0,0 +1,15 @@ +--- +# # config file version +apiVersion: 1 + +providers: + - name: Ceph + folder: Ceph + type: file + options: + path: /var/lib/grafana/dashboards/ceph + - name: OpenStack + folder: OpenStack + type: file + options: + path: /var/lib/grafana/dashboards/openstack diff --git a/etc/kayobe/kolla/config/haproxy/services.d/os_capacity.cfg b/etc/kayobe/kolla/config/haproxy/services.d/os_capacity.cfg new file mode 100644 index 000000000..cebacb98b --- /dev/null +++ b/etc/kayobe/kolla/config/haproxy/services.d/os_capacity.cfg @@ -0,0 +1,25 @@ +{% if stackhpc_enable_os_capacity | bool %} +{% raw %} +frontend os_capacity_frontend + mode http + http-request del-header X-Forwarded-Proto + option httplog + option forwardfor + http-request set-header X-Forwarded-Proto https if { ssl_fc } +{% if kolla_enable_tls_internal | bool %} + bind {{ kolla_internal_vip_address }}:9090 ssl crt /etc/haproxy/certificates/haproxy-internal.pem +{% else %} + bind {{ kolla_internal_vip_address }}:9090 +{% endif %} + default_backend os_capacity_backend + +backend os_capacity_backend + mode http + +{% for host in groups['monitoring'] %} +{% set host_name = hostvars[host].ansible_facts.hostname %} +{% set host_ip = 'api' | kolla_address(host) %} + server {{ host_name }} {{ host_ip }}:9000 check inter 2000 rise 2 fall 5 +{% endfor %} +{% endraw %} +{% endif %} diff --git a/etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg b/etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg new file mode 100644 index 000000000..50e55a283 --- /dev/null +++ b/etc/kayobe/kolla/config/haproxy/services.d/radosgw_usage_exporter.cfg @@ -0,0 +1,25 @@ +{% if stackhpc_enable_radosgw_usage_exporter | bool %} +{% raw %} +frontend radosgw_usage_exporter_frontend + mode http + http-request del-header X-Forwarded-Proto + option httplog + option forwardfor + http-request set-header X-Forwarded-Proto https if { ssl_fc } +{% if kolla_enable_tls_internal | bool %} + bind {{ kolla_internal_vip_address }}:{{ stackhpc_radosgw_usage_exporter_frontend_port }} ssl crt /etc/haproxy/certificates/haproxy-internal.pem +{% else %} + bind {{ kolla_internal_vip_address }}:{{ stackhpc_radosgw_usage_exporter_frontend_port }} +{% endif %} + default_backend radosgw_usage_exporter_backend + +backend radosgw_usage_exporter_backend + mode http + +{% for host in groups['monitoring'] %} +{% set host_name = hostvars[host].ansible_facts.hostname %} +{% set host_ip = 'api' | kolla_address(host) %} + server {{ host_name }} {{ host_ip }}:{{ stackhpc_radosgw_usage_exporter_backend_port }} check inter 2000 rise 2 fall 5 +{% endfor %} +{% endraw %} +{% endif %} diff --git a/etc/kayobe/kolla/config/ironic/policy.yaml b/etc/kayobe/kolla/config/ironic/policy.yaml new file mode 100644 index 000000000..e37f33113 --- /dev/null +++ b/etc/kayobe/kolla/config/ironic/policy.yaml @@ -0,0 +1,7 @@ +--- +# Retrieve multiple Node records +# GET /nodes +# GET /nodes/detail +# Intended scope(s): system, project +# Overridden: added role:admin +"baremetal:node:list_all": "role:admin or (role:reader and system_scope:all) or (role:service and system_scope:all) or rule:service_role" diff --git a/etc/kayobe/kolla/config/nova.conf b/etc/kayobe/kolla/config/nova.conf new file mode 100644 index 000000000..d1000be56 --- /dev/null +++ b/etc/kayobe/kolla/config/nova.conf @@ -0,0 +1,2 @@ +[libvirt] +hw_machine_type = x86_64=q35 diff --git a/etc/kayobe/kolla/config/nova/nova-api.conf b/etc/kayobe/kolla/config/nova/nova-api.conf new file mode 100644 index 000000000..ba96a2ddb --- /dev/null +++ b/etc/kayobe/kolla/config/nova/nova-api.conf @@ -0,0 +1,4 @@ +[pci] +{% for item in (gpu_group_map | default({})) | dict2items | map(attribute='value') | flatten | unique | list %} +alias = { "vendor_id":"{{ stackhpc_gpu_data[item].vendor_id }}", "product_id":"{{ stackhpc_gpu_data[item].product_id }}", "device_type":"{{ stackhpc_gpu_data[item].device_type }}", "name":"{{ stackhpc_gpu_data[item].resource_name }}" } +{% endfor %} diff --git a/etc/kayobe/kolla/config/nova/nova-compute-ironic.conf b/etc/kayobe/kolla/config/nova/nova-compute-ironic.conf new file mode 100644 index 000000000..9f6db7a55 --- /dev/null +++ b/etc/kayobe/kolla/config/nova/nova-compute-ironic.conf @@ -0,0 +1,4 @@ +{% if kolla_enable_ironic|bool and kolla_nova_compute_ironic_host is not none %} +[DEFAULT] +host = {{ kolla_nova_compute_ironic_static_host_name | mandatory('You must set a static host name to help with service failover. See the operations documentation, Ironic section.') }} +{% endif %} diff --git a/etc/kayobe/kolla/config/nova/nova-compute.conf b/etc/kayobe/kolla/config/nova/nova-compute.conf new file mode 100644 index 000000000..ed83c7101 --- /dev/null +++ b/etc/kayobe/kolla/config/nova/nova-compute.conf @@ -0,0 +1,13 @@ +[pci] +{% raw %} +{% set gpu_list = [] %} +{% for gpu_group in (gpu_group_map | default({})) | dict2items %} +{% if gpu_group.key in group_names %} +{% set _ = gpu_list.append(gpu_group.value) %} +{% endif %} +{% endfor %} +{% for item in gpu_list | flatten | unique %} +device_spec = { "vendor_id":"{{ stackhpc_gpu_data[item].vendor_id }}", "product_id":"{{ stackhpc_gpu_data[item].product_id }}" } +alias = { "vendor_id":"{{ stackhpc_gpu_data[item].vendor_id }}", "product_id":"{{ stackhpc_gpu_data[item].product_id }}", "device_type":"{{ stackhpc_gpu_data[item].device_type }}", "name":"{{ stackhpc_gpu_data[item].resource_name }}" } +{% endfor %} +{% endraw %} diff --git a/etc/kayobe/kolla/config/nova/nova-scheduler.conf b/etc/kayobe/kolla/config/nova/nova-scheduler.conf new file mode 100644 index 000000000..f41bd8548 --- /dev/null +++ b/etc/kayobe/kolla/config/nova/nova-scheduler.conf @@ -0,0 +1,7 @@ +[filter_scheduler] +# Default list plus PciPassthroughFilter +# NOTE(Upgrade): defaults may change in each release. Default values can be +# checked here: +# https://docs.openstack.org/nova/latest/configuration/sample-config.html +enabled_filters = ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,PciPassthroughFilter +available_filters = nova.scheduler.filters.all_filters diff --git a/etc/kayobe/kolla/config/prometheus/blackbox.rules b/etc/kayobe/kolla/config/prometheus/blackbox.rules new file mode 100644 index 000000000..59e3247bc --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/blackbox.rules @@ -0,0 +1,79 @@ +# https://awesome-prometheus-alerts.grep.to/rules#blackbox-1 + +{% raw %} + +groups: +- name: Blackbox + rules: + - alert: BlackboxProbeFailed + expr: probe_success == 0 + for: 2m + labels: + severity: critical + annotations: + summary: Blackbox probe failed (instance {{ $labels.instance }}) + description: "Probe failed" + + - alert: BlackboxSlowProbe + expr: avg_over_time(probe_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox slow probe (instance {{ $labels.instance }}) + description: "Blackbox probe took more than 1s to complete" + + - alert: BlackboxProbeHttpFailure + expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400 + for: 2m + labels: + severity: critical + annotations: + summary: Blackbox probe HTTP failure (instance {{ $labels.instance }}) + description: "HTTP status code is not 200-399. Blackbox exporter got status code: {{ $value }}" + + - alert: BlackboxSslCertificateWillExpireSoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30 + for: 0m + labels: + severity: warning + annotations: + summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}) + description: "SSL certificate expires in less than 30 days" + + - alert: BlackboxSslCertificateWillExpireSoon + expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 3 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}) + description: "SSL certificate expires in less than 3 days" + + - alert: BlackboxSslCertificateExpired + expr: probe_ssl_earliest_cert_expiry - time() <= 0 + for: 0m + labels: + severity: critical + annotations: + summary: Blackbox SSL certificate expired (instance {{ $labels.instance }}) + description: "SSL certificate has expired already" + + - alert: BlackboxProbeSlowHttp + expr: avg_over_time(probe_http_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox probe slow HTTP (instance {{ $labels.instance }}) + description: "HTTP request took more than 1s" + + - alert: BlackboxProbeSlowPing + expr: avg_over_time(probe_icmp_duration_seconds[1m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: Blackbox probe slow ping (instance {{ $labels.instance }}) + description: "Blackbox ping took more than 1s" +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/cadvisor.rules b/etc/kayobe/kolla/config/prometheus/cadvisor.rules new file mode 100644 index 000000000..726517452 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/cadvisor.rules @@ -0,0 +1,45 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Cadvisor + rules: + + - alert: ContainerKilled + expr: time() - container_last_seen > 61 + for: 5m + labels: + severity: warning + annotations: + summary: "Container killed (instance {{ $labels.instance }})" + description: "A container has disappeared" + + - alert: ContainerCpuUsage + expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) BY (instance, name) * 100) > 3000 + for: 2m + labels: + severity: warning + annotations: + summary: "Container CPU usage (instance {{ $labels.instance }})" + description: "Container CPU usage is above 300%" + + - alert: ContainerVolumeUsage + expr: (1 - (sum(container_fs_inodes_free{name!=""}) BY (instance) / sum(container_fs_inodes_total{name!=""}) BY (instance)) * 100) > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "Container Volume usage (instance {{ $labels.instance }})" + description: "Container Volume usage is above 80%" + + - alert: ContainerHighThrottleRate + expr: rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Container high throttle rate (instance {{ $labels.instance }})" + description: "Container is being throttled" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/ceph.rules b/etc/kayobe/kolla/config/prometheus/ceph.rules new file mode 100644 index 000000000..88b04f1e6 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/ceph.rules @@ -0,0 +1,207 @@ +# Modified official set of upstream alerts https://github.com/ceph/ceph/blob/octopus/monitoring/prometheus/alerts/ceph_default_alerts.yml + +{% raw %} + +groups: +- name: health + rules: + - alert: HealthError + expr: ceph_health_status == 2 + for: 5m + labels: + severity: critical + annotations: + description: > + Ceph in HEALTH_ERROR state for more than 5 minutes. + Please check "ceph health detail" for more information. + + - alert: HealthWarn + expr: ceph_health_status == 1 + for: 15m + labels: + severity: warning + annotations: + description: > + Ceph has been in HEALTH_WARN for more than 15 minutes. + Please check "ceph health detail" for more information. + +- name: mon + rules: + - alert: LowMonitorQuorumCount + expr: sum(ceph_mon_quorum_status) < 3 + labels: + severity: critical + annotations: + description: | + Monitor count in quorum is below three. + + Only {{ $value }} of {{ with query "count(ceph_mon_quorum_status)" }}{{ . | first | value }}{{ end }} monitors are active. + + The following monitors are down: + {{- range query "(ceph_mon_quorum_status == 0) + on(ceph_daemon) group_left(hostname) (ceph_mon_metadata * 0)" }} + - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} + {{- end }} + +- name: osd + rules: + - alert: 10percentOSDsDown + expr: count(ceph_osd_up == 0) / count(ceph_osd_up) * 100 >= 10 + for: 15m + labels: + severity: critical + annotations: + description: | + {{ $value | humanize }}% or {{ with query "count(ceph_osd_up == 0)" }}{{ . | first | value }}{{ end }} of {{ with query "count(ceph_osd_up)" }}{{ . | first | value }}{{ end }} OSDs are down (≥ 10%). + + The following OSDs are down: + {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0" }} + - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} + {{- end }} + + - alert: OSDDdown + expr: count(ceph_osd_up == 0) > 0 + for: 15m + labels: + severity: warning + annotations: + description: | + {{ $s := "" }}{{ if gt $value 1.0 }}{{ $s = "s" }}{{ end }} + {{ $value }} OSD{{ $s }} down for more than 15 minutes. + + {{ $value }} of {{ query "count(ceph_osd_up)" | first | value }} OSDs are down. + + The following OSD{{ $s }} {{ if eq $s "" }}is{{ else }}are{{ end }} down: + {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0"}} + - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} + {{- end }} + + - alert: OSDsNearFull + expr: | + ( + ((ceph_osd_stat_bytes_used / ceph_osd_stat_bytes) and on(ceph_daemon) ceph_osd_up == 1) + * on(ceph_daemon) group_left(hostname) ceph_osd_metadata + ) * 100 > 90 + for: 5m + labels: + severity: critical + annotations: + description: > + OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} is + dangerously full: {{ $value | humanize }}% + + - alert: FlappingOSD + expr: | + ( + rate(ceph_osd_up[5m]) + * on(ceph_daemon) group_left(hostname) ceph_osd_metadata + ) * 60 > 1 + labels: + severity: warning + annotations: + description: > + OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was + marked down and back up at {{ $value | humanize }} times once a + minute for 5 minutes. + +- name: pgs + rules: + - alert: PgsInactive + expr: ceph_pool_metadata * on(pool_id,instance) group_left() (ceph_pg_total - ceph_pg_active) > 0 + for: 5m + labels: + severity: critical + annotations: + description: > + {{ $value }} PGs have been inactive for more than 5 minutes in pool {{ $labels.name }}. + Inactive placement groups aren't able to serve read/write + requests. + + - alert: PgsUnclean + expr: ceph_pool_metadata * on(pool_id,instance) group_left() (ceph_pg_total - ceph_pg_clean - ceph_pg_backfilling - ceph_pg_backfill_wait) > 0 + for: 15m + labels: + severity: warning + annotations: + description: > + {{ $value }} PGs haven't been clean for more than 15 minutes in pool {{ $labels.name }}. + Unclean PGs haven't been able to completely recover from a + previous failure. + +- name: nodes + rules: + - alert: RootVolumeFull + expr: node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100 < 5 + for: 5m + labels: + severity: critical + annotations: + description: "Root volume (OSD and MON store) is dangerously full: {{ $value | humanize }}% free." + + # alert on nic packet errors and drops rates > alertmanager_packet_drop_threshold packet/s + - alert: NetworkPacketsDropped + expr: irate(node_network_receive_drop_total{device!~"lo|br.*|.*-ovs|tap.*|ha.*|qc.*|qr.*|qg.*"}[5m]) + irate(node_network_transmit_drop_total{device!~"lo|br.*|.*-ovs|tap.*|ha.*|qc.*|qr.*|qg.*"}[5m]) > {% endraw %}{{ alertmanager_packet_drop_threshold }}{% raw %} + labels: + severity: warning + annotations: + description: > + Node {{ $labels.instance }} experiences packet drop > {% endraw %}{{ alertmanager_packet_drop_threshold }}{% raw %} + packet/s on interface {{ $labels.device }}. + + - alert: NetworkPacketErrors + expr: | + irate(node_network_receive_errs_total{device!="lo"}[5m]) + + irate(node_network_transmit_errs_total{device!="lo"}[5m]) > {% endraw %}{{ alertmanager_packet_errors_threshold }}{% raw %} + labels: + severity: warning + annotations: + description: > + Node {{ $labels.instance }} experiences packet errors > {% endraw %}{{ alertmanager_packet_errors_threshold }}{% raw %} + packet/s on interface {{ $labels.device }}. + + - alert: StorageFillingUp + expr: | + predict_linear(node_filesystem_avail_bytes[2d], 3600 * 24 * 5) * + on(instance) group_left(nodename) node_uname_info < 0 + labels: + severity: warning + annotations: + description: > + Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} + will be full in less than 5 days assuming the average fill-up + rate of the past 48 hours. + +- name: pools + rules: + - alert: PoolFull + expr: | + ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail) + * on(pool_id) group_right ceph_pool_metadata * 100 > 90 + labels: + severity: critical + annotations: + description: "Pool {{ $labels.name }} at {{ $value | humanize }}% capacity." + + - alert: PoolFillingUp + expr: | + ( + predict_linear(ceph_pool_stored[2d], 3600 * 24 * 5) + >= ceph_pool_stored + ceph_pool_max_avail + ) * on(pool_id) group_left(name) ceph_pool_metadata + labels: + severity: warning + annotations: + description: > + Pool {{ $labels.name }} will be full in less than 5 days + assuming the average fill-up rate of the past 48 hours. + +- name: healthchecks + rules: + - alert: SlowOSDOps + expr: ceph_healthcheck_slow_ops > 0 + for: 30s + labels: + severity: warning + annotations: + description: "{{ $value }} OSD requests are taking too long to process (osd_op_complaint_time exceeded)" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/elasticsearch.rules b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules new file mode 100644 index 000000000..42a196b9a --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/elasticsearch.rules @@ -0,0 +1,99 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Elasticsearch + rules: + + - alert: ElasticsearchHeapUsageTooHigh + expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90 + for: 2m + labels: + severity: critical + annotations: + summary: "Elasticsearch Heap Usage Too High (instance {{ $labels.instance }})" + description: "The heap usage is over 90%" + + - alert: ElasticsearchHeapUsageWarning + expr: (elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80 + for: 2m + labels: + severity: warning + annotations: + summary: "Elasticsearch Heap Usage warning (instance {{ $labels.instance }})" + description: "The heap usage is over 80%" + + - alert: ElasticsearchDiskOutOfSpace + expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10 + for: 0m + labels: + severity: critical + annotations: + summary: "Elasticsearch disk out of space (instance {{ $labels.instance }})" + description: "The disk usage is over 90%" + + - alert: ElasticsearchDiskSpaceLow + expr: elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20 + for: 2m + labels: + severity: warning + annotations: + summary: "Elasticsearch disk space low (instance {{ $labels.instance }})" + description: "The disk usage is over 80%" + + - alert: ElasticsearchClusterRed + expr: elasticsearch_cluster_health_status{color="red"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Elasticsearch Cluster Red (instance {{ $labels.instance }})" + description: "Elastic Cluster Red status" + + - alert: ElasticsearchClusterYellow + expr: elasticsearch_cluster_health_status{color="yellow"} == 1 + for: 15m + labels: + severity: warning + annotations: + summary: "Elasticsearch Cluster Yellow (instance {{ $labels.instance }})" + description: "Elastic Cluster Yellow status" + + - alert: ElasticsearchRelocatingShardsTooLong + expr: elasticsearch_cluster_health_relocating_shards > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Elasticsearch relocating shards too long (instance {{ $labels.instance }})" + description: "Elasticsearch has been relocating shards for 15min" + + - alert: ElasticsearchInitializingShardsTooLong + expr: elasticsearch_cluster_health_initializing_shards > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Elasticsearch initializing shards too long (instance {{ $labels.instance }})" + description: "Elasticsearch has been initializing shards for 15 min" + + - alert: ElasticsearchUnassignedShards + expr: elasticsearch_cluster_health_unassigned_shards > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Elasticsearch unassigned shards (instance {{ $labels.instance }})" + description: "Elasticsearch has unassigned shards" + + - alert: ElasticsearchPendingTasks + expr: elasticsearch_cluster_health_number_of_pending_tasks > 0 + for: 15m + labels: + severity: warning + annotations: + summary: "Elasticsearch pending tasks (instance {{ $labels.instance }})" + description: "Elasticsearch has pending tasks. Cluster works slowly." + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/fluentd.rules b/etc/kayobe/kolla/config/prometheus/fluentd.rules new file mode 100644 index 000000000..de8f67e65 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/fluentd.rules @@ -0,0 +1,13 @@ +{% raw %} +groups: +- name: Fluentd + rules: + - alert: FluentdBufferTooLarge + expr: (fluentd_output_status_buffer_total_bytes / 1024^2) > 128 + for: 15m + labels: + severity: warning + annotations: + summary: "Fluentd at {{ $labels.instance }} reports large queue buffers" + description: "Fluentd queue buffers on {{ $labels.instance }} are using {{ $value }} MiB." +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/haproxy.rules b/etc/kayobe/kolla/config/prometheus/haproxy.rules new file mode 100644 index 000000000..0e4218023 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/haproxy.rules @@ -0,0 +1,27 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: HAProxy + rules: + + - alert: HaproxyBackendDown + expr: haproxy_backend_status{state="DOWN"} == 1 + for: 1m + labels: + severity: critical + annotations: + summary: "HAProxy backend down (instance {{ $labels.instance }})" + description: "HAProxy backend is down" + + - alert: HaproxyServerDown + expr: haproxy_server_status{state="DOWN"} == 1 + for: 1m + labels: + severity: critical + annotations: + summary: "HAProxy server down at {{ $labels.server }}" + description: "HAProxy server for {{ $labels.proxy }} is down at {{ $labels.server }}" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/loadbalancer.rules b/etc/kayobe/kolla/config/prometheus/loadbalancer.rules new file mode 100644 index 000000000..efabab5be --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/loadbalancer.rules @@ -0,0 +1,35 @@ +{% raw %} + +groups: +- name: Loadbalancer + rules: + + - alert: LoadBalancerError + expr: openstack_loadbalancer_loadbalancer_status{operating_status="ERROR"} > 0 OR openstack_loadbalancer_loadbalancer_status{provisioning_status="ERROR"} > 0 + for: 1m + labels: + severity: alert + annotations: + summary: "Load balancer in error state" + description: "Load balancer {{ $labels.id }} is in an error state" + + - alert: LoadBalancerDegraded + expr: openstack_loadbalancer_loadbalancer_status{operating_status="DEGRADED"} > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Load balancer in degraded state" + description: "Load balancer {{ $labels.id }} is in a degraded state" + + - alert: AmphoraError + expr: openstack_loadbalancer_amphora_status{status="ERROR"} > 0 + for: 1m + labels: + severity: alert + annotations: + summary: "Amphora in error state" + description: "Amphora {{ $labels.id }} is in an error state" + + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/mysql.rules b/etc/kayobe/kolla/config/prometheus/mysql.rules new file mode 100644 index 000000000..86134207b --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/mysql.rules @@ -0,0 +1,81 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Mysql + rules: + + - alert: MysqlDown + expr: mysql_up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "MySQL down (instance {{ $labels.instance }})" + description: "MySQL instance is down on {{ $labels.instance }}" + + - alert: MysqlTooManyConnections + expr: avg by (instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL too many connections (instance {{ $labels.instance }})" + description: "More than 80% of MySQL connections are in use on {{ $labels.instance }}" + + - alert: MysqlHighThreadsRunning + expr: avg by (instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 60 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL high threads running (instance {{ $labels.instance }})" + description: "More than 60% of MySQL connections are in running state on {{ $labels.instance }}" + + - alert: MysqlSlaveIoThreadNotRunning + expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "MySQL Slave IO thread not running (instance {{ $labels.instance }})" + description: "MySQL Slave IO thread not running on {{ $labels.instance }}" + + - alert: MysqlSlaveSqlThreadNotRunning + expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "MySQL Slave SQL thread not running (instance {{ $labels.instance }})" + description: "MySQL Slave SQL thread not running on {{ $labels.instance }}" + + - alert: MysqlSlaveReplicationLag + expr: mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL Slave replication lag (instance {{ $labels.instance }})" + description: "MySQL replication lag on {{ $labels.instance }}" + + - alert: MysqlSlowQueries + expr: increase(mysql_global_status_slow_queries[1m]) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: "MySQL slow queries (instance {{ $labels.instance }})" + description: "MySQL server is having some slow queries." + + - alert: MysqlRestarted + expr: mysql_global_status_uptime < 60 + for: 1m + labels: + severity: warning + annotations: + summary: "MySQL restarted (instance {{ $labels.instance }})" + description: "MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}." + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/notifications.tmpl b/etc/kayobe/kolla/config/prometheus/notifications.tmpl new file mode 100644 index 000000000..b1abadc99 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/notifications.tmpl @@ -0,0 +1,32 @@ +{% raw %} + +{{ define "slack.default.title" -}} + [{{ .Status | toUpper -}} + {{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{- end -}} + ] {{ .CommonLabels.alertname }} +{{- end }} + +{{ define "slack.default.text" -}} + +{{ range .Alerts }} +*Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}` +*Description:* {{ .Annotations.description }} +*Details:* + {{ range .Labels.SortedPairs -}} + • *{{ .Name }}:* `{{ .Value }}` + {{ end }} +{{ end }} + +{{ end }} + +{{ define "__alert_silence_link" -}} + {{ .ExternalURL }}/#/silences/new?filter=%7B + {{- range .CommonLabels.SortedPairs -}} + {{- if ne .Name "alertname" -}} + {{- .Name }}%3D"{{- .Value -}}"%2C%20 + {{- end -}} + {{- end -}} + alertname%3D"{{ .CommonLabels.alertname }}"%7D +{{- end }} + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/openstack.rules b/etc/kayobe/kolla/config/prometheus/openstack.rules new file mode 100644 index 000000000..ba78e76a3 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/openstack.rules @@ -0,0 +1,24 @@ +{% raw %} + +groups: +- name: OpenStack + rules: + - alert: OpenStackServiceDown + expr: (sum({__name__=~"openstack.+_state", job="openstack_exporter"} == 0) by (hostname, service)) + for: 1m + labels: + severity: alert + annotations: + summary: "{{ $labels.service }} at {{ $labels.hostname }} is down" + description: "OpenStack service {{ $labels.service }} at {{ $labels.hostname }} is down" +- name: Routers + rules: + - alert: OpenStackRouterDown + expr: count by (router_id) (openstack_neutron_l3_agent_of_router{ha_state="active"}) != 1 + for: 1m + labels: + severity: alert + annotations: + summary: "The router {{ $labels.router_id }} is not active on exactly one agent" + description: "The router {{ $labels.router_id }} should be active on exactly one agent. It can either be active on multiple agents or not active at all." +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example new file mode 100644 index 000000000..e4575f831 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.msteamsv2.yml.example @@ -0,0 +1,29 @@ +--- +global: + resolve_timeout: 5m + smtp_require_tls: true + +route: + receiver: 'msteamsv2-notifications' + group_by: [alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + + routes: + - matchers: + - severity=~"critical|alert" + receiver: 'msteamvs2-critical-notifications' + +receivers: + - name: 'msteamsv2-notifications' + msteamsv2_configs: + - webhook_url: '{{ secrets_msteams_notification_channel_url | default('https://prod-01.westeurope.logic.azure.com/workflows/') }}' + send_resolved: true + - name: 'msteamsv2-critical-notifications' + msteamsv2_configs: + - webhook_url: '{{ secrets_msteams_notification_critical_channel_url | default('https://prod-01.westeurope.logic.azure.com/workflows/') }}' + send_resolved: true + +templates: + - '/etc/prometheus/*.tmpl' diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example new file mode 100644 index 000000000..0548a8286 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus-alertmanager.yml.example @@ -0,0 +1,47 @@ +--- +global: + resolve_timeout: 5m + smtp_require_tls: true + slack_api_url: 'https://hooks.slack.com/services/example/alerts/webhook' + +route: + receiver: 'slack-notifications' + group_by: [alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + + routes: + - matchers: + - severity=~"critical|alert" + receiver: 'slack-critical-notifications' + + +receivers: + - name: 'slack-notifications' + slack_configs: + - channel: '#notifications' + actions: + - type: button + text: 'Silence 🔕' + url: {{ '{% raw %}' }}{% raw %} '{{ template "__alert_silence_link" . }}' +{% endraw %}{{ '{% endraw %}' }} + send_resolved: true + http_config: + proxy_url: http://1.2.3.4:3128 + - name: 'slack-critical-notifications' + slack_configs: + - channel: '#notifications-critical' + actions: + - type: button + text: 'Silence 🔕' + url: {{ '{% raw %}' }}{% raw %} '{{ template "__alert_silence_link" . }}' +{% endraw %}{{ '{% endraw %}' }} + send_resolved: true + http_config: + proxy_url: http://1.2.3.4:3128 + api_url: 'https://hooks.slack.com/services/example/alerts/webhook-critical' + + +templates: + - '/etc/prometheus/*.tmpl' diff --git a/etc/kayobe/kolla/config/prometheus/prometheus-blackbox-exporter.yml b/etc/kayobe/kolla/config/prometheus/prometheus-blackbox-exporter.yml new file mode 100644 index 000000000..f4cf32737 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus-blackbox-exporter.yml @@ -0,0 +1,56 @@ +# yamllint disable-file +modules: + http_2xx: + prober: http + os_endpoint: + prober: http + http: + valid_status_codes: [200,300] + headers: + Content-Type: application/json + fail_if_body_not_matches_regexp: + - "versions" + http_post_2xx: + prober: http + http: + method: POST + tcp_connect: + prober: tcp + tls_connect: + prober: tcp + tcp: + tls: true + ssh_banner: + prober: tcp + tcp: + query_response: + - expect: "^SSH-2.0-" + icmp: + prober: icmp + http_2xx_opensearch_dashboards: + prober: http + timeout: 5s + http: + basic_auth: +{% raw %} + username: {{ opensearch_dashboards_user }} + password: {{ opensearch_dashboards_password }} +{% endraw %} + http_2xx_prometheus: + prober: http + timeout: 5s + http: + basic_auth: +{% raw %} + username: admin + password: {{ prometheus_password }} +{% endraw %} + http_2xx_alertmanager: + prober: http + timeout: 5s + http: + basic_auth: +{% raw %} + username: {{ prometheus_alertmanager_user }} + password: {{ prometheus_alertmanager_password }} +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.rules b/etc/kayobe/kolla/config/prometheus/prometheus.rules new file mode 100644 index 000000000..20e1b303a --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus.rules @@ -0,0 +1,234 @@ +# Taken from https://awesome-prometheus-alerts.grep.to/rules + +{% raw %} + +groups: +- name: Prometheus + rules: + + - alert: PrometheusTargetMissing + expr: up{job!="redfish-exporter-seed"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Prometheus target missing (instance {{ $labels.instance }})" + description: "A Prometheus target has disappeared. An exporter might have crashed." + + - alert: PrometheusTargetMissing + expr: up{job="redfish-exporter-seed"} == 0 + for: 15m + labels: + severity: critical + annotations: + summary: "Prometheus target missing (instance {{ $labels.instance }})" + description: "A Prometheus target has disappeared. An exporter might have crashed." + + - alert: PrometheusAllTargetsMissing + expr: count by (job) (up) == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus all targets missing (instance {{ $labels.instance }})" + description: "A Prometheus job does not have living target anymore." + + - alert: PrometheusConfigurationReloadFailure + expr: prometheus_config_last_reload_successful != 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus configuration reload failure (instance {{ $labels.instance }})" + description: "Prometheus configuration reload error" + + - alert: PrometheusTooManyRestarts + expr: changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus too many restarts (instance {{ $labels.instance }})" + description: "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping." + + - alert: PrometheusAlertmanagerConfigurationReloadFailure + expr: alertmanager_config_last_reload_successful != 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus AlertManager configuration reload failure (instance {{ $labels.instance }})" + description: "AlertManager configuration reload error" + + - alert: PrometheusAlertmanagerConfigNotSynced + expr: count(count_values("config_hash", alertmanager_config_hash)) > 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus AlertManager config not synced (instance {{ $labels.instance }})" + description: "Configurations of AlertManager cluster instances are out of sync" + + # - alert: PrometheusAlertmanagerE2eDeadManSwitch + # expr: vector(1) + # for: 1m + # labels: + # severity: critical + # annotations: + # summary: "Prometheus AlertManager E2E dead man switch (instance {{ $labels.instance }})" + # description: "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager." + + - alert: PrometheusNotConnectedToAlertmanager + expr: prometheus_notifications_alertmanagers_discovered < 1 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus not connected to alertmanager (instance {{ $labels.instance }})" + description: "Prometheus cannot connect the alertmanager" + + - alert: PrometheusRuleEvaluationFailures + expr: increase(prometheus_rule_evaluation_failures_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus rule evaluation failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts." + + - alert: PrometheusTemplateTextExpansionFailures + expr: increase(prometheus_template_text_expansion_failures_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus template text expansion failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} template text expansion failures" + + - alert: PrometheusRuleEvaluationSlow + expr: prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus rule evaluation slow (instance {{ $labels.instance }})" + description: "Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query." + + - alert: PrometheusNotificationsBacklog + expr: min_over_time(prometheus_notifications_queue_length[10m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus notifications backlog (instance {{ $labels.instance }})" + description: "The Prometheus notification queue has not been empty for 10 minutes" + + - alert: PrometheusAlertmanagerNotificationFailing + expr: rate(alertmanager_notifications_failed_total[1m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus AlertManager notification failing (instance {{ $labels.instance }})" + description: "Alertmanager is failing sending notifications" + + - alert: PrometheusTargetEmpty + expr: prometheus_sd_discovered_targets == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus target empty (instance {{ $labels.instance }})" + description: "Prometheus has no target in service discovery" + + - alert: PrometheusTargetScrapingSlow + expr: prometheus_target_interval_length_seconds{quantile="0.9",interval="1m0s"} > 61 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus target scraping slow (instance {{ $labels.instance }})" + description: "Prometheus is scraping exporters slowly" + + - alert: PrometheusLargeScrape + expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus large scrape (instance {{ $labels.instance }})" + description: "Prometheus has many scrapes that exceed the sample limit" + + - alert: PrometheusTargetScrapeDuplicate + expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus target scrape duplicate (instance {{ $labels.instance }})" + description: "Prometheus has many samples rejected due to duplicate timestamps but different values" + + - alert: PrometheusTsdbCheckpointCreationFailures + expr: increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB checkpoint creation failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} checkpoint creation failures" + + - alert: PrometheusTsdbCheckpointDeletionFailures + expr: increase(prometheus_tsdb_checkpoint_deletions_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB checkpoint deletion failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} checkpoint deletion failures" + + - alert: PrometheusTsdbCompactionsFailed + expr: increase(prometheus_tsdb_compactions_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB compactions failed (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB compactions failures" + + - alert: PrometheusTsdbHeadTruncationsFailed + expr: increase(prometheus_tsdb_head_truncations_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB head truncations failed (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB head truncation failures" + + - alert: PrometheusTsdbReloadFailures + expr: increase(prometheus_tsdb_reloads_failures_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB reload failures (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB reload failures" + + - alert: PrometheusTsdbWalCorruptions + expr: increase(prometheus_tsdb_wal_corruptions_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB WAL corruptions (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB WAL corruptions" + + - alert: PrometheusTsdbWalTruncationsFailed + expr: increase(prometheus_tsdb_wal_truncations_failed_total[3m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Prometheus TSDB WAL truncations failed (instance {{ $labels.instance }})" + description: "Prometheus encountered {{ $value }} TSDB WAL truncation failures" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/60-redfish.yml b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/60-redfish.yml new file mode 100644 index 000000000..300305729 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/60-redfish.yml @@ -0,0 +1,27 @@ +# yamllint disable-file +--- +{% if seed_redfish_exporter_container_enabled | bool %} +scrape_configs: + - job_name: redfish-exporter + params: + collectlogs: ['false'] + metrics_path: /redfish + scrape_timeout: 300s + scrape_interval: {{ stackhpc_redfish_exporter_scrape_interval }}s + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: "{{ lookup('vars', admin_oc_net_name ~ '_ips')[groups.seed.0] }}:9610" + static_configs: +{% for host in (groups.get('redfish_exporter_targets', []) | sort) %} + - targets: + - '{{ hostvars[host]["redfish_exporter_target_address"] }}' + labels: + server: '{{ host }}' + env: "{{ kayobe_environment | default('openstack') }}" + group: "{{ hostvars[host]['redfish_exporter_scrape_group'] | default('overcloud') }}" +{% endfor %} +{% endif %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml new file mode 100644 index 000000000..77e224bbc --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml @@ -0,0 +1,18 @@ +# yamllint disable-file +--- +{% if stackhpc_enable_os_capacity | bool %} +{% raw %} +scrape_configs: + - job_name: os_capacity + static_configs: + - targets: + - '{{ kolla_internal_fqdn | put_address_in_context('url') }}:9090' +{% endraw %} + scrape_interval: "{{ stackhpc_os_capacity_scrape_interval }}s" + scrape_timeout: 10m +{% raw %} +{% if kolla_enable_tls_internal | bool %} + scheme: https +{% endif %} +{% endraw %} +{% endif %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml new file mode 100644 index 000000000..7c3a204bd --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/80-radosgw-exporter.yml @@ -0,0 +1,22 @@ +# yamllint disable-file +--- +{% if stackhpc_enable_radosgw_usage_exporter | bool %} +{% raw %} +scrape_configs: + - job_name: ceph_radosgw_usage_exporter + honor_labels: true + scrape_interval: 15s + metric_relabel_configs: + - replacement: ${1} + source_labels: [owner,user] + target_label: tenant_id + separator: "" + regex: (.+) + static_configs: + - targets: + - "{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ stackhpc_radosgw_usage_exporter_frontend_port }}" +{% if kolla_enable_tls_internal | bool %} + scheme: https +{% endif %} +{% endraw %} +{% endif %} diff --git a/etc/kayobe/kolla/config/prometheus/rabbitmq.rules b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules new file mode 100644 index 000000000..82022d72a --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules @@ -0,0 +1,80 @@ +# https://awesome-prometheus-alerts.grep.to/rules#rabbitmq-1 + +{% raw %} + +groups: +- name: rabbitmq.rules + rules: + - alert: RabbitMQNodeDown + expr: sum(rabbitmq_build_info{instance!=""}) < {% endraw %}{{ alertmanager_number_of_rabbitmq_nodes }}{% raw %} + for: 30m + labels: + severity: critical + annotations: + description: RabbitMQ node down on {{ $labels.instance }} + - alert: RabbitMQConsumersLowUtilization + expr: rabbitmq_queue_consumer_utilisation < 0.4 + for: 5m + labels: + severity: warning + annotations: + description: RabbitMQ consumers message consumption speed is low on {{ $labels.instance }} + - alert: RabbitMQNodeNotDistributed + expr: erlang_vm_dist_node_state < 3 + for: 5m + labels: + severity: critical + annotations: + description: RabbitMQ node not distributed on node {{ $labels.instance }} + - alert: RabbitMQMemoryHigh + expr: rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes + * 100 > 90 + for: 2m + labels: + severity: warning + annotations: + description: RabbitMQ memory too high on {{ $labels.instance }} + - alert: RabbitMQFileDescriptorsUsage + expr: rabbitmq_process_open_fds / rabbitmq_process_max_fds * 100 > 90 + for: 2m + labels: + severity: warning + annotations: + description: RabbitMQ file descriptors usage on {{ $labels.instance }} + - alert: RabbitMQTooMuchReady + expr: rabbitmq_queue_messages_ready > 100 + for: 1m + labels: + severity: warning + annotations: + description: RabbitMQ too much ready on {{ $labels.instance }} + - alert: RabbitMQTooMuchUnack + expr: rabbitmq_queue_messages_unacked > 100 + for: 1m + labels: + severity: warning + annotations: + description: RabbitMQ too much unack on {{ $labels.instance }} + - alert: RabbitMQTooMuchConnections + expr: rabbitmq_connections > {% endraw %}{{ (1500 * groups['controllers'] | length + 50 * groups['compute'] | length) }}{% raw %} + for: 2m + labels: + severity: warning + annotations: + description: RabbitMQ too much connections on {{ $labels.instance }} + - alert: RabbitMQNoQueueConsumer + expr: rabbitmq_queue_consumers < 1 + for: 1m + labels: + severity: warning + annotations: + description: RabbitMQ no queue consumer on {{ $labels.instance }} + - alert: RabbitMQUnroutableMessages + expr: increase(rabbitmq_channel_messages_unroutable_returned_total[1m]) > 0 or increase(rabbitmq_channel_messages_unroutable_dropped_total[1m]) > 0 + for: 2m + labels: + severity: warning + annotations: + description: RabbitMQ unroutable messages on {{ $labels.instance }} + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/smart.rules b/etc/kayobe/kolla/config/prometheus/smart.rules new file mode 100644 index 000000000..cd7dbb3d6 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/smart.rules @@ -0,0 +1,32 @@ +{% raw %} + +groups: +- name: Smart + rules: + + - alert: DiskSmartStatusUnhealthy + expr: smartmon_device_smart_healthy < 1 + for: 10m + labels: + severity: alert + annotations: + summary: "SMART monitor reports bad disk on (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} is reporting unhealthy for the disk at {{ $labels.disk }}. Disk serial number is: {{ $labels.serial_number }}" + + - alert: DWPDTooHigh + expr: (delta(nvme_data_units_written_total[30d])*512000 / nvme_physical_size_bytes) / 30 > nvme_rated_dwpd + labels: + severity: alert + annotations: + summary: "High 30-Day Average DWPD for {{ $labels.instance }}" + description: "The 30-Day average for Disk Writes Per Day for disk {{ $labels.device }} on {{ $labels.instance }} exceeds the rated DWPD" + + - alert: DWPDTooHighWarning + expr: (delta(nvme_data_units_written_total[7d])*512000 / nvme_physical_size_bytes) / 7 > nvme_rated_dwpd + labels: + severity: warning + annotations: + summary: "High 7-Day Average DWPD for {{ $labels.instance }}" + description: "The 7-day average for Disk Writes Per Day for disk {{ $labels.device }} on {{ $labels.instance }} exceeds the rated DWPD" + +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/system.rules b/etc/kayobe/kolla/config/prometheus/system.rules new file mode 100644 index 000000000..f14120b8f --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/system.rules @@ -0,0 +1,169 @@ +# https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware-1 + +{% raw %} + +groups: +- name: Node + rules: + + - alert: LowDiskSpace + expr: ( ( node_filesystem_free_bytes - node_filesystem_avail_bytes ) / node_filesystem_free_bytes ) * 100 >= 80 + for: 1m + labels: + severity: alert + annotations: + summary: "Prometheus exporter at {{ $labels.instance }} reports low disk space" + description: "{{ $labels.device }} is {{ $value }}% full." + + - alert: LowMemory + expr: (node_memory_MemAvailable_bytes / 1024^3) < {% endraw %}{{ alertmanager_low_memory_threshold_gib }}{% raw %} + for: 1m + labels: + severity: alert + annotations: + summary: "Prometheus exporter at {{ $labels.instance }} reports low memory" + description: "Available memory is {{ $value }} GiB." + + - alert: LowSwapSpace + expr: node_memory_SwapTotal_bytes > 0 and (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_warning_threshold_ratio }}{% raw %} + for: 1m + labels: + severity: warning + annotations: + summary: "Swap space at {{ $labels.instance }} reports low memory" + description: "Available swap space is {{ $value | humanizePercentage }}. Running out of swap space causes OOM Kills." + + - alert: LowSwapSpace + expr: node_memory_SwapTotal_bytes > 0 and (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_critical_threshold_ratio }}{% raw %} + for: 1m + labels: + severity: critical + annotations: + summary: "Swap space at {{ $labels.instance }} reports low memory" + description: "Available swap space is {{ $value | humanizePercentage }}. Running out of swap space causes OOM Kills." + + - alert: HostOomKillDetected + expr: increase(node_vmstat_oom_kill[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host OOM kill detected (instance {{ $labels.instance }})" + description: "OOM kill detected" + + - alert: Overheating + expr: node_hwmon_temp_celsius >= node_hwmon_temp_max_celsius + for: 1m + labels: + severity: warning + annotations: + summary: "Prometheus exporter at {{ $labels.instance }} reports overheating" + description: "Sensor {{ $labels.chip }} reports {{ $value }} degrees celcius." + + - alert: HostNodeOvertemperatureAlarm + expr: node_hwmon_temp_crit_alarm_celsius == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Host node overtemperature alarm (instance {{ $labels.instance }})" + description: "Physical node temperature alarm triggered" + + - alert: InstanceDown + expr: up{job="node"} == 0 + for: 5m + labels: + severity: alert + annotations: + summary: "Instance {{$labels.instance}} down" + description: "{{$labels.instance}} has been down for more than 5 minutes." + + - alert: HostEdacCorrectableErrorsDetected + expr: increase(node_edac_correctable_errors_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host EDAC Correctable Errors detected (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes." + + - alert: HostEdacUncorrectableErrorsDetected + expr: node_edac_uncorrectable_errors_total > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes." + + - alert: HostClockSkew + expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0) + for: 2m + labels: + severity: warning + annotations: + summary: Host clock skew (instance {{ $labels.instance }}) + description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host." + + - alert: HostClockNotSynchronising + expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16 + for: 2m + labels: + severity: warning + annotations: + summary: Host clock not synchronising (instance {{ $labels.instance }}) + description: "Clock not synchronising. Ensure NTP is configured on this host." + + - alert: HostNetworkBondDegraded + expr: (node_bonding_active - node_bonding_slaves) != 0 + for: 2m + labels: + severity: warning + annotations: + summary: Host network bond degraded (instance {{ $labels.instance }}) + description: "Bond {{ $labels.master }} degraded on {{ $labels.instance }}" +{% endraw %} + +{% if alertmanager_warn_network_bond_single_link | bool %} +{% raw %} + - alert: HostNetworkBondSingleLink + expr: node_bonding_slaves == 1 + for: 2m + labels: + severity: warning + annotations: + summary: Host network bond with a single link (instance {{ $labels.instance }}) + description: "Bond {{ $labels.master }} configured with a single link on {{ $labels.instance }}" +{% endraw %} +{% endif %} + +{% raw %} + - alert: HostConntrackLimit + expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: Host conntrack limit (instance {{ $labels.instance }}) + description: "The number of conntrack is approaching limit" + + - alert: NodeRAIDDegraded + expr: | + node_md_disks_required{job="node",device!=""} - ignoring (state) (node_md_disks{state="active",job="node",device!=""}) > 0 + for: "15m" + labels: + severity: critical + annotations: + description: "RAID array '{{ $labels.device }}' at {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically." + summary: "RAID Array is degraded." + + - alert: NodeRAIDDiskFailure + expr: | + node_md_disks{state="failed",job="node",device!=""} > 0 + labels: + severity: warning + annotations: + description: "At least one device in RAID array at {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap." + summary: "Failed device in RAID array." + +{% endraw %} diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 358a22f89..1d91da7e6 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,3 +1,4 @@ +# yamllint disable-file --- # Add extra Kolla global configuration here. prechecks_enable_host_ntp_checks: false @@ -23,4 +24,70 @@ keystone_identity_mappings: - name: "ska_mapping" file: "{{ kayobe_config_path }}/kolla/config/keystone/ska_identity-mappings.json" kolla_enable_tls_external: true -keystone_federation_oidc_response_type: code \ No newline at end of file +keystone_federation_oidc_response_type: code + +# kolla_base_distro must be set here to be resolvable on a per-host basis +# This is necessary for os migrations where mixed clouds might be deployed +kolla_base_distro: "{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}" + +# Use facts so this is determined correctly when the control host OS differs +# from os_distribuition. +kolla_base_distro_version: "{% raw %}{{ 'noble' if ansible_facts.distribution_release == 'noble' else kolla_base_distro_version_default_map[kolla_base_distro] }}{% endraw %}" + +# Convenience variable for base distro and version string. +kolla_base_distro_and_version: "{% raw %}{{ kolla_base_distro }}-{{ kolla_base_distro_version }}{% endraw %}" + +kolla_base_distro_version_custom_map: { + "rocky-9": "3.9", + "ubuntu-jammy": "3.10", + "ubuntu-noble": "3.12" +} + +distro_python_version: "{% raw %}{{ kolla_base_distro_version_custom_map[kolla_base_distro_and_version] }}{% endraw %}" + +# Dict of Kolla image tags to deploy for each service. +# Each key is the tag variable prefix name, and the value is another dict, +# where the key is the OS distro and the value is the tag to deploy. +# NOTE: This is defined in etc/kayobe/kolla-image-tags.yml. +kolla_image_tags: +{{ kolla_image_tags | to_nice_yaml | indent(width=4, first=true) }} + +# Variables defining which tag to use for each container's image. +{{ lookup('pipe', 'python3 ' ~ kayobe_config_path ~ '/../../tools/kolla-images.py list-tag-vars') }} + +############################################################################# +# Monitoring and alerting related settings + +opensearch_heap_size: 8g +prometheus_cmdline_extras: "--storage.tsdb.retention.time=30d" + +# Additional command line flags for node exporter to enable texfile collector for disk metrics and create textfile docker volume +prometheus_node_exporter_extra_volumes: + - "textfile:/var/lib/node_exporter/textfile_collector" +prometheus_node_exporter_cmdline_extras: "--collector.textfile.directory=/var/lib/node_exporter/textfile_collector" + +# Set Ceph manager exporter endpoints +{% if groups['mgrs'] | length > 0 %} +prometheus_ceph_mgr_exporter_endpoints: +{% for host in groups['mgrs'] %} + - "{{ admin_oc_net_name | net_ip(host) }}:9283" +{% endfor %} +{% endif %} + +# Use inventory hostnames as labels +prometheus_instance_label: "{% raw %}{{ ansible_facts.hostname }}{% endraw %}" + +# Make openstack-exporter use Nova API version 2.1 to keep metrics the same as +# in Yoga. This is required to include a valid value for the flavor_id label on +# openstack_nova_server_status metrics. +prometheus_openstack_exporter_compute_api_version: "2.1" + +# The Pulp URL must be templated by Kayobe rather than Kolla Ansible. +# The rest of the Prometheus Blackbox exporter configuration can be found in +# the Kolla inventory. +prometheus_blackbox_exporter_endpoints_kayobe: + - endpoints: + - "pulp:http_2xx:{{ pulp_url }}/pulp/api/v3/status/" + enabled: "{{ seed_pulp_container_enabled | bool }}" + +prometheus_openstack_exporter_interval: "{{ stackhpc_prometheus_openstack_exporter_interval }}s" diff --git a/etc/kayobe/kolla/inventory/group_vars/prometheus b/etc/kayobe/kolla/inventory/group_vars/prometheus new file mode 100644 index 000000000..d50b4eb13 --- /dev/null +++ b/etc/kayobe/kolla/inventory/group_vars/prometheus @@ -0,0 +1,210 @@ +--- +######################################## +# Prometheus Blackbox monitoring options +######################################## + +# prometheus_blackbox_exporter_endpoints_kayobe is another set of default +# endpoints that are templated by Kayobe rather than Kolla Ansible. See +# kolla/globals.yml for more details. +prometheus_blackbox_exporter_endpoints: | + {% set endpoints = [] %} + {% for dict_item in (prometheus_blackbox_exporter_endpoints_kayobe | default([]) + prometheus_blackbox_exporter_endpoints_default) %} + {% if dict_item.enabled | bool %} + {% for endpoint in dict_item.endpoints %} + {% set _ = endpoints.append(endpoint) %} + {% endfor %} + {% endif %} + {% endfor %} + {{ (endpoints + prometheus_blackbox_exporter_endpoints_custom) | unique | select | list }} + +# A list of custom prometheus Blackbox exporter endpoints. Each element should +# have the following format: +# - name:module:endpoint +# for example +# - stackhpc:http_2xx:https://stackhpc.com:443 +prometheus_blackbox_exporter_endpoints_custom: [] + +prometheus_blackbox_exporter_endpoints_default: + # OpenStack endpoints + - endpoints: + - "barbican:os_endpoint:{{ barbican_public_endpoint }}" + - "{{ ('barbican_internal:os_endpoint:' + barbican_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_barbican | bool }}" + - endpoints: + - "blazar:os_endpoint:{{ blazar_public_base_endpoint }}" + - "{{ ('blazar_internal:os_endpoint:' + blazar_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_blazar | bool }}" + - endpoints: + - "ceph_rgw:http_2xx:{{ ceph_rgw_public_base_endpoint }}" + - "{{ ('ceph_rgw_internal:http_2xx:' + ceph_rgw_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_ceph_rgw | bool and enable_ceph_rgw_loadbalancer | bool }}" + - endpoints: + - "cinder:os_endpoint:{{ cinder_public_base_endpoint }}" + - "{{ ('cinder_internal:os_endpoint:' + cinder_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_cinder | bool }}" + - endpoints: + - "cloudkitty:os_endpoint:{{ cloudkitty_public_endpoint }}" + - "{{ ('cloudkitty_internal:os_endpoint:' + cloudkitty_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_cloudkitty | bool }}" + - endpoints: + - "designate:os_endpoint:{{ designate_public_endpoint }}" + - "{{ ('designate_internal:os_endpoint:' + designate_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_designate | bool }}" + - endpoints: + - "glance:os_endpoint:{{ glance_public_endpoint }}" + - "{{ ('glance_internal:os_endpoint:' + glance_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_glance | bool }}" + - endpoints: + - "gnocchi:os_endpoint:{{ gnocchi_public_endpoint }}" + - "{{ ('gnocchi_internal:os_endpoint:' + gnocchi_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_gnocchi | bool }}" + - endpoints: + - "heat:os_endpoint:{{ heat_public_base_endpoint }}" + - "{{ ('heat_internal:os_endpoint:' + heat_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + - "heat_cfn:os_endpoint:{{ heat_cfn_public_base_endpoint }}" + - "{{ ('heat_cfn_internal:os_endpoint:' + heat_cfn_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_heat | bool }}" + - endpoints: + - "horizon:http_2xx:{{ horizon_public_endpoint }}" + - "{{ ('horizon_internal:http_2xx:' + horizon_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_horizon | bool }}" + - endpoints: + - "ironic:os_endpoint:{{ ironic_public_endpoint }}" + - "{{ ('ironic_internal:os_endpoint:' + ironic_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + - "ironic_inspector:os_endpoint:{{ ironic_inspector_public_endpoint }}" + - "{{ ('ironic_inspector_internal:os_endpoint:' + ironic_inspector_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_ironic | bool }}" + - endpoints: + - "keystone:os_endpoint:{{ keystone_public_url }}" + - "{{ ('keystone_internal:os_endpoint:' + keystone_internal_url) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_keystone | bool }}" + - endpoints: + - "magnum:os_endpoint:{{ magnum_public_base_endpoint }}" + - "{{ ('magnum_internal:os_endpoint:' + magnum_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_magnum | bool }}" + - endpoints: + - "manila:os_endpoint:{{ manila_public_base_endpoint }}" + - "{{ ('manila_internal:os_endpoint:' + manila_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_manila | bool }}" + - endpoints: + - "neutron:os_endpoint:{{ neutron_public_endpoint }}" + - "{{ ('neutron_internal:os_endpoint:' + neutron_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_neutron | bool }}" + - endpoints: + - "nova:os_endpoint:{{ nova_public_base_endpoint }}" + - "{{ ('nova_internal:os_endpoint:' + nova_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_nova | bool }}" + - endpoints: + - "octavia:os_endpoint:{{ octavia_public_endpoint }}" + - "{{ ('octavia_internal:os_endpoint:' + octavia_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_octavia | bool }}" + - endpoints: + - "placement:os_endpoint:{{ placement_public_endpoint }}" + - "{{ ('placement_internal:os_endpoint:' + placement_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_placement | bool }}" + - endpoints: + - "swift:os_endpoint:{{ swift_public_base_endpoint }}" + - "{{ ('swift_internal:os_endpoint:' + swift_internal_base_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_swift | bool }}" + # Additional service endpoints + - endpoints: "{% set etcd_endpoints = [] %}{% for host in groups.get('etcd', []) %}{{ etcd_endpoints.append('etcd_' + host.replace('-', '') + ':http_2xx:' + hostvars[host]['etcd_protocol'] + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['etcd_client_port'] + '/metrics')}}{% endfor %}{{ etcd_endpoints }}" + enabled: "{{ enable_etcd | bool }}" + - endpoints: + - "grafana:http_2xx:{{ grafana_public_endpoint }}" + enabled: "{{ enable_grafana_external | bool }}" + - endpoints: + - "{{ ('grafana_internal:http_2xx:' + grafana_internal_endpoint) if not kolla_same_external_internal_vip | bool }}" + enabled: "{{ enable_grafana | bool }}" + - endpoints: + - "opensearch:http_2xx:{{ opensearch_internal_endpoint }}" + enabled: "{{ enable_opensearch | bool }}" + - endpoints: + - "opensearch_dashboards_external:http_2xx_opensearch_dashboards:{{ opensearch_dashboards_external_endpoint }}/api/status" + enabled: "{{ enable_opensearch_dashboards_external | bool }}" + - endpoints: + - "opensearch_dashboards:http_2xx_opensearch_dashboards:{{ opensearch_dashboards_internal_endpoint }}/api/status" + enabled: "{{ enable_opensearch_dashboards | bool }}" + - endpoints: + - "prometheus:http_2xx_prometheus:{{ prometheus_public_endpoint if enable_prometheus_server_external else prometheus_internal_endpoint }}/-/healthy" + enabled: "{{ enable_prometheus | bool }}" + - endpoints: + - "prometheus_alertmanager:http_2xx_alertmanager:{{ prometheus_alertmanager_public_endpoint if enable_prometheus_alertmanager_external else prometheus_alertmanager_internal_endpoint }}" + enabled: "{{ enable_prometheus_alertmanager | bool }}" + - endpoints: "{% set rabbitmq_endpoints = [] %}{% for host in groups.get('rabbitmq', []) %}{{ rabbitmq_endpoints.append('rabbitmq_' + host.replace('-', '') + (':tls_connect:' if rabbitmq_enable_tls | bool else ':tcp_connect:') + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['rabbitmq_port'] ) }}{% endfor %}{{ rabbitmq_endpoints }}" + enabled: "{{ enable_rabbitmq | bool }}" + - endpoints: "{% set redis_endpoints = [] %}{% for host in groups.get('redis', []) %}{{ redis_endpoints.append('redis_' + host.replace('-', '') + ':tcp_connect:' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['redis_port']) }}{% endfor %}{{ redis_endpoints }}" + enabled: "{{ enable_redis | bool }}" + # Backend endpoints + - endpoints: "{% set barbican_endpoints = [] %}{% for host in groups.get('barbican-api', []) %}{{ barbican_endpoints.append('barbican_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['barbican_api_port']) }}{% endfor %}{{ barbican_endpoints }}" + enabled: "{{ enable_barbican | bool }}" + - endpoints: "{% set blazar_endpoints = [] %}{% for host in groups.get('blazar-api', []) %}{{ blazar_endpoints.append('blazar_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['blazar_api_port']) }}{% endfor %}{{ blazar_endpoints }}" + enabled: "{{ enable_blazar | bool }}" + - endpoints: "{% set cinder_endpoints = [] %}{% for host in groups.get('cinder-api', []) %}{{ cinder_endpoints.append('cinder_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['cinder_api_port']) }}{% endfor %}{{ cinder_endpoints }}" + enabled: "{{ enable_cinder | bool }}" + - endpoints: "{% set designate_endpoints = [] %}{% for host in groups.get('designate-api', []) %}{{ designate_endpoints.append('designate_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['designate_api_port']) }}{% endfor %}{{ designate_endpoints }}" + enabled: "{{ enable_designate | bool }}" + - endpoints: "{% set glance_endpoints = [] %}{% for host in groups.get('glance-api', []) %}{{ glance_endpoints.append('glance_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if glance_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['glance_api_port']) }}{% endfor %}{{ glance_endpoints }}" + enabled: "{{ enable_glance | bool }}" + - endpoints: "{% set gnocchi_endpoints = [] %}{% for host in groups.get('gnocchi-api', []) %}{{ gnocchi_endpoints.append('gnocchi_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['gnocchi_api_port']) }}{% endfor %}{{ gnocchi_endpoints }}" + enabled: "{{ enable_gnocchi | bool }}" + - endpoints: "{% set heat_endpoints = [] %}{% for host in groups.get('heat-api', []) %}{{ heat_endpoints.append('heat_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['heat_api_port']) }}{% endfor %}{{ heat_endpoints }}" + enabled: "{{ enable_heat | bool }}" + - endpoints: "{% set heat_cfn_endpoints = [] %}{% for host in groups.get('heat-api-cfn', []) %}{{ heat_cfn_endpoints.append('heat_cfn_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['heat_api_cfn_port']) }}{% endfor %}{{ heat_cfn_endpoints }}" + enabled: "{{ enable_heat | bool }}" + - endpoints: "{% set horizon_endpoints = [] %}{% for host in groups.get('horizon', []) %}{{ horizon_endpoints.append('horizon_backend_' + host.replace('-', '') + ':http_2xx:' + ('https' if horizon_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['horizon_listen_port']) }}{% endfor %}{{ horizon_endpoints }}" + enabled: "{{ enable_horizon | bool }}" + - endpoints: "{% set keystone_endpoints = [] %}{% for host in groups.get('keystone-api', []) %}{{ keystone_endpoints.append('keystone_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['keystone_internal_port']) }}{% endfor %}{{ keystone_endpoints }}" + enabled: "{{ enable_keystone | bool }}" + - endpoints: "{% set magnum_endpoints = [] %}{% for host in groups.get('magnum-api', []) %}{{ magnum_endpoints.append('magnum_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['magnum_api_port']) }}{% endfor %}{{ magnum_endpoints }}" + enabled: "{{ enable_magnum | bool }}" + - endpoints: "{% set manila_endpoints = [] %}{% for host in groups.get('manila-api', []) %}{{ manila_endpoints.append('manila_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['manila_api_port']) }}{% endfor %}{{ manila_endpoints }}" + enabled: "{{ enable_manila | bool }}" + - endpoints: "{% set neutron_endpoints = [] %}{% for host in groups.get('neutron-server', []) %}{{ neutron_endpoints.append('neutron_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if neutron_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['neutron_server_port']) }}{% endfor %}{{ neutron_endpoints }}" + enabled: "{{ enable_neutron | bool }}" + - endpoints: "{% set nova_endpoints = [] %}{% for host in groups.get('nova-api', []) %}{{ nova_endpoints.append('nova_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['nova_api_port']) }}{% endfor %}{{ nova_endpoints }}" + enabled: "{{ enable_nova | bool }}" + - endpoints: "{% set octavia_endpoints = [] %}{% for host in groups.get('octavia-api', []) %}{{ octavia_endpoints.append('octavia_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['octavia_api_port']) }}{% endfor %}{{ octavia_endpoints }}" + enabled: "{{ enable_octavia | bool }}" + - endpoints: "{% set placement_endpoints = [] %}{% for host in groups.get('placement-api', []) %}{{ placement_endpoints.append('placement_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['placement_api_port']) }}{% endfor %}{{ placement_endpoints }}" + enabled: "{{ enable_placement | bool }}" + - endpoints: "{% set ironic_endpoints = [] %}{% for host in groups.get('ironic-api', []) %}{{ ironic_endpoints.append('ironic_backend_' + host.replace('-', '') + ':os_endpoint:' + ('https' if kolla_enable_tls_backend | bool else 'http') + '://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['ironic_api_port']) }}{% endfor %}{{ ironic_endpoints }}" + enabled: "{{ enable_ironic | bool }}" + - endpoints: "{% set ironic_inspector_endpoints = [] %}{% for host in groups.get('ironic-inspector', []) %}{{ ironic_inspector_endpoints.append('ironic_inspector_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['ironic_inspector_port']) }}{% endfor %}{{ ironic_inspector_endpoints }}" + enabled: "{{ enable_ironic | bool }}" + - endpoints: "{% set swift_endpoints = [] %}{% for host in groups.get('swift-api', []) %}{{ swift_endpoints.append('swift_backend_' + host.replace('-', '') + ':os_endpoint:' + 'http://' + ('api' | kolla_address(host) | put_address_in_context('url')) + ':' + hostvars[host]['swift_api_port']) }}{% endfor %}{{ swift_endpoints }}" + enabled: "{{ enable_swift | bool }}" + +# Ensure service endpoints are defined +blazar_internal_base_endpoint: "{{ blazar_internal_fqdn | kolla_url(internal_protocol, blazar_api_port) }}" +blazar_public_base_endpoint: "{{ blazar_external_fqdn | kolla_url(public_protocol, blazar_api_public_port) }}" +ceph_rgw_internal_base_endpoint: "{{ ceph_rgw_internal_fqdn | kolla_url(internal_protocol, ceph_rgw_port) }}" +ceph_rgw_public_base_endpoint: "{{ ceph_rgw_external_fqdn | kolla_url(public_protocol, ceph_rgw_public_port) }}" +cinder_internal_base_endpoint: "{{ cinder_internal_fqdn | kolla_url(internal_protocol, cinder_api_port) }}" +cinder_public_base_endpoint: "{{ cinder_external_fqdn | kolla_url(public_protocol, cinder_api_public_port) }}" +cloudkitty_internal_endpoint: "{{ cloudkitty_internal_fqdn | kolla_url(internal_protocol, cloudkitty_api_port) }}" +cloudkitty_public_endpoint: "{{ cloudkitty_external_fqdn | kolla_url(public_protocol, cloudkitty_api_public_port) }}" +gnocchi_internal_endpoint: "{{ gnocchi_internal_fqdn | kolla_url(internal_protocol, gnocchi_api_port) }}" +gnocchi_public_endpoint: "{{ gnocchi_external_fqdn | kolla_url(public_protocol, gnocchi_api_public_port) }}" +grafana_public_endpoint: "{{ grafana_external_fqdn | kolla_url(public_protocol, grafana_server_public_port) }}" +heat_cfn_internal_base_endpoint: "{{ heat_cfn_internal_fqdn | kolla_url(internal_protocol, heat_api_cfn_port) }}" +heat_cfn_public_base_endpoint: "{{ heat_cfn_external_fqdn | kolla_url(public_protocol, heat_api_cfn_public_port) }}" +heat_internal_base_endpoint: "{{ heat_internal_fqdn | kolla_url(internal_protocol, heat_api_port) }}" +heat_public_base_endpoint: "{{ heat_external_fqdn | kolla_url(public_protocol, heat_api_public_port) }}" +horizon_public_endpoint: "{{ horizon_external_fqdn | kolla_url(public_protocol, horizon_tls_port if kolla_enable_tls_external | bool else horizon_port) }}" +ironic_inspector_internal_endpoint: "{{ ironic_inspector_internal_fqdn | kolla_url(internal_protocol, ironic_inspector_port) }}" +ironic_inspector_public_endpoint: "{{ ironic_inspector_external_fqdn | kolla_url(public_protocol, ironic_inspector_public_port) }}" +magnum_internal_base_endpoint: "{{ magnum_internal_fqdn | kolla_url(internal_protocol, magnum_api_port) }}" +magnum_public_base_endpoint: "{{ magnum_external_fqdn | kolla_url(public_protocol, magnum_api_public_port) }}" +manila_internal_base_endpoint: "{{ manila_internal_fqdn | kolla_url(internal_protocol, manila_api_port) }}" +manila_public_base_endpoint: "{{ manila_external_fqdn | kolla_url(public_protocol, manila_api_public_port) }}" +nova_internal_base_endpoint: "{{ nova_internal_fqdn | kolla_url(internal_protocol, nova_api_port) }}" +nova_public_base_endpoint: "{{ nova_external_fqdn | kolla_url(public_protocol, nova_api_public_port) }}" +opensearch_dashboards_external_endpoint: "{{ opensearch_dashboards_external_fqdn | default(kolla_external_fqdn) | kolla_url(public_protocol, opensearch_dashboards_port_external) }}" +opensearch_dashboards_internal_endpoint: "{{ opensearch_dashboards_internal_fqdn | default(kolla_internal_fqdn) | kolla_url(internal_protocol, opensearch_dashboards_port) }}" +placement_internal_endpoint: "{{ placement_internal_fqdn | kolla_url(internal_protocol, placement_api_port) }}" +placement_public_endpoint: "{{ placement_external_fqdn | kolla_url(public_protocol, placement_api_public_port) }}" +prometheus_alertmanager_internal_endpoint: "{{ prometheus_alertmanager_internal_fqdn | kolla_url(internal_protocol, prometheus_alertmanager_port) }}" +prometheus_alertmanager_public_endpoint: "{{ prometheus_alertmanager_external_fqdn | kolla_url(public_protocol, prometheus_alertmanager_public_port) }}" +prometheus_internal_endpoint: "{{ prometheus_internal_fqdn | kolla_url(internal_protocol, prometheus_port) }}" +prometheus_public_endpoint: "{{ prometheus_external_fqdn | kolla_url(public_protocol, prometheus_public_port) }}" +swift_public_base_endpoint: "{{ swift_external_fqdn | kolla_url(public_protocol, swift_proxy_server_port) }}" diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf new file mode 100644 index 000000000..f8daae9eb --- /dev/null +++ b/etc/kayobe/kolla/kolla-build.conf @@ -0,0 +1,35 @@ +[DEFAULT] +{# Packages in the latest upstream Ubuntu base image can be ahead of our package repo #} +{# snapshots, so pin to a specific tag. #} +{# This tag should be updated when Ubuntu package repo snapshot versions are changed. #} +{% if kolla_base_distro == 'ubuntu' and kolla_base_distro_version == 'jammy' %} +base_tag = jammy-20240808 +{% elif kolla_base_distro == 'ubuntu' and kolla_base_distro_version == 'noble' %} +base_tag = noble-20241118.1 +{# Similarly pinning to Rocky 9 minor version used in our repos #} +{% elif kolla_base_distro == 'rocky' %} +base_tag = 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} +{% endif %} +build_args = {{ (kolla_build_args | default({})).items() | map('join', ':') | join(',') }} + +[openstack-base] +type = git +location = https://github.com/stackhpc/requirements +reference = stackhpc/{{ openstack_release }} + +[etcd] +version = 3.5.27 +sha256 = amd64:0aad9a9e4e0817a021e933f9806a2b2960a62f949ad5a3d6436d8886945cb1bc,arm64:1277309f540c5a0329c428f95455c9f76d24f768c8d28fd2753e891c379053fa + +[letsencrypt-lego] +version = v4.23.1 +sha256 = amd64:1fd60b1fd59c239bed22719a5de402cb745d1f933540cb1ec196e2c03e6e8882,arm64:1114745108343286d4bff189b4bdee3cba9d07ebcacc673860d91ab951d31e0d + +# TODO: move to kolla_sources in kolla.yml once https://review.opendev.org/c/openstack/kayobe/+/970268 is available +[prometheus-cadvisor] +version = 0.56.2 +sha256 = amd64:ad92930f16a2f9da15190675e09eeaceb8fd38637d07a686bb0dd68695f692af,arm64:b7a707379496fd7a7b5d2768c5c494427112f534ba5069f889af28ffe6ad11bb + +[prometheus-msteams] +version = 1.5.3 +sha256 = amd64:8eae63d89338f53a990fa2720b3fabf58c916e2648c948ce1e0f29942459a491 diff --git a/etc/kayobe/monitoring.yml b/etc/kayobe/monitoring.yml index 5468936d3..6dc11fe39 100644 --- a/etc/kayobe/monitoring.yml +++ b/etc/kayobe/monitoring.yml @@ -3,7 +3,8 @@ # Monitoring node configuration. # User with which to access the monitoring nodes via SSH during bootstrap, in -# order to setup the Kayobe user account. +# order to setup the Kayobe user account. Default is 'cloud-user' if +# os_distribution is set to centos, otherwise 'os_distribution'. #monitoring_bootstrap_user: ############################################################################### @@ -116,6 +117,13 @@ # - state: enabled #monitoring_firewalld_rules: +############################################################################### +# Monitoring node swap configuration. + +# List of swap devices. Each item is a dict containing a 'device' item. Default +# is an empty list. +# monitoring_swap: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index be9b065ba..8d807b92b 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -68,6 +68,12 @@ storage_mgmt_net_name: data # hosts #cleaning_net_name: +# Name of the network used to manage octavia loadbalancers +#octavia_net_name: + +# Name of the network where Wazuh manager is exposed +#wazuh_dashboard_net_name: + ############################################################################### # Network definitions. diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml new file mode 100644 index 000000000..c439b26b8 --- /dev/null +++ b/etc/kayobe/ofed.yml @@ -0,0 +1,35 @@ +--- +# DOCA OFED configuration. + +############################################################################### +# DOCA host version +stackhpc_pulp_doca_version_matrix: + "6": 2.9.3 + "7": 3.2.1 +stackhpc_pulp_doca_version: "{{ stackhpc_pulp_doca_version_matrix[stackhpc_pulp_repo_rocky_9_minor_version | string] | default('2.9.1') }}" + +# Available and tested versions of the pre-compiled doca-ofed kernel modules +stackhpc_doca_kernel_version_matrix: + "6": 5.14.0.570.21.1.el9.6.x86.64 + "7": 5.14.0.611.13.1.el9.7.x86.64 + +############################################################################### +# Pulp configuration for DOCA OFED + +# Whether to sync OFED repositories into the local Pulp service +stackhpc_pulp_sync_ofed: "{{ groups['mlnx'] | length > 0 }}" + +# DOCA Snapshot lookup vars +doca_rocky_9_minor_dot: "{% if stackhpc_pulp_doca_version is version('3.2.0', '>=') %}9{% else %}9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}{% endif %}" +# eg stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version or stackhpc_pulp_repo_doca_3_2_1_rhel9_version +doca_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel{{ doca_rocky_9_minor_dot | replace('.', '_') }}_version" +doca_modules_version_lookup_var: "stackhpc_pulp_repo_doca_{{ stackhpc_pulp_doca_version | replace('.', '_') }}_rhel9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_modules_version" + +# DOCA Snapshot versions. The defaults use the appropriate version from +# pulp-repo-versions.yml +stackhpc_pulp_repo_rhel9_doca_version: "{{ lookup('vars', doca_version_lookup_var) }}" +stackhpc_pulp_repo_rhel9_doca_modules_version: "{{ lookup('vars', doca_modules_version_lookup_var) }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/openbao.yml b/etc/kayobe/openbao.yml new file mode 100644 index 000000000..0a545eced --- /dev/null +++ b/etc/kayobe/openbao.yml @@ -0,0 +1,47 @@ +--- +############################################################################### +# OpenBao deployment configuration. + +# Registry information for seed. +# seed_openbao_registry_url: +# seed_openbao_registry_username: +# seed_openbao_registry_password: + +# Seed OpenBao container image. +# seed_openbao_docker_image: + +# Seed OpenBao container image tag. +seed_openbao_docker_tag: 2.2.1 + +# Seed OpenBao PKI Role name +# seed_openbao_pki_role_name: + +# Seed OpenBao PKI Roles definition +# seed_openbao_pki_roles: [] + +# Registry information for overcloud. +# overcloud_openbao_registry_url: +# overcloud_openbao_registry_username: +# overcloud_openbao_registry_password: + +# Overcloud OpenBao container image. +# overcloud_openbao_docker_image: + +# Overcloud OpenBao container image tag. +overcloud_openbao_docker_tag: 2.2.1 + +# Overcloud OpenBao PKI Default Role name +# overcloud_openbao_pki_default_role_name: + +# Overcloud OpenBao PKI Internal TLS Role name +# overcloud_openbao_pki_internal_tls_role_name: + +# Overcloud OpenBao PKI Backend TLS Role name +# overcloud_openbao_pki_backend_tls_role_name: + +# Overcloud OpenBao PKI Roles definition +# overcloud_openbao_pki_roles: [] + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/openstack.yml b/etc/kayobe/openstack.yml index 4db759fc7..4d304fa38 100644 --- a/etc/kayobe/openstack.yml +++ b/etc/kayobe/openstack.yml @@ -5,7 +5,11 @@ # Name of the current OpenStack release. Default is "2024.1". #openstack_release: -# Name of the current OpenStack branch. Default is "stable/2024.1". +# Codename of the current OpenStack release. +# NOTE(upgrade): Update to current release codename. +openstack_release_codename: "caracal" + +# Name of the current OpenStack branch. Default is "unmaintained/2024.1". #openstack_branch: ############################################################################### diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index f5f935202..541103f08 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -8,10 +8,11 @@ # Bifrost. Setting it to true disables Bifrost image build and allows images to # be built with the `kayobe overcloud host image build` command. Default value # is true. -#overcloud_dib_build_host_images: +overcloud_dib_build_host_images: "{{ not stackhpc_download_overcloud_host_images | bool }}" # List of additional build host packages to install. Default is an empty list. -#overcloud_dib_host_packages_extra: +overcloud_dib_host_packages_extra: + - "{% if 'ubuntu-minimal' in overcloud_dib_host_images | map(attribute='elements') | flatten | list %}debootstrap{% endif %}" # List of overcloud host disk images to build. Each element is a dict defining # an image in a format accepted by the stackhpc.openstack.os_images role. @@ -63,7 +64,11 @@ # List of additional git repositories containing Diskimage Builder (DIB) # elements. See stackhpc.openstack.os_images role for usage. Default is empty. -#overcloud_dib_git_elements_extra: +overcloud_dib_git_elements_extra: + - repo: "https://github.com/stackhpc/stackhpc-image-elements" + local: "{{ source_checkout_path }}/stackhpc-image-elements" + version: "v1.6.3" + elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See # stackhpc.openstack.os_images role for usage. Default is a combination of diff --git a/etc/kayobe/pip.yml b/etc/kayobe/pip.yml index 10a4817a3..0d8dcc75a 100644 --- a/etc/kayobe/pip.yml +++ b/etc/kayobe/pip.yml @@ -3,7 +3,7 @@ # Upper constraints file for installation of python packages. # Default value is # "https://releases.openstack.org/constraints/upper/{{ openstack_release }}" -pip_upper_constraints_file: "https://opendev.org/openstack/requirements/raw/branch/unmaintained/2023.1/upper-constraints.txt" +pip_upper_constraints_file: "https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/{{ openstack_release }}/upper-constraints.txt" # Use a local PyPi mirror for installing Pip packages #pip_local_mirror: false diff --git a/etc/kayobe/proxy.yml b/etc/kayobe/proxy.yml index 714b9dae5..d7df51e52 100644 --- a/etc/kayobe/proxy.yml +++ b/etc/kayobe/proxy.yml @@ -12,8 +12,9 @@ # List of domains, hostnames, IP addresses and networks for which no proxy is # used. Defaults to ["127.0.0.1", "localhost", "{{ ('http://' ~ -# docker_registry) | urlsplit('hostname') }}"] if docker_registry is set, or -# ["127.0.0.1", "localhost"] otherwise. This is configured only if either +# docker_registry) | urlsplit('hostname') }}","{{ kolla_internal_vip_address +# }}"] if docker_registry is set, or ["127.0.0.1", "localhost","{{ +# kolla_internal_vip_address }}"] otherwise. This is configured only if either # http_proxy or https_proxy is set. #no_proxy: diff --git a/etc/kayobe/pulp-host-image-versions.yml b/etc/kayobe/pulp-host-image-versions.yml new file mode 100644 index 000000000..b7e13a44d --- /dev/null +++ b/etc/kayobe/pulp-host-image-versions.yml @@ -0,0 +1,6 @@ +--- +# Overcloud host image versioning tags +# These images must be in SMS, since they are used by our AIO CI runners +stackhpc_rocky_9_overcloud_host_image_version: "2024.1-20260205T145550" +stackhpc_ubuntu_jammy_overcloud_host_image_version: "2024.1-20250408T082408" +stackhpc_ubuntu_noble_overcloud_host_image_version: "2024.1-20250417T104113" diff --git a/etc/kayobe/pulp-ipa-image-versions.yml b/etc/kayobe/pulp-ipa-image-versions.yml new file mode 100644 index 000000000..ca2781ae2 --- /dev/null +++ b/etc/kayobe/pulp-ipa-image-versions.yml @@ -0,0 +1,5 @@ +--- +# IPA image versioning tags +stackhpc_rocky_9_ipa_image_version: "2024.1-20241231T102920" +stackhpc_ubuntu_jammy_ipa_image_version: "2024.1-20241206T160829" +stackhpc_ubuntu_noble_ipa_image_version: "2024.1-20250402T132932" diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml new file mode 100644 index 000000000..1f5e98107 --- /dev/null +++ b/etc/kayobe/pulp-repo-versions.yml @@ -0,0 +1,104 @@ +--- +# This file is autogenerated by Ansible using the following workflow: +# https://github.com/stackhpc/stackhpc-release-train/actions/workflows/package-update-kayobe.yml +stackhpc_pulp_repo_centos_stream_9_docker_version: 20260202T201529 +stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20260202T213000 +stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20260202T211958 +stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20260202T213000 +stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20250603T020143 +stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20250603T013132 +stackhpc_pulp_repo_centos_stream_9_opstools_version: 20231213T031318 +stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version: 20240927T073838 +stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20250617T023108 +stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20250617T020101 +stackhpc_pulp_repo_doca_2_9_1_rhel9_4_version: 20241211T153620 +stackhpc_pulp_repo_doca_2_9_1_rhel9_4_modules_version: 20241213T112245 +stackhpc_pulp_repo_doca_2_9_1_rhel9_5_version: 20241211T171301 +stackhpc_pulp_repo_doca_2_9_1_rhel9_5_modules_version: 20250115T150314 +stackhpc_pulp_repo_doca_2_9_3_rhel9_6_version: 20250703T135021 +stackhpc_pulp_repo_doca_2_9_3_rhel9_6_modules_version: 20250714T141841 +stackhpc_pulp_repo_doca_3_2_1_rhel9_version: 20251219T132023 +stackhpc_pulp_repo_doca_3_2_1_rhel9_7_modules_version: 20251222T160623 +stackhpc_pulp_repo_ceph_reef_debian_version: 20240925T152022 +stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version: 20241218T154614 +stackhpc_pulp_repo_docker_ce_ubuntu_noble_version: 20250401T001425 +stackhpc_pulp_repo_elrepo_9_version: 20260127T212055 +stackhpc_pulp_repo_elrepo_9_aarch64_version: 20250408T030629 +stackhpc_pulp_repo_epel_9_version: 20260204T220346 +stackhpc_pulp_repo_epel_9_aarch64_version: 20260204T223146 +stackhpc_pulp_repo_grafana_version: 20260214T213531 +stackhpc_pulp_repo_opensearch_2_x_version: 20251106T202313 +stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20251106T202313 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20251130T213625 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26_aarch64_version: 20260112T224827 +stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27_aarch64_version: 20260112T224827 +stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20260122T202504 +stackhpc_pulp_repo_rhel_9_influxdb_version: 20260113T202459 +stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20260112T224827 +stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20260204T213517 +stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20260204T223146 +stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20251219T202129 +stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20251219T213344 +stackhpc_pulp_repo_rocky_9_1_appstream_version: 20231207T013715 +stackhpc_pulp_repo_rocky_9_1_baseos_version: 20231206T014015 +stackhpc_pulp_repo_rocky_9_1_crb_version: 20231211T120328 +stackhpc_pulp_repo_rocky_9_1_extras_version: 20231211T120328 +stackhpc_pulp_repo_rocky_9_1_highavailability_version: 20231208T011702 +stackhpc_pulp_repo_rocky_9_2_appstream_version: 20231211T120328 +stackhpc_pulp_repo_rocky_9_2_baseos_version: 20231211T120328 +stackhpc_pulp_repo_rocky_9_2_crb_version: 20231207T013715 +stackhpc_pulp_repo_rocky_9_2_extras_version: 20231207T013715 +stackhpc_pulp_repo_rocky_9_2_highavailability_version: 20231208T011702 +stackhpc_pulp_repo_rocky_9_3_appstream_version: 20240508T001430 +stackhpc_pulp_repo_rocky_9_3_baseos_version: 20240509T005935 +stackhpc_pulp_repo_rocky_9_3_crb_version: 20240429T003818 +stackhpc_pulp_repo_rocky_9_3_extras_version: 20240506T000343 +stackhpc_pulp_repo_rocky_9_3_highavailability_version: 20240510T001129 +stackhpc_pulp_repo_rocky_9_4_appstream_version: 20240816T002610 +stackhpc_pulp_repo_rocky_9_4_baseos_version: 20240816T002610 +stackhpc_pulp_repo_rocky_9_4_crb_version: 20240816T002610 +stackhpc_pulp_repo_rocky_9_4_extras_version: 20240816T002610 +stackhpc_pulp_repo_rocky_9_4_highavailability_version: 20240816T002610 +stackhpc_pulp_repo_rocky_9_5_appstream_version: 20250202T120931 +stackhpc_pulp_repo_rocky_9_5_appstream_aarch64_version: 20250203T124809 +stackhpc_pulp_repo_rocky_9_5_baseos_version: 20250201T125442 +stackhpc_pulp_repo_rocky_9_5_baseos_aarch64_version: 20250201T123411 +stackhpc_pulp_repo_rocky_9_5_crb_version: 20250204T095037 +stackhpc_pulp_repo_rocky_9_5_crb_aarch64_version: 20250206T042027 +stackhpc_pulp_repo_rocky_9_5_extras_version: 20250122T025402 +stackhpc_pulp_repo_rocky_9_5_extras_aarch64_version: 20250124T030113 +stackhpc_pulp_repo_rocky_9_5_highavailability_version: 20250204T095037 +stackhpc_pulp_repo_rocky_9_5_highavailability_aarch64_version: 20250206T042027 +stackhpc_pulp_repo_rocky_9_6_appstream_version: 20250614T015933 +stackhpc_pulp_repo_rocky_9_6_appstream_aarch64_version: 20250614T024523 +stackhpc_pulp_repo_rocky_9_6_baseos_version: 20250614T030644 +stackhpc_pulp_repo_rocky_9_6_baseos_aarch64_version: 20250614T024523 +stackhpc_pulp_repo_rocky_9_6_crb_version: 20250614T015933 +stackhpc_pulp_repo_rocky_9_6_crb_aarch64_version: 20250614T024523 +stackhpc_pulp_repo_rocky_9_6_extras_version: 20250726T040613 +stackhpc_pulp_repo_rocky_9_6_extras_aarch64_version: 20250726T045704 +stackhpc_pulp_repo_rocky_9_6_highavailability_version: 20250605T150141 +stackhpc_pulp_repo_rocky_9_6_highavailability_aarch64_version: 20250605T150141 +stackhpc_pulp_repo_rocky_9_7_appstream_version: 20260224T210434 +stackhpc_pulp_repo_rocky_9_7_appstream_aarch64_version: 20260224T214928 +stackhpc_pulp_repo_rocky_9_7_appstream_source_version: 20260224T211933 +stackhpc_pulp_repo_rocky_9_7_baseos_version: 20260224T213540 +stackhpc_pulp_repo_rocky_9_7_baseos_aarch64_version: 20260224T214928 +stackhpc_pulp_repo_rocky_9_7_baseos_source_version: 20260224T211933 +stackhpc_pulp_repo_rocky_9_7_crb_version: 20260224T210434 +stackhpc_pulp_repo_rocky_9_7_crb_aarch64_version: 20260224T214928 +stackhpc_pulp_repo_rocky_9_7_crb_source_version: 20260208T212722 +stackhpc_pulp_repo_rocky_9_7_extras_version: 20251229T144024 +stackhpc_pulp_repo_rocky_9_7_extras_aarch64_version: 20251229T153210 +stackhpc_pulp_repo_rocky_9_7_extras_source_version: 20251229T150242 +stackhpc_pulp_repo_rocky_9_7_highavailability_version: 20260213T205816 +stackhpc_pulp_repo_rocky_9_7_highavailability_aarch64_version: 20260212T231318 +stackhpc_pulp_repo_rocky_9_7_highavailability_source_version: 20260212T230226 +stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20251108T211425 +stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20251109T211404 +stackhpc_pulp_repo_rocky_9_sig_security_common_source_version: 20251203T103807 +stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20260506T232713 +stackhpc_pulp_repo_ubuntu_jammy_security_version: 20260506T021915 +stackhpc_pulp_repo_ubuntu_jammy_version: 20260506T021915 +stackhpc_pulp_repo_ubuntu_noble_security_version: 20260507T015832 +stackhpc_pulp_repo_ubuntu_noble_version: 20260507T015832 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml new file mode 100644 index 000000000..0ab2694ab --- /dev/null +++ b/etc/kayobe/pulp.yml @@ -0,0 +1,789 @@ +--- +############################################################################### +# Local Pulp server configuration + +# Base URL of the local Pulp service. +# Default uses the seed node's IP on the admin network. +pulp_url: "{{ 'https' if pulp_enable_tls | bool else 'http' }}://{{ lookup('vars', admin_oc_net_name ~ '_ips')[groups.seed.0] }}:{{ pulp_port }}" + +# Port on the seed node's interface on the admin network that the Pulp service +# listens on. +pulp_port: "{{ '443' if pulp_enable_tls | bool else '80' }}" + +# Whether to enable TLS for Pulp. +pulp_enable_tls: false + +# Path to a TLS certificate to use when TLS is enabled. +#pulp_cert_path: + +# Path to a TLS key to use when TLS is enabled. +#pulp_key_path: + +############################################################################### +# Local Pulp access credentials + +# Credentials used to access the local Pulp REST API. +pulp_username: admin +pulp_password: "{{ secrets_pulp_password }}" + +# Credentials for non-admin user within Pulp. +pulp_stack_username: stack +pulp_stack_password: "{{ secrets_pulp_stack_password | default('') }}" + +# Proxy to use when adding remotes. +pulp_proxy_url: "{{ omit }}" + +############################################################################### +# StackHPC Pulp server + +# The scheme of the StackHPC Pulp service (http or https) +stackhpc_release_pulp_scheme: "https" + +# The domain of the StackHPC Pulp service. +stackhpc_release_pulp_domain: "ark.stackhpc.com" + +# Base URL of the StackHPC Pulp service. +stackhpc_release_pulp_url: "{{ stackhpc_release_pulp_scheme }}://{{ stackhpc_release_pulp_domain }}" + +# Credentials used to access the StackHPC Pulp service. +stackhpc_release_pulp_username: +stackhpc_release_pulp_password: + +# Content URL of the StackHPC Pulp service. +stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content" + +# Content URL of the StackHPC Pulp service, with basic auth. +# NOTE(Alex-Welsh): This may need reworking if it reveals the credentials +# at runtime in the ansible output +stackhpc_release_pulp_content_url_with_auth: "{{ stackhpc_release_pulp_scheme }}://{{ stackhpc_release_pulp_username }}:{{ stackhpc_release_pulp_password }}@{{ stackhpc_release_pulp_domain }}/pulp/content" + +# Sync all repositories required for building Kolla images from the +# StackHPC Ark Pulp service to local Pulp. +# NOTE: Only RPM repositories are supported. +stackhpc_pulp_sync_for_local_container_build: false + +############################################################################### +# Debs + +# Whether to sync Ubuntu packages. +stackhpc_pulp_sync_ubuntu_jammy: "{{ os_release == 'jammy' }}" +stackhpc_pulp_sync_ubuntu_noble: "{{ os_release == 'noble' }}" + +# Default configuration, which each element of stackhpc_pulp_deb_repos +# is combined with +stackhpc_pulp_deb_repos_default_config: + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" + client_cert: "" + client_key: "" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + architectures: amd64 + mirror: true + state: present + +# Deb repositories +stackhpc_pulp_deb_repos: + # Base Ubuntu Jammy repositories + - name: "Ubuntu jammy" + url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/jammy/{{ stackhpc_pulp_repo_ubuntu_jammy_version }}" + distribution_name: "ubuntu-jammy-" + base_path: "ubuntu/jammy/" + components: "main restricted universe multiverse" + distributions: "jammy jammy-updates jammy-backports" + required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + + - name: "Ubuntu jammy security" + url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/jammy-security/{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}" + distribution_name: "ubuntu-jammy-security-" + base_path: "ubuntu/jammy-security/" + components: "main restricted universe multiverse" + distributions: "jammy-security" + required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + + - name: "Ubuntu jammy CVE-2024-6387" + url: "{{ stackhpc_release_pulp_content_url }}/ubuntu-jammy-cve-2024-6387/" + distribution_name: "ubuntu-jammy-cve-2024-6387-" + base_path: "ubuntu-jammy-cve-2024-6387/" + components: "upload" + distributions: "pulp" + required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + + # Base Ubuntu Noble repositories + - name: "Ubuntu Noble" + url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/noble/{{ stackhpc_pulp_repo_ubuntu_noble_version }}" + distribution_name: "ubuntu-noble-" + base_path: "ubuntu/noble/" + components: "main restricted universe multiverse" + distributions: "noble noble-updates noble-backports" + required: "{{ stackhpc_pulp_sync_ubuntu_noble | bool }}" + + - name: "Ubuntu Noble security" + url: "{{ stackhpc_release_pulp_content_url }}/ubuntu/noble-security/{{ stackhpc_pulp_repo_ubuntu_noble_security_version }}" + distribution_name: "ubuntu-noble-security-" + base_path: "ubuntu/noble-security/" + components: "main restricted universe multiverse" + distributions: "noble-security" + required: "{{ stackhpc_pulp_sync_ubuntu_noble | bool }}" + + # Ubuntu Cloud Archive (UCA) repositories + - name: "Ubuntu Cloud Archive" + url: "{{ stackhpc_release_pulp_content_url }}/ubuntu-cloud-archive/{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}" + distribution_name: "ubuntu-cloud-archive-" + base_path: "ubuntu-cloud-archive/" + components: "main" + distributions: "jammy-updates/{{ openstack_release_codename }}" + required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + + # Third-party repositories + - name: "Docker CE for Ubuntu Jammy" + url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/ubuntu-jammy/{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}" + distribution_name: "docker-ce-for-ubuntu-jammy-" + base_path: "docker-ce/ubuntu-jammy/" + distributions: "jammy" + components: "stable" + required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + + - name: "Docker CE for Ubuntu Noble" + url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/ubuntu-noble/{{ stackhpc_pulp_repo_docker_ce_ubuntu_noble_version }}" + distribution_name: "docker-ce-for-ubuntu-noble-" + base_path: "docker-ce/ubuntu-noble/" + distributions: "noble" + components: "stable" + required: "{{ stackhpc_pulp_sync_ubuntu_noble | bool }}" + + - name: "Ceph Reef for Debian" + url: "{{ stackhpc_release_pulp_content_url }}/ceph/debian-reef/{{ stackhpc_pulp_repo_ceph_reef_debian_version }}" + distribution_name: "ceph-reef-debian-" + base_path: "ceph/debian-reef/" + distributions: "jammy" + components: "main" + required: "{{ stackhpc_pulp_sync_ubuntu_jammy | bool }}" + +# Publication format is a subset of distribution. +stackhpc_pulp_publication_deb_development: "{{ stackhpc_pulp_distribution_deb_development }}" + +# Content is made available at well-known distribution names for development, +# production. + +stackhpc_pulp_repository_deb_repos: >- + {%- set deb_repos = [] -%} + {%- for repo in stackhpc_pulp_deb_repos -%} + {%- set deb_repo = {"name": repo.name} -%} + {%- set deb_repo = deb_repo | combine({"url": repo.url}) -%} + {%- set deb_repo = deb_repo | combine({"required": repo.required}) -%} + {%- if "policy" in repo -%} + {%- set deb_repo = deb_repo | combine({"policy": repo.policy}) -%} + {%- endif -%} + {%- if "mirror" in repo -%} + {%- set deb_repo = deb_repo | combine({"mirror": repo.mirror}) -%} + {%- endif -%} + {%- if "distributions" in repo -%} + {%- set deb_repo = deb_repo | combine({"distributions": repo.distributions}) -%} + {%- endif -%} + {%- if "components" in repo -%} + {%- set deb_repo = deb_repo | combine({"components": repo.components}) -%} + {%- endif -%} + {%- if "architectures" in repo -%} + {%- set deb_repo = deb_repo | combine({"architectures": repo.architectures}) -%} + {%- endif -%} + {%- set _ = deb_repos.append(stackhpc_pulp_deb_repos_default_config | combine(deb_repo)) -%} + {%- endfor -%} + {{ deb_repos }} + +stackhpc_pulp_deb_dists_default_config: + state: present + mode: verbatim + +# Development tracks the release repositories. +stackhpc_pulp_distribution_deb_development: >- + {%- set dev_dists = [] -%} + {%- for repo in stackhpc_pulp_deb_repos -%} + {%- set name = repo.distribution_name ~ 'development' -%} + {%- set base_path = repo.base_path ~ 'development' -%} + {%- set dev_dist = { + "name": name, + "repository": repo.name, + "base_path": base_path, + "required": repo.required + } + -%} + {%- if "mode" in repo -%} + {%- set dev_dist = dev_dist | combine({"mode": repo.mode}) -%} + {%- endif -%} + {%- set _ = dev_dists.append(stackhpc_pulp_deb_dists_default_config | combine(dev_dist)) -%} + {%- endfor -%} + {{ dev_dists }} + +# Development gets promoted to production. +stackhpc_pulp_distribution_deb_production: >- + {%- set prod_dists = [] -%} + {%- for repo in stackhpc_pulp_deb_repos -%} + {%- set name = repo.distribution_name ~ 'production' -%} + {%- set base_path = repo.base_path ~ 'production' -%} + {%- set prod_dist = { + "name": name, + "distribution": repo.distribution_name ~ 'development', + "base_path": base_path, + "required": repo.required + } + -%} + {%- if "mode" in repo -%} + {%- set prod_dist = prod_dist | combine({"mode": repo.mode}) -%} + {%- endif -%} + {%- set _ = prod_dists.append(stackhpc_pulp_deb_dists_default_config | combine(prod_dist)) -%} + {%- endfor -%} + {{ prod_dists }} + +############################################################################### +# RPMs + +# Content comes in from release repositories, via pinned repo versions. + +# Whether to sync Rocky Linux 9 packages. +stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}" +# Rocky 9 minor version number. Supported values: 1, 2, 3, 4, 5, 6, 7. Default is 7 +stackhpc_pulp_repo_rocky_9_minor_version: '7' +# Rocky 9 Snapshot versions. The defaults use the appropriate version from +# pulp-repo-versions.yml for the selected minor release. + +# Define the architecture suffix +arch_suffix: "{{ '_aarch64' if kolla_base_arch == 'aarch64' else '' }}" + +# Rocky 9 Snapshot versions +stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_baseos' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_appstream' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_extras' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_crb' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_highavailability' ~ arch_suffix ~ '_version') }}" + +# Rocky 9 Multiarch repositories +#NOTE(bbezak): Versioned Erlang repos (aarch64 only). Fallback to generic Erlang version if not defined. +stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel9_rabbitmq_erlang' ~ arch_suffix ~ '_version', default=stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version) }}" +stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_26_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel9_rabbitmq_erlang_26' ~ arch_suffix ~ '_version', default=stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version) }}" +stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_27_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel9_rabbitmq_erlang_27' ~ arch_suffix ~ '_version', default=stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version) }}" +stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_caracal_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_openstack_caracal' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_opstools' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_reef_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version: "{{ lookup('vars', 'stackhpc_pulp_repo_centos_stream_9_docker' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_treasuredata_5' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_mariadb_10_11' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_influxdb' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_epel_9_version: "{{ lookup('vars', 'stackhpc_pulp_repo_epel_9' ~ arch_suffix ~ '_version') }}" +stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_sig_security_common' ~ arch_suffix ~ '_version') }}" + +# Whether to sync packages common to all RHEL 9 derivatives. +stackhpc_pulp_sync_el_9: "{{ stackhpc_pulp_sync_rocky_9 | bool }}" + +# Default configuration, which each element of stackhpc_pulp_rpm_repos +# is combined with +stackhpc_pulp_rpm_repos_default_config: + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" + client_cert: "" + client_key: "" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + sync_policy: mirror_complete + state: present + +stackhpc_pulp_rpm_repos: + # Grafana and RabbitMQ repositories - version 8/9 agnositic + - name: Grafana + url: "{{ stackhpc_release_pulp_content_url }}/grafana/oss/rpm/{{ stackhpc_pulp_repo_grafana_version }}" + distribution_name: "grafana-" + base_path: "grafana/oss/rpm/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: RabbitMQ - Erlang - RHEL 9 + url: "{{ stackhpc_release_pulp_content_url }}/rabbitmq/erlang/el/9/x86_64/{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}" + distribution_name: "rhel9-rabbitmq-erlang-" + base_path: "rabbitmq/erlang/el/9/x86_64/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: RabbitMQ - Server - RHEL 9 + url: "{{ stackhpc_release_pulp_content_url }}/rabbitmq/rabbitmq-server/el/9/noarch/{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}" + distribution_name: "rhel9-rabbitmq-server-" + base_path: "rabbitmq/rabbitmq-server/el/9/x86_64/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + # Base Rocky 9 repositories + - name: Rocky Linux 9 - AppStream + url: "{{ stackhpc_release_pulp_content_url }}/rocky/9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/AppStream/x86_64/os/{{ stackhpc_pulp_repo_rocky_9_appstream_version }}" + distribution_name: "rocky-9-appstream-" + base_path: "rocky/9/AppStream/x86_64/os/" + required: "{{ stackhpc_pulp_sync_rocky_9 | bool }}" + + - name: Rocky Linux 9 - BaseOS + url: "{{ stackhpc_release_pulp_content_url }}/rocky/9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/BaseOS/x86_64/os/{{ stackhpc_pulp_repo_rocky_9_baseos_version }}" + distribution_name: "rocky-9-baseos-" + base_path: "rocky/9/BaseOS/x86_64/os/" + required: "{{ stackhpc_pulp_sync_rocky_9 | bool }}" + + - name: Rocky Linux 9 - Extras + url: "{{ stackhpc_release_pulp_content_url }}/rocky/9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/extras/x86_64/os/{{ stackhpc_pulp_repo_rocky_9_extras_version }}" + distribution_name: "rocky-9-extras-" + base_path: "rocky/9/extras/x86_64/os/" + required: "{{ stackhpc_pulp_sync_rocky_9 | bool }}" + + - name: Rocky Linux 9 - CRB + url: "{{ stackhpc_release_pulp_content_url }}/rocky/9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/CRB/x86_64/os/{{ stackhpc_pulp_repo_rocky_9_crb_version }}" + distribution_name: "rocky-9-crb-" + base_path: "rocky/9/CRB/x86_64/os/" + required: "{{ stackhpc_pulp_sync_rocky_9 | bool }}" + + - name: Rocky Linux 9 - HighAvailability + url: "{{ stackhpc_release_pulp_content_url }}/rocky/9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/highavailability/x86_64/os/{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}" + distribution_name: "rocky-9-highavailability-" + base_path: "rocky/9/highavailability/x86_64/os/" + required: "{{ stackhpc_pulp_sync_rocky_9 | bool }}" + + - name: Rocky Linux 9 - SIG Security Common + url: "{{ stackhpc_release_pulp_content_url }}/rocky/sig/9/security/x86_64/security-common/{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}" + distribution_name: rocky-9-sig-security-common- + base_path: "rocky/sig/9/security/x86_64/security-common/" + required: "{{ stackhpc_pulp_sync_rocky_9 | bool }}" + + # Additional CentOS Stream 9 repositories + - name: CentOS Stream 9 - NFV OpenvSwitch + url: "{{ stackhpc_release_pulp_content_url }}/centos/9-stream/nfv/x86_64/openvswitch-2/{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}" + distribution_name: "centos-stream-9-nfv-openvswitch-" + base_path: "centos/9-stream/nfv/x86_64/openvswitch-2/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: CentOS Stream 9 - OpenStack Caracal + url: "{{ stackhpc_release_pulp_content_url }}/centos/9-stream/cloud/x86_64/openstack-caracal/{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}" + distribution_name: "centos-stream-9-openstack-caracal-" + base_path: "centos/9-stream/cloud/x86_64/openstack-caracal/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: CentOS 9 - OpsTools - collectd + url: "{{ stackhpc_release_pulp_content_url }}/centos/9-stream/opstools/x86_64/collectd-5/{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}" + distribution_name: "centos-9-opstools-collectd-" + base_path: "centos/9-stream/opstools/x86_64/collectd-5/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: CentOS Stream 9 - Ceph Reef + url: "{{ stackhpc_release_pulp_content_url }}/centos/9-stream/storage/x86_64/ceph-reef/{{ stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version }}" + distribution_name: "centos-stream-9-ceph-reef-" + base_path: "centos/9-stream/storage/x86_64/ceph-reef/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + # EPEL 9 repository + - name: Extra Packages for Enterprise Linux 9 - x86_64 + url: "{{ stackhpc_release_pulp_content_url }}/epel/9/Everything/x86_64/{{ stackhpc_pulp_repo_epel_9_version }}" + distribution_name: "extra-packages-for-enterprise-linux-9-x86_64-" + base_path: "epel/9/Everything/x86_64/" + sync_policy: mirror_content_only + required: "{{ stackhpc_pulp_sync_el_9 | bool }}" + + # ELRepo 9 repository + - name: ELRepo.org Community Enterprise Linux Repository - el9 + url: "{{ stackhpc_release_pulp_content_url }}/elrepo/elrepo/el9/x86_64/{{ stackhpc_pulp_repo_elrepo_9_version }}" + distribution_name: elrepo-el9-x86_64- + base_path: elrepo/elrepo/el9/x86_64/ + sync_policy: mirror_content_only + required: "{{ stackhpc_pulp_sync_el_9 | bool }}" + + # Third-party repositories + - name: Docker CE for CentOS 9 + url: "{{ stackhpc_release_pulp_content_url }}/docker-ce/centos/9/x86_64/stable/{{ stackhpc_pulp_repo_centos_stream_9_docker_version }}" + distribution_name: "docker-ce-for-centos-9-" + base_path: "docker-ce/centos/9/x86_64/stable/" + required: "{{ stackhpc_pulp_sync_el_9 | bool }}" + + # Additional RHEL 9 repositories + - name: TreasureData 5 for RHEL 9 + url: "{{ stackhpc_release_pulp_content_url }}/treasuredata/lts/5/redhat/9/x86_64/{{ stackhpc_pulp_repo_rhel_9_treasuredata_5_version }}" + distribution_name: "rhel-9-treasuredata-5-" + base_path: "treasuredata/5/redhat/9/x86_64/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: MariaDB 10.11 for RHEL 9 + url: "{{ stackhpc_release_pulp_content_url }}/mariadb-10.11/yum/rhel/9/x86_64/{{ stackhpc_pulp_repo_rhel_9_mariadb_10_11_version }}" + distribution_name: "rhel-9-mariadb-10-11-" + base_path: "mariadb-10.11/yum/rhel/9/x86_64/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: InfluxDB for RHEL 9 + url: "{{ stackhpc_release_pulp_content_url }}/influxdb/rhel/9/x86_64/stable/{{ stackhpc_pulp_repo_rhel_9_influxdb_version }}" + distribution_name: "rhel-9-influxdb-" + base_path: "influxdb/rhel/9/x86_64/stable/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: OpenSearch 2.x + url: "{{ stackhpc_release_pulp_content_url }}/opensearch/2.x/yum/{{ stackhpc_pulp_repo_opensearch_2_x_version }}" + distribution_name: "opensearch-2-x-" + base_path: "opensearch/2.x/yum/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: OpenSearch Dashboards 2.x + url: "{{ stackhpc_release_pulp_content_url }}/opensearch-dashboards/2.x/yum/{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}" + distribution_name: "opensearch-dashboards-2-x-" + base_path: "opensearch-dashboards/2.x/yum/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL {{ doca_rocky_9_minor_dot }} + url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_pulp_repo_rhel9_doca_version }}" + distribution_name: "doca-{{ stackhpc_pulp_doca_version }}-rhel{{ doca_rocky_9_minor_dot }}-" + base_path: "doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/" + required: "{{ stackhpc_pulp_sync_ofed | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_version }} - RHEL 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} + url: "{{ stackhpc_release_pulp_content_url }}/doca-modules/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_pulp_repo_rhel9_doca_modules_version }}" + distribution_name: "doca-modules-{{ stackhpc_pulp_doca_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-" + base_path: "doca-modules/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/" + required: "{{ stackhpc_pulp_sync_ofed | bool and stackhpc_pulp_sync_el_9 | bool }}" + +# RPM repositories +stackhpc_pulp_repository_rpm_repos: >- + {%- set rpm_repos = [] -%} + {%- for repo in stackhpc_pulp_rpm_repos -%} + {%- set rpm_repo = {"name": repo.name} -%} + {%- set rpm_repo = rpm_repo | combine({"url": repo.url}) -%} + {%- set rpm_repo = rpm_repo | combine({"required": repo.required}) -%} + {%- if "policy" in repo -%} + {%- set rpm_repo = rpm_repo | combine({"policy": repo.policy}) -%} + {%- endif -%} + {%- if "sync_policy" in repo -%} + {%- set rpm_repo = rpm_repo | combine({"sync_policy": repo.sync_policy}) -%} + {%- endif -%} + {%- set _ = rpm_repos.append(stackhpc_pulp_rpm_repos_default_config | combine(rpm_repo)) -%} + {%- endfor -%} + {{ rpm_repos }} + +# Publication format is a subset of distribution. +stackhpc_pulp_publication_rpm_development: "{{ stackhpc_pulp_distribution_rpm_development }}" + +# Content is made available at well-known distribution names for development, +# production. + +stackhpc_pulp_rpm_dists_default_config: + state: present + +# Development tracks the release repositories. +stackhpc_pulp_distribution_rpm_development: >- + {%- set dev_dists = [] -%} + {%- for repo in stackhpc_pulp_rpm_repos -%} + {%- set name = repo.distribution_name ~ 'development' -%} + {%- set base_path = repo.base_path ~ 'development' -%} + {%- set dev_dist = { + "name": name, + "repository": repo.name, + "base_path": base_path, + "required": repo.required + } + -%} + {%- set _ = dev_dists.append(stackhpc_pulp_rpm_dists_default_config | combine(dev_dist)) -%} + {%- endfor -%} + {{ dev_dists }} + +# Development gets promoted to production. +stackhpc_pulp_distribution_rpm_production: >- + {%- set prod_dists = [] -%} + {%- for repo in stackhpc_pulp_rpm_repos -%} + {%- set name = repo.distribution_name ~ 'production' -%} + {%- set base_path = repo.base_path ~ 'production' -%} + {%- set dev_dist = { + "name": name, + "distribution": repo.distribution_name ~ 'development', + "base_path": base_path, + "required": repo.required + } + -%} + {%- set _ = prod_dists.append(stackhpc_pulp_rpm_dists_default_config | combine(dev_dist)) -%} + {%- endfor -%} + {{ prod_dists }} + +############################################################################### +# Containers + +# URL of the StackHPC Pulp service's container image registry. +stackhpc_release_pulp_registry_url: "{{ stackhpc_release_pulp_url }}" + +# List of all image names. +stackhpc_pulp_images_kolla: + - barbican-api + - barbican-keystone-listener + - barbican-worker + - bifrost-deploy + - blazar-api + - blazar-manager + - caso + - cinder-api + - cinder-backup + - cinder-scheduler + - cinder-volume + - cloudkitty-api + - cloudkitty-processor + - cron + - designate-api + - designate-backend-bind9 + - designate-central + - designate-mdns + - designate-producer + - designate-sink + - designate-worker + - dnsmasq + - etcd + - fluentd + - glance-api + - grafana + - haproxy + - haproxy-ssh + - heat-api + - heat-api-cfn + - heat-engine + - horizon + - influxdb + - ironic-api + - ironic-conductor + - ironic-inspector + - ironic-neutron-agent + - ironic-prometheus-exporter + - ironic-pxe + - iscsid + - keepalived + - keystone + - keystone-fernet + - keystone-ssh + - kolla-toolbox + - letsencrypt-lego + - letsencrypt-webserver + - magnum-api + - magnum-conductor + - manila-api + - manila-data + - manila-scheduler + - manila-share + - mariadb-clustercheck + - mariadb-server + - memcached + - neutron-bgp-dragent + - neutron-dhcp-agent + - neutron-l3-agent + - neutron-metadata-agent + - neutron-mlnx-agent + - neutron-openvswitch-agent + - neutron-server + - neutron-sriov-agent + - nova-api + - nova-compute + - nova-compute-ironic + - nova-conductor + - nova-libvirt + - nova-novncproxy + - nova-scheduler + - nova-serialproxy + - nova-ssh + - octavia-api + - octavia-driver-agent + - octavia-health-manager + - octavia-housekeeping + - octavia-worker + - opensearch + - opensearch-dashboards + - openvswitch-db-server + - openvswitch-vswitchd + - ovn-controller + - ovn-nb-db-server + - ovn-northd + - ovn-sb-db-relay + - ovn-sb-db-server + - placement-api + - prometheus-alertmanager + - prometheus-blackbox-exporter + - prometheus-cadvisor + - prometheus-elasticsearch-exporter + - prometheus-libvirt-exporter + - prometheus-memcached-exporter + - prometheus-msteams + - prometheus-mtail + - prometheus-mysqld-exporter + - prometheus-node-exporter + - prometheus-openstack-exporter + - prometheus-v2-server + - rabbitmq + - rabbitmq-4-1 + - redis + - redis-sentinel + +# List of images for each base distribution which should not/cannot be built. +stackhpc_kolla_unbuildable_images: + # See https://github.com/openstack/kolla/commit/34ca6e35c94ddba1e47722718f3fc81a1f03e28a + ubuntu-jammy: + - bifrost-deploy + ubuntu-noble: + - bifrost-deploy + rocky-9: + - bifrost-deploy + +# Whitespace-separated list of regular expressions matching Kolla image names. +# Usage is similar to kolla-build CLI arguments. +# Example: +# kayobe playbook run ... -e stackhpc_pulp_images_kolla_filter='"^glance nova-compute$"' +stackhpc_pulp_images_kolla_filter: ".*" + +# Filtered list of Kolla container image names. +stackhpc_pulp_images_kolla_filtered: "{{ stackhpc_pulp_images_kolla | select('search', '(' ~ stackhpc_pulp_images_kolla_filter.split() | join('|') ~ ')') | list }}" + +# Common parameters for container image repositories. +stackhpc_pulp_repository_container_repos_kolla_common: + url: "{{ stackhpc_release_pulp_registry_url }}" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + remote_username: "{{ stackhpc_release_pulp_username }}" + remote_password: "{{ stackhpc_release_pulp_password }}" + state: present + required: true + +# List of Kolla container image repositories. +stackhpc_pulp_repository_container_repos_kolla: >- + {%- set repos = [] -%} + {%- set image_tags = lookup('pipe', 'python3 ' ~ kayobe_config_path ~ '/../../tools/kolla-images.py list-tags') | from_yaml -%} + {%- for image in stackhpc_pulp_images_kolla_filtered -%} + {%- if image not in stackhpc_kolla_unbuildable_images[kolla_base_distro_and_version] -%} + {%- set image_repo = kolla_docker_namespace ~ "/" ~ image -%} + {%- set repo = {"name": image_repo, "include_tags": image_tags[image]} -%} + {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%} + {%- endif -%} + {%- endfor -%} + {{ repos }} + +# Common parameters for Kolla container image distributions. +stackhpc_pulp_distribution_container_kolla_common: + state: present + required: true + +# List of Kolla container image distributions. +stackhpc_pulp_distribution_container_kolla: >- + {%- set distributions = [] -%} + {%- for image in stackhpc_pulp_images_kolla_filtered -%} + {%- if image not in stackhpc_kolla_unbuildable_images[kolla_base_distro_and_version]-%} + {%- set image_repo = kolla_docker_namespace ~ "/" ~ image -%} + {%- set distribution = {"name": image_repo, "repository": image_repo, "base_path": image_repo} -%} + {%- set _ = distributions.append(stackhpc_pulp_distribution_container_kolla_common | combine(distribution)) -%} + {%- endif -%} + {%- endfor -%} + {{ distributions }} + +# Whether to sync Ceph container images. +stackhpc_sync_ceph_images: false + +# List of Ceph container image repositories. +stackhpc_pulp_repository_container_repos_ceph: + - name: "ceph/ceph" + url: "https://quay.io" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + state: present + include_tags: "{{ cephadm_image_tag }}" + required: "{{ stackhpc_sync_ceph_images | bool }}" + - name: "ceph/haproxy" + url: "https://quay.io" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + state: present + include_tags: "{{ cephadm_haproxy_image_tag }}" + required: "{{ stackhpc_sync_ceph_images | bool }}" + - name: "ceph/keepalived" + url: "https://quay.io" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + state: present + include_tags: "{{ cephadm_keepalived_image_tag }}" + required: "{{ stackhpc_sync_ceph_images | bool }}" + +# List of Ceph container image distributions. +stackhpc_pulp_distribution_container_ceph: + - name: ceph + repository: ceph/ceph + base_path: ceph/ceph + state: present + required: "{{ stackhpc_sync_ceph_images | bool }}" + - name: ceph/haproxy + repository: ceph/haproxy + base_path: ceph/haproxy + state: present + required: "{{ stackhpc_sync_ceph_images | bool }}" + - name: ceph/keepalived + repository: ceph/keepalived + base_path: ceph/keepalived + state: present + required: "{{ stackhpc_sync_ceph_images | bool }}" + +# Whether to sync HashiCorp container images. +stackhpc_sync_hashicorp_images: false + +# List of HashiCorp container image repositories. +stackhpc_pulp_repository_container_repos_hashicorp: + - name: "hashicorp/consul" + url: "https://registry-1.docker.io" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + state: present + include_tags: "{{ overcloud_consul_docker_tag }}" + required: "{{ stackhpc_sync_hashicorp_images | bool }}" + - name: "hashicorp/vault" + url: "https://registry-1.docker.io" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + state: present + include_tags: "{{ overcloud_vault_docker_tag }}" + required: "{{ stackhpc_sync_hashicorp_images | bool }}" + +# List of HashiCorp container image distributions. +stackhpc_pulp_distribution_container_hashicorp: + - name: consul + repository: hashicorp/consul + base_path: hashicorp/consul + state: present + required: "{{ stackhpc_sync_hashicorp_images | bool }}" + - name: vault + repository: hashicorp/vault + base_path: hashicorp/vault + state: present + required: "{{ stackhpc_sync_hashicorp_images | bool }}" + +stackhpc_sync_openbao_images: false + +# List of OpenBao container image repositories. +stackhpc_pulp_repository_container_repos_openbao: + - name: "openbao/openbao" + url: "https://registry-1.docker.io" + policy: on_demand + proxy_url: "{{ pulp_proxy_url }}" + state: present + include_tags: "{{ overcloud_openbao_docker_tag }}" + required: "{{ stackhpc_sync_openbao_images | bool }}" + +# List of OpenBao container image distributions. +stackhpc_pulp_distribution_container_openbao: + - name: openbao + repository: openbao/openbao + base_path: openbao/openbao + state: present + required: "{{ stackhpc_sync_openbao_images | bool }}" + +# List of extra container image repositories. +stackhpc_pulp_repository_container_repos_extra: [] + +# List of extra container image distributions. +stackhpc_pulp_distribution_container_extra: [] + +# List of container image repositories. +stackhpc_pulp_repository_container_repos: >- + {{ (stackhpc_pulp_repository_container_repos_kolla + + stackhpc_pulp_repository_container_repos_ceph + + stackhpc_pulp_repository_container_repos_hashicorp + + stackhpc_pulp_repository_container_repos_openbao + + stackhpc_pulp_repository_container_repos_extra) | selectattr('required') }} + +# List of container image distributions. +stackhpc_pulp_distribution_container: >- + {{ (stackhpc_pulp_distribution_container_kolla + + stackhpc_pulp_distribution_container_ceph + + stackhpc_pulp_distribution_container_hashicorp + + stackhpc_pulp_distribution_container_openbao + + stackhpc_pulp_distribution_container_extra) | selectattr('required') }} diff --git a/etc/kayobe/seed-hypervisor.yml b/etc/kayobe/seed-hypervisor.yml index dd8fbca23..993095ed2 100644 --- a/etc/kayobe/seed-hypervisor.yml +++ b/etc/kayobe/seed-hypervisor.yml @@ -3,7 +3,8 @@ # Seed hypervisor node configuration. # User with which to access the seed hypervisor via SSH during bootstrap, in -# order to setup the Kayobe user account. Default is {{ os_distribution }}. +# order to setup the Kayobe user account. Default is 'cloud-user' if +# os_distribution is set to centos, otherwise 'os_distribution'. #seed_hypervisor_bootstrap_user: ############################################################################### @@ -135,6 +136,13 @@ # - state: enabled #seed_hypervisor_firewalld_rules: +############################################################################### +# Seed Hypervisor node swap configuration. + +# List of swap devices. Each item is a dict containing a 'device' item. Default +# is an empty list. +# seed_hypervisor_swap: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/seed-vm.yml b/etc/kayobe/seed-vm.yml index 42aea8da5..3554a0bfb 100644 --- a/etc/kayobe/seed-vm.yml +++ b/etc/kayobe/seed-vm.yml @@ -31,13 +31,15 @@ # Base image for the seed VM root volume. Default is # "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img" -# when os_distribution is "ubuntu", +# when os_distribution is "ubuntu" and distribution_major_version is '22', +# "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" +# when os_distribution is "ubuntu" and distribution_major_version is not '22', # "https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" # when os_distribution is "rocky" and seed_vm_boot_firmware is "efi", # "https://dl.rockylinux.org/vault/rocky/9.3/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" # when os_distribution is "rocky" and seed_vm_boot_firmware is not "efi" # (default is "bios"), or -# "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20221206.0.x86_64.qcow2" +# "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2" # otherwise. #seed_vm_root_image: diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 2b385beda..eed70de76 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -3,7 +3,8 @@ # Seed node configuration. # User with which to access the seed via SSH during bootstrap, in order to -# setup the Kayobe user account. Default is {{ os_distribution }}. +# setup the Kayobe user account. Default is 'cloud-user' if os_distribution is +# set to centos, otherwise 'os_distribution'. #seed_bootstrap_user: seed_bootstrap_user: ileigh @@ -95,19 +96,83 @@ seed_bootstrap_user: ileigh ############################################################################### # Seed node additional containers configuration +# Seed container running a "Pulp in one" service. +# https://pulpproject.org/pulp-in-one-container/ +# This can be used as a package mirror, and container image registry. +seed_pulp_container_enabled: true + +seed_pulp_container: + pulp: + image: pulp/pulp + pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" + post: "{{ kayobe_config_path }}/containers/pulp/post.yml" + tag: "3.43.1" + network_mode: host + # Override deploy_containers_defaults.init == true to ensure + # s6-overlay-suexec starts as pid 1 + init: false + env: + PULP_CONTENT_WORKERS: "{{ [ansible_facts.processor_vcpus * 2 + 1, 12] | min }}" + PULP_API_WORKERS: "{{ [ansible_facts.processor_vcpus * 2 + 1, 12] | min }}" + PULP_HTTPS: "{{ 'true' if pulp_enable_tls | bool else 'false' }}" + volumes: + - /opt/kayobe/containers/pulp:/etc/pulp + - pulp_storage:/var/lib/pulp + - pulp_pgsql:/var/lib/pgsql + - pulp_containers:/var/lib/containers + restart_policy: unless-stopped + +# Seed container running a Squid caching proxy. This can be used to proxy +# HTTP(S) requests from control plane hosts. +seed_squid_container_enabled: false + +seed_squid_container: + squid: + image: stackhpc/squid + pre: "{{ kayobe_config_path }}/containers/squid_proxy/pre.yml" + tag: "4.15" + network_mode: host + volumes: + - squid_spool:/var/spool/squid + - /srv/docker/squid/squid.conf:/etc/squid/squid.conf + - /var/log/squid:/var/log/squid + +# Seed container running redfish exporter. +# https://github.com/jenningsloy318/redfish_exporter +# This can be be used for monitoring servers in bifrost. +seed_redfish_exporter_container_enabled: "{{ stackhpc_enable_redfish_exporter }}" + +seed_redfish_exporter_container: + redfish_exporter: + image: ghcr.io/stackhpc/redfish-exporter + pre: "{{ kayobe_config_path }}/containers/redfish_exporter/pre.yml" + post: "{{ kayobe_config_path }}/containers/redfish_exporter/post.yml" + tag: "v2.1.1-stackhpc" + network_mode: host + command: redfish_exporter --config.file /redfish_exporter.yml + volumes: "/opt/kayobe/containers/redfish_exporter/redfish_exporter.yml:/redfish_exporter.yml:ro" + restart_policy: unless-stopped + # Dict of container images to start # Example: # seed_containers: # squid: -# image: "stackhpc/squid:3.5.20-1" +# image: "docker.io/stackhpc/squid" # pre: "{{ kayobe_env_config_path }}/containers/squid/pre.yml" # post: "{{ kayobe_env_config_path }}/containers/squid/post.yml" +# tag: "3.5.20-1" # -#seed_containers: +seed_containers: >- + {{ {} | combine(seed_pulp_container if seed_pulp_container_enabled | bool else {}) + | combine(seed_squid_container if seed_squid_container_enabled | bool else {}) + | combine(seed_redfish_exporter_container if seed_redfish_exporter_container_enabled | bool else {}) + | combine(seed_extra_containers) }} + +seed_extra_containers: {} # Whether to attempt a basic authentication login to a registry when # deploying seed containers -#seed_deploy_containers_registry_attempt_login: +seed_deploy_containers_registry_attempt_login: "{{ not seed_pulp_container_enabled | bool }}" ############################################################################### # Seed node firewalld configuration. @@ -130,6 +195,13 @@ seed_bootstrap_user: ileigh # - state: enabled #seed_firewalld_rules: +############################################################################### +# Seed node swap configuration. + +# List of swap devices. Each item is a dict containing a 'device' item. Default +# is an empty list. +# seed_swap: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/stackhpc-compute.yml b/etc/kayobe/stackhpc-compute.yml new file mode 100644 index 000000000..5e86b0030 --- /dev/null +++ b/etc/kayobe/stackhpc-compute.yml @@ -0,0 +1,103 @@ +--- +# StackHPC compute node configuration + +# Map of inventory groups to GPU types. +# This is used to determine which GPU types each compute node should pass +# through to OpenStack. +# Keys are group names, values are a list of GPU types. +# Groups must be added to kolla_overcloud_inventory_top_level_group_map +# GPU types must be keys in stackhpc_gpu_data. +# Example GPU group map: +# gpu_group_map: +# compute_a100: +# - a100_80 +# compute_v100: +# - v100_32 +# compute_multi_gpu: +# - a100_80 +# - v100_32 +gpu_group_map: {} + +# Dict mapping GPUs to PCI data. +# Resource names are used to identify the device in placement, and can be +# edited to match deployment-specific naming conventions +# The default list covers many common GPUs, but can be extended as needed. +stackhpc_gpu_data: + # Nvidia H100 SXM5 80GB + h100_80_sxm: + resource_name: "{{ h100_80_sxm_resource_name | default('h100_80_sxm')}}" + vendor_id: "10de" + product_id: "2330" + device_type: "type-PF" + # Nvidia A100 SXM5 80GB + a100_80_sxm: + resource_name: "{{ a100_80_sxm_resource_name | default('a100_80_sxm')}}" + vendor_id: "10de" + product_id: "20b2" + device_type: "type-PF" + # Nvidia A100 SXM5 40GB + a100_40_sxm: + resource_name: "{{ a100_40_sxm_resource_name | default('a100_40_sxm')}}" + vendor_id: "10de" + product_id: "20b0" + device_type: "type-PF" + # Nvidia A100 PCI 80GB + a100_80: + resource_name: "{{ a100_80_resource_name | default('a100_80')}}" + vendor_id: "10de" + product_id: "20b5" + device_type: "type-PF" + # Nvidia A100 PCI 40GB + a100_40: + resource_name: "{{ a100_40_resource_name | default('a100_40')}}" + vendor_id: "10de" + product_id: "20f1" + device_type: "type-PF" + # Nvidia V100 SXM3 32GB + v100_32_sxm3: + resource_name: "{{ v100_32_sxm3_resource_name | default('v100_32_sxm3')}}" + vendor_id: "10de" + product_id: "1db8" + device_type: "type-PCI" + # Nvidia V100 SXM2 32GB + v100_32_sxm2: + resource_name: "{{ v100_32_sxm2_resource_name | default('v100_32_sxm2')}}" + vendor_id: "10de" + product_id: "1db5" + device_type: "type-PCI" + # Nvidia V100 PCI 32GB + v100_32: + resource_name: "{{ v100_32_resource_name | default('v100_32')}}" + vendor_id: "10de" + product_id: "1db6" + device_type: "type-PCI" + # Nvidia RTX A6000 + a6000: + resource_name: "{{ a6000_resource_name | default('a6000')}}" + vendor_id: "10de" + product_id: "2230" + device_type: "type-PCI" + # Nvidia A40 + a40: + resource_name: "{{ a40_resource_name | default('a40')}}" + vendor_id: "10de" + product_id: "2235" + device_type: "type-PF" + # Nvidia T4 + t4: + resource_name: "{{ t4_resource_name | default('t4')}}" + vendor_id: "10de" + product_id: "1eb8" + device_type: "type-PF" + # Nvidia L40 + l40: + resource_name: "{{ l40_resource_name | default('l40')}}" + vendor_id: "10de" + product_id: "26b5" + device_type: "type-PF" + # Nvidia L40s + l40s: + resource_name: "{{ l40s_resource_name | default('l40s')}}" + vendor_id: "10de" + product_id: "26b9" + device_type: "type-PF" diff --git a/etc/kayobe/stackhpc-ipa-images.yml b/etc/kayobe/stackhpc-ipa-images.yml new file mode 100644 index 000000000..3c6a0a62c --- /dev/null +++ b/etc/kayobe/stackhpc-ipa-images.yml @@ -0,0 +1,21 @@ +--- +################################# +# Release train IPA image sources + +# Whether to use Release Train IPA images for Bifrost +stackhpc_ipa_image_bifrost_enabled: true + +# Whether to use Release Train IPA images for Overcloud Ironic +stackhpc_ipa_image_overcloud_enabled: true + +# The IPA image source, defined by os_distribution, +# os_release, and the current stable version. +stackhpc_ipa_image_url: "{{ stackhpc_release_pulp_content_url }}/ipa-images/\ + {{ openstack_release }}/{{ os_distribution }}/\ + {{ os_release }}/{{ stackhpc_ipa_image_version }}" + +# IPA image version tag selection +stackhpc_ipa_image_version: >- + {{ stackhpc_rocky_9_ipa_image_version if os_distribution == 'rocky' and os_release == '9' else + stackhpc_ubuntu_jammy_ipa_image_version if os_distribution == 'ubuntu' and os_release == 'jammy' else + stackhpc_ubuntu_noble_ipa_image_version if os_distribution == 'ubuntu' and os_release == 'noble' }} diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml new file mode 100644 index 000000000..f629b153b --- /dev/null +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -0,0 +1,89 @@ +--- +# StackHPC monitoring configuration + +############################################################################### +# Alert configuration + +# Threshold to trigger a LowMemory alert in Gibibytes (GiB). When the amount +# of free memory is lower than this value an alert will be triggered. +alertmanager_low_memory_threshold_gib: 5 + +# Whether to raise an alert if any network bond is configured with a single +# link. Change to false to disable this alert. +alertmanager_warn_network_bond_single_link: true + +# Threshold to trigger an LowSwapSpace alert on swap space depletion (ratio). +# When the ratio of free swap space is lower than each of these values, warning +# and critical alerts will be triggered respectively. +alertmanager_node_free_swap_warning_threshold_ratio: 0.25 +alertmanager_node_free_swap_critical_threshold_ratio: 0.1 + +# Threshold to trigger an alert for dropped packets, measured in packets/s +# averaged over 5 minutes. +alertmanager_packet_drop_threshold: 1 + +# Threshold to trigger an alert for packet receive/transmit errors, measured in +# packets/s averaged over 5 minutes. +alertmanager_packet_errors_threshold: 1 + +# Number of RabbitMQ nodes in the cluster. +alertmanager_number_of_rabbitmq_nodes: "{{ groups['controllers'] | length }}" + +############################################################################### +# Exporter configuration + +# Whether the OpenStack Capacity exporter is enabled. +# Enabling this flag will result in HAProxy configuration and Prometheus scrape +# targets being templated during deployment. +stackhpc_enable_os_capacity: true + +# OpenStack Capacity exporter version +stackhpc_os_capacity_version: v0.5 + +# Path to a CA certificate file to trust in the OpenStack Capacity exporter. +stackhpc_os_capacity_openstack_cacert: "" + +# Whether TLS certificate verification is enabled for the OpenStack Capacity +# exporter during Keystone authentication. +stackhpc_os_capacity_openstack_verify: true +# Redfish exporter + +# How often to scrape the os capacity exporter in seconds. +stackhpc_os_capacity_scrape_interval: 900 + +# Whether the redfish exporter is enabled. +stackhpc_enable_redfish_exporter: false + +# How often to scrape the BMCs in seconds. +stackhpc_redfish_exporter_scrape_interval: "{{ [8 * groups['redfish_exporter_targets'] | length, 300] | max }}" + +# Credentials +redfish_exporter_default_username: "{{ ipmi_username }}" +redfish_exporter_default_password: "{{ ipmi_password }}" + +# The address of the BMC that is used to query redfish metrics. +redfish_exporter_target_address: "{{ ipmi_address }}" + +# How often to scrape OpenStack Exporter in seconds. +stackhpc_prometheus_openstack_exporter_interval: 300 + +############################################################################### + +# Whether the RADOS gateway usage exporter is enabled. +# Enabling this will result in templating radosgw_usage_exporter endpoint as +# Prometheus scrape targets during deployment. +stackhpc_enable_radosgw_usage_exporter: false + +# Port to expose RADOS gateway usage exporter backend. Default is 9242 +stackhpc_radosgw_usage_exporter_backend_port: 9242 + +# Port to expose RADOS gateway usage exporter frontend (via HAProxy). Default is 9240 +stackhpc_radosgw_usage_exporter_frontend_port: 9240 + +# Path to a certificate for internal TLS in the RADOS gateway usage exporter. +stackhpc_radosgw_usage_exporter_cacert: "" + +# Whether TLS certificate verification is enabled for the RADOS gateway usage +# exporter for querying Ceph RADOS gateway APIs. Default follows the condition +# of kolla_enable_tls_internal +stackhpc_radosgw_usage_exporter_verify: "{{ kolla_enable_tls_internal }}" diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml new file mode 100644 index 000000000..99f5ae836 --- /dev/null +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -0,0 +1,236 @@ +--- +# StackHPC overcloud host disk image configuration. + +############################################################################### +# Diskimage-builder configuration for overcloud host disk images. + +# StackHPC overcloud host disk image Diskimage Builder recipe. This may be used +# as an item in the overcloud_dib_host_images list when +# overcloud_dib_build_host_images is true. +# Example: +# overcloud_dib_host_images: +# - "{{ stackhpc_overcloud_dib_host_image }}" +stackhpc_overcloud_dib_host_image: + name: "{{ stackhpc_overcloud_dib_name }}" + elements: "{{ stackhpc_overcloud_dib_elements }}" + env: "{{ stackhpc_overcloud_dib_env_vars }}" + packages: "{{ stackhpc_overcloud_dib_packages }}" + +# StackHPC overcloud DIB image name. +stackhpc_overcloud_dib_name: "deployment_image" + +# StackHPC overcloud DIB image elements. +stackhpc_overcloud_dib_elements: + - "{{ os_distribution }}-{% if os_distribution == 'rocky' %}container-stackhpc{% else %}minimal{% endif %}" + - "cloud-init-datasources" + - "{% if os_distribution == 'rocky' %}selinux-permissive{% endif %}" + - "enable-serial-console" + - "vm" + - "block-device-efi" + - "cloud-init" + - "{% if os_distribution == 'rocky' %}dracut-regenerate{% endif %}" + - "{% if os_distribution == 'ubuntu' %}lvm{% endif %}" + - "openssh-server" + - "{% if os_distribution == 'ubuntu' %}sudoers{% endif %}" + +# StackHPC overcloud DIB image environment variables. +stackhpc_overcloud_dib_env_vars: + DIB_BLOCK_DEVICE_CONFIG: "{{ stackhpc_overcloud_dib_block_device_config_uefi_lvm }}" + DIB_BOOTLOADER_DEFAULT_CMDLINE: "nofb nomodeset gfxpayload=text net.ifnames=1 rd.auto" + DIB_GRUB_TIMEOUT: "5" + DIB_GRUB_TIMEOUT_STYLE: "menu" + DIB_CLOUD_INIT_DATASOURCES: "OpenStack, ConfigDrive" + DIB_CONTAINERFILE_RUNTIME: "docker" + DIB_CONTAINERFILE_NETWORK_DRIVER: "host" + DIB_CONTAINERFILE_DOCKERFILE: "/opt/kayobe/src/stackhpc-image-elements/elements/rocky-container-stackhpc/containerfiles/9-stackhpc" + # NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in + # Ark does not work if the upstream mirror pulls packages (which it does + # sometimes). + # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" + DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}" + DIB_RELEASE: "{{ overcloud_dib_os_release }}" + DIB_SUDOERS_FILENAME: "no-fqdn" + # Avoid DNS queries during sudo commands, since we might not always have working DNS. + DIB_SUDOERS_CONFIG: | + Defaults !fqdn + # FIXME: Support templating repo files. + # DIB_YUM_MINIMAL_BOOTSTRAP_REPOS: /path/to/dir/containing/dib-mirror-*.repo + YUM: dnf + # Workaround for stack user home ownership bug + DIB_IMAGE_CACHE: "/tmp/yum" + +# StackHPC overcloud DIB image packages. +stackhpc_overcloud_dib_packages: + - "ethtool" + - "git" + - "less" + - "logrotate" + - "net-tools" + - "nvme-cli" + - "pciutils" + - "python3" + - "smartmontools" + - "vim" + - "{% if os_distribution == 'ubuntu' %}netbase{% endif %}" + - "{% if os_distribution == 'ubuntu' %}iputils-ping{% endif %}" + - "{% if os_distribution == 'ubuntu' %}curl{% endif %}" + - "{% if os_distribution == 'ubuntu' %}apt-utils{% endif %}" + - "{% if os_distribution == 'rocky' %}NetworkManager-config-server{% endif %}" + - "{% if os_distribution == 'rocky' %}linux-firmware{% endif %}" + - "{% if os_distribution == 'rocky' %}cloud-utils-growpart{% endif %}" + - "{% if os_distribution == 'ubuntu' %}cloud-guest-utils{% endif %}" + +# StackHPC overcloud DIB image block device configuration. +# This image layout conforms to the CIS partition benchmarks. +# This configuration builds a UEFI-compatible image with 3 partitions. +# * p0: EFI ESP bootloader +# * p1: EFI BSP +# * p2: LVM PV (rootpv) +# The rootpv PV is in the rootvg VG, and has the following LVs: +# * lv_root -> / +# * lv_tmp -> /tmp +# * lv_var -> /var +# * lv_var_tmp -> /var/tmp +# * lv_log -> /var/log +# * lv_audit -> /var/log/audit +# * lv_home -> /home +stackhpc_overcloud_dib_block_device_config_uefi_lvm: | + - local_loop: + name: image0 + size: 20GiB + - partitioning: + base: image0 + label: gpt + partitions: + - name: ESP + type: 'EF00' + size: 500MiB + mkfs: + type: vfat + mount: + mount_point: /boot/efi + fstab: + options: "defaults" + fsck-passno: 2 + - name: BSP + type: 'EF02' + size: 8MiB + - name: root + type: '8E00' + flags: [ boot ] + size: 100% + - lvm: + name: lvm + base: [ root ] + pvs: + - name: rootpv + base: root + options: [ "--force" ] + vgs: + - name: rootvg + base: [ "rootpv" ] + options: [ "--force" ] + lvs: + - name: lv_root + base: rootvg + size: 5G + - name: lv_tmp + base: rootvg + size: 1G + - name: lv_var + base: rootvg + size: 1G + - name: lv_var_tmp + base: rootvg + size: 1G + - name: lv_log + base: rootvg + size: 1G + - name: lv_audit + base: rootvg + size: 128M + - name: lv_home + base: rootvg + size: 128M + - mkfs: + name: fs_root + base: lv_root + type: ext4 + label: "rootfs" + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 + - mkfs: + name: fs_tmp + base: lv_tmp + type: ext4 + label: "tmpfs" + mount: + mount_point: /tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_var + base: lv_var + type: ext4 + label: "varfs" + mount: + mount_point: /var + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_var_tmp + base: lv_var_tmp + type: ext4 + label: "vartmpfs" + mount: + mount_point: /var/tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_log + base: lv_log + type: ext4 + label: "logfs" + mount: + mount_point: /var/log + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_audit + base: lv_audit + type: ext4 + label: "auditfs" + mount: + mount_point: /var/log/audit + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_home + base: lv_home + type: ext4 + label: "homefs" + mount: + mount_point: /home + fstab: + options: "rw,nodev" + fsck-passno: 2 + +# StackHPC overcloud DIB image Dracut module configuration. +stackhpc_overcloud_dib_dracut_enabled_modules_default_config: | + - name: crypt + packages: + - cryptsetup + - name: lvm + packages: + - lvm2 + - name: mdraid + packages: + - mdraid diff --git a/etc/kayobe/stackhpc-overcloud-host-images.yml b/etc/kayobe/stackhpc-overcloud-host-images.yml new file mode 100644 index 000000000..b0594e1e0 --- /dev/null +++ b/etc/kayobe/stackhpc-overcloud-host-images.yml @@ -0,0 +1,19 @@ +--- +############################## +# Release train overcloud host image sources + +# Whether or not to download overcloud host images from Ark +stackhpc_download_overcloud_host_images: false + +# The overcloud host image source, defined by os_distribution, os_release, +# and the current stable version. +stackhpc_overcloud_host_image_url: "{{ stackhpc_release_pulp_content_url_with_auth }}/kayobe-images/\ + {{ openstack_release }}/{{ os_distribution }}/{{ os_release }}/\ + {{ stackhpc_overcloud_host_image_version }}/\ + overcloud-{{ os_distribution }}-{{ os_release }}.qcow2" + +# Overcloud host image version tag selection +stackhpc_overcloud_host_image_version: >- + {{ stackhpc_rocky_9_overcloud_host_image_version if os_distribution == 'rocky' and os_release == '9' else + stackhpc_ubuntu_jammy_overcloud_host_image_version if os_distribution == 'ubuntu' and os_release == 'jammy' else + stackhpc_ubuntu_noble_overcloud_host_image_version if os_distribution == 'ubuntu' and os_release == 'noble' }} diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml new file mode 100644 index 000000000..c2ce41826 --- /dev/null +++ b/etc/kayobe/stackhpc.yml @@ -0,0 +1,219 @@ +--- +# Host and port of a package repository mirror. +stackhpc_repo_mirror_url: "{{ pulp_url }}" + +# Username of a package repository mirror. +stackhpc_repo_mirror_username: + +# Password of a package repository mirror. +stackhpc_repo_mirror_password: + +# Whether to use an authenticating reverse proxy to access the package +# repository mirror. This may be used when building container images, to avoid +# injecting package repository mirror credentials into the built images. See +# ansible/roles/pulp_auth_proxy/README.md for details. +stackhpc_repo_mirror_auth_proxy_enabled: false + +# URL of an authenticating reverse proxy used to access the package repository +# mirror. Used during container image builds when +# stackhpc_repo_mirror_auth_proxy_enabled is true. +stackhpc_repo_mirror_auth_proxy_url: "http://localhost" + +# Distribution name. Either 'development' or 'production'. +stackhpc_repo_distribution: "development" + +# Whether or not to include the os minor version in the url for rocky yum repositories +stackhpc_include_os_minor_version_in_repo_url: false + +stackhpc_rocky_9_url_version: "{{ '9.' + stackhpc_pulp_repo_rocky_9_minor_version | string if stackhpc_include_os_minor_version_in_repo_url | bool else '9' }}" + +############################################################################### +# Debs +# Ubuntu jammy +stackhpc_repo_ubuntu_jammy_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ubuntu/jammy/{{ stackhpc_repo_ubuntu_jammy_version }}" +stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_repo_distribution }}" + +# Ubuntu jammy security +stackhpc_repo_ubuntu_jammy_security_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ubuntu/jammy-security/{{ stackhpc_repo_ubuntu_jammy_security_version }}" +stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_repo_distribution }}" + +# Ubuntu jammy CVE-3034-6287 +stackhpc_repo_ubuntu_jammy_cve_2024_6387_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ubuntu-jammy-cve-2024-6387/{{ stackhpc_repo_ubuntu_jammy_cve_2024_6387_version }}" +stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: "{{ stackhpc_repo_distribution }}" + +# Ubuntu noble +stackhpc_repo_ubuntu_noble_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ubuntu/noble/{{ stackhpc_repo_ubuntu_noble_version }}" +stackhpc_repo_ubuntu_noble_version: "{{ stackhpc_repo_distribution }}" + +# Ubuntu noble security +stackhpc_repo_ubuntu_noble_security_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ubuntu/noble-security/{{ stackhpc_repo_ubuntu_noble_security_version }}" +stackhpc_repo_ubuntu_noble_security_version: "{{ stackhpc_repo_distribution }}" + +# Ubuntu Cloud Archive +stackhpc_repo_ubuntu_cloud_archive_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ubuntu-cloud-archive/{{ stackhpc_repo_ubuntu_cloud_archive_version }}" +stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_repo_distribution }}" + +# Docker CE for Ubuntu Jammy +stackhpc_repo_docker_ce_ubuntu_jammy_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/docker-ce/ubuntu-jammy/{{ stackhpc_repo_docker_ce_ubuntu_jammy_version }}" +stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_repo_distribution }}" + +# Docker CE for Ubuntu Noble +stackhpc_repo_docker_ce_ubuntu_noble_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/docker-ce/ubuntu-noble/{{ stackhpc_repo_docker_ce_ubuntu_noble_version }}" +stackhpc_repo_docker_ce_ubuntu_noble_version: "{{ stackhpc_repo_distribution }}" + +# Ceph Reef for Debian +stackhpc_repo_ceph_reef_debian_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/ceph/debian-reef/{{ stackhpc_repo_ceph_reef_debian_version }}" +stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_repo_distribution }}" + +############################################################################### +# RPMs + +# DOCA +stackhpc_repo_rhel9_doca_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca/{{ stackhpc_pulp_doca_version }}/rhel{{ doca_rocky_9_minor_dot }}/x86_64/{{ stackhpc_repo_rhel9_doca_version }}" +stackhpc_repo_rhel9_doca_version: "{{ stackhpc_repo_distribution }}" + +# DOCA Modules +stackhpc_repo_rhel9_doca_modules_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca-modules/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_repo_rhel9_doca_modules_version }}" +stackhpc_repo_rhel9_doca_modules_version: "{{ stackhpc_repo_distribution }}" + +# Grafana +stackhpc_repo_grafana_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/grafana/oss/rpm/{{ stackhpc_repo_grafana_version }}" +stackhpc_repo_grafana_version: "{{ stackhpc_repo_distribution }}" + +# RabbitMQ - Erlang for RHEL 9 +stackhpc_repo_rhel9_rabbitmq_erlang_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/erlang/el/9/{{ kolla_base_arch }}/{{ stackhpc_repo_rhel9_rabbitmq_erlang_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_repo_distribution }}" +# RabbitMQ - Erlang 26 for RHEL 9 (aarch64) +stackhpc_repo_rhel9_rabbitmq_erlang_26_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/erlang-26/el/9/aarch64/{{ stackhpc_repo_rhel9_rabbitmq_erlang_26_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_26_version: "{{ stackhpc_repo_distribution }}" + +# RabbitMQ - Erlang 27 for RHEL 9 (aarch64) +stackhpc_repo_rhel9_rabbitmq_erlang_27_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/erlang-27/el/9/aarch64/{{ stackhpc_repo_rhel9_rabbitmq_erlang_27_version }}" +stackhpc_repo_rhel9_rabbitmq_erlang_27_version: "{{ stackhpc_repo_distribution }}" + +# RabbitMQ for RHEL 9 +stackhpc_repo_rhel9_rabbitmq_server_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rabbitmq/rabbitmq-server/el/9/noarch/{{ stackhpc_repo_rhel9_rabbitmq_server_version }}" +stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_repo_distribution }}" + +# CentOS Stream 9 - NFV OpenvSwitch +stackhpc_repo_centos_stream_9_nfv_openvswitch_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/9-stream/nfv/{{ kolla_base_arch }}/openvswitch-2/{{ stackhpc_repo_centos_stream_9_nfv_openvswitch_version }}" +stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_repo_distribution }}" + +# CentOS Stream 9 - OpenStack Caracal +stackhpc_repo_centos_stream_9_openstack_caracal_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/9-stream/cloud/{{ kolla_base_arch }}/openstack-caracal/{{ stackhpc_repo_centos_stream_9_openstack_caracal_version }}" +stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_repo_distribution }}" + +# CentOS Stream 9 - OpsTools - collectd +stackhpc_repo_centos_stream_9_opstools_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/9-stream/opstools/{{ kolla_base_arch }}/collectd-5/{{ stackhpc_repo_centos_stream_9_opstools_version }}" +stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_repo_distribution }}" + +# CentOS Stream 9 - Ceph Reef +stackhpc_repo_centos_stream_9_storage_ceph_reef_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/centos/9-stream/storage/{{ kolla_base_arch }}/ceph-reef/{{ stackhpc_repo_centos_stream_9_storage_ceph_reef_version }}" +stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_repo_distribution }}" + +# CentOS Stream 9 Docker CE +stackhpc_repo_centos_stream_9_docker_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/docker-ce/centos/9/{{ kolla_base_arch }}/stable/{{ stackhpc_repo_centos_stream_9_docker_version }}" +stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_repo_distribution }}" + +# TreasureData 5 for RHEL 9 +stackhpc_repo_rhel_9_treasuredata_5_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/treasuredata/5/redhat/9/{{ kolla_base_arch }}/{{ stackhpc_repo_rhel_9_treasuredata_5_version }}" +stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_repo_distribution }}" + +# MariaDB 10.11 for RHEL 9 +stackhpc_repo_rhel_9_mariadb_10_11_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/mariadb-10.11/yum/rhel/9/{{ kolla_base_arch }}/{{ stackhpc_repo_rhel_9_mariadb_10_11_version }}" +stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_repo_distribution }}" + +# InfluxDB for RHEL 9 +stackhpc_repo_rhel_9_influxdb_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/influxdb/rhel/9/{{ kolla_base_arch }}/stable/{{ stackhpc_repo_rhel_9_influxdb_version }}" +stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_repo_distribution }}" + +# OpenSearch for RHEL 9 +stackhpc_repo_opensearch_2_x_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/opensearch/2.x/yum/{{ stackhpc_repo_opensearch_2_x_version }}" +stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_repo_distribution }}" + +# OpenSearch Dashboards for RHEL 9 +stackhpc_repo_opensearch_dashboards_2_x_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/opensearch-dashboards/2.x/yum/{{ stackhpc_repo_opensearch_dashboards_2_x_version }}" +stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 9 AppStream +stackhpc_repo_rocky_9_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/{{ stackhpc_rocky_9_url_version }}/AppStream/{{ kolla_base_arch }}/os/{{ stackhpc_repo_rocky_9_appstream_version }}" +stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 9 BaseOS +stackhpc_repo_rocky_9_baseos_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/{{ stackhpc_rocky_9_url_version }}/BaseOS/{{ kolla_base_arch }}/os/{{ stackhpc_repo_rocky_9_baseos_version }}" +stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 9 CRB +stackhpc_repo_rocky_9_crb_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/{{ stackhpc_rocky_9_url_version }}/CRB/{{ kolla_base_arch }}/os/{{ stackhpc_repo_rocky_9_crb_version }}" +stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 9 extras +stackhpc_repo_rocky_9_extras_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/{{ stackhpc_rocky_9_url_version }}/extras/{{ kolla_base_arch }}/os/{{ stackhpc_repo_rocky_9_extras_version }}" +stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 9 highavailability +stackhpc_repo_rocky_9_highavailability_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/{{ stackhpc_rocky_9_url_version }}/highavailability/{{ kolla_base_arch }}/os/{{ stackhpc_repo_rocky_9_highavailability_version }}" +stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_repo_distribution }}" + +# Rocky 9 SIG Security Common +stackhpc_repo_rocky_9_sig_security_common_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/sig/9/security/{{ kolla_base_arch }}/security-common/{{ stackhpc_repo_rocky_9_sig_security_common_version }}" +stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_repo_distribution }}" + +# EPEL 9 +stackhpc_repo_epel_9_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/epel/9/Everything/{{ kolla_base_arch }}/{{ stackhpc_repo_epel_9_version }}" +stackhpc_repo_epel_9_version: "{{ stackhpc_repo_distribution }}" + +# ELRepo 9 +stackhpc_repo_elrepo_9_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/elrepo/elrepo/el9/{{ kolla_base_arch }}/{{ stackhpc_repo_elrepo_9_version }}" +stackhpc_repo_elrepo_9_version: "{{ stackhpc_repo_distribution }}" + +############################################################################### +# Sources + +# Kolla source repository. +stackhpc_kolla_source_url: "https://github.com/stackhpc/kolla" +stackhpc_kolla_source_version: stackhpc/18.8.0.11 + +# Kolla Ansible source repository. +stackhpc_kolla_ansible_source_url: "https://github.com/stackhpc/kolla-ansible" +stackhpc_kolla_ansible_source_version: stackhpc/18.8.0.9 + +############################################################################### +# Container image registry + +# Host and port of container registry. +stackhpc_docker_registry: "{{ pulp_url | regex_replace('^https?://', '') }}" + +# Username and password of container registry. +stackhpc_docker_registry_username: "{{ pulp_stack_username if pulp_stack_password is defined and pulp_stack_password | length > 0 else pulp_username }}" +stackhpc_docker_registry_password: "{{ pulp_stack_password if pulp_stack_password is defined and pulp_stack_password | length > 0 else pulp_password }}" + +############################################################################### +# Feature flags + +# Whether or not to run CIS benchmark hardening playbooks. Default is false. +#stackhpc_enable_cis_benchmark_hardening_hook: + +# Whether or not to check the installed Kayobe version when running Kayobe commands. +# Default is true. +stackhpc_enable_kayobe_check: true + +# Whether or not to check the installed Kolla-Ansible version when running Kayobe commands. +# Default is true. +stackhpc_enable_kolla_ansible_check: true + +############################################################################### +# Octavia Amphora image + +# Whether or not to download Octavia Amphora image from Ark. Default is true. +download_amphora_from_ark: true + +# Octavia Amphora image version +stackhpc_amphora_image_version: "2024.1-20250117T092645" + +################################################################################ +# Certificate Authority + +# Secret store to deploy as a Certificate Authority. +# Valid options are "vault" and "openbao". Default is "openbao". +stackhpc_ca_secret_store: openbao diff --git a/etc/kayobe/storage.yml b/etc/kayobe/storage.yml index ac7b30ff1..9a0ba93c6 100644 --- a/etc/kayobe/storage.yml +++ b/etc/kayobe/storage.yml @@ -2,8 +2,10 @@ ############################################################################### # Storage node configuration. -# User with which to access the storages via SSH during bootstrap, in order -# to setup the Kayobe user account. Default is {{ os_distribution }}. +# User with which to access the storage nodes via SSH during bootstrap, in +# order to setup the Kayobe user account. Default is 'cloud-user' if +# os_distribution is set to centos, otherwise 'os_distribution'. +#storage_bootstrap_user: storage_bootstrap_user: ileigh ############################################################################### @@ -148,6 +150,13 @@ storage_firewalld_default_zone: trusted # - state: enabled storage_firewalld_rules: "{{ stackhpc_firewalld_rules }}" +############################################################################### +# Storage node swap configuration. + +# List of swap devices. Each item is a dict containing a 'device' item. Default +# is an empty list. +# storage_swap: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/time.yml b/etc/kayobe/time.yml index 844251c76..133d9bb5a 100644 --- a/etc/kayobe/time.yml +++ b/etc/kayobe/time.yml @@ -39,6 +39,20 @@ chrony_ntp_servers: # Synchronise hardware clock with system time. Default is true. #chrony_rtcsync_enabled: +# Force synchronisation from NTP sources. This methods may jump the clock by +# large values which can cause issues with some software. Disabled by default. +#ntp_force_sync: + +# Maximum number of tries used by the `chronyc waitsync` command. Only used +# when ntp_force_sync is true. Default is 60 which waits for a maximum of 10 +# minutes (60 times 10 seconds). +#chrony_waitsync_max_tries: + +# Maximum correction used by the `chronyc waitsync` command. Only used when +# ntp_force_sync is true. Default is 0.01 which waits for the remaining +# correction to be less than 10 milliseconds. +#chrony_waitsync_max_correction: + ############################################################################### # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/trivy/allowed-vulnerabilities.yml b/etc/kayobe/trivy/allowed-vulnerabilities.yml new file mode 100644 index 000000000..26ee5977d --- /dev/null +++ b/etc/kayobe/trivy/allowed-vulnerabilities.yml @@ -0,0 +1,98 @@ +--- +############################################################################### +# Trivy allowed vulnerabilities list + +# Example allowed vulnerabilities file setup +# +# global_allowed_vulnerabilities: +# - CVE-2024-36039 +# +# keystone_allowed_vulnerabilities: +# - CVE-2022-2447 +# +# barbican_api_allowed_vulnerabilities: +# - CVE-2023-31047 +fluentd_allowed_vulnerabilities: + - CVE-2024-27280 +grafana_allowed_vulnerabilities: + - CVE-2024-8986 + - CVE-2025-68121 # the opensearch datasource plugin is still vulnerable +influxdb_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +ironic_neutron_agent_allowed_vulnerabilities: + - CVE-2025-68121 +letsencrypt_lego_allowed_vulnerabilities: + - CVE-2025-68121 +magnum_conductor_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +neutron_base_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_bgp_dragent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_dhcp_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_l3_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_linuxbridge_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_metadata_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_mlnx_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_openvswitch_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_ovn_agent_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_server_allowed_vulnerabilities: + - CVE-2025-68121 +neutron_sriov_agent_allowed_vulnerabilities: + - CVE-2025-68121 +opensearch_dashboards_allowed_vulnerabilities: + - CVE-2025-68428 +prometheus_alertmanager_allowed_vulnerabilities: + - CVE-2025-68121 +prometheus_blackbox_exporter_allowed_vulnerabilities: + - CVE-2024-24790 + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_memcached_exporter_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_mysqld_exporter_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_elasticsearch_exporter_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_node_exporter_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_openstack_exporter_allowed_vulnerabilities: + - CVE-2024-24790 + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_ovn_exporter_allowed_vulnerabilities: + - CVE-2024-24790 + - CVE-2025-68121 +prometheus_libvirt_exporter_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_cadvisor_allowed_vulnerabilities: + - CVE-2024-41110 + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_msteams_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 +prometheus_mtail_allowed_vulnerabilities: + - CVE-2024-24790 + - CVE-2025-68121 +prometheus_v2_server_allowed_vulnerabilities: + - CVE-2024-45337 + - CVE-2025-68121 + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/vault.yml b/etc/kayobe/vault.yml new file mode 100644 index 000000000..98e48ce4a --- /dev/null +++ b/etc/kayobe/vault.yml @@ -0,0 +1,59 @@ +--- +############################################################################### +# Hashicorp Vault deployment configuration. + +# Registry information for seed. +# seed_hashicorp_registry_url: +# seed_hashicorp_registry_username: +# seed_hashicorp_registry_password: + +# Seed Consul container image. +# seed_consul_docker_image: + +# Seed Consul container image tag. +# seed_consul_docker_tag: + +# Seed Vault container image. +# seed_vault_docker_image: + +# Seed Vault container image tag. +# seed_vault_docker_tag: + +# Seed Vault PKI Role name +# seed_vault_pki_role_name: + +# Seed Vault PKI Roles definition +# seed_vault_pki_roles: [] + +# Registry information for overcloud. +# overcloud_hashicorp_registry_url: +# overcloud_hashicorp_registry_username: +# overcloud_hashicorp_registry_password: + +# Overcloud Consul container image. +# overcloud_consul_docker_image: + +# Overcloud Consul container image tag. +# overcloud_consul_docker_tag: + +# Overcloud Vault container image. +# overcloud_vault_docker_image: + +# Overcloud Vault container image tag. +# overcloud_vault_docker_tag: + +# Overcloud Vault PKI Default Role name +# overcloud_vault_pki_default_role_name: + +# Overcloud Vault PKI Internal TLS Role name +# overcloud_vault_pki_internal_tls_role_name: + +# Overcloud Vault PKI Backend TLS Role name +# overcloud_vault_pki_backend_tls_role_name: + +# Overcloud Vault PKI Roles definition +# overcloud_vault_pki_roles: [] + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/releasenotes/config.yaml b/releasenotes/config.yaml new file mode 100644 index 000000000..3ec1ba0b3 --- /dev/null +++ b/releasenotes/config.yaml @@ -0,0 +1,5 @@ +--- +# NOTE(upgrade): This needs to be updated to the current release's major version. +release_tag_re: stackhpc/16\.\d+\.\d+\.\d +ignore_null_merges: false +default_branch: 'stackhpc/2025.1' diff --git a/releasenotes/notes/2.11.1-opensearch-bump-11fc566cd57a5f39.yaml b/releasenotes/notes/2.11.1-opensearch-bump-11fc566cd57a5f39.yaml new file mode 100644 index 000000000..758fac264 --- /dev/null +++ b/releasenotes/notes/2.11.1-opensearch-bump-11fc566cd57a5f39.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates OpenSearch to 2.11.1. diff --git a/releasenotes/notes/2023.1-update-packages-and-containers-q2-2024-02ed375b9c73ac75.yaml b/releasenotes/notes/2023.1-update-packages-and-containers-q2-2024-02ed375b9c73ac75.yaml new file mode 100644 index 000000000..263c70230 --- /dev/null +++ b/releasenotes/notes/2023.1-update-packages-and-containers-q2-2024-02ed375b9c73ac75.yaml @@ -0,0 +1,45 @@ +--- +features: + - | + Bumped pulp repo versions for Q2 2024 + Bumped Kolla image tags for Q2 2024 + Bumped prometheus server from 2.38.0 to 2.51.1 + Bumped prometheus alertmanager from 0.24.0 to 0.26.0 + Bumped prometheus blackbox exporter from 0.23.0 to 0.25.0 + Bumped prometheus cadvisor exporter from 0.48.0 to 0.49.1 + Bumped prometheus haproxy exporter from 0.13.0 to 0.15.0 + Bumped prometheus memcached exporter from 0.10.0 to 0.14.3 + Bumped prometheus msteams from 1.5.1 to 1.5.2 + Bumped prometheus mtail from 3.0.0-rc50 to 3.0.0-rc53 + Bumped prometheus mysqld exporter from 0.15.0 to 0.15.1 + Bumped prometheus node exporter from 1.4.0 to 1.7.0 + Bumped prometheus openstack exporter from 1.6.0 to 1.7.0 + Bumped prometheus ovn exporter from 1.0.6 to 1.0.7 + Bumped opensearch from 2.11.1-1 to 2.13.0-1 (Rocky Linux 9) + Bumped opensearch from 2.12.0 to 2.13.0 (Ubuntu Jammy) + Bumped grafana from 10.1.5-1 to 10.4.2-1 (Rocky Linux 9) + Bumped grafana from 10.4.0 to 10.4.2 (Ubuntu Jammy) +security: + - | + Fixed CVE-2023-31047, CVE-2023-23969, CVE-2023-24580, CVE-2023-36053, + CVE-2023-46695, CVE-2023-30861, CVE-2022-4899. CVE-2024-1135, + GHSA-2m57-hf25-phgg, CVE-2023-0286, CVE-2023-50782, CVE-2024-26130 + for openstack services. + + Fixed CVE-2022-41723, CVE-2023-39325 (except prometheus-alertmanager, + prometheus-msteams-exporter, prometheus-haproxy-exporter, + prometheus-openstack-exporter. No patch available.), CVE-2021-43565, + CVE-2022-27191, CVE-2022-27664, CVE-2021-38561, CVE-2022-21698, + CVE-2021-4238, CVE-2022-40083, CVE-2022-41721, CVE-2021-33194, + CVE-2023-2253, CVE-2023-27561, CVE-2023-28840, CVE-2024-21626, + CVE-2022-32149, CVE-2023-45142, GHSA-m425-mq94-257g + for prometheus server and exporters except prometheus-libvirt-exporter + and prometheus-haproxy-exporter. (Source repository of each are archived + and no longer maintained) + + Fixed CVE-2023-39325, CVE-2023-45142, CVE-2023-47108, CVE-2023-49568, + CVE-2023-49569, GHSA-9763-4f94-gfch, GHSA-m425-mq94-257g + for grafana. + + It is advised to redeploy service with current version of images from + StackHPC Release Train. diff --git a/releasenotes/notes/NetworkManager-config-server-5423b1ba9e3bc165.yaml b/releasenotes/notes/NetworkManager-config-server-5423b1ba9e3bc165.yaml new file mode 100644 index 000000000..f1b9fa9ef --- /dev/null +++ b/releasenotes/notes/NetworkManager-config-server-5423b1ba9e3bc165.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Added ``NetworkManager-config-server`` package to Rocky Linux 9 deployment + image. Which prevents NetworkManager from automatically running DHCP on + unconfigured ethernet devices and allows connections with static IP + addresses to be brought up even on ethernet devices with no carrier. diff --git a/releasenotes/notes/OSSA-2024-005-74fc3c536cde645f.yaml b/releasenotes/notes/OSSA-2024-005-74fc3c536cde645f.yaml new file mode 100644 index 000000000..f0a359ff9 --- /dev/null +++ b/releasenotes/notes/OSSA-2024-005-74fc3c536cde645f.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + Rebuilt Neutron containers to include fixes for `OSSA-2024-005 + `_. The + vulnerability allows unauthorised users to change tags on networks in + Neutron. diff --git a/releasenotes/notes/add-alerts-for-swap-availability-75e28ed7f913d1ec.yaml b/releasenotes/notes/add-alerts-for-swap-availability-75e28ed7f913d1ec.yaml new file mode 100644 index 000000000..b63a56530 --- /dev/null +++ b/releasenotes/notes/add-alerts-for-swap-availability-75e28ed7f913d1ec.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Added two alerts (warning and critical) that are triggered when the ratio + of (free_swap_space / total_swap_space) is below thresholds. + Each threshold can be modified by altering value of + ``alertmanager_node_free_swap_warning_threshold_ratio`` and + ``alertmanager_node_free_swap_critical_threshold_ratio``. + + Currently this solution has limitation of having one-size fits all policy. + This can cause unwanted alerts for the hosts which utilise swap heavily + Therefore it is recommended to tune the thresholds or apply silence rules + for the needs. diff --git a/releasenotes/notes/add-blazar-fa6ecce8b21c73b4.yaml b/releasenotes/notes/add-blazar-fa6ecce8b21c73b4.yaml new file mode 100644 index 000000000..029c00e33 --- /dev/null +++ b/releasenotes/notes/add-blazar-fa6ecce8b21c73b4.yaml @@ -0,0 +1,7 @@ +--- + features: + - | + Add ``blazar`` project Kolla container images. ``Blazar`` is a resource + reservation service for OpenStack. ``Blazar`` enables users to reserve a + specific type/amount of resources for a specific time period and it leases + these resources to users based on their reservations. diff --git a/releasenotes/notes/add-caso-f36b98453be10169.yaml b/releasenotes/notes/add-caso-f36b98453be10169.yaml new file mode 100644 index 000000000..ee71bcf1f --- /dev/null +++ b/releasenotes/notes/add-caso-f36b98453be10169.yaml @@ -0,0 +1,8 @@ +--- + features: + - | + Adds ``caso`` container images. ``cASO`` is an is an accounting reporter + that supports Cloud Accounting Usage Records. For more information, see + the `upstream docs `__. Note that + this container does not exist in upstream Kolla and is maintained + downstream by StackHPC. diff --git a/releasenotes/notes/add-ceph-mgr-exporter-endpoints-f7d3c8b1131f6982.yaml b/releasenotes/notes/add-ceph-mgr-exporter-endpoints-f7d3c8b1131f6982.yaml new file mode 100644 index 000000000..6cac3a11b --- /dev/null +++ b/releasenotes/notes/add-ceph-mgr-exporter-endpoints-f7d3c8b1131f6982.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds code to the globals.yml file to add endpoints for the + ceph_mgr_exporter. If ceph is configured correctly, managers will be + under the mgrs inventory group. If this group is empty, then the + variable will just be empty (the KA default). This also requires + setting ``kolla_enable_prometheus_ceph_mgr_exporter`` to ``true``. diff --git a/releasenotes/notes/add-confirmation-prompt-to-reboot.yml-4fd1ae8e8d360e57.yaml b/releasenotes/notes/add-confirmation-prompt-to-reboot.yml-4fd1ae8e8d360e57.yaml new file mode 100644 index 000000000..b696ac7c6 --- /dev/null +++ b/releasenotes/notes/add-confirmation-prompt-to-reboot.yml-4fd1ae8e8d360e57.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + A confirmation prompt has been added to ``reboot.yml`` to help avoid + rebooting the wrong hosts by mistake. This check can be skipped by setting + ``confirm_reboot: true``. diff --git a/releasenotes/notes/add-disk-utils-58b871d9afc9cbe8.yaml b/releasenotes/notes/add-disk-utils-58b871d9afc9cbe8.yaml new file mode 100644 index 000000000..570f725bd --- /dev/null +++ b/releasenotes/notes/add-disk-utils-58b871d9afc9cbe8.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds NVMe and S.M.A.R.T utilities to the overcloud host image built by DIB. diff --git a/releasenotes/notes/add-friendly-interface-names-d95bba2f9af28302.yaml b/releasenotes/notes/add-friendly-interface-names-d95bba2f9af28302.yaml new file mode 100644 index 000000000..64b54e737 --- /dev/null +++ b/releasenotes/notes/add-friendly-interface-names-d95bba2f9af28302.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add optional support for relabelling network devices in Prometheus. + Use network names as defined in kayobe, instead of network device names. + Reuse of device names within an environment is not supported. \ No newline at end of file diff --git a/releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml b/releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml new file mode 100644 index 000000000..e18490003 --- /dev/null +++ b/releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Adds support for deploying GitHub runners and creating GitHub workflows + for use within Kayobe Automation. Two playbooks and their requirements + have been added to `ansible/` in addition to the relevant groups defined + with some useful default variables where appropriate. Finally, + documentation has been added to cover how to deploy these runners and + workflows. diff --git a/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml b/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml new file mode 100644 index 000000000..b4235388b --- /dev/null +++ b/releasenotes/notes/add-grafana-plugins-f4856a30529ac686.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + The grafana image now includes the `gnocchixyz-gnocchi-datasource` and the + `grafana-opensearch-datasource` plugins, which are the default upstream + plugins. diff --git a/releasenotes/notes/add-hotfix-containers-playbook-5a6d3d48067cf0b6.yaml b/releasenotes/notes/add-hotfix-containers-playbook-5a6d3d48067cf0b6.yaml new file mode 100644 index 000000000..98d1e7c10 --- /dev/null +++ b/releasenotes/notes/add-hotfix-containers-playbook-5a6d3d48067cf0b6.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The playbook ``hotfix-containers.yml`` has been added. This allows + arbitrary files to be copied into, and/or arbitrary commands to be executed + within, overcloud containers. diff --git a/releasenotes/notes/add-iscsid-support-e05d5dfe76de235d.yaml b/releasenotes/notes/add-iscsid-support-e05d5dfe76de235d.yaml new file mode 100644 index 000000000..339019340 --- /dev/null +++ b/releasenotes/notes/add-iscsid-support-e05d5dfe76de235d.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add support for building ``iscsid`` Kolla container images. diff --git a/releasenotes/notes/add-jammy-repo-yoga-f58bfb9792a3c6a7.yaml b/releasenotes/notes/add-jammy-repo-yoga-f58bfb9792a3c6a7.yaml new file mode 100644 index 000000000..448bf2ad7 --- /dev/null +++ b/releasenotes/notes/add-jammy-repo-yoga-f58bfb9792a3c6a7.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Support for Ubuntu 22.04 Jammy Jellyfish + repositories have been added to the Yoga Release. \ No newline at end of file diff --git a/releasenotes/notes/add-letsencrypt-eab-support-807bdc4078d3114d.yaml b/releasenotes/notes/add-letsencrypt-eab-support-807bdc4078d3114d.yaml new file mode 100644 index 000000000..ff0ec922b --- /dev/null +++ b/releasenotes/notes/add-letsencrypt-eab-support-807bdc4078d3114d.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds support for Let's Encrypt external account binding (EAB). diff --git a/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml b/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml new file mode 100644 index 000000000..0a985519c --- /dev/null +++ b/releasenotes/notes/add-monitoring-to-multinode-958603f026edbc7c.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Set monitoring services be enabled by default in the ``ci-multinode`` + environment. diff --git a/releasenotes/notes/add-more-tempest-tests-4c162a50378f9c5a.yaml b/releasenotes/notes/add-more-tempest-tests-4c162a50378f9c5a.yaml new file mode 100644 index 000000000..3156a2f2b --- /dev/null +++ b/releasenotes/notes/add-more-tempest-tests-4c162a50378f9c5a.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add additional Tempest tests plugins covering; Barbican, Cinder, + Cloudkitty, Designate, Glance, Ironic, Keystone, Magnum, + Manila, Neutron and Octavia. + Add load lists for the new Tempest plugins. + Use `docker-rally` v2.0.1. diff --git a/releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml b/releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml new file mode 100644 index 000000000..c5b52984f --- /dev/null +++ b/releasenotes/notes/add-nova-compute-ironic-failover-doc-a0c4f45b1fb48c4a.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Adds basic support and a document explaining how to migrate to a single + nova-compute-ironic instance, and how to re-deploy the instance to another + machine in the event of failure. See the operations / nova-compute-ironic + doc for further details. +upgrade: + - | + Ensure that your deployment has only one nova-compute-ironic service running + per conductor group. See the operations / nova-compute-ironic doc for further + details. diff --git a/releasenotes/notes/add-openbao-for-tls-698ae3834ed5c67f.yaml b/releasenotes/notes/add-openbao-for-tls-698ae3834ed5c67f.yaml new file mode 100644 index 000000000..c6c760b40 --- /dev/null +++ b/releasenotes/notes/add-openbao-for-tls-698ae3834ed5c67f.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Add support for deploying ``OpenBao`` across the ``seed`` and ``overcloud`` hosts + for the purpose of internal and backend TLS generation. +deprecations: + - | + Hashicorp Vault for TLS generation is deprecated in favour of OpenBao. + The ``openbao`` role is now used to deploy OpenBao on the seed and overcloud hosts. + New deployments should use OpenBao for TLS generation. + Existing deployments using Hashicorp Vault for TLS generation should be migrated + to OpenBao once migration steps are available. diff --git a/releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml b/releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml new file mode 100644 index 000000000..596a3eada --- /dev/null +++ b/releasenotes/notes/add-openbao-raft-ha-e8d78ffe68913512.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support for highly available Raft when using OpenBao on + overcloud hosts. diff --git a/releasenotes/notes/add-opensearch-bdc4a9d230ffef40.yaml b/releasenotes/notes/add-opensearch-bdc4a9d230ffef40.yaml new file mode 100644 index 000000000..b374d219c --- /dev/null +++ b/releasenotes/notes/add-opensearch-bdc4a9d230ffef40.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + ``OpenSearch`` container images have been added. diff --git a/releasenotes/notes/add-post-reboot-delay-2ac3fa99badf4601.yaml b/releasenotes/notes/add-post-reboot-delay-2ac3fa99badf4601.yaml new file mode 100644 index 000000000..468631aef --- /dev/null +++ b/releasenotes/notes/add-post-reboot-delay-2ac3fa99badf4601.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds an extra delay after reboot in the ``reboot.yml`` playbook to ensure + hosts can be reached reliably. The default delay is 5 seconds and can be + customised with the ``post_reboot_delay_s`` variable. diff --git a/releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml b/releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml new file mode 100644 index 000000000..1f0ac091d --- /dev/null +++ b/releasenotes/notes/add-pre-commit-hooks-07ce3b82bbe1d7a3.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add playbook to install pre-commit hooks and register them with git. + The hooks currently configured to be installed will check yaml syntax, + fix new line at end of file and remove excess whitespace. This is + currently opt-in which can be achieved by running `install-pre-commit-hooks` + playbook. diff --git a/releasenotes/notes/add-rabbitmq-4-1-support-caracal-ddb7271c83b75897.yaml b/releasenotes/notes/add-rabbitmq-4-1-support-caracal-ddb7271c83b75897.yaml new file mode 100644 index 000000000..e7e8c99b8 --- /dev/null +++ b/releasenotes/notes/add-rabbitmq-4-1-support-caracal-ddb7271c83b75897.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Added RabbitMQ 4.1 support. + Added a step to upgrade RabbitMQ from version 3.13 to 4.1 after queue + migration in ``upgrade-prerequisites.sh``. + Multinode upgrade CI will go through this when run with major upgrade + enabled. diff --git a/releasenotes/notes/add-radosgw-usage-exporter-support-93d55c544418b05a.yaml b/releasenotes/notes/add-radosgw-usage-exporter-support-93d55c544418b05a.yaml new file mode 100644 index 000000000..8b6e4d4e3 --- /dev/null +++ b/releasenotes/notes/add-radosgw-usage-exporter-support-93d55c544418b05a.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Adds RADOS Gateway usage exporter support. + + To deploy the exporter, set the variable ``stackhpc_enable_radosgw_usage_exporter`` + to true. Then run playbook ``deploy-radosgw-usage-exporter.yml``. + A certificate path needs to be set to ``stackhpc_radosgw_usage_exporter_cacert`` + if internal TLS is enabled. diff --git a/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml b/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml new file mode 100644 index 000000000..d1edf25a4 --- /dev/null +++ b/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added the ``rekey-hosts.yml`` playbook to automatically rotate the SSH + keys on all hosts. diff --git a/releasenotes/notes/add-rocky-9-packages-43074cb3367171a5.yaml b/releasenotes/notes/add-rocky-9-packages-43074cb3367171a5.yaml new file mode 100644 index 000000000..f9c6bd534 --- /dev/null +++ b/releasenotes/notes/add-rocky-9-packages-43074cb3367171a5.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add the package repository configuration required for Rocky Linux 9 + support. + + Add CI for Rocky 9 hosts. diff --git a/releasenotes/notes/add-rocky-9.2-repos-eff5e5b0b07111f5.yaml b/releasenotes/notes/add-rocky-9.2-repos-eff5e5b0b07111f5.yaml new file mode 100644 index 000000000..b902a7767 --- /dev/null +++ b/releasenotes/notes/add-rocky-9.2-repos-eff5e5b0b07111f5.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added support for Rocky Linux 9.2 repositories and made 9.2 the default + version. diff --git a/releasenotes/notes/add-rocky-9.3-repos-454c2f825e088417.yaml b/releasenotes/notes/add-rocky-9.3-repos-454c2f825e088417.yaml new file mode 100644 index 000000000..9a25d0e1e --- /dev/null +++ b/releasenotes/notes/add-rocky-9.3-repos-454c2f825e088417.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Added support for Rocky Linux 9.3 repositories and Kolla containers. + Made 9.3 the default version for Rocky Linux. + - | + Updated Rocky Linux 9.2 pulp repo versions. + Added Rocky Linux 9.3 pulp repo versions. + Rebuilt Kolla containers with Rocky Linux 9.3. diff --git a/releasenotes/notes/add-rocky-94-repos-27caa21c77252866.yaml b/releasenotes/notes/add-rocky-94-repos-27caa21c77252866.yaml new file mode 100644 index 000000000..0a05455ca --- /dev/null +++ b/releasenotes/notes/add-rocky-94-repos-27caa21c77252866.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Added support for Rocky Linux 9.4 repositories and Kolla containers. + Made 9.4 the default version for Rocky Linux. + - | + Updated Rocky Linux 9.3 pulp repo versions. + Added Rocky Linux pulp repo versions. + Rebuilt Kolla containers with Rocky 9.4. diff --git a/releasenotes/notes/add-stack-user-for-pulp-c96041e82c13aa10.yaml b/releasenotes/notes/add-stack-user-for-pulp-c96041e82c13aa10.yaml new file mode 100644 index 000000000..63b4dae95 --- /dev/null +++ b/releasenotes/notes/add-stack-user-for-pulp-c96041e82c13aa10.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add support for a basic user for Pulp operations instead of using + the admin user for usage. Can be enabled by setting + `pulp_stack_password`. diff --git a/releasenotes/notes/add-ubuntu-noble-support-caracal-1b9e64d4aa2e1ff7.yaml b/releasenotes/notes/add-ubuntu-noble-support-caracal-1b9e64d4aa2e1ff7.yaml new file mode 100644 index 000000000..12079d9af --- /dev/null +++ b/releasenotes/notes/add-ubuntu-noble-support-caracal-1b9e64d4aa2e1ff7.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added support for Ubuntu 24.04 Noble Numbat as a host operating system. + Repositories and configuration for Ubuntu Noble have been added. diff --git a/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml b/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml new file mode 100644 index 000000000..cabf7a891 --- /dev/null +++ b/releasenotes/notes/add-vm-overcloud-support-92baa7c2ce87ac1d.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds support for using a VMs as compute and controller nodes in the + ``ci-multinode`` environment by dynamically setting the MTU of the networks + in networks.yml and removing the static definition of the network + interfaces for the compute and controller groups. diff --git a/releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml b/releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml new file mode 100644 index 000000000..571bc9d28 --- /dev/null +++ b/releasenotes/notes/add_wazuh-be2999448db5cb7c.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add Wazuh deployment playbook. + diff --git a/releasenotes/notes/adds-alert-for-active-routers-91281912213692c9.yaml b/releasenotes/notes/adds-alert-for-active-routers-91281912213692c9.yaml new file mode 100644 index 000000000..1578a4706 --- /dev/null +++ b/releasenotes/notes/adds-alert-for-active-routers-91281912213692c9.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds an alert to check that there is exactly one active router on ML2/OVS based + deployments. diff --git a/releasenotes/notes/adds-amphora-utility-playbooks-21f7c523f4960f89.yaml b/releasenotes/notes/adds-amphora-utility-playbooks-21f7c523f4960f89.yaml new file mode 100644 index 000000000..bcca7d66f --- /dev/null +++ b/releasenotes/notes/adds-amphora-utility-playbooks-21f7c523f4960f89.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds utility playbooks to build and rotate amphora images. For more details + check out the Octavia section of the Operator Guide included in the + documentation. diff --git a/releasenotes/notes/adds-cis-hardening-for-ubuntu-jammy-d9bf23a34c08f5be.yaml b/releasenotes/notes/adds-cis-hardening-for-ubuntu-jammy-d9bf23a34c08f5be.yaml new file mode 100644 index 000000000..66de6e0e8 --- /dev/null +++ b/releasenotes/notes/adds-cis-hardening-for-ubuntu-jammy-d9bf23a34c08f5be.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for Ubuntu Jammy and Rocky 9 to the CIS benchmark hardening playbook: + ``cis.yml``. This playbook will need to be manually applied. diff --git a/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml b/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml new file mode 100644 index 000000000..63c3bd0e0 --- /dev/null +++ b/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a hook to automatically run the CIS benchmark hardening playbooks as + part of host configure. This is guarded by the + ``stackhpc_enable_cis_benchmark_hardening_hook`` configuration option and is + disabled by default. diff --git a/releasenotes/notes/adds-kolla-config-merging-options-6f75d63b4b784d7f.yaml b/releasenotes/notes/adds-kolla-config-merging-options-6f75d63b4b784d7f.yaml new file mode 100644 index 000000000..f9ab145d0 --- /dev/null +++ b/releasenotes/notes/adds-kolla-config-merging-options-6f75d63b4b784d7f.yaml @@ -0,0 +1,19 @@ +--- +features: + - | + Adds kolla config merging options to the ``Kolla custom config generation`` + section of ``etc/kayobe/kolla.yml``. +upgrade: + - | + `Kolla config merging is enabled by default in the Antelope release of Kayobe + `__. + This was quite an extensive change and whilst backwards compatbility was + one of the goals, there may be some situations where refactoring of your + Kolla config will be necessary. Extra care should be taken if you are using + the multiple environments feature. It is recommended that you carefully + check the diff in the resultant Kolla configuration by following `these steps + `__ + to check for missing config or duplicated config options. The + ``kolla_openstack_custom_config_environment_merging_enabled`` option can + be set to ``False`` to revert back to the old behaviour. + diff --git a/releasenotes/notes/adds-mdraid-alerts-309fb79e61389325.yaml b/releasenotes/notes/adds-mdraid-alerts-309fb79e61389325.yaml new file mode 100644 index 000000000..831281609 --- /dev/null +++ b/releasenotes/notes/adds-mdraid-alerts-309fb79e61389325.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds alerts for software raid failures. diff --git a/releasenotes/notes/adds-networking-generic-switch-batching-support-adffe038ea2441d0.yaml b/releasenotes/notes/adds-networking-generic-switch-batching-support-adffe038ea2441d0.yaml new file mode 100644 index 000000000..f68ab23ed --- /dev/null +++ b/releasenotes/notes/adds-networking-generic-switch-batching-support-adffe038ea2441d0.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Brings in new neutron container images to add batching support to + Networking Generic Switch. This is opt in via the ``ngs_batch_requests`` + configuration option and only affects Ironic deployments that use + Networking Generic Switch. See the following `PR + `__ for more + details. diff --git a/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml b/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml new file mode 100644 index 000000000..695ab035c --- /dev/null +++ b/releasenotes/notes/adds-networking-mlnx-05fdc10ea6132145.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds the ``networking-mlnx`` mechanism driver to the Neutron Server + container and ``ebrctl`` utility to the Nova Compute container. This allows + you to use the ``kolla_enable_neutron_mlnx`` feature flag. diff --git a/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml b/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml new file mode 100644 index 000000000..2146c460d --- /dev/null +++ b/releasenotes/notes/adds-support-for-trunk-ports-7301a258dca0c469.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Updates neutron containers to contain a version of + networking-generic-switch with support for trunk ports when using DellOS 10 + or Cisco switches. See this `PR + `__ for more + details. + - | + Updates neutron containers to contain a version of + networking-generic-switch with support for DellOS 10. See this `PR + `__ for more + details. diff --git a/releasenotes/notes/aio-lvm-script-7de0c919f312040b.yaml b/releasenotes/notes/aio-lvm-script-7de0c919f312040b.yaml new file mode 100644 index 000000000..9b049fc0c --- /dev/null +++ b/releasenotes/notes/aio-lvm-script-7de0c919f312040b.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Improvements to the ci-aio automated deployment script + to allow the script to successfully run on LVM-based + images. \ No newline at end of file diff --git a/releasenotes/notes/aio-script-8221b9dcbcc40c55.yaml b/releasenotes/notes/aio-script-8221b9dcbcc40c55.yaml new file mode 100644 index 000000000..b3148b6f3 --- /dev/null +++ b/releasenotes/notes/aio-script-8221b9dcbcc40c55.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added a script to the AIO environment that can be used to quickly + deploy an AIO for testing. diff --git a/releasenotes/notes/allow-admins-to-list-bms-ff0bc40d605d4ca2.yaml b/releasenotes/notes/allow-admins-to-list-bms-ff0bc40d605d4ca2.yaml new file mode 100644 index 000000000..b830aac0d --- /dev/null +++ b/releasenotes/notes/allow-admins-to-list-bms-ff0bc40d605d4ca2.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added a custom policy to Ironic that allows users with the `admin` role to + list all baremetal nodes. This is required at sites where baremetal + provisioning targets a specific node, as we need to look up the node's uuid + to pass as the hypervisor hostname. diff --git a/releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml b/releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml new file mode 100644 index 000000000..7caa578cc --- /dev/null +++ b/releasenotes/notes/ansible-profile-tasks-3d341727a39dadcb.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds time information to tasks using the `ansible.posix.profile_tasks` + callback. diff --git a/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml b/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml new file mode 100644 index 000000000..13e4144e0 --- /dev/null +++ b/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds some basic tuning of Ansible, including use of 20 forks, enabling + SSH pipelining, YAML-formatted output, and disabling fact variable + injection. diff --git a/releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml b/releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml new file mode 100644 index 000000000..4a06e6705 --- /dev/null +++ b/releasenotes/notes/automated-firewalld-a95e7322fd457259.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + A default firewall configuration is now included on an opt-in basis. The + rules are defined under ``etc/kayobe/inventory/group_vars/all/firewall``. + More information can be found `here + `__ diff --git a/releasenotes/notes/backend-tls-network-aa7b09008a2e1914.yaml b/releasenotes/notes/backend-tls-network-aa7b09008a2e1914.yaml new file mode 100644 index 000000000..69cb215f9 --- /dev/null +++ b/releasenotes/notes/backend-tls-network-aa7b09008a2e1914.yaml @@ -0,0 +1,6 @@ +--- +issues: + - | + Generate backend TLS files for network hosts. This fixes backend TLS + configuration for deployments where some API services are running on + network hosts. diff --git a/releasenotes/notes/bifrost-disable-ironic-debug-5427c6f1160779aa.yaml b/releasenotes/notes/bifrost-disable-ironic-debug-5427c6f1160779aa.yaml new file mode 100644 index 000000000..0a945ad5d --- /dev/null +++ b/releasenotes/notes/bifrost-disable-ironic-debug-5427c6f1160779aa.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Bifrost Ironic debug logging is now disabled by default. Change + ``ironic_debug`` to ``true`` to revert. diff --git a/releasenotes/notes/bifrost-fix-ipa-auth-9d0e0c3b948b5850.yaml b/releasenotes/notes/bifrost-fix-ipa-auth-9d0e0c3b948b5850.yaml new file mode 100644 index 000000000..87bb66c90 --- /dev/null +++ b/releasenotes/notes/bifrost-fix-ipa-auth-9d0e0c3b948b5850.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an issue where the IPA images in Ark could not be downloaded by + Bifrost, due to missing authentication parameters. diff --git a/releasenotes/notes/bifrost-update-discovery-fix-220d745e22d6fbe0.yaml b/releasenotes/notes/bifrost-update-discovery-fix-220d745e22d6fbe0.yaml new file mode 100644 index 000000000..d855669f8 --- /dev/null +++ b/releasenotes/notes/bifrost-update-discovery-fix-220d745e22d6fbe0.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Update Bifrost container images to include a fix for + DHCP-based hardware discovery from + https://review.opendev.org/c/openstack/bifrost/+/902233. + diff --git a/releasenotes/notes/blackbox-backend-4415919e10a1aa4e.yaml b/releasenotes/notes/blackbox-backend-4415919e10a1aa4e.yaml new file mode 100644 index 000000000..a16b40622 --- /dev/null +++ b/releasenotes/notes/blackbox-backend-4415919e10a1aa4e.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + Added Blackbox monitoring for backend endpoints by default. Note that this + configuration will only work if the Blackbox exporters have access to the + backend endpoints. +issues: + - | + Backend Blackbox monitoring will not work if the exporter does not have + access to the backend OpenStack endpoints. This usually happens when + separate monitoring nodes are deployed. In this case, move the Blackbox + exporter to the Haproxy group, remove the endpoints from + `etc/kayobe/kolla/inventory/group_vars/prometheus-blackbox-exporter`, or + silence the alerts permanently. diff --git a/releasenotes/notes/bug-2019977-456bbf34fed3f090.yaml b/releasenotes/notes/bug-2019977-456bbf34fed3f090.yaml new file mode 100644 index 000000000..8d91a99cf --- /dev/null +++ b/releasenotes/notes/bug-2019977-456bbf34fed3f090.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Updates the ``nova-compute-ironic`` container image to fix `bug 2019977 + `__, which would cause Ironic + instances to fail to delete. diff --git a/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml new file mode 100644 index 000000000..50edfe142 --- /dev/null +++ b/releasenotes/notes/bugfix-smart-rules-53ff5400e7f681c8.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixed a syntax error in Prometheus SMART monitoring rules. diff --git a/releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml b/releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml new file mode 100644 index 000000000..22b60e653 --- /dev/null +++ b/releasenotes/notes/bump-alertmanager-958f90fa2bc9b562.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Prometheus Alertmanager has been updated to ``0.28.1``. This release + includes support for Microsoft Teams notifications. diff --git a/releasenotes/notes/bump-all-tags-58e916ff669d87e2.yaml b/releasenotes/notes/bump-all-tags-58e916ff669d87e2.yaml new file mode 100644 index 000000000..34099202e --- /dev/null +++ b/releasenotes/notes/bump-all-tags-58e916ff669d87e2.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Rebuilt all kolla and package repo tags to bring in kernel fixes and apply + CentOS image build customisations that were previously being ignored. diff --git a/releasenotes/notes/bump-ansible-collection-cephadm-2a6c988a34b192a6.yaml b/releasenotes/notes/bump-ansible-collection-cephadm-2a6c988a34b192a6.yaml new file mode 100644 index 000000000..dbc6dd22e --- /dev/null +++ b/releasenotes/notes/bump-ansible-collection-cephadm-2a6c988a34b192a6.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates the StackHPC Cephadm Ansible collection from 1.18.0 to 1.19.1. diff --git a/releasenotes/notes/bump-bifrost-xena-c0c65bd823ed15fa.yaml b/releasenotes/notes/bump-bifrost-xena-c0c65bd823ed15fa.yaml new file mode 100644 index 000000000..298f82b1e --- /dev/null +++ b/releasenotes/notes/bump-bifrost-xena-c0c65bd823ed15fa.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Rebuild and bump the Bifrost container for Xena to include fix for + `Error while running update_to_latest_versions: ''BIOSSetting'' + object has no attribute` during `Ironic database migrations on upgrade` + diff --git a/releasenotes/notes/bump-centos8-stream-snapshots-2023-09-04-a473edfd3f3b2298.yaml b/releasenotes/notes/bump-centos8-stream-snapshots-2023-09-04-a473edfd3f3b2298.yaml new file mode 100644 index 000000000..7a187e308 --- /dev/null +++ b/releasenotes/notes/bump-centos8-stream-snapshots-2023-09-04-a473edfd3f3b2298.yaml @@ -0,0 +1,18 @@ +--- +security: + - | + Bumps CentOS Stream 8 snapshots to include fixes for Zenbleed + (CVE-2023-20593) and Downfall (CVE-2022-40982). It is recommended that you + update your OS packages and reboot into the kernel as soon as possible. +upgrade: + - | + CentOS Stream 8 snapshots have been bumped and new container images are + available. Make sure to sync these into your local pulp. The yum repositories + must be reconfigured to exclude a `buggy version of iptables + `__. To do this use: + ``kayobe overcloud service reconfigure -kt none -t dnf``. + - | + CentOS Extras has been replaced with CentOS Extras Common. You may need to + use the ``--allowerasing`` option with DNF if you have packages installed + from the old repo. This is a one time only thing and on the next package + update you can drop this argument. diff --git a/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml new file mode 100644 index 000000000..439f8b842 --- /dev/null +++ b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates default Ceph images to v17.2.7 for Quincy. diff --git a/releasenotes/notes/bump-focal-7d99ewj84r7h70sq1.yaml b/releasenotes/notes/bump-focal-7d99ewj84r7h70sq1.yaml new file mode 100644 index 000000000..1d47cbd5a --- /dev/null +++ b/releasenotes/notes/bump-focal-7d99ewj84r7h70sq1.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Bumped focal package versions due to unmet depenencies diff --git a/releasenotes/notes/bump-glance-to-fix-bug-2086675-303542d3208103b9.yaml b/releasenotes/notes/bump-glance-to-fix-bug-2086675-303542d3208103b9.yaml new file mode 100644 index 000000000..f5b071e48 --- /dev/null +++ b/releasenotes/notes/bump-glance-to-fix-bug-2086675-303542d3208103b9.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix `bug 2086675: `__ + Performance regression for Glance with RBD backend. diff --git a/releasenotes/notes/bump-hashicorp-images-2c23873ad5eeb719.yaml b/releasenotes/notes/bump-hashicorp-images-2c23873ad5eeb719.yaml new file mode 100644 index 000000000..8d52ac986 --- /dev/null +++ b/releasenotes/notes/bump-hashicorp-images-2c23873ad5eeb719.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates Consul to 1.16.4 and Vault to 1.14.8. diff --git a/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml b/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml new file mode 100644 index 000000000..6e25cb43b --- /dev/null +++ b/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates Consul to 1.16.3 and Vault to 1.14.6. diff --git a/releasenotes/notes/bump-horizon-694d426decbf7df3.yaml b/releasenotes/notes/bump-horizon-694d426decbf7df3.yaml new file mode 100644 index 000000000..780797d9e --- /dev/null +++ b/releasenotes/notes/bump-horizon-694d426decbf7df3.yaml @@ -0,0 +1,5 @@ +--- +security: + - | + Update Horizon on Ubuntu to include apache2 package ``2.4.52-1ubuntu4.8`` + which fixes CVE-2023-31122. diff --git a/releasenotes/notes/bump-horizon-cloudkitty-dashboards-e94f706d5a993cc9.yaml b/releasenotes/notes/bump-horizon-cloudkitty-dashboards-e94f706d5a993cc9.yaml new file mode 100644 index 000000000..fd88487ac --- /dev/null +++ b/releasenotes/notes/bump-horizon-cloudkitty-dashboards-e94f706d5a993cc9.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + Brings in a change for cloudkitty-dashboard, which includes the `option to + attach pre/postfix to the rate value + `__. diff --git a/releasenotes/notes/bump-horizon-dae0cd4cfffe7d7e.yaml b/releasenotes/notes/bump-horizon-dae0cd4cfffe7d7e.yaml new file mode 100644 index 000000000..00addea99 --- /dev/null +++ b/releasenotes/notes/bump-horizon-dae0cd4cfffe7d7e.yaml @@ -0,0 +1,5 @@ +--- +security: + - | + Updated Horizon Kolla image to fix `CVE-2024-42005 + `_ diff --git a/releasenotes/notes/bump-keystone-image-ec2token-98caa61ccf86c944.yaml b/releasenotes/notes/bump-keystone-image-ec2token-98caa61ccf86c944.yaml new file mode 100644 index 000000000..3576d6aaa --- /dev/null +++ b/releasenotes/notes/bump-keystone-image-ec2token-98caa61ccf86c944.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Security fixes for `bug 2119646 `__: + Unauthenticated access to EC2/S3 token + endpoints can grant Keystone authorization. diff --git a/releasenotes/notes/bump-magnum-51e03a61ae8aa5a4.yaml b/releasenotes/notes/bump-magnum-51e03a61ae8aa5a4.yaml new file mode 100644 index 000000000..b28764800 --- /dev/null +++ b/releasenotes/notes/bump-magnum-51e03a61ae8aa5a4.yaml @@ -0,0 +1,3 @@ +--- +fixes: + - Updates Magnum CAPI Helm driver version to v0.10.0 diff --git a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml new file mode 100644 index 000000000..7fc3cca1a --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates Magnum CAPI Helm driver version to v0.11.0 diff --git a/releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml b/releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml new file mode 100644 index 000000000..6677583fb --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-6febfe840e81cea5.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates Magnum CAPI Helm driver version to v0.12.0 diff --git a/releasenotes/notes/bump-magnum-capi-helm-7e4ad37d3d9eecce.yaml b/releasenotes/notes/bump-magnum-capi-helm-7e4ad37d3d9eecce.yaml new file mode 100644 index 000000000..883521087 --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-7e4ad37d3d9eecce.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates Magnum CAPI Helm driver version to v0.11.0 diff --git a/releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml b/releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml new file mode 100644 index 000000000..eb1e37640 --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-d766b5956de65d31.yaml @@ -0,0 +1,4 @@ +--- + features: + - | + Updates Magnum CAPI Helm driver version to v0.13.0 diff --git a/releasenotes/notes/bump-nova-ede0cd4cfffe7d4e.yaml b/releasenotes/notes/bump-nova-ede0cd4cfffe7d4e.yaml new file mode 100644 index 000000000..7fba3e9d5 --- /dev/null +++ b/releasenotes/notes/bump-nova-ede0cd4cfffe7d4e.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Updates Nova container images to pick up various bug fixes. One of + these should improve API performance: + `LP#2122036 `__. diff --git a/releasenotes/notes/bump-octavia-273fb50e05836f21.yaml b/releasenotes/notes/bump-octavia-273fb50e05836f21.yaml new file mode 100644 index 000000000..8551fc7f8 --- /dev/null +++ b/releasenotes/notes/bump-octavia-273fb50e05836f21.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + Bumps octavia container versions +fixes: + - | + Fixes bug https://bugs.launchpad.net/neutron/+bug/2004238 diff --git a/releasenotes/notes/bump-ovs-ovn-rocky9-2024.1-eb5a42d9f890d3cd.yaml b/releasenotes/notes/bump-ovs-ovn-rocky9-2024.1-eb5a42d9f890d3cd.yaml new file mode 100644 index 000000000..0268db52b --- /dev/null +++ b/releasenotes/notes/bump-ovs-ovn-rocky9-2024.1-eb5a42d9f890d3cd.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Updated OpenvSwitch to 3.3.4-115 and OVN to 24.03.5-88 for Rocky Linux 9 + in Caracal. For details, see the following changelogs: + + * `OVS changelog `__ + * `OVN changelog `__ + + - | + Kolla Toolbox, Manila, Neutron, Nova, and Octavia containers received + updates on both Rocky Linux 9 and Ubuntu. Only the Rocky Linux 9 images + include the new OVS versions. diff --git a/releasenotes/notes/bump-qemu-kvm-pdcm-e890d516f1efc690.yaml b/releasenotes/notes/bump-qemu-kvm-pdcm-e890d516f1efc690.yaml new file mode 100644 index 000000000..9c4cc6d50 --- /dev/null +++ b/releasenotes/notes/bump-qemu-kvm-pdcm-e890d516f1efc690.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Updates Nova container images on Rocky Linux 9 to resolve the following + issue: ``operation failed: guest CPU doesn't match specification: missing + features: pdcm``. diff --git a/releasenotes/notes/bump-rocky-9-90412343a29bc45b.yaml b/releasenotes/notes/bump-rocky-9-90412343a29bc45b.yaml new file mode 100644 index 000000000..cd8bbb59c --- /dev/null +++ b/releasenotes/notes/bump-rocky-9-90412343a29bc45b.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Bumped rocky 9 package versions due to missing snapshot diff --git a/releasenotes/notes/bump-rocky8-snapshots-2023-09-29-e115427edd3334c7.yaml b/releasenotes/notes/bump-rocky8-snapshots-2023-09-29-e115427edd3334c7.yaml new file mode 100644 index 000000000..f44c44d98 --- /dev/null +++ b/releasenotes/notes/bump-rocky8-snapshots-2023-09-29-e115427edd3334c7.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + The Rocky 8 minor version has been bumped to 8.8 and new snapshots have + been created to include fixes for Zenbleed (CVE-2023-20593), Downfall + (CVE-2022-40982). It is recommended that you update your OS packages and + reboot into the kernel as soon as possible. diff --git a/releasenotes/notes/bump-rocky9-snapshots-2023-09-29-c736c3d37afd7e5c.yaml b/releasenotes/notes/bump-rocky9-snapshots-2023-09-29-c736c3d37afd7e5c.yaml new file mode 100644 index 000000000..83a9f5565 --- /dev/null +++ b/releasenotes/notes/bump-rocky9-snapshots-2023-09-29-c736c3d37afd7e5c.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + The snapshots for Rocky 9.2 have been refreshed to include fixes for + Zenbleed (CVE-2023-20593), Downfall (CVE-2022-40982). It is recommended + that you update your OS packages and reboot into the kernel as soon as + possible. diff --git a/releasenotes/notes/bump-ubuntu-snapshots-2023-09-15-22ca5250d40bd5b6.yaml b/releasenotes/notes/bump-ubuntu-snapshots-2023-09-15-22ca5250d40bd5b6.yaml new file mode 100644 index 000000000..8eba53ab3 --- /dev/null +++ b/releasenotes/notes/bump-ubuntu-snapshots-2023-09-15-22ca5250d40bd5b6.yaml @@ -0,0 +1,8 @@ +--- +security: + - | + Bumps Ubuntu repository snapshots and container images to bring in latest + security patches. This includes the microcode to patch Downfall + (CVE-2022-40982). Zenbleed (CVE-2023-20593) was patched in the previous + snapshot bump. To apply the microcode updates, it is recommended to reboot + each host after upgrading all of the packages. diff --git a/releasenotes/notes/bumps-redfish-exporter-to-v2-11032fb9dde36283.yaml b/releasenotes/notes/bumps-redfish-exporter-to-v2-11032fb9dde36283.yaml new file mode 100644 index 000000000..6ef30e11a --- /dev/null +++ b/releasenotes/notes/bumps-redfish-exporter-to-v2-11032fb9dde36283.yaml @@ -0,0 +1,15 @@ +--- +features: + - Upgrades the redfish exporter container image to the v2.x series. + - Adds support for lenovo hardware to the redfish exporter dashboard. + - | + Adds the ``stackhpcredfish_exporter_scrape_interval``, + ``stackhpc_os_capacity_scrape_interval``, and + ``stackhpc_prometheus_openstack_exporter_interval`` + configuration variables. +fixes: + - Fixes various issues with the redfish exporter dashboard. +upgrade: + - | + Increases default ``os_capacity_scrape_interval`` to ``5m``. If you already customise + this please move to the new ``stackhpc_os_capacity_scrape_interval`` variable. diff --git a/releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml b/releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml new file mode 100644 index 000000000..b1ca452fa --- /dev/null +++ b/releasenotes/notes/cap-pulp-workers-e0f12c0b67d3d0bf.yaml @@ -0,0 +1,4 @@ +fixes: + - | + Caps the number of Pulp API and content workers to 32 each to avoid errors + on hosts with many CPUs. diff --git a/releasenotes/notes/capi-bump-78cd6f89a0eff0d7.yaml b/releasenotes/notes/capi-bump-78cd6f89a0eff0d7.yaml new file mode 100644 index 000000000..5a4aae413 --- /dev/null +++ b/releasenotes/notes/capi-bump-78cd6f89a0eff0d7.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Updated the version of magnum-capi-helm used in Magnum containers. This + resolves an issue stopping non-default node groups from being deleted. See + `#2095539 `__ for more + details. diff --git a/releasenotes/notes/capi-changes-3275e667680af8e1.yaml b/releasenotes/notes/capi-changes-3275e667680af8e1.yaml new file mode 100644 index 000000000..ddf8878b1 --- /dev/null +++ b/releasenotes/notes/capi-changes-3275e667680af8e1.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + container tags for magnum capi changes +features: + - magnum container now has capi driver +other: + - | + deployment guide docs added for new capi driver diff --git a/releasenotes/notes/caracal-bump-container-images-083b691e83816b3b.yaml b/releasenotes/notes/caracal-bump-container-images-083b691e83816b3b.yaml new file mode 100644 index 000000000..e4070f66a --- /dev/null +++ b/releasenotes/notes/caracal-bump-container-images-083b691e83816b3b.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Bumped the base image for Ubuntu 22.04 containers. + - | + Additionally bumped Nova, Neutron and Octavia images for + both Rocky and Ubuntu. diff --git a/releasenotes/notes/ceph-config-append-1cc6146d3241b63e.yaml b/releasenotes/notes/ceph-config-append-1cc6146d3241b63e.yaml new file mode 100644 index 000000000..1d0e534d4 --- /dev/null +++ b/releasenotes/notes/ceph-config-append-1cc6146d3241b63e.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Add ``kolla_ceph_conf_append`` configuration option to specify a string + to be appended to all ceph.conf files gathered from a ceph cluster using + ``kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml``. diff --git a/releasenotes/notes/ceph-ingress-images-e756bde5087460ad.yaml b/releasenotes/notes/ceph-ingress-images-e756bde5087460ad.yaml new file mode 100644 index 000000000..dec3154c0 --- /dev/null +++ b/releasenotes/notes/ceph-ingress-images-e756bde5087460ad.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds support for using Ceph HAProxy and Keepalived images stored in Pulp. + This is enabled automatically if ``stackhpc_sync_ceph_images`` is set to + ``true``. diff --git a/releasenotes/notes/ceph-maintenance-4c4eb0a4f7149665.yaml b/releasenotes/notes/ceph-maintenance-4c4eb0a4f7149665.yaml new file mode 100644 index 000000000..285e396f8 --- /dev/null +++ b/releasenotes/notes/ceph-maintenance-4c4eb0a4f7149665.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Adds two new custom playbooks for placing Ceph hosts into and removing them + from maintenance: + + - ``ceph-enter-maintenance.yml`` + - ``ceph-exit-maintenance.yml`` +upgrade: + - | + Updates the ``stackhpc.cephadm`` collection to version ``1.18.0``. +fixes: + - | + Fixes an issue with idempotency in the ``stackhpc.ceph.cephadm_keys`` + plugin. diff --git a/releasenotes/notes/ceph-pacific-16.2.11-a675ccb4e4d70de7.yaml b/releasenotes/notes/ceph-pacific-16.2.11-a675ccb4e4d70de7.yaml new file mode 100644 index 000000000..33f240fdf --- /dev/null +++ b/releasenotes/notes/ceph-pacific-16.2.11-a675ccb4e4d70de7.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates Ceph Pacific container image to v16.2.11. diff --git a/releasenotes/notes/ceph-quincy-e110f3ba270c3a52.yaml b/releasenotes/notes/ceph-quincy-e110f3ba270c3a52.yaml new file mode 100644 index 000000000..366f5dd81 --- /dev/null +++ b/releasenotes/notes/ceph-quincy-e110f3ba270c3a52.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Automatically install Quincy if the node is running Ubuntu 22.04, else install Pacific. diff --git a/releasenotes/notes/ceph-reef-18.2.7-9d8d3ad3a1ea517b.yaml b/releasenotes/notes/ceph-reef-18.2.7-9d8d3ad3a1ea517b.yaml new file mode 100644 index 000000000..8f7efe343 --- /dev/null +++ b/releasenotes/notes/ceph-reef-18.2.7-9d8d3ad3a1ea517b.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Bumps the default Ceph Reef container image to ``v18.2.7``. diff --git a/releasenotes/notes/ceph-reef-f447dde8c397f91b.yaml b/releasenotes/notes/ceph-reef-f447dde8c397f91b.yaml new file mode 100644 index 000000000..a5c1313b8 --- /dev/null +++ b/releasenotes/notes/ceph-reef-f447dde8c397f91b.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + Updates the default version of Ceph to Reef. The following container tags + are used for Ceph: + + - ``ceph``: ``v18.2.4`` + - ``haproxy``: ``2.6`` + - ``keepalived``: ``2.2.4`` diff --git a/releasenotes/notes/ceph-squid-19.2.3-35c1a6b914d31388.yaml b/releasenotes/notes/ceph-squid-19.2.3-35c1a6b914d31388.yaml new file mode 100644 index 000000000..02d462c3c --- /dev/null +++ b/releasenotes/notes/ceph-squid-19.2.3-35c1a6b914d31388.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Bumps the default Ceph Squid container image to ``v19.2.3``. diff --git a/releasenotes/notes/cephadm-bump-294148e93121da48.yaml b/releasenotes/notes/cephadm-bump-294148e93121da48.yaml new file mode 100644 index 000000000..82f938582 --- /dev/null +++ b/releasenotes/notes/cephadm-bump-294148e93121da48.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Increase `stackhpc.cephadm` collection to version `1.12.2`. diff --git a/releasenotes/notes/cephadm-pre-post-commands-now-have-defaults-b42ce9b13fa6d63b.yaml b/releasenotes/notes/cephadm-pre-post-commands-now-have-defaults-b42ce9b13fa6d63b.yaml new file mode 100644 index 000000000..8d545b690 --- /dev/null +++ b/releasenotes/notes/cephadm-pre-post-commands-now-have-defaults-b42ce9b13fa6d63b.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + The Cephadm pre and post commands now support default commands with the + variables ``cephadm_commands_pre_default`` and + ``cephadm_commands_post_default``. As such, any extra commands should be + added to the variables ``cephadm_commands_pre_extra`` and + ``cephadm_commands_post_extra``. diff --git a/releasenotes/notes/cinder-bug-1823445-5d07997488bc2416.yaml b/releasenotes/notes/cinder-bug-1823445-5d07997488bc2416.yaml new file mode 100644 index 000000000..624d7624d --- /dev/null +++ b/releasenotes/notes/cinder-bug-1823445-5d07997488bc2416.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Updates Cinder container images to fix `bug 1823445 + `__ + (``cinder.exception.MetadataCopyFailure``). diff --git a/releasenotes/notes/cis-do-not-disable-ipv6-98bd79ad86555f51.yaml b/releasenotes/notes/cis-do-not-disable-ipv6-98bd79ad86555f51.yaml new file mode 100644 index 000000000..0034ff808 --- /dev/null +++ b/releasenotes/notes/cis-do-not-disable-ipv6-98bd79ad86555f51.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + IPV6 is no longer disabled by default in the Ubuntu CIS hardening. If + using the old behaviour you may hit `2071443 + `__. +upgrade: + - | + To match the new CIS benchmark defaults on Ubuntu, you should remove + the ``ipv6.disable=1`` kernel command line option. If you wish to carry + on with the current settings, change ``ubtu22cis_ipv6_required`` to + ``false``. diff --git a/releasenotes/notes/cis-hardening-no-longer-sets-permissions-on-logs-81a48ab8ed2d6b5f.yaml b/releasenotes/notes/cis-hardening-no-longer-sets-permissions-on-logs-81a48ab8ed2d6b5f.yaml new file mode 100644 index 000000000..e50b5b62b --- /dev/null +++ b/releasenotes/notes/cis-hardening-no-longer-sets-permissions-on-logs-81a48ab8ed2d6b5f.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + The CIS hardening scripts no longer change permissions of log files by + default. It is preferred to configure these permissions at source i.e on + whatever is creating the files. It also suffered from a time-of-check to + time-of-use race condition. If you want the old behaviour you can change + ``rhel9cis_rule_4_2_3`` and/or ``ubtu22cis_rule_4_2_3`` to ``true``. diff --git a/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml b/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml new file mode 100644 index 000000000..818f415a4 --- /dev/null +++ b/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the bulk API of CloudKitty so that it now supports the migration + from Elasticsearch to OpenSearch. diff --git a/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml new file mode 100644 index 000000000..67a99f9c2 --- /dev/null +++ b/releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Kolla container images created using the + ``stackhpc-container-image-build.yml`` workflow are now automatically + scanned for vulnerablilities. diff --git a/releasenotes/notes/cve-2024-32498-2cbd205129351766.yaml b/releasenotes/notes/cve-2024-32498-2cbd205129351766.yaml new file mode 100644 index 000000000..8dabdaf23 --- /dev/null +++ b/releasenotes/notes/cve-2024-32498-2cbd205129351766.yaml @@ -0,0 +1,7 @@ +--- +critical: + - | + Fixes `CVE-2024-32498 + `__ with + updated container images for Cinder, Glance and Nova services. + diff --git a/releasenotes/notes/deduplicate-pulp-yml-a65b3e949662a284.yaml b/releasenotes/notes/deduplicate-pulp-yml-a65b3e949662a284.yaml new file mode 100644 index 000000000..514354615 --- /dev/null +++ b/releasenotes/notes/deduplicate-pulp-yml-a65b3e949662a284.yaml @@ -0,0 +1,3 @@ +--- +other: + - Reduced verbosity in etc/kayobe/pulp.yml diff --git a/releasenotes/notes/dell-sonic-support-a1dd2edd958bbe9d.yaml b/releasenotes/notes/dell-sonic-support-a1dd2edd958bbe9d.yaml new file mode 100644 index 000000000..91dbdcad9 --- /dev/null +++ b/releasenotes/notes/dell-sonic-support-a1dd2edd958bbe9d.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds a basic NGS driver for Dell SONiC switches. diff --git a/releasenotes/notes/diagnostics-378a6693a64d0b3c.yaml b/releasenotes/notes/diagnostics-378a6693a64d0b3c.yaml new file mode 100644 index 000000000..4e8d45ee7 --- /dev/null +++ b/releasenotes/notes/diagnostics-378a6693a64d0b3c.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + Adds a new ``diagnostics.yml`` playbook that collects diagnostic + information from hosts. The diagnostics are aggregated to a directory + (``$PWD/diagnostics/`` by default) on localhost. The diagnostics include: + + * Docker container logs + * Kolla configuration files + * Log files + + *The collected diagnostic information contains sensitive information such + as passwords in configuration files.* + diff --git a/releasenotes/notes/disable-heat-03f3ac52ac610223.yaml b/releasenotes/notes/disable-heat-03f3ac52ac610223.yaml new file mode 100644 index 000000000..9a5e8eda5 --- /dev/null +++ b/releasenotes/notes/disable-heat-03f3ac52ac610223.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The Heat OpenStack service is now disabled by default. +upgrade: + - | + The Heat service is now disabled by default. This behaviour can be + overriden by setting `kolla_enable_heat: true` in `etc/kayobe/kolla.yml`. + It is recommended that you migrate to the CAPI Helm driver for Magnum + wherever possible. diff --git a/releasenotes/notes/disable-ironic-prometheus-exporter-894d98022a1e926d.yaml b/releasenotes/notes/disable-ironic-prometheus-exporter-894d98022a1e926d.yaml new file mode 100644 index 000000000..7b94110ce --- /dev/null +++ b/releasenotes/notes/disable-ironic-prometheus-exporter-894d98022a1e926d.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The Ironic Prometheus Exporter is now disabled by default as redfish exporter + is preferred. If you need to use the Ironic Prometheus Exporter, you can + enable it by setting the `kolla_enable_ironic_prometheus_exporter` + option to `true`. diff --git a/releasenotes/notes/disable-ironic-prometheus-exporter-daf2dda149466a2e.yaml b/releasenotes/notes/disable-ironic-prometheus-exporter-daf2dda149466a2e.yaml new file mode 100644 index 000000000..8c204d850 --- /dev/null +++ b/releasenotes/notes/disable-ironic-prometheus-exporter-daf2dda149466a2e.yaml @@ -0,0 +1,9 @@ +--- +issues: + - | + The Ironic Prometheus Exporter has been (temporarily) disabled. See bug for + details: https://bugs.launchpad.net/kolla-ansible/+bug/2062401 +upgrade: + - | + The Ironic Prometheus Exporter has been (temporarily) disabled. See bug for + details: https://bugs.launchpad.net/kolla-ansible/+bug/2062401 diff --git a/releasenotes/notes/disable-non-overcloud-apt-config-5b79b5b63c78779c.yaml b/releasenotes/notes/disable-non-overcloud-apt-config-5b79b5b63c78779c.yaml new file mode 100644 index 000000000..7624a632c --- /dev/null +++ b/releasenotes/notes/disable-non-overcloud-apt-config-5b79b5b63c78779c.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Disabled custom APT configuration for non-overcloud hosts (Ubuntu Only). + This resolves the issue of the seed hypervisor attempting to pull packages + from the repository on the seed before it has been deployed. diff --git a/releasenotes/notes/disable-password-expiry-3c296c72d36cd9da.yaml b/releasenotes/notes/disable-password-expiry-3c296c72d36cd9da.yaml new file mode 100644 index 000000000..3ad5e55b9 --- /dev/null +++ b/releasenotes/notes/disable-password-expiry-3c296c72d36cd9da.yaml @@ -0,0 +1,7 @@ +--- +critical: + - | + Disables password expiration and inactivity policies. This caused the kayobe + and kolla service accounts to be locked out of the system. You should re-apply + the CIS benchmark hardening playbook as soon as possible to avoid being locked + out of your system. diff --git a/releasenotes/notes/doca-2-9-3-238838fb78e0c7d9.yaml b/releasenotes/notes/doca-2-9-3-238838fb78e0c7d9.yaml new file mode 100644 index 000000000..054f460a0 --- /dev/null +++ b/releasenotes/notes/doca-2-9-3-238838fb78e0c7d9.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added support for DOCA OFED on Rocky Linux 9.6 at version ``2.9.3``. The + package versions for Rocky 9.4 and 9.5 remain unchanged, using ``2.9.1``. diff --git a/releasenotes/notes/doca-workflow-017931dbfd580b41.yaml b/releasenotes/notes/doca-workflow-017931dbfd580b41.yaml new file mode 100644 index 000000000..8b023c4e9 --- /dev/null +++ b/releasenotes/notes/doca-workflow-017931dbfd580b41.yaml @@ -0,0 +1,16 @@ +--- +features: + - | + Using DOCA LTS 2.9.1. + - | + DOCA workflows updated to build kernel modules only, relying + on Release Train synchronisation of DOCA upstream for userspace + packages. + - | + Improved documentation now including DOCA install playbook. + - | + Local Pulp syncs for DOCA/DOCA kernel module repository. +fixes: + - | + Miscellaneous issues with the package-build-ofed workflow + are resolved in this patchset. diff --git a/releasenotes/notes/docker-bump-dc4e85668ee69a21.yaml b/releasenotes/notes/docker-bump-dc4e85668ee69a21.yaml new file mode 100644 index 000000000..d5c6c3e21 --- /dev/null +++ b/releasenotes/notes/docker-bump-dc4e85668ee69a21.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + The Docker CE package for Ubuntu has been bumped from ``5:24.0.6-1`` to + ``5:25.0.0-1`` This is a side effect of separating out the repos for Docker + CE for Ubuntu Jammy/Focal. +fixes: + - | + Separated out repos for Docker CE for Ubuntu Jammy/Focal. This fixes a Pulp + sync issue where two "identical" repository versions existed with different + checksums. diff --git a/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml b/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml new file mode 100644 index 000000000..8c516d3d0 --- /dev/null +++ b/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Enables Docker live restore by default. This may be disabled by setting + ``docker_daemon_live_restore`` to ``false`` in ``docker.yml``. diff --git a/releasenotes/notes/drop-skyline-support-31d683f58f125335.yaml b/releasenotes/notes/drop-skyline-support-31d683f58f125335.yaml new file mode 100644 index 000000000..60541ce0c --- /dev/null +++ b/releasenotes/notes/drop-skyline-support-31d683f58f125335.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - | + Disabled building of Kolla container images for Skyline diff --git a/releasenotes/notes/dwpd-6b9fb0c8d6d3a570.yaml b/releasenotes/notes/dwpd-6b9fb0c8d6d3a570.yaml new file mode 100644 index 000000000..62d918519 --- /dev/null +++ b/releasenotes/notes/dwpd-6b9fb0c8d6d3a570.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Adds a panel in the Hardware Overview dashboard to show DWPD (Drive writes + per day) for NVMEs. This is calculated by dividing the total bytes written + in the past 24 hours by the drive capacity. This is currently only + supported on NVMEs. + - | + Adds alerts that will fire after 1 DWPD is sustained for 7 days, and a + critical alert if 1 DWPD is sustained for 30 days. diff --git a/releasenotes/notes/elrepo-9-9d88af1dea541155.yaml b/releasenotes/notes/elrepo-9-9d88af1dea541155.yaml new file mode 100644 index 000000000..bdb66215b --- /dev/null +++ b/releasenotes/notes/elrepo-9-9d88af1dea541155.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Provide ELRepo 9, which in turn provides packages to support be2net + and mpt3sas hardware. Configuration of ELRepo 9 is disabled by default + and may be enabled by setting `dnf_install_elrepo_9: true`. diff --git a/releasenotes/notes/enable-build-of-neutron-dragent-14897d9744c7de1f.yaml b/releasenotes/notes/enable-build-of-neutron-dragent-14897d9744c7de1f.yaml new file mode 100644 index 000000000..dc39b7b09 --- /dev/null +++ b/releasenotes/notes/enable-build-of-neutron-dragent-14897d9744c7de1f.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Enable building of the ``Neutron BGP Dragent`` container image. diff --git a/releasenotes/notes/enable-mariabackup-97834ea8d87e14a9.yaml b/releasenotes/notes/enable-mariabackup-97834ea8d87e14a9.yaml new file mode 100644 index 000000000..a2b8662ee --- /dev/null +++ b/releasenotes/notes/enable-mariabackup-97834ea8d87e14a9.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Mariabackup is now enabled by default. diff --git a/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml b/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml new file mode 100644 index 000000000..bced4f825 --- /dev/null +++ b/releasenotes/notes/enable-rabbitmq-ha-and-update-rabbitmq-version-8eaa68b2a9486320.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + The flag ``om_enable_rabbitmq_high_availability`` is now set to ``true``. + Adds tags for new RabbitMQ containers to update to RabbitMQ version 3.9.22. +upgrade: + - | + The flag ``om_enable_rabbitmq_high_availability`` is now set to ``true``. + As this enables durable queues, RabbitMQ will need to be reset, and the + services which use it restarted. + Tags are added to update the RabbitMQ containers to version 3.9.22. diff --git a/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml b/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml new file mode 100644 index 000000000..3267a0447 --- /dev/null +++ b/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml @@ -0,0 +1,10 @@ +--- +upgrade: + - | + Enabled ML2/OVN by default. Checks preventing accidental migration + from ML2/OVS were added in Kolla Ansible. If you are using a Neutron + plugin other than ML2/OVN, set ``kolla_enable_ovn`` to ``false``. + + OVN distributed FIP is disabled, to enable it set + ``neutron_ovn_distributed_fip`` to ``true`` in + ``etc/kayobe/kolla/globals.yml``. diff --git a/releasenotes/notes/erlang-aarch64-repo-change-a77c49c408413857.yaml b/releasenotes/notes/erlang-aarch64-repo-change-a77c49c408413857.yaml new file mode 100644 index 000000000..0bd6a8c62 --- /dev/null +++ b/releasenotes/notes/erlang-aarch64-repo-change-a77c49c408413857.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Switch aarch64 Erlang RPMs to versioned erlang-26 and erlang-27 COPR + repositories to support RabbitMQ 3.13/4.1 and align with Kolla [1]. + x86_64 remains unchanged. + [1] https://review.opendev.org/c/openstack/kolla/+/959692 diff --git a/releasenotes/notes/etcd-e4927551bf5d020c.yaml b/releasenotes/notes/etcd-e4927551bf5d020c.yaml new file mode 100644 index 000000000..93b6344a6 --- /dev/null +++ b/releasenotes/notes/etcd-e4927551bf5d020c.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds an ``etcd`` Kolla container image. This can be used for OpenStack + service coordination as a tooz backend, or for batched processing of switch + configuration in Networking Generic Switch (this requires a downstream NGS + patch). diff --git a/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml new file mode 100644 index 000000000..335691c30 --- /dev/null +++ b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + Updates the Ansible configuration to `fail on any unparsed inventory source + `__. + If you are using a separate Ansible configuration for Kolla Ansible, you + may wish to add this setting in ``etc/kayobe/kolla/ansible.cfg``. diff --git a/releasenotes/notes/feature-smartmon-65cacfe893f0eb47.yaml b/releasenotes/notes/feature-smartmon-65cacfe893f0eb47.yaml new file mode 100644 index 000000000..23b9e5bc8 --- /dev/null +++ b/releasenotes/notes/feature-smartmon-65cacfe893f0eb47.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds drive temperatures to the table on the hardware overview dashboard + and a timeseries to show the temperature over time. + - | + Adds picker to hardware overview dashboard to select a specific host to + show drive information for. diff --git a/releasenotes/notes/fix-2024.1-kolla-image-build-f78a5524381fa4da.yaml b/releasenotes/notes/fix-2024.1-kolla-image-build-f78a5524381fa4da.yaml new file mode 100644 index 000000000..89f02fb87 --- /dev/null +++ b/releasenotes/notes/fix-2024.1-kolla-image-build-f78a5524381fa4da.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fix Kolla container image build workflow failing to find default + sources.list.ubuntu. + The default sources.list for ubuntu now has each for Ubuntu Jammy and + Noble. + This upstream change was brought by `Ubuntu 24.04 support for Caracal + `__. diff --git a/releasenotes/notes/fix-bifrost-httpboot-config-templating-0488dd14d15d65e4.yaml b/releasenotes/notes/fix-bifrost-httpboot-config-templating-0488dd14d15d65e4.yaml new file mode 100644 index 000000000..5ed9cb7c8 --- /dev/null +++ b/releasenotes/notes/fix-bifrost-httpboot-config-templating-0488dd14d15d65e4.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Bumps Bifrost container image tags to fix `bug 2072550 + `__ which adds + leading and trailing whitespace for ``inspection_callback_url`` + in httpboot config when templating. diff --git a/releasenotes/notes/fix-blackbox-exporter-config-with-no-external-grafana-de1db02c540af6d8.yaml b/releasenotes/notes/fix-blackbox-exporter-config-with-no-external-grafana-de1db02c540af6d8.yaml new file mode 100644 index 000000000..bcf30fa2e --- /dev/null +++ b/releasenotes/notes/fix-blackbox-exporter-config-with-no-external-grafana-de1db02c540af6d8.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue where the external Grafana endpoint would be added to + Prometheus Blackbox Exporter config, even when ``enable_grafana_external`` + was disabled. diff --git a/releasenotes/notes/fix-blackbox-opensearch-dashboard-d973b43c557c8103.yaml b/releasenotes/notes/fix-blackbox-opensearch-dashboard-d973b43c557c8103.yaml new file mode 100644 index 000000000..04de31778 --- /dev/null +++ b/releasenotes/notes/fix-blackbox-opensearch-dashboard-d973b43c557c8103.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixed incorrect Opensearch Dashboards Prometheus Blackbox Exporter + configuration. diff --git a/releasenotes/notes/fix-broken-links-dcd128c8e211b2b8.yaml b/releasenotes/notes/fix-broken-links-dcd128c8e211b2b8.yaml new file mode 100644 index 000000000..8d65779e0 --- /dev/null +++ b/releasenotes/notes/fix-broken-links-dcd128c8e211b2b8.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix some broken links in the docs. diff --git a/releasenotes/notes/fix-ceph-pools-top-capacity-used-panel-26d495c45f2678c8.yaml b/releasenotes/notes/fix-ceph-pools-top-capacity-used-panel-26d495c45f2678c8.yaml new file mode 100644 index 000000000..cdf38bb3a --- /dev/null +++ b/releasenotes/notes/fix-ceph-pools-top-capacity-used-panel-26d495c45f2678c8.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes Grafana panel of top Ceph pools by capacity used. This panel was only + showing the most used pool instead of as many pools as configured with the + ``$topk`` variable. diff --git a/releasenotes/notes/fix-ceph-rgw-blackbox-endpoint-7af02679b3fd093d.yaml b/releasenotes/notes/fix-ceph-rgw-blackbox-endpoint-7af02679b3fd093d.yaml new file mode 100644 index 000000000..fc8cce6b5 --- /dev/null +++ b/releasenotes/notes/fix-ceph-rgw-blackbox-endpoint-7af02679b3fd093d.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue where the ``ceph-rgw`` endpoints were added to the + Prometheus blackbox exporter when the Kolla loadbalancer was not in use for + RGWs. diff --git a/releasenotes/notes/fix-cephadm-facts-2ee6dc9a1c617944.yaml b/releasenotes/notes/fix-cephadm-facts-2ee6dc9a1c617944.yaml new file mode 100644 index 000000000..bb0071fae --- /dev/null +++ b/releasenotes/notes/fix-cephadm-facts-2ee6dc9a1c617944.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + The Ceph version is now determined by ``os_release``, rather + than Ansible facts. Using Ansible facts caused playbooks to fail when + facts are not gathered. diff --git a/releasenotes/notes/fix-cloudkitty-opensearch-backend-743501558e69e712.yaml b/releasenotes/notes/fix-cloudkitty-opensearch-backend-743501558e69e712.yaml new file mode 100644 index 000000000..af5735557 --- /dev/null +++ b/releasenotes/notes/fix-cloudkitty-opensearch-backend-743501558e69e712.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + The OpenSearch backend for CloudKitty has been fixed, so the Horizon + ``Rating`` panels work again. diff --git a/releasenotes/notes/fix-cve-2024-40767-24b9b3c35f61a0c8.yaml b/releasenotes/notes/fix-cve-2024-40767-24b9b3c35f61a0c8.yaml new file mode 100644 index 000000000..d272329ea --- /dev/null +++ b/releasenotes/notes/fix-cve-2024-40767-24b9b3c35f61a0c8.yaml @@ -0,0 +1,6 @@ +--- +critical: + - | + Fixes `CVE-2024-40767 + `_ with updated + container images for Nova services. diff --git a/releasenotes/notes/fix-cve-2024-44082-122ef225f674d864.yaml b/releasenotes/notes/fix-cve-2024-44082-122ef225f674d864.yaml new file mode 100644 index 000000000..3066b0758 --- /dev/null +++ b/releasenotes/notes/fix-cve-2024-44082-122ef225f674d864.yaml @@ -0,0 +1,12 @@ +--- +security: + - | + Fixes `CVE-2024-44082 + `_ with updated + container images for Ironic services. Note that Ironic Python Agent images + also need to be updated to fully fix this vulnerability. If this is not + possible, a new configuration option + ``[conductor]conductor_always_validates_images`` is available. See the + `OSSA-2024-003 description + `_ for more + details. diff --git a/releasenotes/notes/fix-cve-2026-33551-e37bc84ad753d740.yaml b/releasenotes/notes/fix-cve-2026-33551-e37bc84ad753d740.yaml new file mode 100644 index 000000000..5f8718d99 --- /dev/null +++ b/releasenotes/notes/fix-cve-2026-33551-e37bc84ad753d740.yaml @@ -0,0 +1,5 @@ +--- +security: + - | + Fixes `CVE-2026-33551 `__ + with updated Keystone images diff --git a/releasenotes/notes/fix-fix-houston-notify-165a7871bda48f68.yaml b/releasenotes/notes/fix-fix-houston-notify-165a7871bda48f68.yaml new file mode 100644 index 000000000..a7e12a0e2 --- /dev/null +++ b/releasenotes/notes/fix-fix-houston-notify-165a7871bda48f68.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an issue where the fix-houston.yml playbook would fail to reload + systemd as the ``notify`` statement was incorrect. diff --git a/releasenotes/notes/fix-fluentd-opensearch-template-14bde9b6d5a723f8.yaml b/releasenotes/notes/fix-fluentd-opensearch-template-14bde9b6d5a723f8.yaml new file mode 100644 index 000000000..b5e98506a --- /dev/null +++ b/releasenotes/notes/fix-fluentd-opensearch-template-14bde9b6d5a723f8.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix missing bracket in fluentd OpenSearch configuration + causing container crashes when caso is enabled. diff --git a/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml b/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml new file mode 100644 index 000000000..f3e4b84d2 --- /dev/null +++ b/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue with the growroot playbook where disks such as 'sdp' would + become 'sd' due to the removal of the trailing 'p' when dealing with nvme + devices. diff --git a/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml b/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml new file mode 100644 index 000000000..6a395888b --- /dev/null +++ b/releasenotes/notes/fix-hardware-overview-dashboard-temp-ecc047ba46305668.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes the hardware overview dashboard to use the correct metric for + displaying drive temps. Now uses an `or` to display whichever metric + is compatible with the drives in the system. The two metrics are + temperature_case_raw_value and temperature_celsius_raw_value. diff --git a/releasenotes/notes/fix-horizon-bug-2055784-3bb729919d345e77.yaml b/releasenotes/notes/fix-horizon-bug-2055784-3bb729919d345e77.yaml new file mode 100644 index 000000000..c2c5994fa --- /dev/null +++ b/releasenotes/notes/fix-horizon-bug-2055784-3bb729919d345e77.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Updates Horizon container images to fix `bug 2055784 + `__, which would cause + Horizon to be unable to retrieve resource providers statistics in + deployments with VGPU resources. diff --git a/releasenotes/notes/fix-houston-tc-mirred-bfb16c89f63b472a.yaml b/releasenotes/notes/fix-houston-tc-mirred-bfb16c89f63b472a.yaml new file mode 100644 index 000000000..64c619cee --- /dev/null +++ b/releasenotes/notes/fix-houston-tc-mirred-bfb16c89f63b472a.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Adds a custom ``fix-houston.yml`` playbook to address dmesg errors, specifically: + "tc mirred to Houston: device bond0-ovs is down". This error typically appears + when OVS HW offloading is enabled, often in conjunction with VF-LAG and ASAP^2. + Detailed usage instructions are provided within the playbook's comments. + Additional context is available at the following links: + `LP#1899364 + `__ + `Kernel Patch + `__ diff --git a/releasenotes/notes/fix-ipa-collectors-f7afc852d5005282.yaml b/releasenotes/notes/fix-ipa-collectors-f7afc852d5005282.yaml new file mode 100644 index 000000000..57c128f9c --- /dev/null +++ b/releasenotes/notes/fix-ipa-collectors-f7afc852d5005282.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Restores the use of extra IPA collectors (including ``extra-hardware``) + when using StackHPC IPA images. diff --git a/releasenotes/notes/fix-issue-with-redfish-exporter-scrape-group-b10eaac6ee1e6af3.yaml b/releasenotes/notes/fix-issue-with-redfish-exporter-scrape-group-b10eaac6ee1e6af3.yaml new file mode 100644 index 000000000..1ee5a9a41 --- /dev/null +++ b/releasenotes/notes/fix-issue-with-redfish-exporter-scrape-group-b10eaac6ee1e6af3.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue where setting ``redfish_exporter_scrape_group`` to a value + other than ``overcloud`` would exclude those nodes from the redfish + exporter scrapes. diff --git a/releasenotes/notes/fix-kayobe-version-checks-d1fb3e09391e4a3e.yaml b/releasenotes/notes/fix-kayobe-version-checks-d1fb3e09391e4a3e.yaml new file mode 100644 index 000000000..f185977a4 --- /dev/null +++ b/releasenotes/notes/fix-kayobe-version-checks-d1fb3e09391e4a3e.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix Kayobe version checks that were failing on multiuser + Ansible control hosts. diff --git a/releasenotes/notes/fix-keystone-multiple-cves-cdb096a2fbb94bba.yaml b/releasenotes/notes/fix-keystone-multiple-cves-cdb096a2fbb94bba.yaml new file mode 100644 index 000000000..0ae767558 --- /dev/null +++ b/releasenotes/notes/fix-keystone-multiple-cves-cdb096a2fbb94bba.yaml @@ -0,0 +1,5 @@ +--- +security: + - | + Fixes CVE-2026-42998, CVE-2026-42999, CVE-2026-43000, CVE-2026-43001 + and CVE-2026-44394 with updated Keystone images. diff --git a/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml b/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml new file mode 100644 index 000000000..dc5f3c2bf --- /dev/null +++ b/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes Neutron so that load balancer FIPs are not broken on Neutron restart. + See `Neutron bug report + `__. diff --git a/releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml b/releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml new file mode 100644 index 000000000..4c71ad67d --- /dev/null +++ b/releasenotes/notes/fix-liblasso-netiq-issue-afec5b2ee7de2a1e.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the `issue `__ with using + SAML2 federation in Keystone against NetIQ IdP. diff --git a/releasenotes/notes/fix-libvirt-dashboard-b33327f804bea5f9.yaml b/releasenotes/notes/fix-libvirt-dashboard-b33327f804bea5f9.yaml new file mode 100644 index 000000000..d06acc3c9 --- /dev/null +++ b/releasenotes/notes/fix-libvirt-dashboard-b33327f804bea5f9.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed broken panels in the project metrics Grafana dashboard by updating + the Prometheus Libvirt exporter and editing the dashboard queries. + + The exporter was updated from ``v1.6.0`` to ``v2.2.0``. A full breakdown of + changes can be found `here + `__ diff --git a/releasenotes/notes/fix-local-image-build-59967df0a2bceddf.yaml b/releasenotes/notes/fix-local-image-build-59967df0a2bceddf.yaml new file mode 100644 index 000000000..accf7f166 --- /dev/null +++ b/releasenotes/notes/fix-local-image-build-59967df0a2bceddf.yaml @@ -0,0 +1,26 @@ +--- +fixes: + - | + Fixes an issue with local image builds where kolla_tag had not been set. + The error had the signature: + + .. code-block: + + Traceback (most recent call last): + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/kolla/image/build.py", line 565, in builder + buildargs=buildargs): + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/docker/api/build.py", line 269, in build + timeout=timeout, + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/docker/utils/decorators.py", line 46, in inner + return f(self, *args, **kwargs) + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/docker/api/client.py", line 233, in _post + return self.post(url, **self._set_request_timeout(kwargs)) + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/requests/sessions.py", line 577, in post + return self.request('POST', url, data=data, json=json, **kwargs) + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/requests/sessions.py", line 529, in request + resp = self.send(prep, **send_kwargs) + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/requests/sessions.py", line 645, in send + r = adapter.send(request, **kwargs) + File "/opt/kayobe/venvs/kolla/lib/python3.6/site-packages/requests/adapters.py", line 501, in send + raise ConnectionError(err, request=request) + requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')) diff --git a/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml b/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml new file mode 100644 index 000000000..007de8a76 --- /dev/null +++ b/releasenotes/notes/fix-multinode-vm-internet-connectivity-49ab64cd75833f09.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes internet connectivity for VMs deployed in the ``ci-multinode`` + environment. diff --git a/releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml b/releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml new file mode 100644 index 000000000..ff8c55bc6 --- /dev/null +++ b/releasenotes/notes/fix-neutron-fip-ovn-lb-attach-1457b0fcc2757ed9.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Bumps Neutron container image tags to fix `bug 2068644 + `__ which could prevent + associating floating IPs with OVN-based load balancers. diff --git a/releasenotes/notes/fix-neutron-keepalived-pid-cleanup-5faae4cf78b07f89.yaml b/releasenotes/notes/fix-neutron-keepalived-pid-cleanup-5faae4cf78b07f89.yaml new file mode 100644 index 000000000..81d65f0cf --- /dev/null +++ b/releasenotes/notes/fix-neutron-keepalived-pid-cleanup-5faae4cf78b07f89.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Update neutron container images to apply `keepalived PID clean up fix + `__ + With this change, Neutron always deletes stale PID file if exists. diff --git a/releasenotes/notes/fix-octavia-tls-terminated-pkcs12-4f7e32a6f5ca0143.yaml b/releasenotes/notes/fix-octavia-tls-terminated-pkcs12-4f7e32a6f5ca0143.yaml new file mode 100644 index 000000000..ee5167ce9 --- /dev/null +++ b/releasenotes/notes/fix-octavia-tls-terminated-pkcs12-4f7e32a6f5ca0143.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes creation and failover of Octavia TLS-terminated load balancers when + storing the certificate and key as a PKCS12 bundle in Barbican. diff --git a/releasenotes/notes/fix-openbao-include-tag-dfef2a0e731674f0.yaml b/releasenotes/notes/fix-openbao-include-tag-dfef2a0e731674f0.yaml new file mode 100644 index 000000000..0b3a889e5 --- /dev/null +++ b/releasenotes/notes/fix-openbao-include-tag-dfef2a0e731674f0.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Ensure that the correct tag is used for ``OpenBao`` repository in ``Pulp``. diff --git a/releasenotes/notes/fix-opensearch-heap-size-0cd31054927d4bbd.yaml b/releasenotes/notes/fix-opensearch-heap-size-0cd31054927d4bbd.yaml new file mode 100644 index 000000000..1a33a997a --- /dev/null +++ b/releasenotes/notes/fix-opensearch-heap-size-0cd31054927d4bbd.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Bumps OpenSearch heap size to 8 GB, to be identical to Elasticsearch. diff --git a/releasenotes/notes/fix-os-capacity-prometheus-job-name-for-azimuth-cloud-metrics-39cf81eebe16b879.yaml b/releasenotes/notes/fix-os-capacity-prometheus-job-name-for-azimuth-cloud-metrics-39cf81eebe16b879.yaml new file mode 100644 index 000000000..da274014f --- /dev/null +++ b/releasenotes/notes/fix-os-capacity-prometheus-job-name-for-azimuth-cloud-metrics-39cf81eebe16b879.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Changed the Prometheus job name of OS Capacity exporter to + ``os_capacity`` which is what Azimuth is expecting to have for + `cloud metrics dashboard `__. diff --git a/releasenotes/notes/fix-ossa-2024-004-f732e58c12e26785.yaml b/releasenotes/notes/fix-ossa-2024-004-f732e58c12e26785.yaml new file mode 100644 index 000000000..ae0bd5b5f --- /dev/null +++ b/releasenotes/notes/fix-ossa-2024-004-f732e58c12e26785.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Fixes `OSSA-2024-004 + `_ with updated + container images for Ironic. diff --git a/releasenotes/notes/fix-ossa-2026-002-65bd525d6aee93cc.yaml b/releasenotes/notes/fix-ossa-2026-002-65bd525d6aee93cc.yaml new file mode 100644 index 000000000..6ae01445e --- /dev/null +++ b/releasenotes/notes/fix-ossa-2026-002-65bd525d6aee93cc.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Fixes `OSSA-2026-002 / CVE-2026-24708 + `_ with updated + ``nova-compute`` container images. diff --git a/releasenotes/notes/fix-over-1TBram-vm-amd-iommu-rl9-6ddb3d7c030a1674.yaml b/releasenotes/notes/fix-over-1TBram-vm-amd-iommu-rl9-6ddb3d7c030a1674.yaml new file mode 100644 index 000000000..5bce3ab04 --- /dev/null +++ b/releasenotes/notes/fix-over-1TBram-vm-amd-iommu-rl9-6ddb3d7c030a1674.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes creation of over 1TB memory VMs on AMD with IOMMU enabled on Rocky + Linux 9. diff --git a/releasenotes/notes/fix-ovn-log-permissions-4b067cc1b1a7c3b8.yaml b/releasenotes/notes/fix-ovn-log-permissions-4b067cc1b1a7c3b8.yaml new file mode 100644 index 000000000..3354ae5fb --- /dev/null +++ b/releasenotes/notes/fix-ovn-log-permissions-4b067cc1b1a7c3b8.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Bumps the OVN container image tags to bring in a fix to the openvswitch + logs permissions. This allows fluentd to scrape these logs. See + patch: https://review.opendev.org/c/openstack/kolla/+/954899 diff --git a/releasenotes/notes/fix-pci-default-template-8660ab2a7a106376.yaml b/releasenotes/notes/fix-pci-default-template-8660ab2a7a106376.yaml new file mode 100644 index 000000000..affc29fd8 --- /dev/null +++ b/releasenotes/notes/fix-pci-default-template-8660ab2a7a106376.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes possible templating error with PCI passthrough configuration. diff --git a/releasenotes/notes/fix-radosgw-usage-exporter-duplicate-metrics-ca96b3e4b9d6afe9.yaml b/releasenotes/notes/fix-radosgw-usage-exporter-duplicate-metrics-ca96b3e4b9d6afe9.yaml new file mode 100644 index 000000000..fb54ae397 --- /dev/null +++ b/releasenotes/notes/fix-radosgw-usage-exporter-duplicate-metrics-ca96b3e4b9d6afe9.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Bumps the radosgw_usage_exporter tag to fix an issue where duplicate + metrics could be presented to Prometheus if S3 store usage was particularly + high. diff --git a/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml b/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml new file mode 100644 index 000000000..bec2cafb8 --- /dev/null +++ b/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes issue where Netmiko devices were sending no commands to the switch + since plug_bond_to_network is overridden in + networking_generic_switch/devices/netmiko_devices/init.py and + PLUG_BOND_TO_NETWORK to set to None. + See `NGS bug report + `__. diff --git a/releasenotes/notes/fix-smart-metrics-permissions-8b1183de434fafcc.yaml b/releasenotes/notes/fix-smart-metrics-permissions-8b1183de434fafcc.yaml new file mode 100644 index 000000000..16a241595 --- /dev/null +++ b/releasenotes/notes/fix-smart-metrics-permissions-8b1183de434fafcc.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes a permission issue with node exporter's textfile collector metrics + for SMART monitoring of disk drives. diff --git a/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml b/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml new file mode 100644 index 000000000..b71a5c439 --- /dev/null +++ b/releasenotes/notes/fix-smartmon-script-db2a38df1b245e58.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes the smartmon script to be case insensitive when checking for + the inital SMART info. This is to ensure that the script works + correctly on systems where the output of `smartctl -i` is not + capitalised as previously expected by the script. This leads to + badly formatted .prom files which lead to node_exporter failing to + scrape the file. diff --git a/releasenotes/notes/fix-squid-issue-11b5f03719ab8b45.yaml b/releasenotes/notes/fix-squid-issue-11b5f03719ab8b45.yaml new file mode 100644 index 000000000..458b1da65 --- /dev/null +++ b/releasenotes/notes/fix-squid-issue-11b5f03719ab8b45.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an issue where Squid proxy could be unable to reach external servers + due to a preference of choosing IPv6 connectivity by default. diff --git a/releasenotes/notes/fix-ssh-in-ubuntu-overcloud-host-image-b395c8afb6d5b820.yaml b/releasenotes/notes/fix-ssh-in-ubuntu-overcloud-host-image-b395c8afb6d5b820.yaml new file mode 100644 index 000000000..d11b74e4e --- /dev/null +++ b/releasenotes/notes/fix-ssh-in-ubuntu-overcloud-host-image-b395c8afb6d5b820.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Bumps the ``stackhpc_ubuntu_jammy_overcloud_host_image_version`` to fix an + issue where ``authorized_keys`` for the ``ubuntu`` user was not populated + on newly provisioned nodes. Ubuntu package snapshots are also bumped to + match those used in the new host image. diff --git a/releasenotes/notes/fix-vault-standby-alert-501b743a40971926.yaml b/releasenotes/notes/fix-vault-standby-alert-501b743a40971926.yaml new file mode 100644 index 000000000..8bcf90f6d --- /dev/null +++ b/releasenotes/notes/fix-vault-standby-alert-501b743a40971926.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes an issue where HashiCorp Vault standby nodes would trigger a + Prometheus alert. To apply this fix to an existing system, the HAProxy + configuration for Vault (``kolla/config/haproxy/services.d/vault.cfg``) + must be manually updated following the `Vault documentation + `__. diff --git a/releasenotes/notes/fix-wazuh-custom-sca-policies-f16c7f0eae8265a5.yaml b/releasenotes/notes/fix-wazuh-custom-sca-policies-f16c7f0eae8265a5.yaml new file mode 100644 index 000000000..b013ade5f --- /dev/null +++ b/releasenotes/notes/fix-wazuh-custom-sca-policies-f16c7f0eae8265a5.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + When using custom SCA policies for Wazuh, the agents are now correctly + configured to allow commands to be executed from the manager. diff --git a/releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml b/releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml new file mode 100644 index 000000000..60be7d77b --- /dev/null +++ b/releasenotes/notes/fixes-InstanceDown-Alert-570a295e3d5006f7.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes the InstanceDown alerting rule wait time to be consistent with + the alert message. The alert message says "for 5 minutes" but the rule + was set to wait for 1 minute. + diff --git a/releasenotes/notes/fixes-blackbox-with-custom-placement-e98f1621dc5c873d.yaml b/releasenotes/notes/fixes-blackbox-with-custom-placement-e98f1621dc5c873d.yaml new file mode 100644 index 000000000..5cb4d5f4d --- /dev/null +++ b/releasenotes/notes/fixes-blackbox-with-custom-placement-e98f1621dc5c873d.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an issue with blackbox exporter endpoint generation when prometheus + server and blackbox exporter are not colocated. diff --git a/releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml b/releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml new file mode 100644 index 000000000..415b4d76d --- /dev/null +++ b/releasenotes/notes/fixes-file-descriptor-leak-3d5c9340e5983e9c.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes a file descriptor leak in networking-mlnx which prevented VMs using + Infiniband virtual functions from provisioning after a period of time. + - | + Fixes ``KeyError: ip_version`` in networking-mlnx when used in conjuction + with OVN mechanism driver. diff --git a/releasenotes/notes/fixes-growroot-for-software-raid-3852bdea5415a0be.yaml b/releasenotes/notes/fixes-growroot-for-software-raid-3852bdea5415a0be.yaml new file mode 100644 index 000000000..0f66c6934 --- /dev/null +++ b/releasenotes/notes/fixes-growroot-for-software-raid-3852bdea5415a0be.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes a regression when using ``growroot.yml`` and software raid where the + playbook would fail to identify the correct disk. diff --git a/releasenotes/notes/fixes-ironic-rebuild-api-version-2.93-4fb297328d55bff8.yaml b/releasenotes/notes/fixes-ironic-rebuild-api-version-2.93-4fb297328d55bff8.yaml new file mode 100644 index 000000000..41cc41c76 --- /dev/null +++ b/releasenotes/notes/fixes-ironic-rebuild-api-version-2.93-4fb297328d55bff8.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes an issue with Ironic rebuilds when the client was using Nova Compute + API version >= 2.93. This caused servers to enter into the error state when + a rebuild was requested. See `LP#2127017 + `_ for more details. diff --git a/releasenotes/notes/fixes-mdev-uuid-parsing-4c9e972c3c424760.yaml b/releasenotes/notes/fixes-mdev-uuid-parsing-4c9e972c3c424760.yaml new file mode 100644 index 000000000..305771ab3 --- /dev/null +++ b/releasenotes/notes/fixes-mdev-uuid-parsing-4c9e972c3c424760.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Updates nova image to bring in a fix for parsing mdev uuids when using + libvirt>=7.7. See `bug `__ + for more details. diff --git a/releasenotes/notes/fixes-osd-size-summary-9924ef4aac61d2b6.yaml b/releasenotes/notes/fixes-osd-size-summary-9924ef4aac61d2b6.yaml new file mode 100644 index 000000000..d2c9fd1d3 --- /dev/null +++ b/releasenotes/notes/fixes-osd-size-summary-9924ef4aac61d2b6.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix an issue with the OSD summary pie chart not showing any data. diff --git a/releasenotes/notes/fluentd-buffer-too-large-alert-682893410855387f.yaml b/releasenotes/notes/fluentd-buffer-too-large-alert-682893410855387f.yaml new file mode 100644 index 000000000..5f44eaa69 --- /dev/null +++ b/releasenotes/notes/fluentd-buffer-too-large-alert-682893410855387f.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds a new Prometheus alert ``FluentdBufferTooLarge`` which is raised when + the total size of queue buffers grows above 128 MiB. diff --git a/releasenotes/notes/generic-blackbox-588859073c7441ca.yaml b/releasenotes/notes/generic-blackbox-588859073c7441ca.yaml new file mode 100644 index 000000000..5ad296aa4 --- /dev/null +++ b/releasenotes/notes/generic-blackbox-588859073c7441ca.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Added a templated set of default Prometheus Blackbox exporter endpoints. diff --git a/releasenotes/notes/glance-s3-multipart-bug-2043740-e7a122a6453c45e4.yaml b/releasenotes/notes/glance-s3-multipart-bug-2043740-e7a122a6453c45e4.yaml new file mode 100644 index 000000000..0efc34916 --- /dev/null +++ b/releasenotes/notes/glance-s3-multipart-bug-2043740-e7a122a6453c45e4.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Updates Glance image to provide a way to force multipart upload to S3 + backends. `LP#2043740 `__. diff --git a/releasenotes/notes/grafana-elasticsearch-cluster-health-154275e8d39dd89f.yaml b/releasenotes/notes/grafana-elasticsearch-cluster-health-154275e8d39dd89f.yaml new file mode 100644 index 000000000..1a45bc9e0 --- /dev/null +++ b/releasenotes/notes/grafana-elasticsearch-cluster-health-154275e8d39dd89f.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes display of the OpenSearch cluster health in Grafana + when in yellow state. diff --git a/releasenotes/notes/grafana-openstack-dashboard-e99a6a44e927363f.yaml b/releasenotes/notes/grafana-openstack-dashboard-e99a6a44e927363f.yaml new file mode 100644 index 000000000..c47c6d034 --- /dev/null +++ b/releasenotes/notes/grafana-openstack-dashboard-e99a6a44e927363f.yaml @@ -0,0 +1,3 @@ +--- +features: + - Openstack dashboard is now used as default Grafana dashboard. diff --git a/releasenotes/notes/grafana-remove-angular-plugins-1c10e83ddb6556f4.yaml b/releasenotes/notes/grafana-remove-angular-plugins-1c10e83ddb6556f4.yaml new file mode 100644 index 000000000..6eb2ccb47 --- /dev/null +++ b/releasenotes/notes/grafana-remove-angular-plugins-1c10e83ddb6556f4.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Grafana now refuses to load AngularJS plugins. As such the + ``grafana-piechart-panel`` and ``gnocchixyz-gnocchi-datasource`` plugins + have been removed from the Grafana image. diff --git a/releasenotes/notes/grub_timeout_values-c39b8b1a2c34a602.yaml b/releasenotes/notes/grub_timeout_values-c39b8b1a2c34a602.yaml new file mode 100644 index 000000000..7c7060d8e --- /dev/null +++ b/releasenotes/notes/grub_timeout_values-c39b8b1a2c34a602.yaml @@ -0,0 +1,8 @@ +--- +upgrade: + - | + Change default values for ``DIB_GRUB_TIMEOUT_STYLE`` and + ``DIB_GRUB_TIMEOUT``. The default value for ``DIB_GRUB_TIMEOUT_STYLE`` + will be ``menu`` and for ``DIB_GRUB_TIMEOUT`` will be ``5``. + Adding ConfigDrive to ``DIB_CLOUD_INIT_DATASOURCES`` var list to fix + cloud-init issue. diff --git a/releasenotes/notes/haproxy-dashboard-instance-label-836b93921e964680.yaml b/releasenotes/notes/haproxy-dashboard-instance-label-836b93921e964680.yaml new file mode 100644 index 000000000..5e2e00c68 --- /dev/null +++ b/releasenotes/notes/haproxy-dashboard-instance-label-836b93921e964680.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix Grafana HAProxy dashboard when non-default Prometheus instance labels + are used. diff --git a/releasenotes/notes/hashicorp-sync-images-96dc726e85323104.yaml b/releasenotes/notes/hashicorp-sync-images-96dc726e85323104.yaml new file mode 100644 index 000000000..e8aee8925 --- /dev/null +++ b/releasenotes/notes/hashicorp-sync-images-96dc726e85323104.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds support for synchronising HashiCorp Consul and Vault images to a local + Pulp registry. diff --git a/releasenotes/notes/hcp-2.5.0-8e30c7b1910f2bd2.yaml b/releasenotes/notes/hcp-2.5.0-8e30c7b1910f2bd2.yaml new file mode 100644 index 000000000..d56f6593e --- /dev/null +++ b/releasenotes/notes/hcp-2.5.0-8e30c7b1910f2bd2.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Updates the ``stackhpc.hashicorp`` Ansible collection to 2.5.0. This brings + in an idempotency fix for generating certificates. diff --git a/releasenotes/notes/horizon-fix-multiple-domains-51086e9ba3d197b7.yaml b/releasenotes/notes/horizon-fix-multiple-domains-51086e9ba3d197b7.yaml new file mode 100644 index 000000000..123d02a1c --- /dev/null +++ b/releasenotes/notes/horizon-fix-multiple-domains-51086e9ba3d197b7.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Bumps the Horizon container image tag to fix the Identity->Domains panel + for admins when multiple Domains are in use. See patch: + https://review.opendev.org/c/openstack/horizon/+/940461 diff --git a/releasenotes/notes/host-image-build-sms-upload-4829088864b15c5b.yaml b/releasenotes/notes/host-image-build-sms-upload-4829088864b15c5b.yaml new file mode 100644 index 000000000..27aa23f78 --- /dev/null +++ b/releasenotes/notes/host-image-build-sms-upload-4829088864b15c5b.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + The overcloud host image build workflow now uploads the built image to SMS + as well as ARK, allowing it to be tested both manually and through AIO CI + jobs. diff --git a/releasenotes/notes/host-image-rebuild-957b0a69de1044fc.yaml b/releasenotes/notes/host-image-rebuild-957b0a69de1044fc.yaml new file mode 100644 index 000000000..a8bed0df6 --- /dev/null +++ b/releasenotes/notes/host-image-rebuild-957b0a69de1044fc.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + The StackHPC overcloud host images have been rebuilt using new packages for + the Caracal release. diff --git a/releasenotes/notes/improve-ceph-pgs-unclean-alert-98306d397344b572.yaml b/releasenotes/notes/improve-ceph-pgs-unclean-alert-98306d397344b572.yaml new file mode 100644 index 000000000..cbf008554 --- /dev/null +++ b/releasenotes/notes/improve-ceph-pgs-unclean-alert-98306d397344b572.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Prevents raising a Ceph ``PgsUnclean`` alert because of backfilling which + can frequently happen because of normal rebalancing activities, such as use + of the Ceph balancer or OSD addition. diff --git a/releasenotes/notes/increase-pulp-retries-79cc258da9aabb4f.yaml b/releasenotes/notes/increase-pulp-retries-79cc258da9aabb4f.yaml new file mode 100644 index 000000000..b388672c7 --- /dev/null +++ b/releasenotes/notes/increase-pulp-retries-79cc258da9aabb4f.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Increase the number of retries when waiting for Pulp to become ready. + This is to avoid issues with Pulp taking longer than expected to start up. diff --git a/releasenotes/notes/increase-tls-ttl-c1eba5cca7767d0f.yaml b/releasenotes/notes/increase-tls-ttl-c1eba5cca7767d0f.yaml new file mode 100644 index 000000000..848fa40c8 --- /dev/null +++ b/releasenotes/notes/increase-tls-ttl-c1eba5cca7767d0f.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Increase the ``ttl`` of the ``PKI`` role to two years providing + the opportunity to replace ``internal`` and ``backend`` certificates + during the annual upgrade. diff --git a/releasenotes/notes/infra-vms-ntp-group-45ade04fe885c3f8.yaml b/releasenotes/notes/infra-vms-ntp-group-45ade04fe885c3f8.yaml new file mode 100644 index 000000000..7f254ded5 --- /dev/null +++ b/releasenotes/notes/infra-vms-ntp-group-45ade04fe885c3f8.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fix missing NTP configuration for infrastructure VMs. Operators should run + ``kayobe infra vm host configure --tags ntp``. diff --git a/releasenotes/notes/ipa-build-workflow-cde24f3b9e3e0667.yaml b/releasenotes/notes/ipa-build-workflow-cde24f3b9e3e0667.yaml new file mode 100644 index 000000000..2b1612b41 --- /dev/null +++ b/releasenotes/notes/ipa-build-workflow-cde24f3b9e3e0667.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds a new build workflow to Release Train for building IPA + images, and deployment for Release Train IPA images to Bifrost + and Ironic inspector. + - | + Release Train IPA is enabled by default. diff --git a/releasenotes/notes/ipa-inspection-settings-133fe91b1d855fa0.yaml b/releasenotes/notes/ipa-inspection-settings-133fe91b1d855fa0.yaml new file mode 100644 index 000000000..cfb761290 --- /dev/null +++ b/releasenotes/notes/ipa-inspection-settings-133fe91b1d855fa0.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Configures the Ironic Python Agent with useful settings for inspection, + such as the ``extra-hardware`` and ``mellanox`` elements. diff --git a/releasenotes/notes/ironic-expand-check_cipher_suite_errors-c8915956be17a28c.yaml b/releasenotes/notes/ironic-expand-check_cipher_suite_errors-c8915956be17a28c.yaml new file mode 100644 index 000000000..d07efe471 --- /dev/null +++ b/releasenotes/notes/ironic-expand-check_cipher_suite_errors-c8915956be17a28c.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Bumped Ironic container image tag to fix a bug where some cipher suite + errors for SuperMicro hardware were not caught. See patch: + https://review.opendev.org/c/openstack/ironic/+/940957 diff --git a/releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml b/releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml new file mode 100644 index 000000000..b7e0f9de7 --- /dev/null +++ b/releasenotes/notes/ironic-fix-online-upgrades-36e86b3ab3ac081c.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Adds Ironic images tags to fix a bug with online upgrades for + Bios/Traits. See patch: + https://review.opendev.org/c/openstack/ironic/+/877409 diff --git a/releasenotes/notes/ironic-inspector-blackbox-backend-ff1bd68297435a36.yaml b/releasenotes/notes/ironic-inspector-blackbox-backend-ff1bd68297435a36.yaml new file mode 100644 index 000000000..bc23a3e60 --- /dev/null +++ b/releasenotes/notes/ironic-inspector-blackbox-backend-ff1bd68297435a36.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the Prometheus Blackbox Exporter backend endpoint for + ironic-inspector, as this service does not support backend TLS. diff --git a/releasenotes/notes/ironic-patch-cve-2024-44082-87c70b333bc28ea0.yaml b/releasenotes/notes/ironic-patch-cve-2024-44082-87c70b333bc28ea0.yaml new file mode 100644 index 000000000..ee8c7cf28 --- /dev/null +++ b/releasenotes/notes/ironic-patch-cve-2024-44082-87c70b333bc28ea0.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Bumps Ironic containers to patch CVE-2024-44082, see `OSSA-2024-003 + `__ for details. diff --git a/releasenotes/notes/ironic-prometheus-exporter-f9301ef233d47291.yaml b/releasenotes/notes/ironic-prometheus-exporter-f9301ef233d47291.yaml new file mode 100644 index 000000000..d8e64e889 --- /dev/null +++ b/releasenotes/notes/ironic-prometheus-exporter-f9301ef233d47291.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + ``ironic-prometheus-exporter`` container image has been added and enabled + by default when kolla_enable_ironic and kolla_enable_prometheus are true + (following upstream defaults). diff --git a/releasenotes/notes/kayobe-image-f1227adacb28f40b.yaml b/releasenotes/notes/kayobe-image-f1227adacb28f40b.yaml new file mode 100644 index 000000000..610f9e5e8 --- /dev/null +++ b/releasenotes/notes/kayobe-image-f1227adacb28f40b.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + StackHPC Kayobe Configuration container images for CI/CD with `Kayobe + Automation `__ are now + published to GitHub Container Registry (GHCR) at + ghcr.io/stackhpc/stackhpc-kayobe-config. The image is tagged with the name + of the release branch, e.g. ``stackhpc/yoga``. diff --git a/releasenotes/notes/kolla-bump-cve-2024-36039-07f18e18b5c86980.yaml b/releasenotes/notes/kolla-bump-cve-2024-36039-07f18e18b5c86980.yaml new file mode 100644 index 000000000..bfddbb74d --- /dev/null +++ b/releasenotes/notes/kolla-bump-cve-2024-36039-07f18e18b5c86980.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + Addresses critical vulnerability CVE-2024-36039 by + bumping the PyMySQL library to 1.1.1 in all affected + Kolla images. This vulnerability allows SQL injection + through untrusted JSON objects. diff --git a/releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml b/releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml new file mode 100644 index 000000000..acbe45077 --- /dev/null +++ b/releasenotes/notes/kolla-dependency-workflow-6ff5520ee0ab8e15.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Workflow to update Kolla dependencies (Kayobe, Kolla and Kolla-Ansible) + to the latest tag available in the StackHPC branch via CI. diff --git a/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml b/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml new file mode 100644 index 000000000..4d1c38a91 --- /dev/null +++ b/releasenotes/notes/kolla-kayobe-version-check-f44d43c9c34d1b89.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + Added playbooks to check the installed Kayobe/Kolla-Ansible versions + against the expected versions in Kayobe configuration. These checks + will run on Kayobe bootstrap, host and service operations. +upgrade: + - | + Kolla Ansible and Kayobe version checks are enabled by default which + may fail on existing deployments using custom forks or branches for + Kayobe and Kolla-Ansible. To disable version checks in configuration + set ``stackhpc_enable_kayobe_check`` and + ``stackhpc_enable_kolla_ansible_check`` to false. diff --git a/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml b/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml new file mode 100644 index 000000000..810295354 --- /dev/null +++ b/releasenotes/notes/kolla-local-builds-eca0277a969803fd.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a new variable ``stackhpc_pulp_sync_for_local_container_build`` which, + when set to ``true``, configures the local Pulp server to sync all package + repositories required for building kolla containers on a local kolla build + host. diff --git a/releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml b/releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml new file mode 100644 index 000000000..0c5e9ddcc --- /dev/null +++ b/releasenotes/notes/kolla-rocky9.3-images-a3ff235485093191.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Rocky images have been rebuilt and are now based on Rocky 9.3. diff --git a/releasenotes/notes/kolla-update-fixes-5ff55225ce85441f.yaml b/releasenotes/notes/kolla-update-fixes-5ff55225ce85441f.yaml new file mode 100644 index 000000000..1d0b34d4b --- /dev/null +++ b/releasenotes/notes/kolla-update-fixes-5ff55225ce85441f.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Allow Kolla dependency updates on non-default branches. diff --git a/releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml b/releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml new file mode 100644 index 000000000..a6340bca7 --- /dev/null +++ b/releasenotes/notes/local-pulp-tls-a8e7464d8cb0d114.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Enable TLS for the Seed Pulp service. Set ``pulp_enable_tls: true`` and + provide paths to a TLS certificate and key using ``pulp_cert_path`` and + ``pulp_key_path`` respectively. diff --git a/releasenotes/notes/logs-in-openstack-dashboard-6e345ff7f16c0658.yaml b/releasenotes/notes/logs-in-openstack-dashboard-6e345ff7f16c0658.yaml new file mode 100644 index 000000000..0176ac636 --- /dev/null +++ b/releasenotes/notes/logs-in-openstack-dashboard-6e345ff7f16c0658.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + The Openstack Dashboard in Grafana now includes logs from Openstack + services. diff --git a/releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml b/releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml new file mode 100644 index 000000000..4732bb759 --- /dev/null +++ b/releasenotes/notes/low-memory-alert-units-a6fde380ff9b7839.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Add unit to LowMemory alert description. diff --git a/releasenotes/notes/lvm-3c785770722f768a.yaml b/releasenotes/notes/lvm-3c785770722f768a.yaml new file mode 100644 index 000000000..5818c3e23 --- /dev/null +++ b/releasenotes/notes/lvm-3c785770722f768a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds a standard LVM configuration that is compatible with the new overcloud + host image. diff --git a/releasenotes/notes/magnum-bump-tags-42u298jfj93r3ok32b.yaml b/releasenotes/notes/magnum-bump-tags-42u298jfj93r3ok32b.yaml new file mode 100644 index 000000000..8a63500c0 --- /dev/null +++ b/releasenotes/notes/magnum-bump-tags-42u298jfj93r3ok32b.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes Octavia health monitors not being created on cluster spawn. diff --git a/releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml b/releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml new file mode 100644 index 000000000..4850e4995 --- /dev/null +++ b/releasenotes/notes/magnum-bump-tags-7841edcbd3da4b9c.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Fixes CoreDNS for Magnum clusters crashing on startup. + - | + Allows cinder-csi nodeplugin to start on the same Magnum cluster host as + cinder-csi controllerplugin. + - | + Corrects ClusterRole rules for Magnum cluster-autoscaler, and sets + cluster-autoscaler pods to use hostNetwork. diff --git a/releasenotes/notes/magnum-capi-1-1-0-68f14759413316c4.yaml b/releasenotes/notes/magnum-capi-1-1-0-68f14759413316c4.yaml new file mode 100644 index 000000000..8290ba39c --- /dev/null +++ b/releasenotes/notes/magnum-capi-1-1-0-68f14759413316c4.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + ``magnum-capi-helm`` driver has been updated to 1.1.0. + Please see `magnum-capi-helm release notes `_ + for changes. diff --git a/releasenotes/notes/magnum-capi-driver-2dd8cf6e68a1919c.yaml b/releasenotes/notes/magnum-capi-driver-2dd8cf6e68a1919c.yaml new file mode 100644 index 000000000..4c3f1902f --- /dev/null +++ b/releasenotes/notes/magnum-capi-driver-2dd8cf6e68a1919c.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds the new magnum-capi-helm out of tree driver (see here + https://github.com/stackhpc/magnum-capi-helm) into release train magnum + containers diff --git a/releasenotes/notes/magnum-container-footer-dcf6dca0667bcb94.yaml b/releasenotes/notes/magnum-container-footer-dcf6dca0667bcb94.yaml new file mode 100644 index 000000000..aada93f51 --- /dev/null +++ b/releasenotes/notes/magnum-container-footer-dcf6dca0667bcb94.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + adds helm client into magnum container diff --git a/releasenotes/notes/magnum-pin-958b39182e565697.yaml b/releasenotes/notes/magnum-pin-958b39182e565697.yaml new file mode 100644 index 000000000..e13bcbdd6 --- /dev/null +++ b/releasenotes/notes/magnum-pin-958b39182e565697.yaml @@ -0,0 +1,6 @@ +--- +issues: + - | + Magnum has been pinned to the Antelope release due to an issue upstream in + Caracal. See `here `__ for + more details. diff --git a/releasenotes/notes/magnum-remove-cert-append-8797b640f25644ea.yaml b/releasenotes/notes/magnum-remove-cert-append-8797b640f25644ea.yaml new file mode 100644 index 000000000..20d195596 --- /dev/null +++ b/releasenotes/notes/magnum-remove-cert-append-8797b640f25644ea.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes appending to ca.crt in make-cert-client.sh causing multiple identical + ca certs being added into /etc/kubernetes/certs/ca.crt. diff --git a/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml b/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml new file mode 100644 index 000000000..6952fe9bc --- /dev/null +++ b/releasenotes/notes/manila-for-multinode-df2186276c6b434a.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds support for Manila in the ci-multinode environment using the CephFS + native backend. This is disabled by default, but can be enabled by setting + the following variables in the kayobe configuration: + `kolla_enable_manila: true` + `kolla_enable_manila_backend_cephfs_native: true` diff --git a/releasenotes/notes/merge-openbao-vault-playbooks-0c78364f4a86f6e0.yaml b/releasenotes/notes/merge-openbao-vault-playbooks-0c78364f4a86f6e0.yaml new file mode 100644 index 000000000..ef377665b --- /dev/null +++ b/releasenotes/notes/merge-openbao-vault-playbooks-0c78364f4a86f6e0.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Created new set of playbooks with name prefix of "secret-store". + These playbooks merge OpenBao playbooks and Hashicorp Vault + playbooks. + By default, the playbooks will deploy OpenBao. + To use Hashicorp Vault, set ``stackhpc_ca_secret_store: vault``. diff --git a/releasenotes/notes/moved-wazuh-certs-b5173d264f543b14.yaml b/releasenotes/notes/moved-wazuh-certs-b5173d264f543b14.yaml new file mode 100644 index 000000000..340a7ac68 --- /dev/null +++ b/releasenotes/notes/moved-wazuh-certs-b5173d264f543b14.yaml @@ -0,0 +1,15 @@ +--- +upgrade: + - | + The path used to store Wazuh certificates has changed. ``local_certs_path`` is + now set to the environment directory e.g + ``$KAYOBE_CONFIG_PATH/environments//wazuh`` or + ``$KAYOBE_CONFIG_PATH/wazuh/`` if not using environments. The contents of + ``$KAYOBE_CONFIG_PATH/ansible/wazuh/certificates`` should be moved to the + new location and the empty directory should be removed. + - | + The ``local_custom_certs_path`` variable has been removed. Custom wazuh + certificates should be moved to + ``$KAYOBE_CONFIG_PATH/environments//wazuh/wazuh-certificates/`` + if using environments, or + ``$KAYOBE_CONFIG_PATH/wazuh/wazuh-certificates`` if not. diff --git a/releasenotes/notes/multiarch-manifests-pipefail-9b860f83d0f93f1e.yaml b/releasenotes/notes/multiarch-manifests-pipefail-9b860f83d0f93f1e.yaml new file mode 100644 index 000000000..c1b685c80 --- /dev/null +++ b/releasenotes/notes/multiarch-manifests-pipefail-9b860f83d0f93f1e.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes CI multiarch manifest creation by setting ``pipefail`` in the + manifest script. Failures from ``docker manifest create`` or + ``docker manifest push`` now fail the job even when piped to ``tee``. diff --git a/releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml b/releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml new file mode 100644 index 000000000..004e3b1f0 --- /dev/null +++ b/releasenotes/notes/multinode-magnum-docs-00607f139dc2b53e.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Updated the documentation for the ci-multinode to include instructions on + how to set up and test Magnum. diff --git a/releasenotes/notes/multinode-wazuh-c9edd65ab5cdbcb8.yaml b/releasenotes/notes/multinode-wazuh-c9edd65ab5cdbcb8.yaml new file mode 100644 index 000000000..fc1e12cd3 --- /dev/null +++ b/releasenotes/notes/multinode-wazuh-c9edd65ab5cdbcb8.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Added support for Wazuh in the ci-multinode environment. diff --git a/releasenotes/notes/network-bond-degraded-alert-d2a0b05002609ac1.yaml b/releasenotes/notes/network-bond-degraded-alert-d2a0b05002609ac1.yaml new file mode 100644 index 000000000..c987c7959 --- /dev/null +++ b/releasenotes/notes/network-bond-degraded-alert-d2a0b05002609ac1.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds a new Prometheus alert ``HostNetworkBondDegraded`` which will be + raised when at least one bond member is down. diff --git a/releasenotes/notes/network-bond-single-link-766adf41a3c2fd4e.yaml b/releasenotes/notes/network-bond-single-link-766adf41a3c2fd4e.yaml new file mode 100644 index 000000000..66d66f40b --- /dev/null +++ b/releasenotes/notes/network-bond-single-link-766adf41a3c2fd4e.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds a new Prometheus alert ``HostNetworkBondSingleLink`` which will be + raised when a bond is configured with only one member. This can happen when + NetworkManager detects that a bond member is down at boot time. This alert + can be disabled by setting ``alertmanager_warn_network_bond_single_link`` + to ``false``. diff --git a/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml b/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml new file mode 100644 index 000000000..cf0a68153 --- /dev/null +++ b/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Neutron containers are now built from our StackHPC fork. diff --git a/releasenotes/notes/neutron-dadfailed-workaround-9cd82d29ee1fba59.yaml b/releasenotes/notes/neutron-dadfailed-workaround-9cd82d29ee1fba59.yaml new file mode 100644 index 000000000..4c0b59cd2 --- /dev/null +++ b/releasenotes/notes/neutron-dadfailed-workaround-9cd82d29ee1fba59.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Disables metadata proxy over IPv6 inside Neutron DHCP agent to work around + `bug 1953165 `__. diff --git a/releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml b/releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml new file mode 100644 index 000000000..2683e94b4 --- /dev/null +++ b/releasenotes/notes/node-exporter-1.5.0-5e3764790a5a0190.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Updates Prometheus Node exporter to version 1.5.0. +fixes: + - | + Fixes a `Prometheus Node exporter crash + `__ which may + affect nodes with AMD processors (first seen on HPE DL385). diff --git a/releasenotes/notes/nova-bug-2091033-4f54893f67af5c7c.yaml b/releasenotes/notes/nova-bug-2091033-4f54893f67af5c7c.yaml new file mode 100644 index 000000000..d7d9e28aa --- /dev/null +++ b/releasenotes/notes/nova-bug-2091033-4f54893f67af5c7c.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Updates the ``nova-compute`` container image to fix `bug 2091033 + `__. This bug would cause + ``nova-compute`` to freeze, which would result in frequent monitoring + alerts. diff --git a/releasenotes/notes/nova-machine-type-q35-ab363c5f7f3f9807.yaml b/releasenotes/notes/nova-machine-type-q35-ab363c5f7f3f9807.yaml new file mode 100644 index 000000000..8ca2b917f --- /dev/null +++ b/releasenotes/notes/nova-machine-type-q35-ab363c5f7f3f9807.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Configure Nova to use more modern 'q35' libvirt machine type rather than + 'pc' which is considered legacy. diff --git a/releasenotes/notes/nova-pci-request-resize-a429c115ba2818e8.yaml b/releasenotes/notes/nova-pci-request-resize-a429c115ba2818e8.yaml new file mode 100644 index 000000000..6a5bad11d --- /dev/null +++ b/releasenotes/notes/nova-pci-request-resize-a429c115ba2818e8.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix for nova resize API not parsing the new flavor on resize - bug 1805969. diff --git a/releasenotes/notes/nova-uefi-edk2-ovmf-d636c299e76728e8.yaml b/releasenotes/notes/nova-uefi-edk2-ovmf-d636c299e76728e8.yaml new file mode 100644 index 000000000..e75630ff0 --- /dev/null +++ b/releasenotes/notes/nova-uefi-edk2-ovmf-d636c299e76728e8.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix creation of VM instances with UEFI enabled and Secure Boot disabled. diff --git a/releasenotes/notes/nova-vflag-noswitchdev-needed-79aaa276bc799923.yaml b/releasenotes/notes/nova-vflag-noswitchdev-needed-79aaa276bc799923.yaml new file mode 100644 index 000000000..cbab83a2e --- /dev/null +++ b/releasenotes/notes/nova-vflag-noswitchdev-needed-79aaa276bc799923.yaml @@ -0,0 +1,18 @@ +--- +fixes: + - | + Previously ``switchdev`` capabilities should be configured manually by a + user with admin privileges using port's binding profile. This blocked + regular users from managing ports with Open vSwitch hardware offloading + as providing write access to a port's binding profile to non-admin users + introduces security risks. For example, a binding profile may contain a + ``pci_slot`` definition, which denotes the host PCI address of the + device attached to the VM. A malicious user can use this parameter to + passthrough any host device to a guest, so it is impossible to provide + write access to a binding profile to regular users in many scenarios. + + This patch fixes this situation by translating VF capabilities reported + by Libvirt to Neutron port binding profiles. Other VF capabilities are + translated as well for possible future use. + `LP#2008238 `__. + `LP#2020813 `__. diff --git a/releasenotes/notes/ntp-alerts-0d110b4979457165.yaml b/releasenotes/notes/ntp-alerts-0d110b4979457165.yaml new file mode 100644 index 000000000..62d932cc2 --- /dev/null +++ b/releasenotes/notes/ntp-alerts-0d110b4979457165.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds NTP alerts to prometheus alertmanager. \ No newline at end of file diff --git a/releasenotes/notes/nvmemon-update-a326f1b46d95b037.yaml b/releasenotes/notes/nvmemon-update-a326f1b46d95b037.yaml new file mode 100644 index 000000000..e6f1bf5cf --- /dev/null +++ b/releasenotes/notes/nvmemon-update-a326f1b46d95b037.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Nvmemon now reports physical size of the disk. diff --git a/releasenotes/notes/octavia-alerts-a7f1fe041f5450be.yaml b/releasenotes/notes/octavia-alerts-a7f1fe041f5450be.yaml new file mode 100644 index 000000000..491997ea7 --- /dev/null +++ b/releasenotes/notes/octavia-alerts-a7f1fe041f5450be.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds alerts for Octavia load balancers and amphorae. Alerts are triggered + when load balancers enter the ERROR or DEGRADED states, or when amphorae + enter the ERROR state. diff --git a/releasenotes/notes/octavia-dashboard-a5f904b20b8423f5.yaml b/releasenotes/notes/octavia-dashboard-a5f904b20b8423f5.yaml new file mode 100644 index 000000000..1cb651c65 --- /dev/null +++ b/releasenotes/notes/octavia-dashboard-a5f904b20b8423f5.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds a new Grafana dashboard for Octavia. This dashboard is used to monitor + the load balancers as well as the amphorae. diff --git a/releasenotes/notes/octavia-sync-healthmonitor-bug-8602034e2185d12c.yaml b/releasenotes/notes/octavia-sync-healthmonitor-bug-8602034e2185d12c.yaml new file mode 100644 index 000000000..29cd53c7d --- /dev/null +++ b/releasenotes/notes/octavia-sync-healthmonitor-bug-8602034e2185d12c.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Neutron ovn db sync operation will no longer removes OVN metadata ports in + networks with Octavia OVN Load balancers health monitors. + A maintenance task process has been added to update the existing OVN LB HM + ports to the new behaviour defined. Specifically, the "device_owner" field + will be updated from network:distributed to ovn-lb-hm:distributed. + Additionally, the "device_id" will be populated during update action. + `LP#2038091 `__. diff --git a/releasenotes/notes/ofed-workflow-119225f36a9265c3.yaml b/releasenotes/notes/ofed-workflow-119225f36a9265c3.yaml new file mode 100644 index 000000000..b371b1a17 --- /dev/null +++ b/releasenotes/notes/ofed-workflow-119225f36a9265c3.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Implement an OFED workflow that builds kernel modules + to support OFED drivers in release train kernels and + upload OFED kernel/userspace drivers to Ark. diff --git a/releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml b/releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml new file mode 100644 index 000000000..0941df773 --- /dev/null +++ b/releasenotes/notes/openstack-dash-ff7bf179e37c8e4b.yaml @@ -0,0 +1,5 @@ +--- +other: + - | + Changes the Grafana OpenStack dashboard to show HTTP status 300 as green + instead of orange. diff --git a/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml b/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml new file mode 100644 index 000000000..e4e2f43ac --- /dev/null +++ b/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Restores valid value for the ``flavor_id`` label on + ``openstack_nova_server_status`` Prometheus metrics. diff --git a/releasenotes/notes/os-capacity-94006f03f16583e4.yaml b/releasenotes/notes/os-capacity-94006f03f16583e4.yaml new file mode 100644 index 000000000..ca317682b --- /dev/null +++ b/releasenotes/notes/os-capacity-94006f03f16583e4.yaml @@ -0,0 +1,28 @@ +--- +features: + - | + This patch adds OpenStack Capacity metrics and exporters to + StackHPC Kayobe Config. This includes a deployment playbook, + Prometheus scrape jobs and HAProxy configurations to support + this change. +upgrade: + - | + To deploy the OpenStack Capacity Grafana dashboard, you must + define OpenStack application credential variables: + ``secrets_os_capacity_credential_id`` and + ``secrets_os_capacity_credential_secret`` as laid out in the + 'Monitoring' documentation. + + You must also enable the ``stackhpc_enable_os_capacity`` + flag for OpenStack Capacity HAProxy and Prometheus configuration + to be templated. + + You may also change the default authentication URL from the + kolla_internal_fqdn and change the default OpenStack region + from RegionOne with the variables: + ``stackhpc_os_capacity_auth_url`` and + ``stackhpc_os_capacity_openstack_region_name``. + + To disable certificate verification for the OpenStack Capacity + exporter, you can set ``stackhpc_os_capacity_openstack_verify`` + to false. diff --git a/releasenotes/notes/os-capacity-cacert-8b800b22d84ae0b1.yaml b/releasenotes/notes/os-capacity-cacert-8b800b22d84ae0b1.yaml new file mode 100644 index 000000000..e20ee0714 --- /dev/null +++ b/releasenotes/notes/os-capacity-cacert-8b800b22d84ae0b1.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds support for providing a CA certificate for OpenStack Capacity exporter. diff --git a/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml new file mode 100644 index 000000000..547939199 --- /dev/null +++ b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Automatic deployment for OpenStack Capacity via a Kayobe service + deploy hook using kolla admin credentials. +upgrade: + - | + OpenStack Capacity no longer uses application credentials. Please + delete any previously generated application credentials. \ No newline at end of file diff --git a/releasenotes/notes/osd-usage-in-dashboards-597ed10af4fc1aa1.yaml b/releasenotes/notes/osd-usage-in-dashboards-597ed10af4fc1aa1.yaml new file mode 100644 index 000000000..26de135f5 --- /dev/null +++ b/releasenotes/notes/osd-usage-in-dashboards-597ed10af4fc1aa1.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Per OSD usage metrics are now available in the OSDs dashboard. The + dashboard now includes a new section that displays a histogram of of the + utilization of each OSD in the cluster. This can be useful for identifying + OSDs that are outliers in terms of utilization, and may need to be + rebalanced. Additionally, there is a histogram displaying the usage of the + bluestoreDB for each OSD. diff --git a/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml b/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml new file mode 100644 index 000000000..c8178302a --- /dev/null +++ b/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds a standard overcloud Diskimage Builder (DIB) host image configuration. diff --git a/releasenotes/notes/overcloud-dib-ethtool-pciutils-5b3404b146c9f2a0.yaml b/releasenotes/notes/overcloud-dib-ethtool-pciutils-5b3404b146c9f2a0.yaml new file mode 100644 index 000000000..ad337ac47 --- /dev/null +++ b/releasenotes/notes/overcloud-dib-ethtool-pciutils-5b3404b146c9f2a0.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds ``ethtool`` and ``pciutils`` to the overcloud host disk image. diff --git a/releasenotes/notes/ovn-24-03-47c1eb9846f261b0.yaml b/releasenotes/notes/ovn-24-03-47c1eb9846f261b0.yaml new file mode 100644 index 000000000..bb75e9984 --- /dev/null +++ b/releasenotes/notes/ovn-24-03-47c1eb9846f261b0.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + ``OVN`` version in Rocky Linux 9 container images has been updated to + ``24.03`` (latest LTS). diff --git a/releasenotes/notes/ovn-fix-chassis-leader-check-551d0e94cbb94ac4.yaml b/releasenotes/notes/ovn-fix-chassis-leader-check-551d0e94cbb94ac4.yaml new file mode 100644 index 000000000..8d47f49c6 --- /dev/null +++ b/releasenotes/notes/ovn-fix-chassis-leader-check-551d0e94cbb94ac4.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Updated the OVN chassis priority fix playbook to detect the northbound + database leader via ``ovs-appctl cluster/status``, ensuring only the true + leader runs the priority alignment. diff --git a/releasenotes/notes/ovn-octavia-provider-bug-2049551-c95d69965733cd78.yaml b/releasenotes/notes/ovn-octavia-provider-bug-2049551-c95d69965733cd78.yaml new file mode 100644 index 000000000..ec9037395 --- /dev/null +++ b/releasenotes/notes/ovn-octavia-provider-bug-2049551-c95d69965733cd78.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Updates Octavia images to fix Neutron endpoint selection in the OVN + provider. `LP#2049551 `__. diff --git a/releasenotes/notes/ovn-octavia-provider-bug-2072754-5cbd7dc9c366668d.yaml b/releasenotes/notes/ovn-octavia-provider-bug-2072754-5cbd7dc9c366668d.yaml new file mode 100644 index 000000000..dc290a9ea --- /dev/null +++ b/releasenotes/notes/ovn-octavia-provider-bug-2072754-5cbd7dc9c366668d.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Updates Octavia container images to fix a maintenance task that was + breaking OVN IPv4 load balancers with health monitors. + `LP#2072754 `__. diff --git a/releasenotes/notes/ovn-sb-db-relay-81f57bc173299913.yaml b/releasenotes/notes/ovn-sb-db-relay-81f57bc173299913.yaml new file mode 100644 index 000000000..9cd500576 --- /dev/null +++ b/releasenotes/notes/ovn-sb-db-relay-81f57bc173299913.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add support for deploying OVN SB Relays, this can be enabled by setting + ``kolla_enable_ovn_sb_db_relay`` in ``kolla.yml`` to ``true``. + To use this feature ``stackhpc/16.4.0.5`` version of ``stackhpc/kayobe`` + and ``stackhpc/18.4.0.6`` version of ``stackhpc/kolla-ansible`` is required. diff --git a/releasenotes/notes/ovn_pkg_up-cfb09b7fd17ea786.yaml b/releasenotes/notes/ovn_pkg_up-cfb09b7fd17ea786.yaml new file mode 100644 index 000000000..88ee44845 --- /dev/null +++ b/releasenotes/notes/ovn_pkg_up-cfb09b7fd17ea786.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updated OVN package version from 22.06 to 22.09. diff --git a/releasenotes/notes/ovs-cve-2023-1668-298a2fc907fc07b9.yaml b/releasenotes/notes/ovs-cve-2023-1668-298a2fc907fc07b9.yaml new file mode 100644 index 000000000..3363affea --- /dev/null +++ b/releasenotes/notes/ovs-cve-2023-1668-298a2fc907fc07b9.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + ``openvswitch`` version has been updated to ~2.17.5 on all distributions + (CentOS/Rocky9 are two patches ahead of 2.17.5). + Images include fixes for CVE-2023-1668. + + Ubuntu repository versions for focal and ubuntu cloud archive have been + updated to `20230515`. diff --git a/releasenotes/notes/pci-passthrough-support-0c7e62585aaf2c23.yaml b/releasenotes/notes/pci-passthrough-support-0c7e62585aaf2c23.yaml new file mode 100644 index 000000000..eae2d774b --- /dev/null +++ b/releasenotes/notes/pci-passthrough-support-0c7e62585aaf2c23.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Added templates and a playbook to simplify configuration of PCI passthrough + GPUs. GPU types can be mapped to inventory groups with the + ``gpu_group_map`` variable, which will configure the host and Nova + automatically. A list of supported GPUs can be found in + ``etc/kayobe/stackhpc-compute.yml`` under ``stackhpc_gpu_data``. diff --git a/releasenotes/notes/pin-baseimage-3d224df5b6f56b10.yaml b/releasenotes/notes/pin-baseimage-3d224df5b6f56b10.yaml new file mode 100644 index 000000000..b03d5c247 --- /dev/null +++ b/releasenotes/notes/pin-baseimage-3d224df5b6f56b10.yaml @@ -0,0 +1,7 @@ +fixes: + - | + Pin the OCI image tag used for the base-image of Kolla image + builds. This prevents packages in the image with + the latest tag getting in front of StackHPC release-train + package repositories. Ubuntu tag should be bumped when new + packages are available in StackHPC release-train. diff --git a/releasenotes/notes/pin-deps-dbe52c49e945daf5.yaml b/releasenotes/notes/pin-deps-dbe52c49e945daf5.yaml new file mode 100644 index 000000000..db4e5383f --- /dev/null +++ b/releasenotes/notes/pin-deps-dbe52c49e945daf5.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + All Ansible dependencies are now pinned to specific versions. The + ``stackhpc.vxlan`` role is pinned to 1.1.0, and ``ansible-role-docker`` + is pinned to stackhpc/7.0.1.1. diff --git a/releasenotes/notes/pin-pulp-squeezer-bd3a3c53d9804010.yaml b/releasenotes/notes/pin-pulp-squeezer-bd3a3c53d9804010.yaml new file mode 100644 index 000000000..7de111226 --- /dev/null +++ b/releasenotes/notes/pin-pulp-squeezer-bd3a3c53d9804010.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an issue with Ansible Pulp modules depending on the ``pulp_glue`` + Python library since the ``pulp.squeezer`` 0.0.14 release. diff --git a/releasenotes/notes/prometheus-hostname-labels-82346cb0914680e9.yaml b/releasenotes/notes/prometheus-hostname-labels-82346cb0914680e9.yaml new file mode 100644 index 000000000..ee5db2c7d --- /dev/null +++ b/releasenotes/notes/prometheus-hostname-labels-82346cb0914680e9.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Instance labels in prometheus now use inventory hostnames rather than IPs. diff --git a/releasenotes/notes/pull-stackhpc-host-images-a623e4ab2d2a9e5b.yaml b/releasenotes/notes/pull-stackhpc-host-images-a623e4ab2d2a9e5b.yaml new file mode 100644 index 000000000..5869eec04 --- /dev/null +++ b/releasenotes/notes/pull-stackhpc-host-images-a623e4ab2d2a9e5b.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Prebuilt overcloud host images can now be pulled from Ark using the + `stackhpc_download_overcloud_host_images` variable. The image is selected + based on `os_distribution` and `os_release`. diff --git a/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml new file mode 100644 index 000000000..e9d54f989 --- /dev/null +++ b/releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a custom playbook (``pulp-auth-proxy.yml``) for deploying an + authenticating proxy for Pulp. This can be used when building container + images to avoid leaking credentials for package repositories into the built + images or their metadata. diff --git a/releasenotes/notes/pulp-container-extra-9379806192900d22.yaml b/releasenotes/notes/pulp-container-extra-9379806192900d22.yaml new file mode 100644 index 000000000..a725a4863 --- /dev/null +++ b/releasenotes/notes/pulp-container-extra-9379806192900d22.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Allows to synchronise a custom list of containers to Pulp using the + ``stackhpc_pulp_repository_container_repos_extra`` and + ``stackhpc_pulp_distribution_container_extra`` variables. diff --git a/releasenotes/notes/pulp-sync-publish-promote-playbook-b638be7c77a649dd.yaml b/releasenotes/notes/pulp-sync-publish-promote-playbook-b638be7c77a649dd.yaml new file mode 100644 index 000000000..41ac1167b --- /dev/null +++ b/releasenotes/notes/pulp-sync-publish-promote-playbook-b638be7c77a649dd.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added a new playbook `pulp_sync_publish_promote` that can be used to sync, + publish and promote all repositories in a single step, as well as sync and + publish container repos. If you do not want to promote repos then run with + ``-e repo_promote_production=false``. diff --git a/releasenotes/notes/rabbimq-too-many-ready-c18ecea9a13c8f2a.yaml b/releasenotes/notes/rabbimq-too-many-ready-c18ecea9a13c8f2a.yaml new file mode 100644 index 000000000..000528f54 --- /dev/null +++ b/releasenotes/notes/rabbimq-too-many-ready-c18ecea9a13c8f2a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Raises an alert when the count of RabbitMQ ready messages increases above a + threshold. diff --git a/releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml b/releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml new file mode 100644 index 000000000..b1a66e16f --- /dev/null +++ b/releasenotes/notes/rabbitmq-cloudsmith-repos-78ba8e4958e0a3ee.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + RabbitMQ and Erlang packages are now all installed from the Cloudsmith + ``rabbitmq.com`` mirrors since the RabbitMQ ``packagecloud.io`` is getting + shut down August 18st, 2024: + https://www.rabbitmq.com/blog/2024/08/11/package-repository-updates#packagecloud-will-be-discontinued-on-aug-18th-2024 diff --git a/releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml b/releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml new file mode 100644 index 000000000..dffc8b9ef --- /dev/null +++ b/releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adapt threshold of RabbitMQ connection alert based on the size of the + deployment to avoid spurious alerts. diff --git a/releasenotes/notes/rabbitmq-queue-migration-script-b6d3abebbebf8087.yaml b/releasenotes/notes/rabbitmq-queue-migration-script-b6d3abebbebf8087.yaml new file mode 100644 index 000000000..a475dbc7a --- /dev/null +++ b/releasenotes/notes/rabbitmq-queue-migration-script-b6d3abebbebf8087.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Added a new script, ``rabbitmq-queue-migration.sh``, which will migrate to + the new RabbitMQ durable queues. This is intended for use prior to an + upgrade to Epoxy. diff --git a/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml b/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml new file mode 100644 index 000000000..338cde866 --- /dev/null +++ b/releasenotes/notes/radosgw-usage-exporter-behind-haproxy-e4371d2732f2a081.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + The radosgw-usage-exporter is now put behind HAProxy. To facilitate this, + ``stackhpc_radosgw_usage_exporter_port`` had been renamed to + ``stackhpc_radosgw_usage_exporter_backend_port`` (it remains 9242) and + ``stackhpc_radosgw_usage_exporter_frontend_port`` (defaults to 9240) has + been introduced. +fixes: + - | + Fixes an issue where object storage metrics were missing from Prometheus by + putting the radosgw-usage-exporter behind HAProxy. diff --git a/releasenotes/notes/rated-dwpd-40526e85e24ef7ea.yaml b/releasenotes/notes/rated-dwpd-40526e85e24ef7ea.yaml new file mode 100644 index 000000000..5b8eb5065 --- /dev/null +++ b/releasenotes/notes/rated-dwpd-40526e85e24ef7ea.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add support of the operator supplying the rated DWPD value for NVMe drives. + There is a playbook ``get-nvme-drives.yml`` that will populate a new + section in the ``stackhpc-monitoring.yml`` file with drive model names for + NVMes in the cloud. The operator can then fill in the rated DWPD values for + each drive. diff --git a/releasenotes/notes/re-enable-pulp-ubuntu-repos-d46ff8eca4a5aac3.yaml b/releasenotes/notes/re-enable-pulp-ubuntu-repos-d46ff8eca4a5aac3.yaml new file mode 100644 index 000000000..52345decb --- /dev/null +++ b/releasenotes/notes/re-enable-pulp-ubuntu-repos-d46ff8eca4a5aac3.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Re-enable Pulp Ubuntu repositories. diff --git a/releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml b/releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml new file mode 100644 index 000000000..2445b5cbc --- /dev/null +++ b/releasenotes/notes/readthedocs-configuration-17cf3e6a63f571fa.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes documentation builds on Read the Docs. diff --git a/releasenotes/notes/reboot-default-serial-5944a2a648da71c7.yaml b/releasenotes/notes/reboot-default-serial-5944a2a648da71c7.yaml new file mode 100644 index 000000000..7eb2e28cd --- /dev/null +++ b/releasenotes/notes/reboot-default-serial-5944a2a648da71c7.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + The ``reboot.yml`` custom Ansible playbook now defaults to reboot only one + host at a time. Existing behaviour can be retained by setting + ANSIBLE_SERIAL=0. diff --git a/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml new file mode 100644 index 000000000..da3cb5cbb --- /dev/null +++ b/releasenotes/notes/rebuild-heat-with-yaql-3.0.0-4415d8232bc547df.yaml @@ -0,0 +1,7 @@ +--- +security: + - | + The Heat container images are rebuilt with yaql 3.0.0 to include patch for + vulnerability OSSN/OSSN-0093. It is recommended that you redeploy Heat + services in your system with the current version of Heat images from + StackHPC Release Train. diff --git a/releasenotes/notes/redfish-dashboard-fix-inlet-temp-91a7018adb2e1763.yaml b/releasenotes/notes/redfish-dashboard-fix-inlet-temp-91a7018adb2e1763.yaml new file mode 100644 index 000000000..73f587172 --- /dev/null +++ b/releasenotes/notes/redfish-dashboard-fix-inlet-temp-91a7018adb2e1763.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes "Max Inlet Temp" time series chart in Redfish dashboard. This chart + could wrongly display CPU2 temperature instead of inlet temperature. diff --git a/releasenotes/notes/redfish-exporter-36cbdabf103ca93d.yaml b/releasenotes/notes/redfish-exporter-36cbdabf103ca93d.yaml new file mode 100644 index 000000000..7f447ad4d --- /dev/null +++ b/releasenotes/notes/redfish-exporter-36cbdabf103ca93d.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds support for deploying a Prometheus Redfish exporter container on the + seed. This can be used to query the overcloud BMCs via their redfish + interfaces to produce various metrics relating to the hardware, and system + health. diff --git a/releasenotes/notes/redfish-uuid-grafana-2f50c76bbff6eba9.yaml b/releasenotes/notes/redfish-uuid-grafana-2f50c76bbff6eba9.yaml new file mode 100644 index 000000000..50ba6aa05 --- /dev/null +++ b/releasenotes/notes/redfish-uuid-grafana-2f50c76bbff6eba9.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes a bug where the Redfish Dashboard's UUID changed, causing Grafana to + fail to update the dashboard on deployment where the old dashboard UUID had + been used previously. diff --git a/releasenotes/notes/reduce-seed-pulp-workers-b085c86bfe7ab5b6.yaml b/releasenotes/notes/reduce-seed-pulp-workers-b085c86bfe7ab5b6.yaml new file mode 100644 index 000000000..387e75b04 --- /dev/null +++ b/releasenotes/notes/reduce-seed-pulp-workers-b085c86bfe7ab5b6.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Reduce the maximum number of PULP_CONTENT_WORKERS and PULP_API_WORKERS + to 12 to prevent connection starvation on seed hosts with more than 12 + cpu threads. diff --git a/releasenotes/notes/reduces-sensitivity-of-redfish-target-alerts-a3d77a3f0c3dac8a.yaml b/releasenotes/notes/reduces-sensitivity-of-redfish-target-alerts-a3d77a3f0c3dac8a.yaml new file mode 100644 index 000000000..0ba59ea7a --- /dev/null +++ b/releasenotes/notes/reduces-sensitivity-of-redfish-target-alerts-a3d77a3f0c3dac8a.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Changes the duration for which redfish exporter must continually fail + scrapes before triggering an alert to 15 minutes. This should hopefully + reduce some alert spam. diff --git a/releasenotes/notes/refactor-tag-overrides-1ad66680fd801345.yaml b/releasenotes/notes/refactor-tag-overrides-1ad66680fd801345.yaml new file mode 100644 index 000000000..1edeb175c --- /dev/null +++ b/releasenotes/notes/refactor-tag-overrides-1ad66680fd801345.yaml @@ -0,0 +1,6 @@ +--- +upgrade: + - | + Kolla tag overrides have been refactored to allow kolla-ansible to + resolve them individually by host. This means that mixed clouds can be + deployed which allows for migration between distributions. \ No newline at end of file diff --git a/releasenotes/notes/release-neutron-bgp-dragent-tags-39b9c2595ae11872.yaml b/releasenotes/notes/release-neutron-bgp-dragent-tags-39b9c2595ae11872.yaml new file mode 100644 index 000000000..bf9efd40a --- /dev/null +++ b/releasenotes/notes/release-neutron-bgp-dragent-tags-39b9c2595ae11872.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Add support for syncing ``neutron-bgp-dragent`` image into ``Pulp``. diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml new file mode 100644 index 000000000..b8c79e5e2 --- /dev/null +++ b/releasenotes/notes/release-train-02-2023-package-refresh-9de8049f8cb08a5e.yaml @@ -0,0 +1,25 @@ +--- +features: + - | + Package repositories and container images for CentOS Stream based deployments have been updated. + Key packages to note are: + + * Kernel + + * version: 4.18.0 + * release: 448.el8 + + * Libvirt + + * version: 8.0.0 + * release: 6.module_el8.7.0+1140+ff0772f9 + + * OVS + + * version: 2.17.0 + * release: 71.el8s + + * OVN + + * version: 22.09.0 + * release: 11.el8s diff --git a/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml new file mode 100644 index 000000000..1895cc2a1 --- /dev/null +++ b/releasenotes/notes/release-train-02-2023-package-refresh-ubuntu-a9fe8a1c3c2f2796.yaml @@ -0,0 +1,20 @@ +--- +features: + - | + Container images for Ubuntu based deployments have been updated. + Key packages to note are: + + * Libvirt + + * version: 8.0.0 + * release: 1ubuntu7.4~cloud0 + + * OVS + + * version: 2.17.3 + * release: 0ubuntu0.22.04.1~cloud0 + + * OVN (unchanged since last container build) + + * version: 22.03.0 + * release: 0ubuntu1~cloud0 diff --git a/releasenotes/notes/remove-swap-playbook-f62a41945e29816d.yaml b/releasenotes/notes/remove-swap-playbook-f62a41945e29816d.yaml new file mode 100644 index 000000000..0e7173b68 --- /dev/null +++ b/releasenotes/notes/remove-swap-playbook-f62a41945e29816d.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + The ``swap.yml`` custom playbook has been removed in favour of Kayobe's + support for configuring swap. See the `Kayobe documentation + `__ + for details. diff --git a/releasenotes/notes/remove-tmout-for-dev-env-0778550b353dce03.yaml b/releasenotes/notes/remove-tmout-for-dev-env-0778550b353dce03.yaml new file mode 100644 index 000000000..59e8d472c --- /dev/null +++ b/releasenotes/notes/remove-tmout-for-dev-env-0778550b353dce03.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Disable the CIS hardening rule ``*_rule_5_4_3_2`` to prevent ``TMOUT`` + from being applied which can disrupt a development environment as it + closes ``TMUX`` panes and servers and may close active ssh session. diff --git a/releasenotes/notes/removes-bogus-ContainerVolumeUsage-alert-c973b61f598d85e5.yaml b/releasenotes/notes/removes-bogus-ContainerVolumeUsage-alert-c973b61f598d85e5.yaml new file mode 100644 index 000000000..f1c76c545 --- /dev/null +++ b/releasenotes/notes/removes-bogus-ContainerVolumeUsage-alert-c973b61f598d85e5.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Removes bogus ContainerVolumeUsage alert. This rule wasn't correctly measuring + container volume IO and could cause spurious alerts. diff --git a/releasenotes/notes/reset-bls-entries-b2bded62c5887937.yaml b/releasenotes/notes/reset-bls-entries-b2bded62c5887937.yaml new file mode 100644 index 000000000..6279b6a00 --- /dev/null +++ b/releasenotes/notes/reset-bls-entries-b2bded62c5887937.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Add a new ``reset-bls-entries.yml`` custom playbook which will rename + existing Boot Loader Specification (BLS) entries using the current machine + ID for each host. This should fix an issue with Grub not selecting the most + recent kernel during boot. diff --git a/releasenotes/notes/restore-kolla-mirrors-9c0c01735d7f43df.yaml b/releasenotes/notes/restore-kolla-mirrors-9c0c01735d7f43df.yaml new file mode 100644 index 000000000..4b948381a --- /dev/null +++ b/releasenotes/notes/restore-kolla-mirrors-9c0c01735d7f43df.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Upstream package repository mirrors are now restored in Kolla container + images. This makes it possible to install or update packages for debugging + purposes. diff --git a/releasenotes/notes/revert-track-all-interfaces-in-keepalived-41ef4834b7659af4.yaml b/releasenotes/notes/revert-track-all-interfaces-in-keepalived-41ef4834b7659af4.yaml new file mode 100644 index 000000000..ef70e21ab --- /dev/null +++ b/releasenotes/notes/revert-track-all-interfaces-in-keepalived-41ef4834b7659af4.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Reverts "Track all interfaces in Keepalived" so only HA interfaces are + tracked. This prevents L3 HA router flapping when detaching floating IP + addresses, because non-HA router interfaces did not include "no_track". + Closes-Bug: #2097770 diff --git a/releasenotes/notes/rgw-usage-exporter-deployment-fixes-0196167326dbe456.yaml b/releasenotes/notes/rgw-usage-exporter-deployment-fixes-0196167326dbe456.yaml new file mode 100644 index 000000000..46a591480 --- /dev/null +++ b/releasenotes/notes/rgw-usage-exporter-deployment-fixes-0196167326dbe456.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixed RADOS gateway usage exporter deployment failing + to generate ec2 credentials for the ceph_rgw user. + - | + Fixed RADOS gateway usage exporter not using the system + trust root as its CA bundle. diff --git a/releasenotes/notes/rl9-image-minimal-cloud-init-83576172be8febca.yaml b/releasenotes/notes/rl9-image-minimal-cloud-init-83576172be8febca.yaml new file mode 100644 index 000000000..ea3ee865e --- /dev/null +++ b/releasenotes/notes/rl9-image-minimal-cloud-init-83576172be8febca.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Rocky Linux 9 image has been rebuilt with missing base packages (e.g. + microcode_ctl) by installing 'Minimal Install' DNF group. + Also cloud-init from CentOS 9 Stream has been installed with NetworkManager + support. diff --git a/releasenotes/notes/rl9-security-common-cve-2024-6409-d36de799a29c3f74.yaml b/releasenotes/notes/rl9-security-common-cve-2024-6409-d36de799a29c3f74.yaml new file mode 100644 index 000000000..077f9c862 --- /dev/null +++ b/releasenotes/notes/rl9-security-common-cve-2024-6409-d36de799a29c3f74.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Updates the Rocky Linux 9 SIG Security Common repository to address + `CVE-2024-6409 `__ + in OpenSSH. diff --git a/releasenotes/notes/rl9.6-f46ea1230214a657.yaml b/releasenotes/notes/rl9.6-f46ea1230214a657.yaml new file mode 100644 index 000000000..3213dd820 --- /dev/null +++ b/releasenotes/notes/rl9.6-f46ea1230214a657.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Added support for Rocky Linux 9.6, including host packages and a full + container image refresh. +upgrade: + - | + 9.6 is now the default release for Rocky Linux. diff --git a/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml new file mode 100644 index 000000000..6b6fa315f --- /dev/null +++ b/releasenotes/notes/rl9.7-c8d033f11097b76b.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Added support for Rocky Linux 9.7, including host packages and a full + container image refresh. + - | + 9.7 is now the default release for Rocky Linux. + +upgrade: + - | + The upgrade to DOCA 3.2.1 is required for Rocky Linux 9.7. diff --git a/releasenotes/notes/rl95-6cbdf902e30502bf.yaml b/releasenotes/notes/rl95-6cbdf902e30502bf.yaml new file mode 100644 index 000000000..b27feadd7 --- /dev/null +++ b/releasenotes/notes/rl95-6cbdf902e30502bf.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added support for Rocky Linux 9.5, including host packages and a full + container image refresh. + - | + Made 9.5 the default release for Rocky Linux. diff --git a/releasenotes/notes/rmq-dashboard-5bb5eb36ff72100c.yaml b/releasenotes/notes/rmq-dashboard-5bb5eb36ff72100c.yaml new file mode 100644 index 000000000..39fdbba23 --- /dev/null +++ b/releasenotes/notes/rmq-dashboard-5bb5eb36ff72100c.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Minor issue with RabbitMQ dashboard panel showing no data with + default scrape settings. diff --git a/releasenotes/notes/rocky-87-rpms-999565263dacde42.yaml b/releasenotes/notes/rocky-87-rpms-999565263dacde42.yaml new file mode 100644 index 000000000..4f9db394b --- /dev/null +++ b/releasenotes/notes/rocky-87-rpms-999565263dacde42.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Sync Rocky Linux 8.7 RPM repositories to local Pulp servers. diff --git a/releasenotes/notes/rocky-crb-b6a6cecfdf4d93a4.yaml b/releasenotes/notes/rocky-crb-b6a6cecfdf4d93a4.yaml new file mode 100644 index 000000000..9c44abcfc --- /dev/null +++ b/releasenotes/notes/rocky-crb-b6a6cecfdf4d93a4.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes synchronisation and DNF configuration of the Rocky Linux 9 CRB repository. diff --git a/releasenotes/notes/security-common-openssh-6fbd5a1e95fd66ae.yaml b/releasenotes/notes/security-common-openssh-6fbd5a1e95fd66ae.yaml new file mode 100644 index 000000000..7c2892c80 --- /dev/null +++ b/releasenotes/notes/security-common-openssh-6fbd5a1e95fd66ae.yaml @@ -0,0 +1,6 @@ +--- +security: + - | + Enables the Rocky Linux 9 SIG Security Common repository, which provides + updated OpenSSH packages addressing CVE-2024-6387 (regreSSHion). Other + packages available in this repository are currently ignored. diff --git a/releasenotes/notes/selinux-host-image-ff0fed2583cae7b0.yaml b/releasenotes/notes/selinux-host-image-ff0fed2583cae7b0.yaml new file mode 100644 index 000000000..b545668f4 --- /dev/null +++ b/releasenotes/notes/selinux-host-image-ff0fed2583cae7b0.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Enables SELinux in permissive mode in the overcloud host image. This + matches the default configuration for SELinux in StackHPC Kayobe Configuration. diff --git a/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml b/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml new file mode 100644 index 000000000..0d4e0ace3 --- /dev/null +++ b/releasenotes/notes/smart-mon-db8fa642c3af74b1.yaml @@ -0,0 +1,3 @@ +--- +features: + - Enables SMART monitoring. Manual action is required, please see the monitoring documentation for the procedure. diff --git a/releasenotes/notes/smartmontools-bc8176f45d58a75d.yaml b/releasenotes/notes/smartmontools-bc8176f45d58a75d.yaml new file mode 100644 index 000000000..ac3451347 --- /dev/null +++ b/releasenotes/notes/smartmontools-bc8176f45d58a75d.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The smartmon-tools playbook now ensures that the cron service is running as + in some cases it may not be running by default. + diff --git a/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml b/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml new file mode 100644 index 000000000..c39ffa43a --- /dev/null +++ b/releasenotes/notes/split-cephadm-commands-pre-post-bddd25b8bd171ab0.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Split `cephadm_commands` into `cephadm_commands_pre` and + `cephadm_commands_post` commands. This allows the user to run commands that + must be run before the rest of the post-deployment configuration, as well + as commands that rely on resources created by the post-deployment config. + diff --git a/releasenotes/notes/stackhpc-openstack-tests-0224908453ae72cc.yaml b/releasenotes/notes/stackhpc-openstack-tests-0224908453ae72cc.yaml new file mode 100644 index 000000000..eb3c21266 --- /dev/null +++ b/releasenotes/notes/stackhpc-openstack-tests-0224908453ae72cc.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Adds a new ``stackhpc-openstack-tests.yml`` playbook that executes tests in + the `StackHPC OpenStack Tests + `__ repository. Both + the playbook and tests are currently experimental, and are currently + targeting only an all-in-one CI use case. diff --git a/releasenotes/notes/stackhpc-repos-enabled-10c8a698991e53c2.yaml b/releasenotes/notes/stackhpc-repos-enabled-10c8a698991e53c2.yaml new file mode 100644 index 000000000..fe21296c9 --- /dev/null +++ b/releasenotes/notes/stackhpc-repos-enabled-10c8a698991e53c2.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Added a new group variable - ``stackhpc_repos_enabled`` - for unified + control over usage of StackHPC Release Train package repositories. This + makes it easier to set which hosts do or do not pull packages from release + train. diff --git a/releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml b/releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml new file mode 100644 index 000000000..6516f8ec4 --- /dev/null +++ b/releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added the ``stop-openstack-services.yml`` playbook, which can be used to + stop OpenStack services across the overcloud. diff --git a/releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml b/releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml new file mode 100644 index 000000000..83ddb8102 --- /dev/null +++ b/releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Supports adding CA certificates to the Tempest container trust store. diff --git a/releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml b/releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml new file mode 100644 index 000000000..db9658694 --- /dev/null +++ b/releasenotes/notes/tempest-concurrency-1c86c0ef90e104f6.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + The default Tempest concurrency has been increased from 2 to 16. + This is often easily achievable in production systems. diff --git a/releasenotes/notes/ubuntu-fix-cve-2024-6387-648efedaeb288023.yaml b/releasenotes/notes/ubuntu-fix-cve-2024-6387-648efedaeb288023.yaml new file mode 100644 index 000000000..8de5f7a7f --- /dev/null +++ b/releasenotes/notes/ubuntu-fix-cve-2024-6387-648efedaeb288023.yaml @@ -0,0 +1,5 @@ +--- +security: + - | + Adds a custom Apt repository to address `CVE-2024-6387 + `__ in OpenSSH. diff --git a/releasenotes/notes/ubuntu-image-early-repos-08b4da6165f5fa11.yaml b/releasenotes/notes/ubuntu-image-early-repos-08b4da6165f5fa11.yaml new file mode 100644 index 000000000..2535d9e0f --- /dev/null +++ b/releasenotes/notes/ubuntu-image-early-repos-08b4da6165f5fa11.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue with Kolla container image builds for Ubuntu where the + release train package repositories could be behind the container image, + leading to image build failures. diff --git a/releasenotes/notes/ubuntu-jammy-cd109d12954d7626.yaml b/releasenotes/notes/ubuntu-jammy-cd109d12954d7626.yaml new file mode 100644 index 000000000..4265655c1 --- /dev/null +++ b/releasenotes/notes/ubuntu-jammy-cd109d12954d7626.yaml @@ -0,0 +1,8 @@ +--- +upgrade: + - | + Dont pull apt packages from pulp for Ubuntu Jammy until Jammy packages + are published. + - | + Dont pull ceph packages from ceph official repos for Ubuntu Jammy until + Jammy packages are published. diff --git a/releasenotes/notes/ubuntu-refresh-december-998f4c2d2bd0032b.yaml b/releasenotes/notes/ubuntu-refresh-december-998f4c2d2bd0032b.yaml new file mode 100644 index 000000000..a98acb505 --- /dev/null +++ b/releasenotes/notes/ubuntu-refresh-december-998f4c2d2bd0032b.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Refreshed all Ubuntu host package versions and contianer images for + December 2024. diff --git a/releasenotes/notes/update-blazar-image-d176c27d55716469.yaml b/releasenotes/notes/update-blazar-image-d176c27d55716469.yaml new file mode 100644 index 000000000..7e53b3543 --- /dev/null +++ b/releasenotes/notes/update-blazar-image-d176c27d55716469.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Use the StackHPC fork for building Blazar images with customizations to support + flavor-based reservation. diff --git a/releasenotes/notes/update-cis-roles-8c73975c37188071.yaml b/releasenotes/notes/update-cis-roles-8c73975c37188071.yaml new file mode 100644 index 000000000..c1705edf0 --- /dev/null +++ b/releasenotes/notes/update-cis-roles-8c73975c37188071.yaml @@ -0,0 +1,9 @@ +--- +upgrade: + - | + Updates the Ansible Lockdown roles for Ubuntu and Rocky Linux to 1.4.1 and + 1.3.1 respectively. See `UBUNTU22-CIS + `__ + and `RHEL9-CIS + `__ for + release notes. diff --git a/releasenotes/notes/update-grafana-version-ed01e7bd04782ff7.yaml b/releasenotes/notes/update-grafana-version-ed01e7bd04782ff7.yaml new file mode 100644 index 000000000..9b05facd8 --- /dev/null +++ b/releasenotes/notes/update-grafana-version-ed01e7bd04782ff7.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates Grafana to 9.4.7 version. diff --git a/releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml b/releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml new file mode 100644 index 000000000..e327a45b1 --- /dev/null +++ b/releasenotes/notes/update-haproxy-alerting-rules-5594547666659c0f.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + HAProxy alerting rules have been updated to use the server name that is + down, rather than the name of the instance that reported the down server. diff --git a/releasenotes/notes/update-haproxy-dashboard-b8807c2a0645ca5e.yaml b/releasenotes/notes/update-haproxy-dashboard-b8807c2a0645ca5e.yaml new file mode 100644 index 000000000..e9ae31c57 --- /dev/null +++ b/releasenotes/notes/update-haproxy-dashboard-b8807c2a0645ca5e.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The HAProxy dashboard and alerts have been updated to use the new metrics + source in Kolla Ansible. For further details see `here + `__. diff --git a/releasenotes/notes/update-hcp-collection-39acaafe76c95b3f.yaml b/releasenotes/notes/update-hcp-collection-39acaafe76c95b3f.yaml new file mode 100644 index 000000000..9a561a3f7 --- /dev/null +++ b/releasenotes/notes/update-hcp-collection-39acaafe76c95b3f.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes the issue with interface names containing dashes in Hashicorp collection. diff --git a/releasenotes/notes/update-kayobe-automation-55fc9c5c380d819c.yaml b/releasenotes/notes/update-kayobe-automation-55fc9c5c380d819c.yaml new file mode 100644 index 000000000..f8dee5b4d --- /dev/null +++ b/releasenotes/notes/update-kayobe-automation-55fc9c5c380d819c.yaml @@ -0,0 +1,23 @@ +--- +features: + - | + Upgrades kayobe-automation submodule to ``7676aa8``. + + Upgrades kayobe-workflows collection to ``v1.1.0``. + + Kayobe-automation config-diff now runs in parallel and generates both + the old and new configuration at the same time. This should improve + config-diff wait times. + + Add support for the `pulp-sync-content` run book. +deprecations: + - | + Kayobe-automation will now automatically detect vaulted files for the + purpose of config-diff therefore, ``KAYOBE_CONFIG_SECRET_PATHS_EXTRA`` and + ``KAYOBE_CONFIG_VAULTED_FILES_PATHS_EXTRA`` are no longer used +security: + - | + The upgraded kayobe-workflows collection increases the version of various + Actions and containers used within GitHub based workflows, including increasing + Docker in Docker to version ``27.3.1`` thus removing the vunerabilities present + in ``24.0-git``. diff --git a/releasenotes/notes/update-magnum-capi-driver-c85135340731a86a.yaml b/releasenotes/notes/update-magnum-capi-driver-c85135340731a86a.yaml new file mode 100644 index 000000000..8c3eaccec --- /dev/null +++ b/releasenotes/notes/update-magnum-capi-driver-c85135340731a86a.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Updated the version of magnum-capi-helm driver to 1.3.0. + For the list of features and fixes see its + `release notes `__. diff --git a/releasenotes/notes/update-magnum-capi-helm-129ea84a8903eb10.yaml b/releasenotes/notes/update-magnum-capi-helm-129ea84a8903eb10.yaml new file mode 100644 index 000000000..34fb10c25 --- /dev/null +++ b/releasenotes/notes/update-magnum-capi-helm-129ea84a8903eb10.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates Magnum CAPI Helm driver version to OpenDev v1.0.0 diff --git a/releasenotes/notes/update-magnum-containers-8814b1425ec45fc0.yaml b/releasenotes/notes/update-magnum-containers-8814b1425ec45fc0.yaml new file mode 100644 index 000000000..9f6a18282 --- /dev/null +++ b/releasenotes/notes/update-magnum-containers-8814b1425ec45fc0.yaml @@ -0,0 +1,5 @@ +fixes: + - | + Updates Magnum to a proper Caracal release version following critical bug + upstream merged & backported. Also, updates CAPI Helm driver version to the + latest (1.2.0). diff --git a/releasenotes/notes/update-ovs-ovn-rocky9-2024.1-a1025a7130d9c647.yaml b/releasenotes/notes/update-ovs-ovn-rocky9-2024.1-a1025a7130d9c647.yaml new file mode 100644 index 000000000..e4a947fca --- /dev/null +++ b/releasenotes/notes/update-ovs-ovn-rocky9-2024.1-a1025a7130d9c647.yaml @@ -0,0 +1,12 @@ +features: + - | + Updated OpenvSwitch to 3.3.0-88 and OVN to 24.03.5-14 for Rocky Linux 9 + in Caracal. For details, see the following changelogs: + + * `OVS changelog `__ + * `OVN changelog `__ + + - | + Kolla Toolbox, Manila, Neutron, Nova, and Octavia containers received + updates on both Rocky Linux 9 and Ubuntu. Only the Rocky Linux 9 images + include the new OVS versions. diff --git a/releasenotes/notes/update-prometheus-3519aa260d5f24ef.yaml b/releasenotes/notes/update-prometheus-3519aa260d5f24ef.yaml new file mode 100644 index 000000000..db9c0159d --- /dev/null +++ b/releasenotes/notes/update-prometheus-3519aa260d5f24ef.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates Prometheus to version 2.55.1. diff --git a/releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml b/releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml new file mode 100644 index 000000000..d1235e62f --- /dev/null +++ b/releasenotes/notes/update-pulp-3.22-aa485b7e619cd380.yaml @@ -0,0 +1,7 @@ +--- +features: + - Upgrades Pulp from ``3.21`` to ``3.22``. + - Disables Pulp analytics. + - | + Sets Pulp worker based on available CPU cores. This may improve performance + when pulling container images to many hosts simultaneously. diff --git a/releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml b/releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml new file mode 100644 index 000000000..f3019561c --- /dev/null +++ b/releasenotes/notes/update-pulp-3.23-b4193a417a219d26.yaml @@ -0,0 +1,3 @@ +--- +features: + - Upgrades Pulp from ``3.22`` to ``3.23``. diff --git a/releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml b/releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml new file mode 100644 index 000000000..32138adb6 --- /dev/null +++ b/releasenotes/notes/update-pulp-3.24-d770db48deaf6076.yaml @@ -0,0 +1,3 @@ +--- +features: + - Upgrades Pulp from ``3.23`` to ``3.24``. diff --git a/releasenotes/notes/update-smartmon-playbook-6bdf9624c579d2e2.yaml b/releasenotes/notes/update-smartmon-playbook-6bdf9624c579d2e2.yaml new file mode 100644 index 000000000..37312d04a --- /dev/null +++ b/releasenotes/notes/update-smartmon-playbook-6bdf9624c579d2e2.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Updates the smartmon-tools.yml playbook to ensure that cron is installed + before attempting to configure crontab. diff --git a/releasenotes/notes/update-ubuntu-repos-4fd1b3083ee7b245.yaml b/releasenotes/notes/update-ubuntu-repos-4fd1b3083ee7b245.yaml new file mode 100644 index 000000000..0153a796f --- /dev/null +++ b/releasenotes/notes/update-ubuntu-repos-4fd1b3083ee7b245.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updated Ubuntu package repository versions. diff --git a/releasenotes/notes/use-include-tags-for-client-pulp-cf46d328b30162be.yaml b/releasenotes/notes/use-include-tags-for-client-pulp-cf46d328b30162be.yaml new file mode 100644 index 000000000..7103ef604 --- /dev/null +++ b/releasenotes/notes/use-include-tags-for-client-pulp-cf46d328b30162be.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Restrict the content that is synced to the client by using include tags. + This feature ensures that the tags as defined within + ``kolla-image-tags.yml`` are synced. diff --git a/releasenotes/notes/use-length-for-rabbitmq-node-down-rule-c9e9c6b09f57954d.yaml b/releasenotes/notes/use-length-for-rabbitmq-node-down-rule-c9e9c6b09f57954d.yaml new file mode 100644 index 000000000..e7b7b2f9a --- /dev/null +++ b/releasenotes/notes/use-length-for-rabbitmq-node-down-rule-c9e9c6b09f57954d.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Allow for easy customisation of the number of expected `RabbitMQ` + nodes when evaluating the alert `RabbitMQNodeDown`. It is set by + the `alertmanager_number_of_rabbitmq_nodes` which defaults to the + number of `controllers`. This is benefical for deployments that + do not use a standard three node setup. diff --git a/releasenotes/notes/vault-without-haproxy-db55707e1cc53896.yaml b/releasenotes/notes/vault-without-haproxy-db55707e1cc53896.yaml new file mode 100644 index 000000000..d6d6dd607 --- /dev/null +++ b/releasenotes/notes/vault-without-haproxy-db55707e1cc53896.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + The overcloud HashiCorp Vault playbooks have been modified to use the + local Vault service rather than via HAProxy. This makes it possible to + deploy and use Vault without HAProxy. This eliminates the previous + bootstrapping issue where HAProxy needed to be deployed without TLS + enabled while generating initial certificates. diff --git a/releasenotes/notes/wallaby-c3c55c39c7b118f2.yaml b/releasenotes/notes/wallaby-c3c55c39c7b118f2.yaml new file mode 100644 index 000000000..37f514563 --- /dev/null +++ b/releasenotes/notes/wallaby-c3c55c39c7b118f2.yaml @@ -0,0 +1,25 @@ +--- +features: + - | + Adds support for package repository snapshots via `Pulp + `__. A local Pulp server is deployed on the seed, + which syncs package repositories and container images from the StackHPC + `Ark `__ Pulp server. Control plane servers pull + packages and container images from the local Pulp server. + - | + The EPEL package repository is disabled by default. It may be enabled by + setting ``dnf_enable_epel`` to ``true``. + - | + Uses StackHPC source code repositories for kolla, kolla-ansible, and bifrost. + - | + Supports Kolla CentOS Stream 8 source container images. + - | + Adds custom playbooks for compute host maintenance: + + * ``nova-compute-drain.yml`` + * ``nova-compute-disable.yml`` + * ``nova-compute-enable.yml`` + * ``reboot.yml`` +other: + - Adds a ``ci-aio`` environment for CI testing. + - Adds a ``ci-builder`` environment for building Kolla container images in CI. diff --git a/releasenotes/notes/wazuh-ansible-v4.10.0-ed5209199194cddf.yaml b/releasenotes/notes/wazuh-ansible-v4.10.0-ed5209199194cddf.yaml new file mode 100644 index 000000000..ef0dc0387 --- /dev/null +++ b/releasenotes/notes/wazuh-ansible-v4.10.0-ed5209199194cddf.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Upgrades the version of wazuh-ansible to v4.10.0. This brings in the SCA CIS + checks for Rocky Linux 9 by default. diff --git a/releasenotes/notes/wazuh-custom-sca-policies-a5f15818948928b0.yaml b/releasenotes/notes/wazuh-custom-sca-policies-a5f15818948928b0.yaml new file mode 100644 index 000000000..902b5b1c5 --- /dev/null +++ b/releasenotes/notes/wazuh-custom-sca-policies-a5f15818948928b0.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Wazuh can now de deployed with additional custom SCA policies. Just add the + policy file(s) to the directory + ``{{ kayobe_env_config_path }}/wazuh/custom_sca_policies``. diff --git a/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml b/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml new file mode 100644 index 000000000..62fd19380 --- /dev/null +++ b/releasenotes/notes/xena-batch-bc1da4e4d0f6257e.yaml @@ -0,0 +1,21 @@ +--- +features: + - | + Adds a custom playbook to run the `Anomaly Detection Visualiser (ADVise) + `__, ``advise-run.yml``. + - | + Adds a custom playbook to reset the RabbitMQ cluster and restart OpenStack + services that use it, ``rabbitmq-reset.yml``. + - | + Adds a custom playbook to configure swap, ``swap.yml``. + - | + Adds the `Kayobe Automation + `__ Git repository as a + submodule, and provides some basic configuration for it in an + ``.automation.conf`` directory. + - | + Adds support for deploying a Squid caching proxy as a custom container on + the seed. + - | + Enables Elasticsearch, Grafana, Kibana, Prometheus by default. Provides + standard dashboards for Grafana and alerting rules for Prometheus. diff --git a/releasenotes/requirements.txt b/releasenotes/requirements.txt new file mode 100644 index 000000000..22e2f6c3a --- /dev/null +++ b/releasenotes/requirements.txt @@ -0,0 +1,4 @@ +reno>=4.1.0 # Apache-2.0 +sphinx>=2.0.0,!=2.1.0 # BSD +sphinxcontrib-svg2pdfconverter>=0.1.0 # BSD +sphinx-material diff --git a/releasenotes/source/2023.1.rst b/releasenotes/source/2023.1.rst new file mode 100644 index 000000000..f1ace6626 --- /dev/null +++ b/releasenotes/source/2023.1.rst @@ -0,0 +1,6 @@ +==================================== +2023.1 Antelope Series Release Notes +==================================== + +.. release-notes:: + :branch: stackhpc/2023.1 diff --git a/releasenotes/source/2024.1.rst b/releasenotes/source/2024.1.rst new file mode 100644 index 000000000..7f16f1b53 --- /dev/null +++ b/releasenotes/source/2024.1.rst @@ -0,0 +1,6 @@ +=================================== +2024.1 Caracal Series Release Notes +=================================== + +.. release-notes:: + :branch: stackhpc/2024.1 diff --git a/releasenotes/source/_static/.placeholder b/releasenotes/source/_static/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/releasenotes/source/_templates/.placeholder b/releasenotes/source/_templates/.placeholder new file mode 100644 index 000000000..e69de29bb diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py new file mode 100644 index 000000000..3db9e147d --- /dev/null +++ b/releasenotes/source/conf.py @@ -0,0 +1,225 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'reno.sphinxext', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'StackHPC Kayobe Configuration Release Notes' +copyright = '2024 StackHPC Ltd' + +# Release notes do not need a version number in the title, they +# cover multiple releases. +# The short X.Y version. +version = '' +# The full version, including alpha/beta/rc tags. +release = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'native' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_material' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'StackHPCKayobeConfigReleaseNotesdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +# latex_documents = [] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +# texinfo_documents = [] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst new file mode 100644 index 000000000..295e4b525 --- /dev/null +++ b/releasenotes/source/index.rst @@ -0,0 +1,22 @@ +Welcome to StackHPC Kayobe Configuration Release Notes +====================================================== + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + 2024.1 + 2023.1 + zed + yoga + xena + wallaby + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/releasenotes/source/wallaby.rst b/releasenotes/source/wallaby.rst new file mode 100644 index 000000000..c6f9153f0 --- /dev/null +++ b/releasenotes/source/wallaby.rst @@ -0,0 +1,6 @@ +============================ +Wallaby Series Release Notes +============================ + +.. release-notes:: + :branch: stackhpc/wallaby diff --git a/releasenotes/source/xena.rst b/releasenotes/source/xena.rst new file mode 100644 index 000000000..e5cdefbdb --- /dev/null +++ b/releasenotes/source/xena.rst @@ -0,0 +1,6 @@ +========================= +Xena Series Release Notes +========================= + +.. release-notes:: + :branch: stackhpc/xena diff --git a/releasenotes/source/yoga.rst b/releasenotes/source/yoga.rst new file mode 100644 index 000000000..c2181c53a --- /dev/null +++ b/releasenotes/source/yoga.rst @@ -0,0 +1,6 @@ +========================= +Yoga Series Release Notes +========================= + +.. release-notes:: + :branch: stackhpc/yoga diff --git a/releasenotes/source/zed.rst b/releasenotes/source/zed.rst new file mode 100644 index 000000000..f370b7312 --- /dev/null +++ b/releasenotes/source/zed.rst @@ -0,0 +1,6 @@ +========================= +Zed Series Release Notes +========================= + +.. release-notes:: + :branch: stackhpc/zed diff --git a/requirements.txt b/requirements.txt index 791e87348..d87bad2f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ +kayobe@git+https://github.com/stackhpc/kayobe@stackhpc/16.8.0.18 ansible-modules-hashivault>=5.2.1 +jmespath diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 000000000..43dec19d5 --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,2 @@ +**/.terraform* +**/terraform.tfstate* diff --git a/terraform/aio/.terraform.lock.hcl b/terraform/aio/.terraform.lock.hcl new file mode 100644 index 000000000..73173a31a --- /dev/null +++ b/terraform/aio/.terraform.lock.hcl @@ -0,0 +1,23 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/terraform-provider-openstack/openstack" { + version = "1.47.0" + hashes = [ + "h1:6GKN5WfUZVVJ9cBHwN74RVOzMK2xlZx1qwGo8CUtPPk=", + "zh:110dfeb02d47af0d1c71d41deac7bb0b23d0d5f9f2ac95f81fee7be1b7131852", + "zh:164c141ca8d1d1b43b866150797f5e15855d48aaddd50a80ca320e638cdbbd3a", + "zh:1be4fab5de93f2947c35df6676c67bf2ea410ec71f29e4a57661119c2f262d6a", + "zh:40dc8f1ffc3521786b38427fe2c0f2ec0b102fed4284c08b2d092ebd1617a603", + "zh:6335190faaeba5bd5a859df7075c6a820a1f492d1f2258296a4fb3170b4f0643", + "zh:6460c8c651ba96d434e0db0e09cb02b6188861a8a683b1f9a488d0e119deeb71", + "zh:6ba87e36384c8c165e0853366252451a20a2c1aeec1a2280eb208db200b5ce33", + "zh:6ebbb699c8673ebbaa5fc880c7b3a8dcf69d51bf242c72b986cf6b919bac4969", + "zh:719dc962699d17d03f017d27809c68162a2a849e478e04b858cd798be737a26a", + "zh:7d865928a695dc3558f671da12dbb134d822968d2f0cb4b600ce79c41a0d1d4b", + "zh:b5c50a6ab05cb18780f4d4b4d48f502d60f4121810127105f5f7d34d2dda9569", + "zh:df3adcaacfdd3696a9df768beb3ec997410724c5dbd1cc5fd023e6644686bd74", + "zh:f836c285e5e36ddaa3d0d13f4e89fcb9047356cfb09c77651cdf97b6394af7fa", + "zh:f91081a4f26b69b8325149353d3199838466e971853a2d1e9c2917051a6fdd6d", + ] +} diff --git a/terraform/aio/README.rst b/terraform/aio/README.rst new file mode 100644 index 000000000..0e173ad24 --- /dev/null +++ b/terraform/aio/README.rst @@ -0,0 +1,191 @@ +========================== +Terraform All in one (aio) +========================== + +This Terraform configuration deploys a single VM on an OpenStack cloud, to be +used as an all-in-one Kayobe test environment. + +This configuration is used in the GitHub Actions all-in-one.yml workflow for CI +testing. + +Usage +===== + +These instructions show how to use this Terraform configuration manually. They +assume you are running an Ubuntu host that will be used to run Terraform. The +machine should have network access to the VM that will be created by this +configuration. + +Install Terraform: + +.. code-block:: console + + wget -qO - terraform.gpg https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/terraform-archive-keyring.gpg + sudo echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/terraform-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/terraform.list + sudo apt update + sudo apt install docker.io terraform + +Clone and initialise the Kayobe config: + +.. code-block:: console + + git clone https://github.com/stackhpc/stackhpc-kayobe-config + cd stackhpc-kayobe-config + git submodule init + git submodule update + +Change to the terraform/aio directory: + +.. code-block:: console + + cd terraform/aio + +Initialise Terraform: + +.. code-block:: console + + terraform init + +Generate an SSH keypair: + +.. code-block:: console + + ssh-keygen -f id_rsa -N '' + +Create an OpenStack clouds.yaml file with your credentials to access an +OpenStack cloud. Alternatively, download one from Horizon. + +.. code-block:: console + + cat << EOF > clouds.yaml + --- + clouds: + sms-lab: + auth: + auth_url: https://api.sms-lab.cloud:5000 + username: + project_name: + domain_name: default + interface: public + EOF + +Export environment variables to use the correct cloud and provide a password: + +.. code-block:: console + + export OS_CLOUD=sms-lab + read -p OS_PASSWORD -s OS_PASSWORD + export OS_PASSWORD + +Generate Terraform variables: + +.. code-block:: console + + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + aio_vm_name = "kayobe-aio" + aio_vm_image = "overcloud-rocky-9-zed-20231013T123933" + aio_vm_flavor = "general.v1.medium" + aio_vm_network = "stackhpc-ipv4-geneve" + aio_vm_subnet = "stackhpc-ipv4-geneve-subnet" + EOF + +Generate a plan: + +.. code-block:: console + + terraform plan + +Apply the changes: + +.. code-block:: console + + terraform apply -auto-approve + +Write Terraform outputs to a Kayobe config file: + +.. code-block:: console + + terraform output -json > ../../etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml + +Change to the repository root: + +.. code-block:: console + + cd ../../ + +Write Terraform network config: + +.. code-block:: console + + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml + + admin_oc_net_name: admin + admin_cidr: "{{ access_cidr.value }}" + admin_allocation_pool_start: 0.0.0.0 + admin_allocation_pool_end: 0.0.0.0 + admin_gateway: "{{ access_gw.value }}" + admin_bootproto: dhcp + admin_ips: + controller0: "{{ access_ip_v4.value }}" + EOF + +Write Terraform network interface config: + +.. code-block:: console + + cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces + admin_interface: "{{ access_interface.value }}" + EOF + +Build a Kayobe image: + +.. code-block:: console + + sudo DOCKER_BUILDKIT=1 docker build --file .automation/docker/kayobe/Dockerfile --tag kayobe:latest . + +Use the ci-aio environment: + +.. code-block:: console + + export KAYOBE_ENVIRONMENT=ci-aio + +Set the Kayobe Vault password env var: + +.. code-block:: console + + read -p KAYOBE_VAULT_PASSWORD -s KAYOBE_VAULT_PASSWORD + export KAYOBE_VAULT_PASSWORD + +Set the Kayobe SSH private key env var: + +.. code-block:: console + + export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=$(cat terraform/aio/id_rsa) + +Host configure: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh + +Service deploy: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh + +Configure aio resources: + +.. code-block:: console + + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml + +Run Tempest: + +.. code-block:: console + + mkdir -p tempest-artifacts + sudo -E docker run -it --rm -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY kayobe:latest /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack + +Tempest results are in tempest-artifacts. diff --git a/terraform/aio/outputs.tf b/terraform/aio/outputs.tf new file mode 100644 index 000000000..8d184e3a9 --- /dev/null +++ b/terraform/aio/outputs.tf @@ -0,0 +1,15 @@ +output "access_ip_v4" { + value = openstack_compute_instance_v2.kayobe-aio.access_ip_v4 +} + +output "access_cidr" { + value = data.openstack_networking_subnet_v2.network.cidr +} + +output "access_gw" { + value = data.openstack_networking_subnet_v2.network.gateway_ip +} + +output "access_interface" { + value = var.aio_vm_interface +} diff --git a/terraform/aio/provider.tf b/terraform/aio/provider.tf new file mode 100644 index 000000000..0e7cbdb05 --- /dev/null +++ b/terraform/aio/provider.tf @@ -0,0 +1,14 @@ +#provider "openstack" { +# use environment variables +#} + +terraform { + required_version = ">= 0.14" + backend "local" { + } + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + } + } +} diff --git a/terraform/aio/templates/userdata.cfg.tpl b/terraform/aio/templates/userdata.cfg.tpl new file mode 100644 index 000000000..b91b0e741 --- /dev/null +++ b/terraform/aio/templates/userdata.cfg.tpl @@ -0,0 +1,17 @@ +#cloud-config +# Don't automatically mount ephemeral disk +mounts: + - [/dev/vdb, null] +# Configure SSH keys here, to avoid creating an ephemeral keypair. +# This means only the instance needs to be cleaned up if the destroy fails. +ssh_authorized_keys: + - ${ssh_public_key} + +write_files: + # WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409 + - content: | + #!/bin/bash + docker exec openvswitch_vswitchd ovs-vsctl "$@" + owner: root:root + path: /usr/bin/ovs-vsctl + permissions: '0755' diff --git a/terraform/aio/vm.tf b/terraform/aio/vm.tf new file mode 100644 index 000000000..8f3bbe31d --- /dev/null +++ b/terraform/aio/vm.tf @@ -0,0 +1,99 @@ +variable "ssh_public_key" { + type = string +} + +variable "ssh_username" { + type = string +} + +variable "aio_vm_name" { + type = string + default = "kayobe-aio" +} + +variable "aio_vm_image" { + type = string + default = "Rocky9" +} + +variable "aio_vm_interface" { + type = string + default = "eth0" +} + +variable "aio_vm_flavor" { + type = string +} + +variable "aio_vm_network" { + type = string +} + +variable "aio_vm_subnet" { + type = string +} + +variable "aio_vm_volume_size" { + type = number + default = 50 +} + +variable "aio_vm_tags" { + type = list(string) + default = [] +} + +locals { + image_is_uuid = length(regexall("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.aio_vm_image)) > 0 +} + +data "openstack_images_image_v2" "image" { + name = var.aio_vm_image + most_recent = true + count = local.image_is_uuid ? 0 : 1 +} + +data "openstack_networking_subnet_v2" "network" { + name = var.aio_vm_subnet +} + +resource "openstack_compute_instance_v2" "kayobe-aio" { + name = var.aio_vm_name + flavor_name = var.aio_vm_flavor + config_drive = true + user_data = templatefile("templates/userdata.cfg.tpl", {ssh_public_key = file(var.ssh_public_key)}) + security_groups = ["kayobe-runner"] + network { + name = var.aio_vm_network + } + + block_device { + uuid = local.image_is_uuid ? var.aio_vm_image: data.openstack_images_image_v2.image[0].id + source_type = "image" + volume_size = var.aio_vm_volume_size + boot_index = 0 + destination_type = "volume" + delete_on_termination = true + } + + tags = var.aio_vm_tags +} + +# Wait for the instance to be accessible via SSH before progressing. +resource "null_resource" "kayobe-aio" { + provisioner "remote-exec" { + connection { + host = openstack_compute_instance_v2.kayobe-aio.access_ip_v4 + user = var.ssh_username + private_key = file("id_rsa") + # Terraform will run the start script from /tmp by default. For the + # current images, /tmp is noexec, so the path must be changed + script_path = "/home/${var.ssh_username}/start.sh" + } + + inline = [ + "#!/bin/sh", + "echo 'connected!'" + ] + } +} diff --git a/test-requirements.txt b/test-requirements.txt index 333d31836..528bb2001 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,5 @@ # process, which may cause wedges in the gate later. yamllint # GPLv3 +doc8 # Apache-2.0 +reno>=4.1.0 # Apache-2.0 diff --git a/tools/diagnostics.sh b/tools/diagnostics.sh new file mode 100644 index 000000000..73d61775a --- /dev/null +++ b/tools/diagnostics.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# NOTE(mgoddard): This has been adapted from +# roles/kayobe-diagnostics/files/get_logs.sh in Kayobe. + +# Environment variables: +# $LOG_DIR is the directory to copy logs to. + +# TODO: Make this script more robust and use set -e. +set +o errexit +set -u + +copy_logs() { + mkdir -p ${LOG_DIR}/{docker_logs,kolla_node_configs,system_logs} + + cp -rnL /etc/kolla/* ${LOG_DIR}/kolla_node_configs + # Don't save the IPA images. + rm ${LOG_DIR}/kolla_node_configs/ironic-http/ironic-agent.{kernel,initramfs} + rm ${LOG_DIR}/kolla_node_configs/ironic-tftp/ironic-agent.{kernel,initramfs} + + if [[ -d /opt/kayobe/etc/kolla ]]; then + mkdir -p ${LOG_DIR}/kolla_build_configs + cp -rnL /opt/kayobe/etc/kolla/* ${LOG_DIR}/kolla_build_configs/ + fi + + cp -rvnL /var/log/* ${LOG_DIR}/system_logs/ + + journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.log + journalctl --no-pager -u docker.service > ${LOG_DIR}/system_logs/docker.log + journalctl --no-pager -u vbmcd.service > ${LOG_DIR}/system_logs/vbmcd.log + journalctl --no-pager -u NetworkManager.service > ${LOG_DIR}/system_logs/NetworkManager.log + + if [[ -d /etc/sysconfig/network-scripts/ ]]; then + cp -r /etc/sysconfig/network-scripts/ ${LOG_DIR}/system_logs/ + fi + + if [[ -d /etc/NetworkManager/system-connections/ ]]; then + cp -r /etc/NetworkManager/system-connections/ ${LOG_DIR}/system_logs/ + fi + + if [[ -d /etc/yum.repos.d/ ]]; then + cp -r /etc/yum.repos.d/ ${LOG_DIR}/system_logs/ + fi + + if [[ -d /etc/apt/sources.list.d/ ]]; then + cp -r /etc/apt/sources.list.d/ ${LOG_DIR}/system_logs/ + fi + + if [[ -d /etc/systemd/ ]]; then + cp -rL /etc/systemd/ ${LOG_DIR}/system_logs/ + fi + + df -h > ${LOG_DIR}/system_logs/df.txt + # Gather disk usage statistics for files and directories larger than 1MB + du -d 5 -hx / | sort -hr | grep '^[0-9\.]*[MGT]' > ${LOG_DIR}/system_logs/du.txt + free > ${LOG_DIR}/system_logs/free.txt + cat /etc/hosts > ${LOG_DIR}/system_logs/hosts.txt + parted -l > ${LOG_DIR}/system_logs/parted-l.txt + mount > ${LOG_DIR}/system_logs/mount.txt + env > ${LOG_DIR}/system_logs/env.txt + ip address > ${LOG_DIR}/system_logs/ip-address.txt + ip route > ${LOG_DIR}/system_logs/ip-route.txt + ip route show table all > ${LOG_DIR}/system_logs/ip-route-all-tables.txt + ip rule list > ${LOG_DIR}/system_logs/ip-rule-list.txt + pvs > ${LOG_DIR}/system_logs/pvs.txt + vgs > ${LOG_DIR}/system_logs/vgs.txt + lvs > ${LOG_DIR}/system_logs/lvs.txt + + iptables-save > ${LOG_DIR}/system_logs/iptables.txt + + if [ `command -v dpkg` ]; then + dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt + fi + if [ `command -v rpm` ]; then + rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt + fi + + # final memory usage and process list + ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt + + # available entropy + cat /proc/sys/kernel/random/entropy_avail > ${LOG_DIR}/system_logs/entropy_avail.txt + + # docker related information + (docker info && docker images && docker ps -a) > ${LOG_DIR}/system_logs/docker-info.txt + + for container in $(docker ps -a --format "{{.Names}}"); do + docker logs --tail all ${container} &> ${LOG_DIR}/docker_logs/${container}.txt + done + + # Bifrost: grab config files and logs from the container. + if [[ $(docker ps -q -f name=bifrost_deploy) ]]; then + mkdir -p ${LOG_DIR}/bifrost + for service in dnsmasq ironic-api ironic-conductor ironic-inspector mariadb nginx rabbitmq-server; do + mkdir -p ${LOG_DIR}/bifrost/$service + docker exec bifrost_deploy \ + systemctl status $service -l -n 10000 > ${LOG_DIR}/bifrost/$service/${service}-systemd-status.txt + docker exec bifrost_deploy \ + journalctl -u $service --no-pager > ${LOG_DIR}/bifrost/$service/${service}-journal.txt + done + docker exec -it bifrost_deploy \ + journalctl --no-pager > ${LOG_DIR}/bifrost/bifrost-journal.log + for d in dnsmasq.conf ironic ironic-inspector nginx/nginx.conf; do + docker cp bifrost_deploy:/etc/$d ${LOG_DIR}/kolla_node_configs/bifrost/ + done + docker cp bifrost_deploy:/var/log/mariadb/mariadb.log ${LOG_DIR}/bifrost/mariadb/ + fi + + # IPA build logs + if [[ -f /opt/kayobe/images/ipa/ipa.stderr ]] || [[ -f /opt/kayobe/images/ipa/ipa.stdout ]]; then + mkdir -p ${LOG_DIR}/ipa + cp /opt/kayobe/images/ipa/ipa.stderr /opt/kayobe/images/ipa/ipa.stdout ${LOG_DIR}/ipa/ + fi + + # Overcloud host image build logs + if [[ -f /opt/kayobe/images/deployment_image/deployment_image.stderr ]] || [[ -f /opt/kayobe/images/deployment_image/deployment_image.stdout ]]; then + mkdir -p ${LOG_DIR}/deployment_image + cp /opt/kayobe/images/deployment_image/deployment_image.stderr /opt/kayobe/images/deployment_image/deployment_image.stdout ${LOG_DIR}/deployment_image/ + fi + + chown -R stack: ${LOG_DIR} +} + +copy_logs diff --git a/tools/kolla-images.py b/tools/kolla-images.py new file mode 100755 index 000000000..6ca651a67 --- /dev/null +++ b/tools/kolla-images.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 + +""" +Script to manage Kolla container image tags. + +Background: +In Kolla Ansible each container is deployed using a specific image. +Typically the image is named the same as the container, with underscores +replaced by dashes, however there are some exceptions. Sometimes multiple +containers use the same image. + +The image tag deployed by each container is defined by a Kolla Ansible variable +named _tag. There are also intermediate tag variables to make it +easier to set the tag for all containers in a service, e.g. nova_tag is the +default for nova_api_tag, nova_compute, etc. There is a global default tag +defined by openstack_tag. This setup forms a hierarchy of tag variables. + +This script captures this logic, as well as exceptions to these rules. +""" + +import argparse +import inspect +import json +import os +import pathlib +import re +import subprocess +import sys +from typing import Dict, List, Optional + +import yaml + + +# Dict of Kolla image tags to deploy for each service. +# Each key is the tag variable prefix name, and the value is another dict, +# where the key is the OS distro and the value is the tag to deploy. +# This is the content of etc/kayobe/kolla-image-tags.yml. +KollaImageTags = Dict[str, Dict[str, str]] + +# Maps a Kolla image to a list of containers that use the image. +IMAGE_TO_CONTAINERS_EXCEPTIONS: Dict[str, List[str]] = { + "dnsmasq": [ + "ironic_dnsmasq", + ], + "haproxy": [ + "glance_tls_proxy", + "haproxy", + "neutron_tls_proxy", + ], + "mariadb-server": [ + "mariadb", + "mariabackup", + ], + "neutron-mlnx-agent": [ + "neutron_eswitchd", + "neutron_mlnx_agent", + ], + "neutron-metadata-agent": [ + "neutron_metadata_agent", + "neutron_ovn_metadata_agent", + ], + "nova-conductor": [ + "nova_super_conductor", + "nova_conductor", + ], + "openvswitch-db-server": [ + "openvswitch_db", + ], + "ovn-nb-db-server": [ + "ovn_nb_db", + ], + "ovn-sb-db-server": [ + "ovn_sb_db", + ], + "prometheus-v2-server": [ + "prometheus_server", + ], +} + +# Maps a container to the parent tag variable in the hierarchy. +CONTAINER_TO_PREFIX_VAR_EXCEPTIONS: Dict[str, str] = { + "cron": "common", + "fluentd": "common", + "glance_tls_proxy": "haproxy", + "hacluster_corosync": "openstack", + "hacluster_pacemaker": "openstack", + "hacluster_pacemaker_remote": "openstack", + "heat_api_cfn": "heat", + "ironic_neutron_agent": "neutron", + "kolla_toolbox": "common", + "neutron_eswitchd": "neutron_mlnx_agent", + "neutron_tls_proxy": "haproxy", + "nova_compute_ironic": "nova", + "ovn_sb_db_relay": "ovn", + "redis_sentinel": "openstack", + "swift_object_expirer": "swift", + "tgtd": "iscsi", +} + +# List of supported base distributions and versions. +SUPPORTED_BASE_DISTROS = [ + "rocky-9", + "ubuntu-jammy", + "ubuntu-noble", +] + + +def parse_args() -> argparse.Namespace: + """Parse command line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument("--base-distros", default=",".join(SUPPORTED_BASE_DISTROS), choices=SUPPORTED_BASE_DISTROS) + subparsers = parser.add_subparsers(dest="command", required=True) + + subparser = subparsers.add_parser("check-image-map", help="Check image mapping against kolla-ansible") + subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch") + + subparser = subparsers.add_parser("check-hierarchy", help="Check tag variable hierarchy against kolla-ansible") + subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch") + + subparser = subparsers.add_parser("check-tags", help="Check specified tags for each image exist in the Ark registry") + subparser.add_argument("--registry", required=True, help="Hostname of container image registry") + subparser.add_argument("--namespace", required=True, help="Namespace in container image registry") + + subparsers.add_parser("list-containers", help="List supported containers based on pulp.yml") + + subparsers.add_parser("list-images", help="List supported images based on pulp.yml") + + subparsers.add_parser("list-tags", help="List tags for each image based on kolla-image-tags.yml") + + subparsers.add_parser("list-tag-vars", help="List Kolla Ansible tag variables") + + return parser.parse_args() + + +def get_abs_path(relative_path: str) -> pathlib.Path: + """Return the absolute path of a file in SKC.""" + script_path = pathlib.Path(inspect.getfile(inspect.currentframe())) + return script_path.parent.parent / relative_path + + +def read_images(images_file: str) -> List[str]: + """Read image list from pulp.yml config file.""" + with open(get_abs_path(images_file), "r") as f: + variables = yaml.safe_load(f) + return variables["stackhpc_pulp_images_kolla"] + + +def read_unbuildable_images(images_file: str) -> Dict[str, List[str]]: + """Read unbuildable image list from pulp.yml config file.""" + with open(get_abs_path(images_file), "r") as f: + variables = yaml.safe_load(f) + return variables["stackhpc_kolla_unbuildable_images"] + + +def read_kolla_image_tags(tags_file: str) -> KollaImageTags: + """Read kolla image tags kolla-image-tags.yml config file.""" + with open(get_abs_path(tags_file), "r") as f: + variables = yaml.safe_load(f) + return variables["kolla_image_tags"] + + +def get_containers(image): + """Return a list of containers that use the specified image.""" + default_container = image.replace('-', '_') + return IMAGE_TO_CONTAINERS_EXCEPTIONS.get(image, [default_container]) + + +def get_parent_tag_name(kolla_image_tags: KollaImageTags, base_distro: Optional[str], container: str) -> str: + """Return the parent tag variable for a container in the tag variable hierarchy.""" + + if container in CONTAINER_TO_PREFIX_VAR_EXCEPTIONS: + prefix_var = CONTAINER_TO_PREFIX_VAR_EXCEPTIONS[container] + if prefix_var in kolla_image_tags and (base_distro is None or base_distro in kolla_image_tags[prefix_var]): + return prefix_var + else: + prefix_var = container + + def tag_key(tag): + """Return a sort key to order the tags.""" + if tag == "openstack": + # This is the default tag. + return 0 + elif tag != prefix_var and prefix_var.startswith(tag) and (base_distro is None or base_distro in kolla_image_tags[tag]): + # Prefix match - sort by the longest match. + return -len(tag) + else: + # No match. + return 1 + + return sorted(kolla_image_tags.keys(), key=tag_key)[0] + + +def get_parent_tag(kolla_image_tags: KollaImageTags, base_distro: str, container: str) -> str: + """Return the tag used by the parent in the hierarchy.""" + parent_tag_name = get_parent_tag_name(kolla_image_tags, base_distro, container) + return kolla_image_tags[parent_tag_name][base_distro] + + +def get_tag(kolla_image_tags: KollaImageTags, base_distro: str, container: str) -> str: + """Return the tag for a container.""" + container_tag = kolla_image_tags.get(container, {}).get(base_distro) + if container_tag: + return container_tag + + return get_parent_tag(kolla_image_tags, base_distro, container) + + +def get_tags(base_distros: List[str], kolla_image_tags: KollaImageTags) -> Dict[str, List[str]]: + """Return a list of tags used for each image.""" + images = read_images("etc/kayobe/pulp.yml") + unbuildable_images = read_unbuildable_images("etc/kayobe/pulp.yml") + image_tags: Dict[str, List[str]] = {} + for base_distro in base_distros: + for image in images: + if image not in unbuildable_images[base_distro]: + for container in get_containers(image): + tag = get_tag(kolla_image_tags, base_distro, container) + tags = image_tags.setdefault(image, []) + if tag not in tags: + tags.append(tag) + return image_tags + + +def get_openstack_release() -> str: + """Return the OpenStack release.""" + with open(get_abs_path(".gitreview"), "r") as f: + gitreview = f.readlines() + for line in gitreview: + if "=" not in line: + continue + key, value = line.split("=") + if key.strip() == "defaultbranch": + value = value.strip() + for prefix in ("stable/", "unmaintained/"): + if value.startswith(prefix): + return value[len(prefix):] + raise Exception("Failed to determine OpenStack release") + + +def validate(kolla_image_tags: KollaImageTags): + """Validate the kolla_image_tags variable.""" + tag_var_re = re.compile(r"^[a-z0-9_]+$") + openstack_release = get_openstack_release() + tag_res = { + base_distro: re.compile(f"^{openstack_release}-{base_distro}-[\d]{{8}}T[\d]{{6}}$") + for base_distro in SUPPORTED_BASE_DISTROS + } + errors = [] + if "openstack" not in kolla_image_tags: + errors.append("Missing default openstack tag") + for tag_var, base_distros in kolla_image_tags.items(): + if not tag_var_re.match(tag_var): + errors.append(f"Key {tag_var} does not match expected pattern. It should match {tag_var_re.pattern}") + for base_distro, tag in base_distros.items(): + if base_distro not in SUPPORTED_BASE_DISTROS: + errors.append(f"{tag_var}: base distro {base_distro} not supported. Options: {SUPPORTED_BASE_DISTROS}") + continue + if not tag_res[base_distro].match(tag): + errors.append(f"{tag_var}: {base_distro}: tag {tag} does not match expected pattern. It should match {tag_res[base_distro].pattern}") + if errors: + print("Errors in kolla_image_tags variable:") + for error in errors: + print(error) + sys.exit(1) + + +def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, registry: str, namespace: str): + """Check whether expected tags are present in container image registry.""" + try: + subprocess.check_output("type skopeo", shell=True) + except subprocess.CalledProcessError: + print("Failed to find skopeo. Please install it.") + sys.exit(1) + image_tags = get_tags(base_distros, kolla_image_tags) + + missing = {} + for image, tags in image_tags.items(): + for _ in range(3): + try: + output = subprocess.check_output(f"skopeo list-tags docker://{registry}/{namespace}/{image}", shell=True) + except Exception as e: + exc = e + else: + break + else: + raise exc + ark_tags = json.loads(output)["Tags"] + missing_tags = set(tags) - set(ark_tags) + if missing_tags: + missing[image] = list(missing_tags) + + if missing: + print(f"ERROR: Some expected tags not found in {namespace} namespace") + print(yaml.dump(missing, indent=2)) + sys.exit(1) + + +def check_image_map(kolla_ansible_path: str): + """Check the image mapping against Kolla Ansible variables. + + The *_image variables in Kolla Ansible define the mapping between + containers and images. Ensure that the mapping defined in this script + matches the one in Kolla Ansible. + """ + supported_images = read_images("etc/kayobe/pulp.yml") + assert supported_images + # Build a map from container to image name. + cmd = """git grep -h '^[a-z0-9_]*_image:' ansible/roles/*/defaults/main.yml""" + image_map_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path)) + image_map = yaml.safe_load(image_map_str) + image_var_re = re.compile(r"^([a-z0-9_]+)_image$") + image_map = { + image_var_re.match(image_var).group(1): image.split("/")[-1] + for image_var, image in image_map.items() + } + # Filter out unsupported images. + image_map = { + container: image + for container, image in image_map.items() + if image in supported_images + } + assert image_map + errors = [] + # Check that our mapping is correct. + for container, image in image_map.items(): + containers = get_containers(image) + if container not in containers: + errors.append((container, image)) + if errors: + print("Errors:") + for tag_var, image in errors: + print(f"Expected {tag_var} container to use {image} image") + if errors: + sys.exit(1) + + +def check_hierarchy(kolla_ansible_path: str): + """Check the tag variable hierarchy against Kolla Ansible variables.""" + cmd = """git grep -h '^[a-z0-9_]*_tag:' ansible/roles/*/defaults/main.yml""" + hierarchy_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path)) + hierarchy = yaml.safe_load(hierarchy_str) + # This one is not a container: + hierarchy.pop("octavia_amp_image_tag") + tag_var_re = re.compile(r"^([a-z0-9_]+)_tag$") + parent_re = re.compile(r"{{[\s]*([a-z0-9_]+)_tag[\s]*}}") + hierarchy = { + tag_var_re.match(tag_var).group(1): parent_re.match(parent).group(1) + for tag_var, parent in hierarchy.items() + } + kolla_image_tags: KollaImageTags = {image: {} for image in hierarchy} + kolla_image_tags["openstack"] = {} + errors = [] + for tag_var, expected in hierarchy.items(): + parent = get_parent_tag_name(kolla_image_tags, None, tag_var) + if parent != expected: + errors.append((tag_var, parent, expected)) + if errors: + print("Errors:") + for tag_var, parent, expected in errors: + print(f"{tag_var} -> {parent} != {expected}") + if errors: + sys.exit(1) + + +def list_containers(base_distros: List[str]): + """List supported containers.""" + images = read_images("etc/kayobe/pulp.yml") + unbuildable_images = read_unbuildable_images("etc/kayobe/pulp.yml") + containers = set() + for base_distro in base_distros: + for image in images: + if image not in unbuildable_images[base_distro]: + containers |= set(get_containers(image)) + print(yaml.dump(sorted(containers))) + + +def list_images(base_distros: List[str]): + """List supported images.""" + images = read_images("etc/kayobe/pulp.yml") + print(yaml.dump(images)) + + +def list_tags(base_distros: List[str], kolla_image_tags: KollaImageTags): + """List tags used by each image.""" + image_tags = get_tags(base_distros, kolla_image_tags) + + print(yaml.dump(image_tags)) + + +def list_tag_vars(kolla_image_tags: KollaImageTags): + """List tag variables.""" + tag_vars = [] + for tag_var in kolla_image_tags: + if tag_var == "openstack": + default = "" + else: + parent_tag_name = get_parent_tag_name(kolla_image_tags, None, tag_var) + default = f" | default({parent_tag_name}_tag)" + tag_vars.append(f"{tag_var}_tag: \"{{{{ kolla_image_tags['{tag_var}'][kolla_base_distro_and_version]{default} }}}}\"") + + for tag_var in tag_vars: + print(tag_var) + + +def main(): + args = parse_args() + kolla_image_tags = read_kolla_image_tags("etc/kayobe/kolla-image-tags.yml") + base_distros = args.base_distros.split(",") + + validate(kolla_image_tags) + + if args.command == "check-image-map": + check_image_map(args.kolla_ansible_path) + elif args.command == "check-hierarchy": + check_hierarchy(args.kolla_ansible_path) + elif args.command == "check-tags": + check_tags(base_distros, kolla_image_tags, args.registry, args.namespace) + elif args.command == "list-containers": + list_containers(base_distros) + elif args.command == "list-images": + list_images(base_distros) + elif args.command == "list-tags": + list_tags(base_distros, kolla_image_tags) + elif args.command == "list-tag-vars": + list_tag_vars(kolla_image_tags) + else: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/loadbalancer-failover.sh b/tools/loadbalancer-failover.sh new file mode 100755 index 000000000..e8c2bccc7 --- /dev/null +++ b/tools/loadbalancer-failover.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Fail over octavia loadbalancers to the latest amphora image. + +set -ex + +expected_image=$(openstack image show amphora-x64-haproxy.qcow2 -f value -c id) + +openstack loadbalancer amphora list --status ALLOCATED -f value -c id | while read a; do + image=$(openstack loadbalancer amphora show $a -f value -c image_id) + if [[ $image != "None" ]] && [[ $image != $expected_image ]]; then + lb_id=$(openstack loadbalancer amphora show $a -f value -c loadbalancer_id) + echo "Failing over loadbalancer $lb_id (amphora $a)" + if ! openstack loadbalancer failover $lb_id --wait; then + echo "Failed failing over loadbalancer $lb_id" + fi + fi +done diff --git a/tools/merge.py b/tools/merge.py new file mode 100755 index 000000000..216614b55 --- /dev/null +++ b/tools/merge.py @@ -0,0 +1,116 @@ +#!/usr/bin/python3 + +DESCRIPTION = """ +This script merges one release branch of SKC into another. + +Example 1: Merge stackhpc/yoga into stackhpc/zed: + + merge.py yoga zed + +Example 2: Merge the branch created in example 1 into stackhpc/2023.1: + + merge.py zed 2023.1 zed-yoga-merge + +Example 3: Continue after manually resolving merge conflicts seen in example 2: + + merge.py zed 2023.1 zed-yoga-merge --continue + +""" + +import argparse +import os +from subprocess import check_call, check_output +import sys + + +def command(cmd): + print("Running:", cmd) + check_call(cmd) + + +def parse_args(): + parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) + #"Merge one branch of SKC into the next") + parser.add_argument("previous", type=str, help="The previous version") + parser.add_argument("current", type=str, help="The current version") + parser.add_argument("previous_branch", type=str, nargs="?", default=None, help="Optional branch to use as the previous release. Allows merging multiple branches in parallel.") + parser.add_argument("--continue", dest="cont", action="store_true", help="Continue after merge conflicts have been resolved.") + parser.add_argument("--remote", type=str, default="origin", help="Git remote") + return parser.parse_args() + + +def fetch(args): + command(["git", "fetch", args.remote]) + + +def checkout(args): + merge_branch = f"{args.current}-{args.previous}-merge" + current_branch = f"{args.remote}/stackhpc/{args.current}" + command(["git", "checkout", "-B", merge_branch, current_branch]) + + +def update_submodules(): + command(["git", "submodule", "update"]) + + +def merge_in_progress(): + repo_root = check_output(["git", "rev-parse", "--show-toplevel"]) + repo_root = repo_root.decode().strip() + return os.path.isfile(os.path.join(repo_root, ".git", "MERGE_HEAD")) + + +def uncommitted_changes(): + unstaged = check_output(["git", "diff"]) + staged = check_output(["git", "diff", "--cached"]) + return unstaged or staged + + +def continue_merge(): + if merge_in_progress(): + command(["git", "merge", "--continue"]) + else: + print("No merge in progress") + + +def merge(args): + if args.previous_branch: + previous_branch = args.previous_branch + else: + previous_branch = f"{args.remote}/stackhpc/{args.previous}" + commit_message = f"Merge stackhpc/{args.previous} into stackhpc/{args.current}" + command(["git", "merge", previous_branch, "-m", commit_message]) + + +def show_diff(args): + print("Proposed changes:") + current_branch = f"{args.remote}/stackhpc/{args.current}" + command(["git", "diff", current_branch]) + + +def create_pr(args): + current_branch = f"stackhpc/{args.current}" + pr_title = f"{args.current}: {args.previous} merge" + command(["gh", "pr", "create", "-f", "-a", "@me", "-B", current_branch, "-t", pr_title]) + + +def main(): + args = parse_args() + if args.cont: + continue_merge() + else: + if merge_in_progress(): + print("Merge in progress - did you miss the --continue argument?") + sys.exit(1) + if uncommitted_changes(): + print("You have uncommitted changes - aborting") + sys.exit(1) + fetch(args) + checkout(args) + update_submodules() + merge(args) + show_diff(args) + create_pr(args) + + +if __name__ == "__main__": + main() diff --git a/tools/multiarch-manifests.sh b/tools/multiarch-manifests.sh new file mode 100755 index 000000000..ec08c4b39 --- /dev/null +++ b/tools/multiarch-manifests.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Script used by the Image build workflow to build & push multiarch manifests. + +set -o errexit +set -o xtrace +set -o pipefail + +mkdir -p logs +images=$(cat all-pushed-images.txt | sort | uniq) + +# Filter out Ubuntu and Rocky Bifrost images +manifest_images=$(echo "$images" \ + | sed -nE 's/-(amd64|aarch64)$//p' \ + | sort | uniq) + +if [ -z "$manifest_images" ]; then + echo "No Rocky overcloud images found. Skipping manifest creation." | tee -a logs/manifest-creation.log + exit 0 +fi + +for base_image in $manifest_images; do + arch_images="" + for arch in amd64 aarch64; do + arch_image="${base_image}-${arch}" + # Check if the image exists in the registry + if docker manifest inspect "$arch_image" > /dev/null 2>&1; then + arch_images="$arch_images $arch_image" + fi + done + if [ -n "$arch_images" ]; then + echo "Creating manifest for $base_image with images:$arch_images" | tee -a logs/manifest-creation.log + docker manifest create "$base_image" $arch_images | tee -a logs/manifest-creation.log + docker manifest push "$base_image" | tee -a logs/manifest-creation.log + else + echo "No images found for $base_image, skipping." | tee -a logs/manifest-creation.log + fi +done diff --git a/tools/rabbitmq-queue-migration.sh b/tools/rabbitmq-queue-migration.sh new file mode 100755 index 000000000..08b51ae6d --- /dev/null +++ b/tools/rabbitmq-queue-migration.sh @@ -0,0 +1,77 @@ +#! /usr/bin/bash + +set -ex + +RED='\033[0;31m' +GREEN='\033[0;32m' + +RABBITMQ_SERVICES_TO_RESTART=barbican,blazar,cinder,cloudkitty,heat,ironic,keystone,magnum,manila,neutron,nova,octavia # Stop Designate separately +RABBITMQ_CONTAINER_NAME=rabbitmq + +if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "${RED}Environment variable \$KAYOBE_CONFIG_PATH is not defined" + echo "${RED}Ensure your environment is set up to run kayobe commands" + exit 2 +fi + +if [[ ! "$1" = "--skip-checks" ]]; then + # Fail if clocks are not synced + if ! ( kayobe overcloud host command run -l controllers -b --command "timedatectl status | grep 'synchronized: yes'" ); then + echo "${RED}Failed precheck: Time not synced on controllers" + echo "${RED}Use 'timedatectl status' to check sync state" + echo "${RED}Either wait for sync or use 'chronyc makestep'" + exit 1 + fi + kayobe overcloud service configuration generate --node-config-dir /tmp/rabbit-migration --kolla-tags none + # Fail if any new feature flags are not set + if ! ( grep 'om_enable_queue_manager: true' $KOLLA_CONFIG_PATH/globals.yml && \ + grep 'om_enable_rabbitmq_quorum_queues: true' $KOLLA_CONFIG_PATH/globals.yml && \ + grep 'om_enable_rabbitmq_transient_quorum_queue: true' $KOLLA_CONFIG_PATH/globals.yml && \ + grep 'om_enable_rabbitmq_stream_fanout: true' $KOLLA_CONFIG_PATH/globals.yml ); then + echo "${RED}Failed precheck: The following must be enabled: om_enable_queue_manager, om_enable_rabbitmq_quorum_queues, om_enable_rabbitmq_transient_quorum_queue, om_enable_rabbitmq_stream_fanout" + exit 1 + fi +fi + +# Generate new config, stop services using rabbit, and reset rabbit state +kayobe overcloud service configuration generate --node-config-dir /etc/kolla --kolla-skip-tags rabbitmq-ha-precheck +kayobe kolla ansible run "stop --yes-i-really-really-mean-it" -kt $RABBITMQ_SERVICES_TO_RESTART +# Stop Designate services except for ``designate_backend_bind`` containers +kayobe overcloud host command run -b -l controllers --command "systemctl list-units --all --type=service --no-legend --plain | awk '/kolla-designate/ && !/backend_bind9/ {print \$1}' | xargs -r systemctl stop || true" +kayobe kolla ansible run rabbitmq-reset-state + +if [[ ! "$1" = "--skip-checks" ]]; then + # Fail if any queues still exist + sleep 20 + # Note(mattcrees): We turn the text grey here so the failed Ansible calls don't freak anyone out + CURRENTTERM=${TERM} + export TERM=xterm-mono + if ( kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_queues name --silent | grep -v '^$'" ); then + export TERM=${CURRENTTERM} + echo -e "${RED}Failed check: RabbitMQ has not stopped properly, queues still exist" + exit 1 + fi + # Fail if any exchanges still exist (excluding those starting with 'amq.') + if ( kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_exchanges name --silent | grep -v '^$' | grep -v '^amq.'" ); then + export TERM=${CURRENTTERM} + echo -e "${RED}Failed check: RabbitMQ has not stopped properly, exchanges still exist" + exit 1 + fi + export TERM=${CURRENTTERM} +fi + +# Include designate services to kolla deployment +RABBITMQ_SERVICES_TO_RESTART=$RABBITMQ_SERVICES_TO_RESTART,designate + +# Redeploy with all durable-type queues enabled +kayobe kolla ansible run deploy-containers -kt $RABBITMQ_SERVICES_TO_RESTART + +if [[ ! "$1" = "--skip-checks" ]]; then + sleep 60 + # Assert that all queues are durable + if ! ( kayobe overcloud host command run -l controllers -b --command "docker exec $RABBITMQ_CONTAINER_NAME rabbitmqctl list_queues durable --silent | grep false" > /dev/null 2>&1 ); then + echo -e "${GREEN}Queues migrated successfully" + else + echo -e "${RED}Failed post-check: A controller has non-durable queues" + fi +fi diff --git a/tools/scan-images.sh b/tools/scan-images.sh new file mode 100755 index 000000000..9736b3bc7 --- /dev/null +++ b/tools/scan-images.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Disable telemetry and version check: +# https://github.com/aquasecurity/trivy/discussions/8945 +export TRIVY_DISABLE_TELEMETRY=true +export TRIVY_SKIP_VERSION_CHECK=true + +# Global variables +scan_common_args=" \ + --exit-code 1 \ + --scanners vuln \ + --format json \ + --severity HIGH,CRITICAL \ + --ignore-unfixed \ + --db-repository ghcr.io/aquasecurity/trivy-db:2 \ + --db-repository public.ecr.aws/aquasecurity/trivy-db \ + --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \ + --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db " + +# Print usage instructions and error with wrong inputs +usage() { + echo "Usage: scan-images.sh [--sbom]" + exit 2 +} + +# Check dependencies are installed, print installation instructions otherwise +check_deps_installed() { + if ! trivy --version > /dev/null; then + echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.69.2' + exit 1 + fi + if ! yq --version > /dev/null; then + echo 'Please install yq: sudo dnf/apt install yq' + exit 1 + fi +} + +# Prepare output files +file_prep() { + rm -rf image-scan-output + mkdir -p image-scan-output + touch image-scan-output/clean-images.txt image-scan-output/high-images.txt image-scan-output/critical-images.txt +} + +# Gather image lists +get_images() { + local output_file="$1-scanned-container-images.txt" + + docker image ls \ + --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2*" \ + --format "{{.Repository}}:{{.Tag}}" \ + > "$output_file" + + cat "$output_file" +} + +# Generate ignored vulnerabilities file +generate_trivy_ignore() { + local imagename=$1 + local global_vulnerabilities + global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + local image_vulnerabilities + image_vulnerabilities=$(yq ."$imagename"'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null) + + truncate -s 0 .trivyignore # ensure we start from a clean slate + for vulnerability in $global_vulnerabilities; do + echo "$vulnerability" >> .trivyignore + done + for vulnerability in $image_vulnerabilities; do + echo "$vulnerability" >> .trivyignore + done +} + +# Put results into CSV +generate_summary_csv() { + local scan="$1" + local summary="$2" + + echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > "$summary" + + jq -r '.Results[] + | select(.Vulnerabilities) + | .Vulnerabilities + | map(select(.PkgName | test("kernel") | not )) + | group_by(.VulnerabilityID) + | map( + [ + (map(.PkgName) | unique | join(";")), + (map(.PkgPath | select( . != null )) | join(";")), + .[0].PkgID, + .[0].VulnerabilityID, + .[0].FixedVersion, + .[0].PrimaryURL, + .[0].Severity + ] + ) + | .[] + | @csv' "$scan" >> "$summary" +} + +# Categorise images based on severity +categorise_image() { + local summary="$1" + local image="$2" + + if [ "$(grep "CRITICAL" "$summary" -c)" -gt 0 ]; then + echo "${image}" >> image-scan-output/critical-images.txt + else + echo "${image}" >> image-scan-output/high-images.txt + fi +} + +# Generate SBOM, return correct scan command for SBOM +generate_sbom() { + local sbom="$1" + local scan="$2" + local image="$3" + trivy image \ + --debug \ + --format spdx-json \ + --output "$sbom" \ + "$image" &> "$sbom.log" + if [ ! -e "$sbom" ]; then + ( + echo "ERROR: trivy image didn't produce the sbom file $sbom for $image" 1>&2 + echo "==== trivy log ====" + cat "$sbom.log" + ) 1>&2 + exit 1 + elif grep -q FATAL "$sbom.log"; then + ( + echo "ERROR: trivy image encountered a fatal error producing $sbom for $image" + echo "==== trivy log ====" + cat "$sbom.log" + echo "==== sbom.json ====" + cat "$sbom" + ) 1>&2 + exit 1 + else + echo "trivy sbom $scan_common_args --output $scan $sbom" + fi +} + +# Scan images, generate SBOMs if requested +scan_image() { + local image=$1 + local filename + filename=$(basename "$image" | sed 's/:/\./g') + local imagename + imagename=$(echo "$filename" | cut -d "." -f 1 | sed 's/-/_/g') + local sbom="image-scan-output/${imagename}/${filename}-sbom.json" + local scan="image-scan-output/${imagename}/${filename}-scan.json" + local summary="image-scan-output/${imagename}/${filename}-summary.csv" + + mkdir -p "image-scan-output/$imagename" + generate_trivy_ignore "$imagename" + + # If SBOM is required, generate it first and scan the results, otherwise we + # scan the image directly. + if $generate_sbom; then + echo "Generating SBOM for $imagename" + scan_command="$(generate_sbom "$sbom" "$scan" "$image")" + else + scan_command="trivy image $scan_common_args --output $scan $image" + fi + + # Run scan against image or SBOM, format output. If no results, delete files. + echo "Scanning $imagename for vulnerabilities" + if $scan_command >& "$scan.log"; then + rm -f "$scan" + echo "${image}" >> image-scan-output/clean-images.txt + elif [ ! -f "$scan" ]; then + ( + echo "ERROR: trivy scan encountered an error producing $scan" + echo "Command: $scan_command" + echo "==== trivy log ====" + cat "$scan.log" + if $generate_sbom; then + echo "==== sbom.json ====" + cat "$sbom" + fi + ) 1>&2 + exit 1 + else + generate_summary_csv "$scan" "$summary" + categorise_image "$summary" "$image" + fi +} + +# Main function +main() { + if [[ ! $2 ]]; then + usage + fi + + generate_sbom=false + if [[ "$3" == "--sbom" ]]; then + generate_sbom=true + fi + + set -u + + check_deps_installed + file_prep + + images=$(get_images "$1" "$2") + for image in $images; do + scan_image "$image" + done +} + +main "$@" diff --git a/tools/ubuntu-upgrade-infra-vm.sh b/tools/ubuntu-upgrade-infra-vm.sh new file mode 100755 index 000000000..ceba43e2f --- /dev/null +++ b/tools/ubuntu-upgrade-infra-vm.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -e + +if [[ -z "$1" ]]; then + echo "Usage: infra-vm-ubuntu-upgrade.sh " + exit 2 +fi + +if [[ -z "$KAYOBE_PATH" ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ -z "$KAYOBE_CONFIG_PATH" ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ -z "$ANSIBLE_ROLES_PATH" ]]; then + set -x + export ANSIBLE_ROLES_PATH="$KAYOBE_PATH/ansible/roles" + set +x +else + set -x + export ANSIBLE_ROLES_PATH="$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles" + set +x +fi + +set -x + +kayobe playbook run "$KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml" \ + -e os_release=noble --limit "$1" + +kayobe infra vm host configure --limit "$1" -e os_release=noble diff --git a/tools/ubuntu-upgrade-overcloud.sh b/tools/ubuntu-upgrade-overcloud.sh new file mode 100755 index 000000000..46dfd2e3a --- /dev/null +++ b/tools/ubuntu-upgrade-overcloud.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e + +if [[ -z "$1" ]]; then + echo "Usage: overcloud-ubuntu-upgrade.sh " + exit 2 +fi + +if [[ -z "$KAYOBE_PATH" ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ -z "$KAYOBE_CONFIG_PATH" ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ -z "$ANSIBLE_ROLES_PATH" ]]; then + set -x + export ANSIBLE_ROLES_PATH="$KAYOBE_PATH/ansible/roles" + set +x +else + set -x + export ANSIBLE_ROLES_PATH="$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles" + set +x +fi + +set -x + +kayobe playbook run "$KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml" \ + -e os_release=noble --limit "$1" + +kayobe playbook run "$KAYOBE_CONFIG_PATH/ansible/ovn-fix-chassis-priorities.yml" + +kayobe overcloud host configure --limit "$1" -e os_release=noble + diff --git a/tools/ubuntu-upgrade-seed-hypervisor.sh b/tools/ubuntu-upgrade-seed-hypervisor.sh new file mode 100755 index 000000000..84350b8f7 --- /dev/null +++ b/tools/ubuntu-upgrade-seed-hypervisor.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +if [[ -z "$KAYOBE_PATH" ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ -z "$KAYOBE_CONFIG_PATH" ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ -z "$ANSIBLE_ROLES_PATH" ]]; then + set -x + export ANSIBLE_ROLES_PATH="$KAYOBE_PATH/ansible/roles" + set +x +else + set -x + export ANSIBLE_ROLES_PATH="$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles" + set +x +fi + +set -x + +kayobe playbook run "$KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml" \ + -e os_release=noble --limit seed-hypervisor + +kayobe seed hypervisor host configure -e os_release=noble diff --git a/tools/ubuntu-upgrade-seed.sh b/tools/ubuntu-upgrade-seed.sh new file mode 100755 index 000000000..0c78cd250 --- /dev/null +++ b/tools/ubuntu-upgrade-seed.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -e + +if [[ -z "$KAYOBE_PATH" ]]; then + echo "Environment variable \$KAYOBE_PATH is not defined" + exit 2 +fi + +if [[ -z "$KAYOBE_CONFIG_PATH" ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + exit 2 +fi + +if [[ -z "$ANSIBLE_ROLES_PATH" ]]; then + set -x + export ANSIBLE_ROLES_PATH="$KAYOBE_PATH/ansible/roles" + set +x +else + set -x + export ANSIBLE_ROLES_PATH="$ANSIBLE_ROLES_PATH:$KAYOBE_PATH/ansible/roles" + set +x +fi + +set -x + +kayobe playbook run "$KAYOBE_CONFIG_PATH/ansible/ubuntu-upgrade.yml" \ + -e os_release=noble --limit seed + +kayobe seed host configure -e os_release=noble diff --git a/tools/upgrade-prerequisites.sh b/tools/upgrade-prerequisites.sh new file mode 100755 index 000000000..5c931dcff --- /dev/null +++ b/tools/upgrade-prerequisites.sh @@ -0,0 +1,53 @@ +#! /usr/bin/bash + +# This script is intended to be run in CI to test upgrades. +# It executes any preparation steps that must be perfomed before upgrading +# OpenStack services. + +# NOTE(upgrade): This script is unique to each release. It may not be required +# for some releases. + +set -ex + +function prechecks() { + if [[ ! $KAYOBE_CONFIG_PATH ]]; then + echo "Environment variable \$KAYOBE_CONFIG_PATH is not defined" + echo "Ensure your environment is set up to run kayobe commands" + exit 2 + fi +} + +function rabbit_upgrade() { + # Ensure RabbitMQ is upgraded to 4.1 + if kayobe overcloud host command run -l controllers -b --command "docker exec rabbitmq rabbitmqctl --version | grep -F 3.13." --show-output; then + kayobe kolla ansible run "rabbitmq-upgrade 4.1" --kolla-skip-tags rabbitmq-version-check + fi +} + +function rabbit_migration() { + if kayobe overcloud host command run -l controllers -b --command "docker exec rabbitmq rabbitmqctl list_queues durable | grep false"; then + # Set feature flaga, execute RabbitMQ queue migration script, unset feature flags (to avoid git conflicts) + KOLLA_GLOBALS_PATH=$KAYOBE_CONFIG_PATH/kolla/globals.yml + if [[ $KAYOBE_ENVIRONMENT ]]; then + KOLLA_GLOBALS_PATH=$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml + fi + sed -i -e '$aom_enable_queue_manager: True' \ + -e '$aom_enable_rabbitmq_quorum_queues: True' \ + -e '$aom_enable_rabbitmq_transient_quorum_queue: True' \ + -e '$aom_enable_rabbitmq_stream_fanout: True' \ + $KOLLA_GLOBALS_PATH + + $KAYOBE_CONFIG_PATH/../../tools/rabbitmq-queue-migration.sh + + sed -i -e '/om_enable_queue_manager: True/d' \ + -e '/om_enable_rabbitmq_quorum_queues: True/d' \ + -e '/om_enable_rabbitmq_transient_quorum_queue: True/d' \ + -e '/om_enable_rabbitmq_stream_fanout: True/d' \ + $KOLLA_GLOBALS_PATH + + fi +} + +prechecks +rabbit_migration +rabbit_upgrade diff --git a/tox.ini b/tox.ini index 3b00c00af..69eab534d 100644 --- a/tox.ini +++ b/tox.ini @@ -6,9 +6,32 @@ skipsdist = True [testenv] install_command = pip install {opts} {packages} deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2024.1} + -c{env:TOX_CONSTRAINTS_FILE:https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/2024.1/upper-constraints.txt} -r{toxinidir}/test-requirements.txt [testenv:pep8] commands = yamllint etc/kayobe + reno lint + # secret-rotation must be skipped because it includes purposeful whitespace + doc8 README.rst doc/source --ignore D001 --ignore-path-errors doc/source/operations/secret-rotation.rst;D002 +# StackHPC Kayobe configuration release notes: +[testenv:releasenotes] +allowlist_externals = rm +skip_install = true +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/2024.1/upper-constraints.txt} + -r{toxinidir}/releasenotes/requirements.txt +commands = + rm -rf releasenotes/build/html + sphinx-build -n -W --keep-going -b html releasenotes/source releasenotes/build/html + +[testenv:docs] +allowlist_externals = rm +skip_install = true +deps = + -c{env:TOX_CONSTRAINTS_FILE:https://raw.githubusercontent.com/stackhpc/requirements/refs/heads/stackhpc/2024.1/upper-constraints.txt} + -r{toxinidir}/doc/requirements.txt +commands = + rm -rf doc/build/html + sphinx-build -W --keep-going -b html doc/source doc/build/html From 5f45501dcdc7a22d0b6057c36b92c9ee9a2828ec Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 7 Jul 2026 08:20:47 +0100 Subject: [PATCH 094/103] Removed controller groups and updated flavour names --- .../hugepages | 0 .../nova | 0 .../hugepages | 0 .../nova | 0 .../group_vars/controllers/network-interfaces | 10 ++----- .../controllers_r650/network-interfaces | 30 ------------------- .../controllers_t620/network-interfaces | 24 --------------- etc/kayobe/inventory/hosts | 16 +++------- etc/kayobe/network-allocation.yml | 16 ++++++---- 9 files changed, 17 insertions(+), 79 deletions(-) rename etc/kayobe/inventory/group_vars/{compute_flavor_r660 => compute_flavor_r660_1}/hugepages (100%) rename etc/kayobe/inventory/group_vars/{compute_flavor_r660 => compute_flavor_r660_1}/nova (100%) rename etc/kayobe/inventory/group_vars/{compute_flavor_sr650 => compute_flavor_sr650v2}/hugepages (100%) rename etc/kayobe/inventory/group_vars/{compute_flavor_sr650 => compute_flavor_sr650v2}/nova (100%) delete mode 100644 etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces delete mode 100644 etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces diff --git a/etc/kayobe/inventory/group_vars/compute_flavor_r660/hugepages b/etc/kayobe/inventory/group_vars/compute_flavor_r660_1/hugepages similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_flavor_r660/hugepages rename to etc/kayobe/inventory/group_vars/compute_flavor_r660_1/hugepages diff --git a/etc/kayobe/inventory/group_vars/compute_flavor_r660/nova b/etc/kayobe/inventory/group_vars/compute_flavor_r660_1/nova similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_flavor_r660/nova rename to etc/kayobe/inventory/group_vars/compute_flavor_r660_1/nova diff --git a/etc/kayobe/inventory/group_vars/compute_flavor_sr650/hugepages b/etc/kayobe/inventory/group_vars/compute_flavor_sr650v2/hugepages similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_flavor_sr650/hugepages rename to etc/kayobe/inventory/group_vars/compute_flavor_sr650v2/hugepages diff --git a/etc/kayobe/inventory/group_vars/compute_flavor_sr650/nova b/etc/kayobe/inventory/group_vars/compute_flavor_sr650v2/nova similarity index 100% rename from etc/kayobe/inventory/group_vars/compute_flavor_sr650/nova rename to etc/kayobe/inventory/group_vars/compute_flavor_sr650v2/nova diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 59fd35a19..94c1b7663 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -15,15 +15,9 @@ # Further information on the available network attributes is provided in the # network configuration reference in the Kayobe documentation. -prod_interface: eno12399 +prod_interface: enp65s0 -data_interface: brens2f0np0 -data_bridge_ports: - - ens2f0np0 - -jbo_interface: breno8303 -jbo_bridge_ports: - - eno8303 +jbo_interface: eno2 ############################################################################## # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces b/etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces deleted file mode 100644 index 59fd35a19..000000000 --- a/etc/kayobe/inventory/group_vars/controllers_r650/network-interfaces +++ /dev/null @@ -1,30 +0,0 @@ ---- -############################################################################### -# Network interface definitions for the controller group. - -# NOTE: The content of this section is very deployment-specific, since it -# depends on the names and types of networks in the deployment. It should -# define the group-specific attributes of networks. The following example shows -# a basic configuration for a network called "example": -# -# example_interface: eth0 -# -# Global network attributes such as subnet CIDRs are typically configured in -# etc/kayobe/networks.yml. -# -# Further information on the available network attributes is provided in the -# network configuration reference in the Kayobe documentation. - -prod_interface: eno12399 - -data_interface: brens2f0np0 -data_bridge_ports: - - ens2f0np0 - -jbo_interface: breno8303 -jbo_bridge_ports: - - eno8303 - -############################################################################## -# Dummy variable to allow Ansible to accept this file. -workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces b/etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces deleted file mode 100644 index 94c1b7663..000000000 --- a/etc/kayobe/inventory/group_vars/controllers_t620/network-interfaces +++ /dev/null @@ -1,24 +0,0 @@ ---- -############################################################################### -# Network interface definitions for the controller group. - -# NOTE: The content of this section is very deployment-specific, since it -# depends on the names and types of networks in the deployment. It should -# define the group-specific attributes of networks. The following example shows -# a basic configuration for a network called "example": -# -# example_interface: eth0 -# -# Global network attributes such as subnet CIDRs are typically configured in -# etc/kayobe/networks.yml. -# -# Further information on the available network attributes is provided in the -# network configuration reference in the Kayobe documentation. - -prod_interface: enp65s0 - -jbo_interface: eno2 - -############################################################################## -# Dummy variable to allow Ansible to accept this file. -workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index db6193d02..2c187f519 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -3,21 +3,13 @@ localhost ansible_connection=local [seed] -# srcseed +srcseed [controllers] -[controllers:children] -controllers_t620 -controllers_r650 - -[controllers_t620] ctl-t620-01 ctl-t620-02 ctl-t620-03 -[controllers_r650] -ctl-r650-01 - [compute_lenovo] c-1 c-2 @@ -39,7 +31,7 @@ c-13 [compute_flavor_general] c-1 -[compute_flavor_sr650] +[compute_flavor_sr650v2] c-2 c-3 c-4 @@ -48,7 +40,7 @@ c-6 c-7 c-14 -[compute_flavor_r660] +[compute_flavor_r660_1] c-8 c-9 c-10 @@ -75,7 +67,7 @@ ceph-hdd-08 [storage_r760_2] ceph-hdd-01 ceph-hdd-02 -ceph-hdd-09 +# ceph-hdd-09 no route to host ceph-hdd-10 ceph-hdd-11 ceph-hdd-12 diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index 5bc7264eb..da7c76ca9 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -15,8 +15,6 @@ data_ips: c-9: 192.168.100.9 ceph-adm-01: 192.168.100.202 ceph-adm-02: 192.168.100.203 - ceph-mds-01: 192.168.100.205 - ceph-mds-02: 192.168.100.206 ceph-hdd-01: 192.168.100.211 ceph-hdd-02: 192.168.100.212 ceph-hdd-03: 192.168.100.213 @@ -25,14 +23,19 @@ data_ips: ceph-hdd-06: 192.168.100.216 ceph-hdd-07: 192.168.100.217 ceph-hdd-08: 192.168.100.218 - ceph-hdd-09: 192.168.100.219 + # ceph-hdd-09: 192.168.100.219 no route to host ceph-hdd-10: 192.168.100.220 ceph-hdd-11: 192.168.100.221 ceph-hdd-12: 192.168.100.222 + ceph-mds-01: 192.168.100.205 + ceph-mds-02: 192.168.100.206 srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 jbo_ips: + ctl-t620-01: 130.88.9.101 + ctl-t620-02: 130.88.9.102 + ctl-t620-03: 130.88.9.103 srcseed: 130.88.9.201 prod_ips: c-1: 192.168.10.1 @@ -51,8 +54,6 @@ prod_ips: c-9: 192.168.10.9 ceph-adm-01: 192.168.10.202 ceph-adm-02: 192.168.10.203 - ceph-mds-01: 192.168.10.205 - ceph-mds-02: 192.168.10.206 ceph-hdd-01: 192.168.10.211 ceph-hdd-02: 192.168.10.212 ceph-hdd-03: 192.168.10.213 @@ -65,4 +66,9 @@ prod_ips: ceph-hdd-10: 192.168.10.220 ceph-hdd-11: 192.168.10.221 ceph-hdd-12: 192.168.10.222 + ceph-mds-01: 192.168.10.205 + ceph-mds-02: 192.168.10.206 + ctl-t620-01: 192.168.10.101 + ctl-t620-02: 192.168.10.102 + ctl-t620-03: 192.168.10.103 srcseed: 192.168.10.201 From 48ae5013a5003622e12d8a8812f213586507dcf7 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Tue, 7 Jul 2026 12:47:10 +0100 Subject: [PATCH 095/103] Updated hosts and network-allocation.yml with new host names --- etc/kayobe/inventory/hosts | 57 +++++++++++++++-------------- etc/kayobe/network-allocation.yml | 60 +++++++++++++++---------------- 2 files changed, 58 insertions(+), 59 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 2c187f519..e49b0006d 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -11,42 +11,42 @@ ctl-t620-02 ctl-t620-03 [compute_lenovo] -c-1 -c-2 -c-3 -c-4 -c-5 -c-6 -c-7 -c-14 +c-sr650v2-01 +c-sr650v2-02 +c-sr650v2-03 +c-sr650v2-04 +c-sr650v2-05 +c-sr650v2-06 +c-sr650v2-07 +c-sr650v2-08 [compute_dell] -c-8 -c-9 -c-10 -c-11 -c-12 -c-13 +c-r660-01 +c-r660-02 +c-r660-03 +c-r660-04 +c-r660-05 +c-r660-06 [compute_flavor_general] -c-1 +c-sr650v2-01 [compute_flavor_sr650v2] -c-2 -c-3 -c-4 -c-5 -c-6 -c-7 -c-14 +c-sr650v2-02 +c-sr650v2-03 +c-sr650v2-04 +c-sr650v2-05 +c-sr650v2-06 +c-sr650v2-07 +c-sr650v2-08 [compute_flavor_r660_1] -c-8 -c-9 -c-10 -c-11 -c-12 -c-13 +c-r660-01 +c-r660-02 +c-r660-03 +c-r660-04 +c-r660-05 +c-r660-06 [compute:children] compute_lenovo @@ -88,4 +88,3 @@ mons [osds:children] storage - diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index da7c76ca9..f12a2026a 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -1,20 +1,22 @@ data_ips: - c-1: 192.168.100.1 - c-10: 192.168.100.10 - c-11: 192.168.100.11 - c-12: 192.168.100.12 - c-13: 192.168.100.13 - c-14: 192.168.100.14 - c-2: 192.168.100.2 - c-3: 192.168.100.3 - c-4: 192.168.100.4 - c-5: 192.168.100.5 - c-6: 192.168.100.6 - c-7: 192.168.100.7 - c-8: 192.168.100.8 - c-9: 192.168.100.9 + c-sr650v2-01: 192.168.100.1 + c-sr650v2-02: 192.168.100.2 + c-sr650v2-03: 192.168.100.3 + c-sr650v2-04: 192.168.100.4 + c-sr650v2-05: 192.168.100.5 + c-sr650v2-06: 192.168.100.6 + c-sr650v2-07: 192.168.100.7 + c-sr650v2-08: 192.168.100.8 + c-r660-01: 192.168.100.51 + c-r660-02: 192.168.100.52 + c-r660-03: 192.168.100.53 + c-r660-04: 192.168.100.54 + c-r660-05: 192.168.100.55 + c-r660-06: 192.168.100.56 ceph-adm-01: 192.168.100.202 ceph-adm-02: 192.168.100.203 + ceph-mds-01: 192.168.100.205 + ceph-mds-02: 192.168.100.206 ceph-hdd-01: 192.168.100.211 ceph-hdd-02: 192.168.100.212 ceph-hdd-03: 192.168.100.213 @@ -27,8 +29,6 @@ data_ips: ceph-hdd-10: 192.168.100.220 ceph-hdd-11: 192.168.100.221 ceph-hdd-12: 192.168.100.222 - ceph-mds-01: 192.168.100.205 - ceph-mds-02: 192.168.100.206 srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 @@ -38,20 +38,20 @@ jbo_ips: ctl-t620-03: 130.88.9.103 srcseed: 130.88.9.201 prod_ips: - c-1: 192.168.10.1 - c-10: 192.168.10.10 - c-11: 192.168.10.11 - c-12: 192.168.10.12 - c-13: 192.168.10.13 - c-14: 192.168.10.14 - c-2: 192.168.10.2 - c-3: 192.168.10.3 - c-4: 192.168.10.4 - c-5: 192.168.10.5 - c-6: 192.168.10.6 - c-7: 192.168.10.7 - c-8: 192.168.10.8 - c-9: 192.168.10.9 + c-sr650v2-01: 192.168.10.1 + c-sr650v2-02: 192.168.10.2 + c-sr650v2-03: 192.168.10.3 + c-sr650v2-04: 192.168.10.4 + c-sr650v2-05: 192.168.10.5 + c-sr650v2-06: 192.168.10.6 + c-sr650v2-07: 192.168.10.7 + c-sr650v2-08: 192.168.10.8 + c-r660-01: 192.168.10.51 + c-r660-02: 192.168.10.52 + c-r660-03: 192.168.10.53 + c-r660-04: 192.168.10.54 + c-r660-05: 192.168.10.55 + c-r660-06: 192.168.10.56 ceph-adm-01: 192.168.10.202 ceph-adm-02: 192.168.10.203 ceph-hdd-01: 192.168.10.211 From 3c37bf0335e1014ab30c45271f1ba1aad1d861ba Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Wed, 8 Jul 2026 10:08:43 +0100 Subject: [PATCH 096/103] Add ceph-hdd-09 --- etc/kayobe/inventory/hosts | 2 +- etc/kayobe/network-allocation.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index e49b0006d..cdcea46fc 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -67,7 +67,7 @@ ceph-hdd-08 [storage_r760_2] ceph-hdd-01 ceph-hdd-02 -# ceph-hdd-09 no route to host +ceph-hdd-09 ceph-hdd-10 ceph-hdd-11 ceph-hdd-12 diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index f12a2026a..cc23f2b52 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -25,7 +25,7 @@ data_ips: ceph-hdd-06: 192.168.100.216 ceph-hdd-07: 192.168.100.217 ceph-hdd-08: 192.168.100.218 - # ceph-hdd-09: 192.168.100.219 no route to host + ceph-hdd-09: 192.168.100.219 ceph-hdd-10: 192.168.100.220 ceph-hdd-11: 192.168.100.221 ceph-hdd-12: 192.168.100.222 From 096d3e920ba7bbfb9b16b8987b83dc45b88af8e4 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Wed, 8 Jul 2026 12:06:24 +0100 Subject: [PATCH 097/103] Overide data network --- etc/kayobe/bifrost.yml | 1 + .../group_vars/controllers/network-interfaces | 18 +++++++++++ etc/kayobe/network-allocation.yml | 30 ++++++++++--------- etc/kayobe/networks.yml | 10 +++---- 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/etc/kayobe/bifrost.yml b/etc/kayobe/bifrost.yml index aa75405b4..dd246a874 100644 --- a/etc/kayobe/bifrost.yml +++ b/etc/kayobe/bifrost.yml @@ -4,6 +4,7 @@ ############################################################################### # Bifrost installation. +kolla_enable_bifrost: false # URL of Bifrost source code repository. #kolla_bifrost_source_url: diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 94c1b7663..29f27dec3 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -19,6 +19,24 @@ prod_interface: enp65s0 jbo_interface: eno2 +# List of default networks to which controller nodes are attached. +controller_default_network_interfaces: > + {{ [admin_oc_net_name, + oob_wl_net_name, + provision_wl_net_name, + inspection_net_name, + internal_net_name, + swift_storage_net_name, + cleaning_net_name] | select | unique | list }} + +# List of default network interfaces to which network nodes are attached. +controller_network_host_default_network_interfaces: > + {{ ([admin_oc_net_name, + internal_net_name, + public_net_name, + tunnel_net_name] + + external_net_names) | select | unique | list }} + ############################################################################## # Dummy variable to allow Ansible to accept this file. workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index cc23f2b52..32f650551 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -1,4 +1,10 @@ data_ips: + c-r660-01: 192.168.100.51 + c-r660-02: 192.168.100.52 + c-r660-03: 192.168.100.53 + c-r660-04: 192.168.100.54 + c-r660-05: 192.168.100.55 + c-r660-06: 192.168.100.56 c-sr650v2-01: 192.168.100.1 c-sr650v2-02: 192.168.100.2 c-sr650v2-03: 192.168.100.3 @@ -7,16 +13,8 @@ data_ips: c-sr650v2-06: 192.168.100.6 c-sr650v2-07: 192.168.100.7 c-sr650v2-08: 192.168.100.8 - c-r660-01: 192.168.100.51 - c-r660-02: 192.168.100.52 - c-r660-03: 192.168.100.53 - c-r660-04: 192.168.100.54 - c-r660-05: 192.168.100.55 - c-r660-06: 192.168.100.56 ceph-adm-01: 192.168.100.202 ceph-adm-02: 192.168.100.203 - ceph-mds-01: 192.168.100.205 - ceph-mds-02: 192.168.100.206 ceph-hdd-01: 192.168.100.211 ceph-hdd-02: 192.168.100.212 ceph-hdd-03: 192.168.100.213 @@ -29,6 +27,10 @@ data_ips: ceph-hdd-10: 192.168.100.220 ceph-hdd-11: 192.168.100.221 ceph-hdd-12: 192.168.100.222 + ceph-mds-01: 192.168.100.205 + ceph-mds-02: 192.168.100.206 + ctl-t620-01: 192.168.100.9 + ctl-t620-02: 192.168.100.10 srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 @@ -38,6 +40,12 @@ jbo_ips: ctl-t620-03: 130.88.9.103 srcseed: 130.88.9.201 prod_ips: + c-r660-01: 192.168.10.51 + c-r660-02: 192.168.10.52 + c-r660-03: 192.168.10.53 + c-r660-04: 192.168.10.54 + c-r660-05: 192.168.10.55 + c-r660-06: 192.168.10.56 c-sr650v2-01: 192.168.10.1 c-sr650v2-02: 192.168.10.2 c-sr650v2-03: 192.168.10.3 @@ -46,12 +54,6 @@ prod_ips: c-sr650v2-06: 192.168.10.6 c-sr650v2-07: 192.168.10.7 c-sr650v2-08: 192.168.10.8 - c-r660-01: 192.168.10.51 - c-r660-02: 192.168.10.52 - c-r660-03: 192.168.10.53 - c-r660-04: 192.168.10.54 - c-r660-05: 192.168.10.55 - c-r660-06: 192.168.10.56 ceph-adm-01: 192.168.10.202 ceph-adm-02: 192.168.10.203 ceph-hdd-01: 192.168.10.211 diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 8d807b92b..652ab603e 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -26,7 +26,7 @@ oob_oc_net_name: ipmi # Name of the network used to expose the internal OpenStack API endpoints. #internal_net_name: -internal_net_name: data +internal_net_name: prod # List of names of networks used to provide external network access via # Neutron. @@ -34,9 +34,7 @@ internal_net_name: data # If external_net_name is defined, external_net_names will default to a list # containing one item, external_net_name. #external_net_names: -external_net_names: -- jbo -- data +external_net_names: jbo # Name of the network used to expose the public OpenStack API endpoints. #public_net_name: @@ -44,7 +42,7 @@ public_net_name: jbo # Name of the network used by Neutron to carry tenant overlay network traffic. #tunnel_net_name: -tunnel_net_name: data +tunnel_net_name: prod # Name of the network used to carry storage data traffic. #storage_net_name: @@ -110,6 +108,8 @@ jbo_allocation_pool_end: 130.88.9.131 data_cidr: 192.168.100.0/24 data_vip_address: 192.168.100.200 +prod_vip_address: 192.168.10.200 + ipmi_cidr: 192.168.1.0/24 ############################################################################### From ea015c79a1ea01d80483f50c95b30abc83cef879 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 9 Jul 2026 10:15:28 +0100 Subject: [PATCH 098/103] prod_vip_address: 192.168.10.200 --- etc/kayobe/network-allocation.yml | 1 + etc/kayobe/networks.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index 32f650551..f795e5b86 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -31,6 +31,7 @@ data_ips: ceph-mds-02: 192.168.100.206 ctl-t620-01: 192.168.100.9 ctl-t620-02: 192.168.100.10 + ctl-t620-03: 192.168.100.11 srcseed: 192.168.100.201 ipmi_ips: srcseed: 192.168.1.201 diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index 652ab603e..c8460bb1f 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -34,7 +34,8 @@ internal_net_name: prod # If external_net_name is defined, external_net_names will default to a list # containing one item, external_net_name. #external_net_names: -external_net_names: jbo +external_net_names: +- jbo # Name of the network used to expose the public OpenStack API endpoints. #public_net_name: From 3702393f265208d213e82989cc097c171fa68621 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 9 Jul 2026 11:08:53 +0100 Subject: [PATCH 099/103] Adding pulp creds --- .../group_vars/controllers/network-interfaces | 1 - etc/kayobe/inventory/hosts | 4 ++-- etc/kayobe/network-allocation.yml | 6 +++--- etc/kayobe/pulp.yml | 4 ++-- etc/kayobe/secrets.yml | 10 ++++++++++ 5 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 etc/kayobe/secrets.yml diff --git a/etc/kayobe/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/inventory/group_vars/controllers/network-interfaces index 29f27dec3..1d8b8ac80 100644 --- a/etc/kayobe/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/inventory/group_vars/controllers/network-interfaces @@ -26,7 +26,6 @@ controller_default_network_interfaces: > provision_wl_net_name, inspection_net_name, internal_net_name, - swift_storage_net_name, cleaning_net_name] | select | unique | list }} # List of default network interfaces to which network nodes are attached. diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index cdcea46fc..0f19cc8fa 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -16,9 +16,9 @@ c-sr650v2-02 c-sr650v2-03 c-sr650v2-04 c-sr650v2-05 -c-sr650v2-06 +# c-sr650v2-06: 192.168.100.6 Interface ens3f0np0 is not active c-sr650v2-07 -c-sr650v2-08 +# c-sr650v2-08: 192.168.100.8 Interface ens3f0np0 does not exist [compute_dell] c-r660-01 diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index f795e5b86..c177bd3ed 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -10,9 +10,9 @@ data_ips: c-sr650v2-03: 192.168.100.3 c-sr650v2-04: 192.168.100.4 c-sr650v2-05: 192.168.100.5 - c-sr650v2-06: 192.168.100.6 - c-sr650v2-07: 192.168.100.7 - c-sr650v2-08: 192.168.100.8 + # c-sr650v2-06: 192.168.100.6 Interface ens3f0np0 is not active + c-sr650v2-07: 192.168.100.7 + # c-sr650v2-08: 192.168.100.8 Interface ens3f0np0 does not exist ceph-adm-01: 192.168.100.202 ceph-adm-02: 192.168.100.203 ceph-hdd-01: 192.168.100.211 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 0ab2694ab..0888a3699 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -46,8 +46,8 @@ stackhpc_release_pulp_domain: "ark.stackhpc.com" stackhpc_release_pulp_url: "{{ stackhpc_release_pulp_scheme }}://{{ stackhpc_release_pulp_domain }}" # Credentials used to access the StackHPC Pulp service. -stackhpc_release_pulp_username: -stackhpc_release_pulp_password: +stackhpc_release_pulp_username: uksrc +stackhpc_release_pulp_password: "{{ secrets_release_pulp_password }}" # Content URL of the StackHPC Pulp service. stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content" diff --git a/etc/kayobe/secrets.yml b/etc/kayobe/secrets.yml new file mode 100644 index 000000000..58df029b6 --- /dev/null +++ b/etc/kayobe/secrets.yml @@ -0,0 +1,10 @@ +$ANSIBLE_VAULT;1.1;AES256 +39316237313738613332336163393364313238343939326637646439646331656630643130353738 +3434373233313061323933313966656461323761353864360a303830623636393837393964663338 +62646638636536653237346161633065396333363264366339636632653038313035653037326561 +3935333035363761650a373536316665363164663636656234623864343435306435386636336130 +36626537616437656162303966306638326137616566636362313562396366383832303434353335 +34373564306361323931383031376333633039336239396633363764656533663761326265373064 +30323664636638376564316339663130626165306138363635303439653736396266343934376437 +38336433303636646165376432383934393832393566363036626239326537363163653963643364 +61626662323939323439643135623363373363626232346530316131623061386137 From b618dd493832852b91f0b19e1a44cd492f55ad93 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Thu, 9 Jul 2026 12:20:01 +0100 Subject: [PATCH 100/103] commented out c-sr650v2-07 after sdb was dd'd --- etc/kayobe/inventory/hosts | 12 ++++++------ etc/kayobe/network-allocation.yml | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 0f19cc8fa..4937ab440 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -16,9 +16,9 @@ c-sr650v2-02 c-sr650v2-03 c-sr650v2-04 c-sr650v2-05 -# c-sr650v2-06: 192.168.100.6 Interface ens3f0np0 is not active -c-sr650v2-07 -# c-sr650v2-08: 192.168.100.8 Interface ens3f0np0 does not exist +# c-sr650v2-06 Interface ens3f0np0 is not active +# c-sr650v2-07 sdb dd'd +# c-sr650v2-08 Interface ens3f0np0 does not exist [compute_dell] c-r660-01 @@ -36,9 +36,9 @@ c-sr650v2-02 c-sr650v2-03 c-sr650v2-04 c-sr650v2-05 -c-sr650v2-06 -c-sr650v2-07 -c-sr650v2-08 +# c-sr650v2-06 Interface ens3f0np0 is not active +# c-sr650v2-07 sdb dd'd +# c-sr650v2-08 Interface ens3f0np0 does not exist [compute_flavor_r660_1] c-r660-01 diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index c177bd3ed..ae7b45439 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -11,7 +11,7 @@ data_ips: c-sr650v2-04: 192.168.100.4 c-sr650v2-05: 192.168.100.5 # c-sr650v2-06: 192.168.100.6 Interface ens3f0np0 is not active - c-sr650v2-07: 192.168.100.7 + # c-sr650v2-07: 192.168.100.7 sdb dd'd # c-sr650v2-08: 192.168.100.8 Interface ens3f0np0 does not exist ceph-adm-01: 192.168.100.202 ceph-adm-02: 192.168.100.203 @@ -52,9 +52,9 @@ prod_ips: c-sr650v2-03: 192.168.10.3 c-sr650v2-04: 192.168.10.4 c-sr650v2-05: 192.168.10.5 - c-sr650v2-06: 192.168.10.6 - c-sr650v2-07: 192.168.10.7 - c-sr650v2-08: 192.168.10.8 + # c-sr650v2-06: 192.168.10.6 Interface ens3f0np0 is not active + # c-sr650v2-07: 192.168.10.7 sdb dd'd + # c-sr650v2-08: 192.168.10.8 Interface ens3f0np0 does not exist ceph-adm-01: 192.168.10.202 ceph-adm-02: 192.168.10.203 ceph-hdd-01: 192.168.10.211 From 17a67ddbe53ab2fd64bc79c4be469387ca8b0c0c Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 10 Jul 2026 06:54:24 +0100 Subject: [PATCH 101/103] Moved prod_vip_address after prod_cidr --- etc/kayobe/networks.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/kayobe/networks.yml b/etc/kayobe/networks.yml index c8460bb1f..468202246 100644 --- a/etc/kayobe/networks.yml +++ b/etc/kayobe/networks.yml @@ -94,6 +94,7 @@ storage_mgmt_net_name: data # network configuration reference in the Kayobe documentation. prod_cidr: 192.168.10.0/24 +prod_vip_address: 192.168.10.200 # Gateway is set in group vars # so controllers are not confused # prod_gateway: 192.168.10.201 @@ -109,8 +110,6 @@ jbo_allocation_pool_end: 130.88.9.131 data_cidr: 192.168.100.0/24 data_vip_address: 192.168.100.200 -prod_vip_address: 192.168.10.200 - ipmi_cidr: 192.168.1.0/24 ############################################################################### From c9259088aa138ceb6c7b7e4b660f16726cb4360f Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 10 Jul 2026 08:31:28 +0100 Subject: [PATCH 102/103] Address problem with disk labels not being persistent. Targetted local storage by-path. --- etc/kayobe/inventory/group_vars/compute_lenovo/storage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/inventory/group_vars/compute_lenovo/storage b/etc/kayobe/inventory/group_vars/compute_lenovo/storage index a187f802e..5d333e994 100644 --- a/etc/kayobe/inventory/group_vars/compute_lenovo/storage +++ b/etc/kayobe/inventory/group_vars/compute_lenovo/storage @@ -1,2 +1,2 @@ --- -compute_nova_disk: /dev/sdb +compute_nova_disk: /dev/disk/by-path/pci-0000:00:17.0-ata-6 From 49376fe27d814b51b78b62a8ce6e865fc54beb27 Mon Sep 17 00:00:00 2001 From: Ian Leigh Date: Fri, 10 Jul 2026 11:41:45 +0000 Subject: [PATCH 103/103] In compute.yml, changed compute_lvm_groups mount point to /var/lib/docker. In seed/network-inferfaces, added provision_oc_net_interface: eno12300 (Prod). Commented c-r660-01 in hosts and network-allocations.yml. --- etc/kayobe/compute.yml | 2 +- etc/kayobe/inventory/group_vars/seed/network-interfaces | 1 + etc/kayobe/inventory/hosts | 4 ++-- etc/kayobe/network-allocation.yml | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/etc/kayobe/compute.yml b/etc/kayobe/compute.yml index a5ed68157..d2b13e89b 100644 --- a/etc/kayobe/compute.yml +++ b/etc/kayobe/compute.yml @@ -80,7 +80,7 @@ compute_lvm_groups: create: true mount: true filesystem: ext4 - mntp: /var/lib/nova + mntp: /var/lib/docker # Default list of compute volume groups. See mrlesmithjr.manage_lvm role for # format. diff --git a/etc/kayobe/inventory/group_vars/seed/network-interfaces b/etc/kayobe/inventory/group_vars/seed/network-interfaces index 04ccd742c..06d1b2a2a 100644 --- a/etc/kayobe/inventory/group_vars/seed/network-interfaces +++ b/etc/kayobe/inventory/group_vars/seed/network-interfaces @@ -18,6 +18,7 @@ prod_interface: eno12399 data_interface: ens2f0np0 ipmi_interface: enp177s0 +provision_oc_net_interface: eno12399 ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/inventory/hosts b/etc/kayobe/inventory/hosts index 4937ab440..aab99e569 100644 --- a/etc/kayobe/inventory/hosts +++ b/etc/kayobe/inventory/hosts @@ -26,7 +26,7 @@ c-r660-02 c-r660-03 c-r660-04 c-r660-05 -c-r660-06 +# c-r660-06 removed for reinstall of OS on BOSS card and NVMEs work [compute_flavor_general] c-sr650v2-01 @@ -46,7 +46,7 @@ c-r660-02 c-r660-03 c-r660-04 c-r660-05 -c-r660-06 +# c-r660-06 removed for reinstall of OS on BOSS card and NVMEs work [compute:children] compute_lenovo diff --git a/etc/kayobe/network-allocation.yml b/etc/kayobe/network-allocation.yml index ae7b45439..57499c98b 100644 --- a/etc/kayobe/network-allocation.yml +++ b/etc/kayobe/network-allocation.yml @@ -4,7 +4,7 @@ data_ips: c-r660-03: 192.168.100.53 c-r660-04: 192.168.100.54 c-r660-05: 192.168.100.55 - c-r660-06: 192.168.100.56 + # c-r660-06: 192.168.10.56 removed for reinstall of OS on BOSS card and NVMEs work c-sr650v2-01: 192.168.100.1 c-sr650v2-02: 192.168.100.2 c-sr650v2-03: 192.168.100.3 @@ -46,7 +46,7 @@ prod_ips: c-r660-03: 192.168.10.53 c-r660-04: 192.168.10.54 c-r660-05: 192.168.10.55 - c-r660-06: 192.168.10.56 + # c-r660-06: 192.168.10.56 removed for reinstall of OS on BOSS card and NVMEs work c-sr650v2-01: 192.168.10.1 c-sr650v2-02: 192.168.10.2 c-sr650v2-03: 192.168.10.3