Skip to content

Commit ad7a316

Browse files
korydraughnkorydraughn
committed
[26] Add workflows for testing plugins against pre-release versions of iRODS
1 parent b6b04ce commit ad7a316

10 files changed

Lines changed: 1579 additions & 0 deletions

‎.github/workflows/build-and-test-plugin-with-irods-packages-debian.yml‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
name: build-and-test-plugin-with-irods-packages-debian
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
docker_image:
7+
description: The docker image and tag identifying the OS used to compile the plugin (e.g. debian:13).
8+
required: true
9+
type: string
10+
build_artifact_suffix:
11+
description: The name used to uniquely identify build output across workflow jobs (e.g. debian_13-main).
12+
required: true
13+
type: string
14+
plugin_package_directory:
15+
description: The directory name which is expected to hold the plugin packages (e.g. 'Debian gnu_linux_13'). Required by the irods_python_ci_utilities module.
16+
required: true
17+
type: string
18+
irods_testing_environment_project_os:
19+
description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. debian-13).
20+
required: true
21+
type: string
22+
irods_testing_environment_project_db:
23+
description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16).
24+
required: true
25+
type: string
26+
cache_key_irods_packages:
27+
description: The key which potentially identifies a cached build of the irods/irods source code.
28+
required: true
29+
type: string
30+
cache_key_icommands_packages:
31+
description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code.
32+
required: true
33+
type: string
34+
35+
defaults:
36+
run:
37+
shell: bash
38+
39+
jobs:
40+
build_plugin:
41+
name: Build plugin - ${{ inputs.docker_image }}
42+
43+
runs-on: ubuntu-latest
44+
container: ${{ inputs.docker_image }}
45+
46+
steps:
47+
- name: Checkout repository
48+
uses: actions/checkout@v5
49+
50+
- name: Install prerequisite packages
51+
run: |
52+
apt-get update -qq
53+
apt-get install -qq \
54+
build-essential \
55+
cmake \
56+
git \
57+
gnupg \
58+
lsb-release \
59+
python3-distro \
60+
python3-packaging \
61+
python3-pip \
62+
python3-setuptools \
63+
wget
64+
65+
- name: Set up iRODS externals package repository
66+
run: |
67+
mkdir -p /etc/apt/keyrings
68+
wget -qO - https://unstable.irods.org/irods-unstable-signing-key.asc | \
69+
gpg \
70+
--no-options \
71+
--no-default-keyring \
72+
--no-auto-check-trustdb \
73+
--homedir /dev/null \
74+
--no-keyring \
75+
--import-options import-export \
76+
--output /etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp \
77+
--import
78+
echo "deb [signed-by=/etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp arch=amd64] https://unstable.irods.org/apt/ $(lsb_release -sc) main" | \
79+
tee /etc/apt/sources.list.d/renci-irods-unstable.list
80+
81+
- name: Check cache for iRODS packages
82+
uses: actions/cache/restore@v4
83+
with:
84+
key: ${{ inputs.cache_key_irods_packages }}
85+
path: _build_irods/*.deb
86+
fail-on-cache-miss: 'true'
87+
88+
- name: Check cache for iCommands packages
89+
uses: actions/cache/restore@v4
90+
with:
91+
key: ${{ inputs.cache_key_icommands_packages }}
92+
path: _build_icommands/*.deb
93+
fail-on-cache-miss: 'true'
94+
95+
- name: Install iRODS development packages
96+
run: |
97+
apt-get update -qq
98+
apt-get install -qq \
99+
./_build_irods/irods-dev*.deb \
100+
./_build_irods/irods-runtime*.deb
101+
102+
- name: Install iRODS python CI utilities
103+
run: |
104+
# TODO(#18): Investigate how to remove the need for --break-system-packages.
105+
python3 -m pip install --break-system-packages git+https://github.com/irods/irods_python_ci_utilities.git@main
106+
107+
- name: Build and package
108+
run: |
109+
mkdir _build
110+
python3 irods_consortium_continuous_integration_build_hook.py --build_directory _build
111+
112+
- name: List contents of build directory
113+
run: ls -l _build
114+
115+
# The server and icommands packages were built and cached while inside of a docker container.
116+
# We cannot use actions/cache to get the server and icommands packages in the next job because
117+
# GHA ties the cache to the runner's OS. For this workflow, the run_tests job does not run inside
118+
# of a container. This leads to a cache miss, causing the workflow to fail. To get around that, we
119+
# use upload-artifact and download-artifact.
120+
- name: Upload iRODS packages for dependent jobs
121+
uses: actions/upload-artifact@v6
122+
with:
123+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
124+
path: |
125+
_build/*.deb
126+
_build_irods/*.deb
127+
_build_icommands/*.deb
128+
overwrite: true
129+
130+
run_tests:
131+
needs: build_plugin
132+
133+
name: Test plugin - ${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}
134+
135+
runs-on: ubuntu-latest
136+
137+
steps:
138+
- name: Checkout repository
139+
uses: actions/checkout@v5
140+
141+
- name: Set up python for iRODS testing environment
142+
uses: actions/setup-python@v6
143+
with:
144+
python-version: '3.8'
145+
146+
- name: Download iRODS packages
147+
uses: actions/download-artifact@v7
148+
with:
149+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
150+
path: build_artifacts
151+
152+
- name: Set up docker compose
153+
uses: docker/setup-compose-action@v1
154+
155+
- name: Set up iRODS testing environment
156+
run: |
157+
git clone -b main --depth 1 --single-branch https://github.com/irods/irods_testing_environment
158+
cd irods_testing_environment
159+
python3 -m venv venv_testing_env
160+
. venv_testing_env/bin/activate
161+
python --version
162+
pip install docker-compose GitPython
163+
pip install docker'<7' requests==2.26.0
164+
pip freeze
165+
166+
- name: Run tests
167+
run: |
168+
# Create a symlink to the directory containing the recently built plugin packages.
169+
# This is required by the irods_python_irods_ci_utilities module.
170+
ln -s build_artifacts/_build "${{ inputs.plugin_package_directory }}"
171+
172+
# Move the icommands package into the same directory containing the server packages.
173+
# This is required by the testing environment.
174+
mv build_artifacts/_build_icommands/*.deb build_artifacts/_build_irods
175+
176+
cd irods_testing_environment
177+
. venv_testing_env/bin/activate
178+
project_dir="${{ inputs.irods_testing_environment_project_os }}/${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}"
179+
python run_plugin_tests.py \
180+
"${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" \
181+
--project-directory "projects/${project_dir}" \
182+
--irods-package-directory ../build_artifacts/_build_irods \
183+
--plugin-package-directory .. \
184+
--discard-logs \
185+
-v

‎.github/workflows/build-and-test-plugin-with-irods-packages-enterprise-linux.yml‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
name: build-and-test-plugin-with-irods-packages-enterprise-linux
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
docker_image:
7+
description: The docker image and tag identifying the OS used to compile the plugin (e.g. rockylinux/rockylinux:10).
8+
required: true
9+
type: string
10+
build_artifact_suffix:
11+
description: The name used to uniquely identify build output across workflow jobs (e.g. rockylinux_10-main).
12+
required: true
13+
type: string
14+
plugin_package_directory:
15+
description: The directory name which is expected to hold the plugin packages (e.g. 'Rocky linux_10'). Required by the irods_python_ci_utilities module.
16+
required: true
17+
type: string
18+
irods_testing_environment_project_os:
19+
description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. rockylinux-10).
20+
required: true
21+
type: string
22+
irods_testing_environment_project_db:
23+
description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16).
24+
required: true
25+
type: string
26+
cache_key_irods_packages:
27+
description: The key which potentially identifies a cached build of the irods/irods source code.
28+
required: true
29+
type: string
30+
cache_key_icommands_packages:
31+
description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code.
32+
required: true
33+
type: string
34+
35+
defaults:
36+
run:
37+
shell: bash
38+
39+
jobs:
40+
build_plugin:
41+
name: Build plugin - ${{ inputs.docker_image }}
42+
43+
runs-on: ubuntu-latest
44+
container: ${{ inputs.docker_image }}
45+
46+
steps:
47+
- name: Checkout repository
48+
uses: actions/checkout@v5
49+
50+
- name: Install prerequisite packages
51+
run: |
52+
dnf update -y
53+
dnf install -y \
54+
dnf-plugins-core \
55+
epel-release \
56+
gcc-c++ \
57+
git \
58+
rpm-build \
59+
sudo \
60+
wget
61+
dnf install -y \
62+
cmake \
63+
python3-devel \
64+
python3-distro \
65+
python3-packaging \
66+
python3-pip
67+
68+
- name: Set up iRODS externals package repository
69+
run: |
70+
rpm --import https://unstable.irods.org/irods-unstable-signing-key.asc
71+
wget -qO - https://unstable.irods.org/renci-irods-unstable.yum.repo | tee /etc/yum.repos.d/renci-irods-unstable.yum.repo
72+
73+
- name: Check cache for iRODS packages
74+
uses: actions/cache/restore@v4
75+
with:
76+
key: ${{ inputs.cache_key_irods_packages }}
77+
path: _build_irods/*.rpm
78+
fail-on-cache-miss: 'true'
79+
80+
- name: Check cache for iCommands packages
81+
uses: actions/cache/restore@v4
82+
with:
83+
key: ${{ inputs.cache_key_icommands_packages }}
84+
path: _build_icommands/*.rpm
85+
fail-on-cache-miss: 'true'
86+
87+
- name: Install iRODS development packages
88+
run: |
89+
dnf update -y
90+
dnf install -y \
91+
./_build_irods/irods-devel*.rpm \
92+
./_build_irods/irods-runtime*.rpm
93+
94+
- name: Install iRODS python CI utilities
95+
run: |
96+
python3 -m pip install git+https://github.com/irods/irods_python_ci_utilities.git@main
97+
98+
- name: Build and package
99+
run: |
100+
mkdir _build
101+
python3 irods_consortium_continuous_integration_build_hook.py --build_directory _build
102+
103+
- name: List contents of build directory
104+
run: ls -l _build
105+
106+
# The server and icommands packages were built and cached while inside of a docker container.
107+
# We cannot use actions/cache to get the server and icommands packages in the next job because
108+
# GHA ties the cache to the runner's OS. For this workflow, the run_tests job does not run inside
109+
# of a container. This leads to a cache miss, causing the workflow to fail. To get around that, we
110+
# use upload-artifact and download-artifact.
111+
- name: Upload iRODS packages for dependent jobs
112+
uses: actions/upload-artifact@v6
113+
with:
114+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
115+
path: |
116+
_build/*.rpm
117+
_build_irods/*.rpm
118+
_build_icommands/*.rpm
119+
overwrite: true
120+
121+
run_tests:
122+
needs: build_plugin
123+
124+
name: Test plugin - ${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}
125+
126+
runs-on: ubuntu-latest
127+
128+
steps:
129+
- name: Checkout repository
130+
uses: actions/checkout@v5
131+
132+
- name: Set up python for iRODS testing environment
133+
uses: actions/setup-python@v6
134+
with:
135+
python-version: '3.8'
136+
137+
- name: Download iRODS packages
138+
uses: actions/download-artifact@v7
139+
with:
140+
name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }}
141+
path: build_artifacts
142+
143+
- name: Set up docker compose
144+
uses: docker/setup-compose-action@v1
145+
146+
- name: Set up iRODS testing environment
147+
run: |
148+
git clone -b main --depth 1 --single-branch https://github.com/irods/irods_testing_environment
149+
cd irods_testing_environment
150+
python3 -m venv venv_testing_env
151+
. venv_testing_env/bin/activate
152+
python --version
153+
pip install docker-compose GitPython
154+
pip install docker'<7' requests==2.26.0
155+
pip freeze
156+
157+
- name: Run tests
158+
run: |
159+
# Create a symlink to the directory containing the recently built plugin packages.
160+
# This is required by the irods_python_irods_ci_utilities module.
161+
ln -s build_artifacts/_build "${{ inputs.plugin_package_directory }}"
162+
163+
# Move the icommands package into the same directory containing the server packages.
164+
# This is required by the testing environment.
165+
mv build_artifacts/_build_icommands/*.rpm build_artifacts/_build_irods
166+
167+
cd irods_testing_environment
168+
. venv_testing_env/bin/activate
169+
project_dir="${{ inputs.irods_testing_environment_project_os }}/${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}"
170+
python run_plugin_tests.py \
171+
"${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" \
172+
--project-directory "projects/${project_dir}" \
173+
--irods-package-directory ../build_artifacts/_build_irods \
174+
--plugin-package-directory .. \
175+
--discard-logs \
176+
-v

0 commit comments

Comments
 (0)