From 892d15547e76a08c702a7f13c78c97e6a1dd721a Mon Sep 17 00:00:00 2001 From: Kory Draughn Date: Fri, 23 Jan 2026 19:26:58 -0500 Subject: [PATCH] [#26] Add workflows for testing plugins against pre-release versions of iRODS --- ...test-plugin-with-irods-packages-debian.yml | 192 +++++++++++++++ ...n-with-irods-packages-enterprise-linux.yml | 183 ++++++++++++++ ...test-plugin-with-irods-packages-matrix.yml | 164 +++++++++++++ ...in-with-irods-packages-pipeline-debian.yml | 72 ++++++ ...ods-packages-pipeline-enterprise-linux.yml | 72 ++++++ ...in-with-irods-packages-pipeline-ubuntu.yml | 72 ++++++ ...test-plugin-with-irods-packages-ubuntu.yml | 196 +++++++++++++++ ...ld-irods-and-icommands-packages-debian.yml | 226 +++++++++++++++++ ...nd-icommands-packages-enterprise-linux.yml | 218 +++++++++++++++++ ...ld-irods-and-icommands-packages-ubuntu.yml | 229 ++++++++++++++++++ 10 files changed, 1624 insertions(+) create mode 100644 .github/workflows/build-and-test-plugin-with-irods-packages-debian.yml create mode 100644 .github/workflows/build-and-test-plugin-with-irods-packages-enterprise-linux.yml create mode 100644 .github/workflows/build-and-test-plugin-with-irods-packages-matrix.yml create mode 100644 .github/workflows/build-and-test-plugin-with-irods-packages-pipeline-debian.yml create mode 100644 .github/workflows/build-and-test-plugin-with-irods-packages-pipeline-enterprise-linux.yml create mode 100644 .github/workflows/build-and-test-plugin-with-irods-packages-pipeline-ubuntu.yml create mode 100644 .github/workflows/build-and-test-plugin-with-irods-packages-ubuntu.yml create mode 100644 .github/workflows/build-irods-and-icommands-packages-debian.yml create mode 100644 .github/workflows/build-irods-and-icommands-packages-enterprise-linux.yml create mode 100644 .github/workflows/build-irods-and-icommands-packages-ubuntu.yml diff --git a/.github/workflows/build-and-test-plugin-with-irods-packages-debian.yml b/.github/workflows/build-and-test-plugin-with-irods-packages-debian.yml new file mode 100644 index 0000000..48deee0 --- /dev/null +++ b/.github/workflows/build-and-test-plugin-with-irods-packages-debian.yml @@ -0,0 +1,192 @@ +# This workflow builds a plugin using pre-release iRODS packages and runs the +# plugin tests using the irods/irods_testing_environment GitHub repository. +# +# This particular workflow targets Debian platforms. + +name: build-and-test-plugin-with-irods-packages-debian + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile the plugin (e.g. debian:13). + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. debian_13-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. 'Debian gnu_linux_13'). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. debian-13). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + cache_key_irods_packages: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + +defaults: + run: + shell: bash + +jobs: + build_plugin: + name: Build plugin - ${{ inputs.docker_image }} + + runs-on: ubuntu-latest + container: ${{ inputs.docker_image }} + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Install prerequisite packages + run: | + apt-get update -qq + apt-get install -qq \ + build-essential \ + cmake \ + git \ + gnupg \ + lsb-release \ + python3-distro \ + python3-packaging \ + python3-pip \ + python3-setuptools \ + wget + + - name: Set up iRODS externals package repository + run: | + mkdir -p /etc/apt/keyrings + wget -qO - https://unstable.irods.org/irods-unstable-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp \ + --import + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp arch=amd64] https://unstable.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods-unstable.list + + - name: Check cache for iRODS packages + uses: actions/cache/restore@v4 + with: + key: ${{ inputs.cache_key_irods_packages }} + path: _build_irods/*.deb + fail-on-cache-miss: 'true' + + - name: Check cache for iCommands packages + uses: actions/cache/restore@v4 + with: + key: ${{ inputs.cache_key_icommands_packages }} + path: _build_icommands/*.deb + fail-on-cache-miss: 'true' + + - name: Install iRODS development packages + run: | + apt-get update -qq + apt-get install -qq \ + ./_build_irods/irods-dev*.deb \ + ./_build_irods/irods-runtime*.deb + + - name: Install iRODS python CI utilities + run: | + # TODO(#18): Investigate how to remove the need for --break-system-packages. + python3 -m pip install --break-system-packages git+https://github.com/irods/irods_python_ci_utilities.git@main + + - name: Build and package + run: | + mkdir _build + python3 irods_consortium_continuous_integration_build_hook.py --build_directory _build + + - name: List contents of build directory + run: ls -l _build + + # The iRODS and iCommands packages were built and cached while inside of a docker container. + # We cannot use actions/cache to get the iRODS and iCommands packages in the next job because + # GitHub Actions ties the cache to the environment. For this workflow, the "run_tests" job does + # not run inside of a container. This leads to a cache miss, causing the workflow to fail. To + # get around that, we use actions/upload-artifact and actions/download-artifact. + - name: Upload iRODS packages for dependent jobs + uses: actions/upload-artifact@v6 + with: + name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }} + path: | + _build/*.deb + _build_irods/*.deb + _build_icommands/*.deb + overwrite: true + + run_tests: + needs: build_plugin + + name: Test plugin - ${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }} + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + # TODO(irods/irods_testing_environment#264): Remove this block once the testing environment + # no longer relies on the docker compose python library. + - name: Set up python for iRODS testing environment + uses: actions/setup-python@v6 + with: + python-version: '3.8' + + - name: Download iRODS packages + uses: actions/download-artifact@v7 + with: + name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }} + path: build_artifacts + + - name: Set up docker compose + uses: docker/setup-compose-action@v1 + + - name: Set up iRODS testing environment + run: | + git clone -b main --depth 1 --single-branch https://github.com/irods/irods_testing_environment + cd irods_testing_environment + python3 -m venv venv_testing_env + . venv_testing_env/bin/activate + python --version + pip install docker-compose GitPython + pip install docker'<7' requests==2.26.0 + pip freeze + + - name: Run tests + run: | + # Create a symlink to the directory containing the recently built plugin packages. + # This is required by the irods_python_irods_ci_utilities module. + ln -s build_artifacts/_build "${{ inputs.plugin_package_directory }}" + + # Move the icommands package into the same directory containing the server packages. + # This is required by the testing environment. + mv build_artifacts/_build_icommands/*.deb build_artifacts/_build_irods + + cd irods_testing_environment + . venv_testing_env/bin/activate + project_dir="${{ inputs.irods_testing_environment_project_os }}/${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}" + python run_plugin_tests.py \ + "${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" \ + --project-directory "projects/${project_dir}" \ + --irods-package-directory ../build_artifacts/_build_irods \ + --plugin-package-directory .. \ + --discard-logs \ + -v diff --git a/.github/workflows/build-and-test-plugin-with-irods-packages-enterprise-linux.yml b/.github/workflows/build-and-test-plugin-with-irods-packages-enterprise-linux.yml new file mode 100644 index 0000000..a5ec64d --- /dev/null +++ b/.github/workflows/build-and-test-plugin-with-irods-packages-enterprise-linux.yml @@ -0,0 +1,183 @@ +# This workflow builds a plugin using pre-release iRODS packages and runs the +# plugin tests using the irods/irods_testing_environment GitHub repository. +# +# This particular workflow targets Enterprise Linux platforms. + +name: build-and-test-plugin-with-irods-packages-enterprise-linux + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile the plugin (e.g. rockylinux/rockylinux:10). + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. rockylinux_10-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. 'Rocky linux_10'). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. rockylinux-10). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + cache_key_irods_packages: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + +defaults: + run: + shell: bash + +jobs: + build_plugin: + name: Build plugin - ${{ inputs.docker_image }} + + runs-on: ubuntu-latest + container: ${{ inputs.docker_image }} + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Install prerequisite packages + run: | + dnf update -y + dnf install -y \ + dnf-plugins-core \ + epel-release \ + gcc-c++ \ + git \ + rpm-build \ + sudo \ + wget + dnf install -y \ + cmake \ + python3-devel \ + python3-distro \ + python3-packaging \ + python3-pip + + - name: Set up iRODS externals package repository + run: | + rpm --import https://unstable.irods.org/irods-unstable-signing-key.asc + wget -qO - https://unstable.irods.org/renci-irods-unstable.yum.repo | tee /etc/yum.repos.d/renci-irods-unstable.yum.repo + + - name: Check cache for iRODS packages + uses: actions/cache/restore@v4 + with: + key: ${{ inputs.cache_key_irods_packages }} + path: _build_irods/*.rpm + fail-on-cache-miss: 'true' + + - name: Check cache for iCommands packages + uses: actions/cache/restore@v4 + with: + key: ${{ inputs.cache_key_icommands_packages }} + path: _build_icommands/*.rpm + fail-on-cache-miss: 'true' + + - name: Install iRODS development packages + run: | + dnf update -y + dnf install -y \ + ./_build_irods/irods-devel*.rpm \ + ./_build_irods/irods-runtime*.rpm + + - name: Install iRODS python CI utilities + run: | + python3 -m pip install git+https://github.com/irods/irods_python_ci_utilities.git@main + + - name: Build and package + run: | + mkdir _build + python3 irods_consortium_continuous_integration_build_hook.py --build_directory _build + + - name: List contents of build directory + run: ls -l _build + + # The iRODS and iCommands packages were built and cached while inside of a docker container. + # We cannot use actions/cache to get the iRODS and iCommands packages in the next job because + # GitHub Actions ties the cache to the environment. For this workflow, the "run_tests" job does + # not run inside of a container. This leads to a cache miss, causing the workflow to fail. To + # get around that, we use actions/upload-artifact and actions/download-artifact. + - name: Upload iRODS packages for dependent jobs + uses: actions/upload-artifact@v6 + with: + name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }} + path: | + _build/*.rpm + _build_irods/*.rpm + _build_icommands/*.rpm + overwrite: true + + run_tests: + needs: build_plugin + + name: Test plugin - ${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }} + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + # TODO(irods/irods_testing_environment#264): Remove this block once the testing environment + # no longer relies on the docker compose python library. + - name: Set up python for iRODS testing environment + uses: actions/setup-python@v6 + with: + python-version: '3.8' + + - name: Download iRODS packages + uses: actions/download-artifact@v7 + with: + name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }} + path: build_artifacts + + - name: Set up docker compose + uses: docker/setup-compose-action@v1 + + - name: Set up iRODS testing environment + run: | + git clone -b main --depth 1 --single-branch https://github.com/irods/irods_testing_environment + cd irods_testing_environment + python3 -m venv venv_testing_env + . venv_testing_env/bin/activate + python --version + pip install docker-compose GitPython + pip install docker'<7' requests==2.26.0 + pip freeze + + - name: Run tests + run: | + # Create a symlink to the directory containing the recently built plugin packages. + # This is required by the irods_python_irods_ci_utilities module. + ln -s build_artifacts/_build "${{ inputs.plugin_package_directory }}" + + # Move the icommands package into the same directory containing the server packages. + # This is required by the testing environment. + mv build_artifacts/_build_icommands/*.rpm build_artifacts/_build_irods + + cd irods_testing_environment + . venv_testing_env/bin/activate + project_dir="${{ inputs.irods_testing_environment_project_os }}/${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}" + python run_plugin_tests.py \ + "${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" \ + --project-directory "projects/${project_dir}" \ + --irods-package-directory ../build_artifacts/_build_irods \ + --plugin-package-directory .. \ + --discard-logs \ + -v diff --git a/.github/workflows/build-and-test-plugin-with-irods-packages-matrix.yml b/.github/workflows/build-and-test-plugin-with-irods-packages-matrix.yml new file mode 100644 index 0000000..233d292 --- /dev/null +++ b/.github/workflows/build-and-test-plugin-with-irods-packages-matrix.yml @@ -0,0 +1,164 @@ +# This workflow defines the matrix used to build and test iRODS plugins using +# pre-release iRODS server and iCommands packages, for all supported platforms. +# +# To use this workflow, repositories must meet the requirements for plugin +# testing, as documented at the following URLs. +# +# - https://github.com/irods/irods_development_environment#how-to-build-an-irods-plugin +# - https://github.com/irods/irods_testing_environment#run-irods-plugin-tests + +name: build-and-test-plugin-with-irods-packages-matrix + +on: + workflow_call: + inputs: + test_against_mysql: + description: Run tests against MySQL. + required: false + type: boolean + default: false + test_against_mariadb: + description: Run tests against MariaDB. + required: false + type: boolean + default: false + +defaults: + run: + shell: bash + +jobs: + debian: + strategy: + matrix: + include: + # Debian 12 + - docker_image: debian:12 + cache_key_irods_packages_prefix: cache-irods-debian-12 + cache_key_icommands_packages_prefix: cache-icommands-debian-12 + ccache_key: ccache-irods-debian-12 + build_artifact_suffix: debian_12-main + plugin_package_directory: 'Debian gnu_linux_12' + irods_testing_environment_project_os: debian-12 + irods_testing_environment_project_db: postgres-16 + + # Debian 13 + - docker_image: debian:13 + cache_key_irods_packages_prefix: cache-irods-debian-13 + cache_key_icommands_packages_prefix: cache-icommands-debian-13 + ccache_key: ccache-irods-debian-13 + build_artifact_suffix: debian_13-main + plugin_package_directory: 'Debian gnu_linux_13' + irods_testing_environment_project_os: debian-13 + irods_testing_environment_project_db: postgres-16 + + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-debian.yml@main + with: + docker_image: ${{ matrix.docker_image }} + cache_key_irods_packages_prefix: ${{ matrix.cache_key_irods_packages_prefix }} + cache_key_icommands_packages_prefix: ${{ matrix.cache_key_icommands_packages_prefix }} + ccache_key: ${{ matrix.ccache_key }} + build_artifact_suffix: ${{ matrix.build_artifact_suffix }} + plugin_package_directory: ${{ matrix.plugin_package_directory }} + irods_testing_environment_project_os: ${{ matrix.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ matrix.irods_testing_environment_project_db }} + + enterprise_linux: + strategy: + matrix: + include: + # EL9 + - docker_image: rockylinux/rockylinux:9 + cache_key_irods_packages_prefix: cache-irods-el9 + cache_key_icommands_packages_prefix: cache-icommands-el9 + ccache_key: ccache-irods-el9 + build_artifact_suffix: rockylinux_9-main + plugin_package_directory: 'Rocky linux_9' + irods_testing_environment_project_os: rockylinux-9 + irods_testing_environment_project_db: postgres-16 + + # EL10 + - docker_image: rockylinux/rockylinux:10 + cache_key_irods_packages_prefix: cache-irods-el10 + cache_key_icommands_packages_prefix: cache-icommands-el10 + ccache_key: ccache-irods-el10 + build_artifact_suffix: rockylinux_10-main + plugin_package_directory: 'Rocky linux_10' + irods_testing_environment_project_os: rockylinux-10 + irods_testing_environment_project_db: postgres-16 + + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-enterprise-linux.yml@main + with: + docker_image: ${{ matrix.docker_image }} + cache_key_irods_packages_prefix: ${{ matrix.cache_key_irods_packages_prefix }} + cache_key_icommands_packages_prefix: ${{ matrix.cache_key_icommands_packages_prefix }} + ccache_key: ${{ matrix.ccache_key }} + build_artifact_suffix: ${{ matrix.build_artifact_suffix }} + plugin_package_directory: ${{ matrix.plugin_package_directory }} + irods_testing_environment_project_os: ${{ matrix.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ matrix.irods_testing_environment_project_db }} + + ubuntu: + strategy: + matrix: + include: + # Ubuntu 22.04 + - docker_image: ubuntu:22.04 + cache_key_irods_packages_prefix: cache-irods-ubuntu-22.04 + cache_key_icommands_packages_prefix: cache-icommands-ubuntu-22.04 + ccache_key: ccache-irods-ubuntu-22.04 + build_artifact_suffix: ubuntu_22-main + plugin_package_directory: Ubuntu_22 + irods_testing_environment_project_os: ubuntu-22.04 + irods_testing_environment_project_db: postgres-16 + + # Ubuntu 24.04 + - docker_image: ubuntu:24.04 + cache_key_irods_packages_prefix: cache-irods-ubuntu-24.04 + cache_key_icommands_packages_prefix: cache-icommands-ubuntu-24.04 + ccache_key: ccache-irods-ubuntu-24.04 + build_artifact_suffix: ubuntu_24-main + plugin_package_directory: Ubuntu_24 + irods_testing_environment_project_os: ubuntu-24.04 + irods_testing_environment_project_db: postgres-16 + + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-ubuntu.yml@main + with: + docker_image: ${{ matrix.docker_image }} + cache_key_irods_packages_prefix: ${{ matrix.cache_key_irods_packages_prefix }} + cache_key_icommands_packages_prefix: ${{ matrix.cache_key_icommands_packages_prefix }} + ccache_key: ${{ matrix.ccache_key }} + build_artifact_suffix: ${{ matrix.build_artifact_suffix }} + plugin_package_directory: ${{ matrix.plugin_package_directory }} + irods_testing_environment_project_os: ${{ matrix.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ matrix.irods_testing_environment_project_db }} + + mysql: + needs: ubuntu + + if: ${{ inputs.test_against_mysql }} + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-ubuntu.yml@main + with: + docker_image: ubuntu:22.04 + cache_key_irods_packages_prefix: cache-irods-ubuntu-22.04 + cache_key_icommands_packages_prefix: cache-icommands-ubuntu-22.04 + ccache_key: ccache-irods-ubuntu-22.04 + build_artifact_suffix: ubuntu_22-main + plugin_package_directory: Ubuntu_22 + irods_testing_environment_project_os: ubuntu-22.04 + irods_testing_environment_project_db: mysql-8.0 + + mariadb: + needs: ubuntu + + if: ${{ inputs.test_against_mariadb }} + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-ubuntu.yml@main + with: + docker_image: ubuntu:22.04 + cache_key_irods_packages_prefix: cache-irods-ubuntu-22.04 + cache_key_icommands_packages_prefix: cache-icommands-ubuntu-22.04 + ccache_key: ccache-irods-ubuntu-22.04 + build_artifact_suffix: ubuntu_22-main + plugin_package_directory: Ubuntu_22 + irods_testing_environment_project_os: ubuntu-22.04 + irods_testing_environment_project_db: mariadb-10.11 diff --git a/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-debian.yml b/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-debian.yml new file mode 100644 index 0000000..71fe33f --- /dev/null +++ b/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-debian.yml @@ -0,0 +1,72 @@ +# This workflow is used to sequence the building of iRODS/iCommands packages, +# plugins, and testing of plugins. +# +# This particular workflow targets Debian platforms. + +name: build-and-test-plugin-with-irods-packages-pipeline-debian + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile source code (e.g. debian:13). + required: true + type: string + cache_key_irods_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + ccache_key: + description: The key which potentially identifies ccache state from a previous build. + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. debian_13-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. 'Debian gnu_linux_13'). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. debian-13). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + +defaults: + run: + shell: bash + +jobs: + build_irods_packages: + uses: irods/irods_reusable_github_workflows/.github/workflows/build-irods-and-icommands-packages-debian.yml@main + with: + docker_image: ${{ inputs.docker_image }} + cache_key_irods_packages_prefix: ${{ inputs.cache_key_irods_packages_prefix }} + cache_key_icommands_packages_prefix: ${{ inputs.cache_key_icommands_packages_prefix }} + ccache_key: ${{ inputs.ccache_key }} + build_artifact_suffix: ${{ inputs.build_artifact_suffix }} + plugin_package_directory: ${{ inputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ inputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ inputs.irods_testing_environment_project_db }} + + build_and_test_plugin_with_irods_packages: + needs: build_irods_packages + + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-debian.yml@main + with: + docker_image: ${{ needs.build_irods_packages.outputs.docker_image }} + build_artifact_suffix: ${{ needs.build_irods_packages.outputs.build_artifact_suffix }} + plugin_package_directory: ${{ needs.build_irods_packages.outputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ needs.build_irods_packages.outputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ needs.build_irods_packages.outputs.irods_testing_environment_project_db }} + cache_key_irods_packages: ${{ needs.build_irods_packages.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: ${{ needs.build_irods_packages.outputs.cache_key_icommands_packages }} diff --git a/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-enterprise-linux.yml b/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-enterprise-linux.yml new file mode 100644 index 0000000..9f9d654 --- /dev/null +++ b/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-enterprise-linux.yml @@ -0,0 +1,72 @@ +# This workflow is used to sequence the building of iRODS/iCommands packages, +# plugins, and testing of plugins. +# +# This particular workflow targets Enterprise Linux platforms. + +name: build-and-test-plugin-with-irods-packages-pipeline-enterprise-linux + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile source code (e.g. rockylinux/rockylinux:10). + required: true + type: string + cache_key_irods_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + ccache_key: + description: The key which potentially identifies ccache state from a previous build. + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. rockylinux_10-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. 'Rocky linux_10'). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. rockylinux-10). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + +defaults: + run: + shell: bash + +jobs: + build_irods_packages: + uses: irods/irods_reusable_github_workflows/.github/workflows/build-irods-and-icommands-packages-enterprise-linux.yml@main + with: + docker_image: ${{ inputs.docker_image }} + cache_key_irods_packages_prefix: ${{ inputs.cache_key_irods_packages_prefix }} + cache_key_icommands_packages_prefix: ${{ inputs.cache_key_icommands_packages_prefix }} + ccache_key: ${{ inputs.ccache_key }} + build_artifact_suffix: ${{ inputs.build_artifact_suffix }} + plugin_package_directory: ${{ inputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ inputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ inputs.irods_testing_environment_project_db }} + + build_and_test_plugin_with_irods_packages: + needs: build_irods_packages + + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-enterprise-linux.yml@main + with: + docker_image: ${{ needs.build_irods_packages.outputs.docker_image }} + build_artifact_suffix: ${{ needs.build_irods_packages.outputs.build_artifact_suffix }} + plugin_package_directory: ${{ needs.build_irods_packages.outputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ needs.build_irods_packages.outputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ needs.build_irods_packages.outputs.irods_testing_environment_project_db }} + cache_key_irods_packages: ${{ needs.build_irods_packages.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: ${{ needs.build_irods_packages.outputs.cache_key_icommands_packages }} diff --git a/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-ubuntu.yml b/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-ubuntu.yml new file mode 100644 index 0000000..8768066 --- /dev/null +++ b/.github/workflows/build-and-test-plugin-with-irods-packages-pipeline-ubuntu.yml @@ -0,0 +1,72 @@ +# This workflow is used to sequence the building of iRODS/iCommands packages, +# plugins, and testing of plugins. +# +# This particular workflow targets Ubuntu platforms. + +name: build-and-test-plugin-with-irods-packages-pipeline-ubuntu + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile source code (e.g. ubuntu:24.04). + required: true + type: string + cache_key_irods_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + ccache_key: + description: The key which potentially identifies ccache state from a previous build. + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. ubuntu_24-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. Ubuntu_24). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. ubuntu-24.04). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + +defaults: + run: + shell: bash + +jobs: + build_irods_packages: + uses: irods/irods_reusable_github_workflows/.github/workflows/build-irods-and-icommands-packages-ubuntu.yml@main + with: + docker_image: ${{ inputs.docker_image }} + cache_key_irods_packages_prefix: ${{ inputs.cache_key_irods_packages_prefix }} + cache_key_icommands_packages_prefix: ${{ inputs.cache_key_icommands_packages_prefix }} + ccache_key: ${{ inputs.ccache_key }} + build_artifact_suffix: ${{ inputs.build_artifact_suffix }} + plugin_package_directory: ${{ inputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ inputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ inputs.irods_testing_environment_project_db }} + + build_and_test_plugin_with_irods_packages: + needs: build_irods_packages + + uses: irods/irods_reusable_github_workflows/.github/workflows/build-and-test-plugin-with-irods-packages-ubuntu.yml@main + with: + docker_image: ${{ needs.build_irods_packages.outputs.docker_image }} + build_artifact_suffix: ${{ needs.build_irods_packages.outputs.build_artifact_suffix }} + plugin_package_directory: ${{ needs.build_irods_packages.outputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ needs.build_irods_packages.outputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ needs.build_irods_packages.outputs.irods_testing_environment_project_db }} + cache_key_irods_packages: ${{ needs.build_irods_packages.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: ${{ needs.build_irods_packages.outputs.cache_key_icommands_packages }} diff --git a/.github/workflows/build-and-test-plugin-with-irods-packages-ubuntu.yml b/.github/workflows/build-and-test-plugin-with-irods-packages-ubuntu.yml new file mode 100644 index 0000000..23e8517 --- /dev/null +++ b/.github/workflows/build-and-test-plugin-with-irods-packages-ubuntu.yml @@ -0,0 +1,196 @@ +# This workflow builds a plugin using pre-release iRODS packages and runs the +# plugin tests using the irods/irods_testing_environment GitHub repository. +# +# This particular workflow targets Ubuntu platforms. + +name: build-and-test-plugin-with-irods-packages-ubuntu + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile the plugin (e.g. ubuntu:24.04). + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. ubuntu_24-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. Ubuntu_24). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. ubuntu-24.04). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + cache_key_irods_packages: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + +defaults: + run: + shell: bash + +jobs: + build_plugin: + name: Build plugin - ${{ inputs.docker_image }} + + runs-on: ubuntu-latest + container: ${{ inputs.docker_image }} + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Install prerequisite packages + run: | + apt-get update -qq + apt-get install -qq \ + build-essential \ + cmake \ + git \ + gnupg \ + lsb-release \ + python3-distro \ + python3-packaging \ + python3-pip \ + python3-setuptools \ + wget + + - name: Set up iRODS externals package repository + run: | + mkdir -p /etc/apt/keyrings + wget -qO - https://unstable.irods.org/irods-unstable-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp \ + --import + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp arch=amd64] https://unstable.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods-unstable.list + + - name: Check cache for iRODS packages + uses: actions/cache/restore@v4 + with: + key: ${{ inputs.cache_key_irods_packages }} + path: _build_irods/*.deb + fail-on-cache-miss: 'true' + + - name: Check cache for iCommands packages + uses: actions/cache/restore@v4 + with: + key: ${{ inputs.cache_key_icommands_packages }} + path: _build_icommands/*.deb + fail-on-cache-miss: 'true' + + - name: Install iRODS development packages + run: | + apt-get update -qq + apt-get install -qq \ + ./_build_irods/irods-dev*.deb \ + ./_build_irods/irods-runtime*.deb + + - name: Install iRODS python CI utilities + run: | + if [[ "${{ inputs.docker_image }}" == "ubuntu:22.04" ]]; then + python3 -m pip install git+https://github.com/irods/irods_python_ci_utilities.git@main + else + # TODO(#18): Investigate how to remove the need for --break-system-packages. + python3 -m pip install --break-system-packages git+https://github.com/irods/irods_python_ci_utilities.git@main + fi + + - name: Build and package + run: | + mkdir _build + python3 irods_consortium_continuous_integration_build_hook.py --build_directory _build + + - name: List contents of build directory + run: ls -l _build + + # The iRODS and iCommands packages were built and cached while inside of a docker container. + # We cannot use actions/cache to get the iRODS and iCommands packages in the next job because + # GitHub Actions ties the cache to the environment. For this workflow, the "run_tests" job does + # not run inside of a container. This leads to a cache miss, causing the workflow to fail. To + # get around that, we use actions/upload-artifact and actions/download-artifact. + - name: Upload iRODS packages for dependent jobs + uses: actions/upload-artifact@v6 + with: + name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }} + path: | + _build/*.deb + _build_irods/*.deb + _build_icommands/*.deb + overwrite: true + + run_tests: + needs: build_plugin + + name: Test plugin - ${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }} + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + # TODO(irods/irods_testing_environment#264): Remove this block once the testing environment + # no longer relies on the docker compose python library. + - name: Set up python for iRODS testing environment + uses: actions/setup-python@v6 + with: + python-version: '3.8' + + - name: Download iRODS packages + uses: actions/download-artifact@v7 + with: + name: irods-packages-prerelease-${{ inputs.build_artifact_suffix }} + path: build_artifacts + + - name: Set up docker compose + uses: docker/setup-compose-action@v1 + + - name: Set up iRODS testing environment + run: | + git clone -b main --depth 1 --single-branch https://github.com/irods/irods_testing_environment + cd irods_testing_environment + python3 -m venv venv_testing_env + . venv_testing_env/bin/activate + python --version + pip install docker-compose GitPython + pip install docker'<7' requests==2.26.0 + pip freeze + + - name: Run tests + run: | + # Create a symlink to the directory containing the recently built plugin packages. + # This is required by the irods_python_irods_ci_utilities module. + ln -s build_artifacts/_build "${{ inputs.plugin_package_directory }}" + + # Move the icommands package into the same directory containing the server packages. + # This is required by the testing environment. + mv build_artifacts/_build_icommands/*.deb build_artifacts/_build_irods + + cd irods_testing_environment + . venv_testing_env/bin/activate + project_dir="${{ inputs.irods_testing_environment_project_os }}/${{ inputs.irods_testing_environment_project_os }}-${{ inputs.irods_testing_environment_project_db }}" + python run_plugin_tests.py \ + "${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" \ + --project-directory "projects/${project_dir}" \ + --irods-package-directory ../build_artifacts/_build_irods \ + --plugin-package-directory .. \ + --discard-logs \ + -v diff --git a/.github/workflows/build-irods-and-icommands-packages-debian.yml b/.github/workflows/build-irods-and-icommands-packages-debian.yml new file mode 100644 index 0000000..6a9dbba --- /dev/null +++ b/.github/workflows/build-irods-and-icommands-packages-debian.yml @@ -0,0 +1,226 @@ +# This workflow compiles the main branches of the irods/irods and +# irods/irods_client_icommands GitHub repositories. +# +# Newly built iRODS packages are stored as artifacts for other jobs and +# workflows to use. This workflow also produces various pieces of information +# which other workflows can use. +# +# This particular workflow targets Debian platforms. + +name: build-irods-and-icommands-packages-debian + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile source code (e.g. debian:13). + required: true + type: string + cache_key_irods_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + ccache_key: + description: The key which potentially identifies ccache state from a previous build. + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. debian_13-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. 'Debian gnu_linux_13'). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. debian-13). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + default: postgres-16 + + # Make various pieces of information available to other workflows. + outputs: + docker_image: + value: ${{ jobs.build_irods_and_icommands.outputs.docker_image }} + build_artifact_suffix: + value: ${{ jobs.build_irods_and_icommands.outputs.build_artifact_suffix }} + plugin_package_directory: + value: ${{ jobs.build_irods_and_icommands.outputs.plugin_package_directory }} + irods_testing_environment_project_os: + value: ${{ jobs.build_irods_and_icommands.outputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: + value: ${{ jobs.build_irods_and_icommands.outputs.irods_testing_environment_project_db }} + build_sha_irods: + value: ${{ jobs.build_irods_and_icommands.outputs.sha_irods }} + build_sha_icommands: + value: ${{ jobs.build_irods_and_icommands.outputs.sha_icommands }} + cache_key_irods_packages: + value: ${{ jobs.build_irods_and_icommands.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: + value: ${{ jobs.build_irods_and_icommands.outputs.cache_key_icommands_packages }} + +defaults: + run: + shell: bash + +jobs: + build_irods_and_icommands: + name: Build iRODS and iCommands packages - ${{ inputs.docker_image }} + + runs-on: ubuntu-latest + container: ${{ inputs.docker_image }} + + # Make various pieces of information available to other jobs. + outputs: + docker_image: ${{ inputs.docker_image }} + build_artifact_suffix: ${{ inputs.build_artifact_suffix }} + plugin_package_directory: ${{ inputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ inputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ inputs.irods_testing_environment_project_db }} + build_sha_irods: ${{ steps.init.outputs.sha_irods }} + build_sha_icommands: ${{ steps.init.outputs.sha_icommands }} + cache_key_irods_packages: ${{ steps.init.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: ${{ steps.init.outputs.cache_key_icommands_packages }} + + steps: + - name: Generate keys for caching newly built iRODS packages + id: init + run: | + apt-get update -qq + apt-get install -qq git + + git clone -b main --depth 1 --single-branch https://github.com/irods/irods + git clone -b main --depth 1 --single-branch https://github.com/irods/irods_client_icommands + + # Make SHAs available to other steps and jobs within the workflow. + sha_irods=$(git -C irods rev-parse HEAD) + sha_icmds=$(git -C irods_client_icommands rev-parse HEAD) + echo "sha_irods=$sha_irods" >> $GITHUB_OUTPUT + echo "sha_icommands=$sha_icmds" >> $GITHUB_OUTPUT + + # Initialize keys for caching unreleased packages. + echo "cache_key_irods_packages=${{ inputs.cache_key_irods_packages_prefix }}-$sha_irods" >> $GITHUB_OUTPUT + echo "cache_key_icommands_packages=${{ inputs.cache_key_icommands_packages_prefix }}-$sha_icmds" >> $GITHUB_OUTPUT + + - name: Check cache for iRODS packages + id: irods + uses: actions/cache@v4 + with: + key: ${{ steps.init.outputs.cache_key_irods_packages }} + path: _build_irods/*.deb + + - name: Check cache for iCommands package + id: icommands + uses: actions/cache@v4 + with: + key: ${{ steps.init.outputs.cache_key_icommands_packages }} + path: _build_icommands/*.deb + + - name: Install prerequisite packages + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + apt-get update -qq + apt-get install -qq \ + bison \ + ca-certificates \ + catch2 \ + cmake \ + curl \ + flex \ + g++ \ + gnupg \ + libarchive-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libfmt-dev \ + libpam0g-dev \ + libspdlog-dev \ + libssl-dev \ + libsystemd-dev \ + libxml2-dev \ + ninja-build \ + nlohmann-json3-dev \ + odbc-postgresql \ + python3-dev \ + python3-distro \ + python3-psutil \ + unixodbc \ + unixodbc-dev \ + wget \ + zlib1g-dev + + - name: Set up iRODS externals package repository + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + mkdir -p /etc/apt/keyrings + wget -qO - https://unstable.irods.org/irods-unstable-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp \ + --import + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp arch=amd64] https://unstable.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods-unstable.list + apt-get update -qq + apt-get install -qq \ + irods-externals-boost1.81.0-2 \ + irods-externals-clang16.0.6-0 \ + irods-externals-jsoncons0.178.0-0 \ + irods-externals-nanodbc2.13.0-3 + + - name: Configure ccache + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + apt-get install -qq ccache + echo "CCACHE_DIR=/irods_build_cache" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV + + - name: Restore ccache state + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + key: ${{ inputs.ccache_key }} + path: /irods_build_cache/ + + - name: Build iRODS packages + if: steps.irods.outputs.cache-hit != 'true' + run: | + mkdir _build_irods + cmake \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja \ + -B _build_irods \ + -S irods + cmake --build _build_irods --target package + + - name: Build iCommands packages + # The iRODS cache hit condition (i.e. steps.irods.outputs.cache-hit) exists to make sure + # the iCommands are always compiled against the newly built iRODS packages. This avoids + # ABI issues on the iRODS code level. If ABI stability ever becomes a requirement for iRODS, + # then we can consider removing the condition for the iRODS cache hit condition. + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + apt-get install -qq ./_build_irods/irods-{dev,runtime}*.deb + mkdir _build_icommands + cmake \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja \ + -B _build_icommands \ + -S irods_client_icommands + cmake --build _build_icommands --target package diff --git a/.github/workflows/build-irods-and-icommands-packages-enterprise-linux.yml b/.github/workflows/build-irods-and-icommands-packages-enterprise-linux.yml new file mode 100644 index 0000000..f2b67ca --- /dev/null +++ b/.github/workflows/build-irods-and-icommands-packages-enterprise-linux.yml @@ -0,0 +1,218 @@ +# This workflow compiles the main branches of the irods/irods and +# irods/irods_client_icommands GitHub repositories. +# +# Newly built iRODS packages are stored as artifacts for other jobs and +# workflows to use. This workflow also produces various pieces of information +# which other workflows can use. +# +# This particular workflow targets Enterprise Linux platforms. + +name: build-irods-and-icommands-packages-enterprise-linux + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile source code (e.g. rockylinux/rockylinux:10). + required: true + type: string + cache_key_irods_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + ccache_key: + description: The key which potentially identifies ccache state from a previous build. + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. rockylinux_10-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. 'Rocky linux_10'). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. rockylinux-10). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + default: postgres-16 + + # Make various pieces of information available to other workflows. + outputs: + docker_image: + value: ${{ jobs.build_irods_and_icommands.outputs.docker_image }} + build_artifact_suffix: + value: ${{ jobs.build_irods_and_icommands.outputs.build_artifact_suffix }} + plugin_package_directory: + value: ${{ jobs.build_irods_and_icommands.outputs.plugin_package_directory }} + irods_testing_environment_project_os: + value: ${{ jobs.build_irods_and_icommands.outputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: + value: ${{ jobs.build_irods_and_icommands.outputs.irods_testing_environment_project_db }} + build_sha_irods: + value: ${{ jobs.build_irods_and_icommands.outputs.sha_irods }} + build_sha_icommands: + value: ${{ jobs.build_irods_and_icommands.outputs.sha_icommands }} + cache_key_irods_packages: + value: ${{ jobs.build_irods_and_icommands.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: + value: ${{ jobs.build_irods_and_icommands.outputs.cache_key_icommands_packages }} + +defaults: + run: + shell: bash + +jobs: + build_irods_and_icommands: + name: Build iRODS and iCommands packages - ${{ inputs.docker_image }} + + runs-on: ubuntu-latest + container: ${{ inputs.docker_image }} + + # Make various pieces of information available to other jobs. + outputs: + docker_image: ${{ inputs.docker_image }} + build_artifact_suffix: ${{ inputs.build_artifact_suffix }} + plugin_package_directory: ${{ inputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ inputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ inputs.irods_testing_environment_project_db }} + build_sha_irods: ${{ steps.init.outputs.sha_irods }} + build_sha_icommands: ${{ steps.init.outputs.sha_icommands }} + cache_key_irods_packages: ${{ steps.init.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: ${{ steps.init.outputs.cache_key_icommands_packages }} + + steps: + - name: Generate keys for caching newly built iRODS packages + id: init + run: | + dnf update -y + dnf install -y git + + git clone -b main --depth 1 --single-branch https://github.com/irods/irods + git clone -b main --depth 1 --single-branch https://github.com/irods/irods_client_icommands + + # Make SHAs available to other steps and jobs within the workflow. + sha_irods=$(git -C irods rev-parse HEAD) + sha_icmds=$(git -C irods_client_icommands rev-parse HEAD) + echo "sha_irods=$sha_irods" >> $GITHUB_OUTPUT + echo "sha_icommands=$sha_icmds" >> $GITHUB_OUTPUT + + # Initialize keys for caching unreleased packages. + echo "cache_key_irods_packages=${{ inputs.cache_key_irods_packages_prefix }}-$sha_irods" >> $GITHUB_OUTPUT + echo "cache_key_icommands_packages=${{ inputs.cache_key_icommands_packages_prefix }}-$sha_icmds" >> $GITHUB_OUTPUT + + - name: Check cache for iRODS packages + id: irods + uses: actions/cache@v4 + with: + key: ${{ steps.init.outputs.cache_key_irods_packages }} + path: _build_irods/*.rpm + + - name: Check cache for iCommands package + id: icommands + uses: actions/cache@v4 + with: + key: ${{ steps.init.outputs.cache_key_icommands_packages }} + path: _build_icommands/*.rpm + + - name: Install prerequisite packages + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + dnf update -y + dnf install -y \ + bind-utils \ + ca-certificates \ + dnf-plugins-core \ + epel-release \ + wget + dnf config-manager --set-enabled crb + dnf install -y \ + bison \ + bzip2-devel \ + catch2-devel \ + cmake \ + curl-devel \ + fakeroot \ + flex \ + fmt-devel \ + gcc \ + gcc-c++ \ + libarchive-devel \ + ninja-build \ + nlohmann_json-devel \ + openssl-devel \ + pam-devel \ + python3-devel \ + python3-distro \ + rpm-build \ + spdlog-devel \ + systemd-devel \ + unixODBC \ + unixODBC-devel \ + zlib-devel + + - name: Set up iRODS externals package repository + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + rpm --import https://unstable.irods.org/irods-unstable-signing-key.asc + wget -qO - https://unstable.irods.org/renci-irods-unstable.yum.repo | tee /etc/yum.repos.d/renci-irods-unstable.yum.repo + dnf update -y + dnf install -y \ + irods-externals-boost1.81.0-2 \ + irods-externals-clang16.0.6-0 \ + irods-externals-jsoncons0.178.0-0 \ + irods-externals-nanodbc2.13.0-3 + + - name: Configure ccache + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + dnf install -y ccache + echo "CCACHE_DIR=/irods_build_cache" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV + + - name: Restore ccache state + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + key: ${{ inputs.ccache_key }} + path: /irods_build_cache/ + + - name: Build iRODS packages + if: steps.irods.outputs.cache-hit != 'true' + run: | + mkdir _build_irods + cmake \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja \ + -B _build_irods \ + -S irods + cmake --build _build_irods --target package + + - name: Build iCommands packages + # The iRODS cache hit condition (i.e. steps.irods.outputs.cache-hit) exists to make sure + # the iCommands are always compiled against the newly built iRODS packages. This avoids + # ABI issues on the iRODS code level. If ABI stability ever becomes a requirement for iRODS, + # then we can consider removing the condition for the iRODS cache hit condition. + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + dnf install -y ./_build_irods/irods-{dev,runtime}*.rpm + mkdir _build_icommands + cmake \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja \ + -B _build_icommands \ + -S irods_client_icommands + cmake --build _build_icommands --target package diff --git a/.github/workflows/build-irods-and-icommands-packages-ubuntu.yml b/.github/workflows/build-irods-and-icommands-packages-ubuntu.yml new file mode 100644 index 0000000..753d1a6 --- /dev/null +++ b/.github/workflows/build-irods-and-icommands-packages-ubuntu.yml @@ -0,0 +1,229 @@ +# This workflow compiles the main branches of the irods/irods and +# irods/irods_client_icommands GitHub repositories. +# +# Newly built iRODS packages are stored as artifacts for other jobs and +# workflows to use. This workflow also produces various pieces of information +# which other workflows can use. +# +# This particular workflow targets Ubuntu platforms. + +name: build-irods-and-icommands-packages-ubuntu + +on: + workflow_call: + inputs: + docker_image: + description: The docker image and tag identifying the OS used to compile source code (e.g. ubuntu:24.04). + required: true + type: string + cache_key_irods_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods source code. + required: true + type: string + cache_key_icommands_packages_prefix: + description: The key which potentially identifies a cached build of the irods/irods_client_icommands source code. + required: true + type: string + ccache_key: + description: The key which potentially identifies ccache state from a previous build. + required: true + type: string + build_artifact_suffix: + description: The name used to uniquely identify build output across workflow jobs (e.g. ubuntu_24-main). + required: true + type: string + plugin_package_directory: + description: The directory name which is expected to hold the plugin packages (e.g. Ubuntu_24). Required by the irods_python_ci_utilities module. + required: true + type: string + irods_testing_environment_project_os: + description: The OS component of the project directory to test against. When joined with 'irods_testing_environment_project_db', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. ubuntu-24.04). + required: true + type: string + irods_testing_environment_project_db: + description: The DB component of the project directory to test against. When joined with 'irods_testing_environment_project_os', it must match a directory under the projects directory within the iRODS Testing Environment repository (e.g. postgres-16). + required: true + type: string + default: postgres-16 + + # Make various pieces of information available to other workflows. + outputs: + docker_image: + value: ${{ jobs.build_irods_and_icommands.outputs.docker_image }} + build_artifact_suffix: + value: ${{ jobs.build_irods_and_icommands.outputs.build_artifact_suffix }} + plugin_package_directory: + value: ${{ jobs.build_irods_and_icommands.outputs.plugin_package_directory }} + irods_testing_environment_project_os: + value: ${{ jobs.build_irods_and_icommands.outputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: + value: ${{ jobs.build_irods_and_icommands.outputs.irods_testing_environment_project_db }} + build_sha_irods: + value: ${{ jobs.build_irods_and_icommands.outputs.sha_irods }} + build_sha_icommands: + value: ${{ jobs.build_irods_and_icommands.outputs.sha_icommands }} + cache_key_irods_packages: + value: ${{ jobs.build_irods_and_icommands.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: + value: ${{ jobs.build_irods_and_icommands.outputs.cache_key_icommands_packages }} + +defaults: + run: + shell: bash + +jobs: + build_irods_and_icommands: + name: Build iRODS and iCommands packages - ${{ inputs.docker_image }} + + runs-on: ubuntu-latest + container: ${{ inputs.docker_image }} + + # Make various pieces of information available to other jobs. + outputs: + docker_image: ${{ inputs.docker_image }} + build_artifact_suffix: ${{ inputs.build_artifact_suffix }} + plugin_package_directory: ${{ inputs.plugin_package_directory }} + irods_testing_environment_project_os: ${{ inputs.irods_testing_environment_project_os }} + irods_testing_environment_project_db: ${{ inputs.irods_testing_environment_project_db }} + build_sha_irods: ${{ steps.init.outputs.sha_irods }} + build_sha_icommands: ${{ steps.init.outputs.sha_icommands }} + cache_key_irods_packages: ${{ steps.init.outputs.cache_key_irods_packages }} + cache_key_icommands_packages: ${{ steps.init.outputs.cache_key_icommands_packages }} + + steps: + - name: Generate keys for caching newly built iRODS packages + id: init + run: | + apt-get update -qq + apt-get install -qq git + + git clone -b main --depth 1 --single-branch https://github.com/irods/irods + git clone -b main --depth 1 --single-branch https://github.com/irods/irods_client_icommands + + # Make SHAs available to other steps and jobs within the workflow. + sha_irods=$(git -C irods rev-parse HEAD) + sha_icmds=$(git -C irods_client_icommands rev-parse HEAD) + echo "sha_irods=$sha_irods" >> $GITHUB_OUTPUT + echo "sha_icommands=$sha_icmds" >> $GITHUB_OUTPUT + + # Initialize keys for caching unreleased packages. + echo "cache_key_irods_packages=${{ inputs.cache_key_irods_packages_prefix }}-$sha_irods" >> $GITHUB_OUTPUT + echo "cache_key_icommands_packages=${{ inputs.cache_key_icommands_packages_prefix }}-$sha_icmds" >> $GITHUB_OUTPUT + + - name: Check cache for iRODS packages + id: irods + uses: actions/cache@v4 + with: + key: ${{ steps.init.outputs.cache_key_irods_packages }} + path: _build_irods/*.deb + + - name: Check cache for iCommands package + id: icommands + uses: actions/cache@v4 + with: + key: ${{ steps.init.outputs.cache_key_icommands_packages }} + path: _build_icommands/*.deb + + - name: Install prerequisite packages + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + apt-get update -qq + apt-get install -qq \ + apt-transport-https \ + ca-certificates \ + dnsutils + apt-get install -qq \ + bison \ + build-essential \ + catch2 \ + cmake \ + curl \ + flex \ + g++ \ + libarchive-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libfmt-dev \ + libpam0g-dev \ + libspdlog-dev \ + libssl-dev \ + libsystemd-dev \ + libxml2-dev \ + ninja-build \ + nlohmann-json3-dev \ + odbc-postgresql \ + python3-dev \ + python3-distro \ + python3-psutil \ + unixodbc \ + unixodbc-dev \ + wget \ + zlib1g-dev + + - name: Set up iRODS externals package repository + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + mkdir -p /etc/apt/keyrings + wget -qO - https://unstable.irods.org/irods-unstable-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp \ + --import + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-unstable-archive-keyring.pgp arch=amd64] https://unstable.irods.org/apt/ $(lsb_release -sc) main" | \ + tee /etc/apt/sources.list.d/renci-irods-unstable.list + apt-get update -qq + apt-get install -qq \ + irods-externals-boost1.81.0-2 \ + irods-externals-clang16.0.6-0 \ + irods-externals-jsoncons0.178.0-0 \ + irods-externals-nanodbc2.13.0-3 + + - name: Configure ccache + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + apt-get install -qq ccache + echo "CCACHE_DIR=/irods_build_cache" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV + + - name: Restore ccache state + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + uses: actions/cache@v4 + with: + key: ${{ inputs.ccache_key }} + path: /irods_build_cache/ + + - name: Build iRODS packages + if: steps.irods.outputs.cache-hit != 'true' + run: | + mkdir _build_irods + cmake \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja \ + -B _build_irods \ + -S irods + cmake --build _build_irods --target package + + - name: Build iCommands packages + # The iRODS cache hit condition (i.e. steps.irods.outputs.cache-hit) exists to make sure + # the iCommands are always compiled against the newly built iRODS packages. This avoids + # ABI issues on the iRODS code level. If ABI stability ever becomes a requirement for iRODS, + # then we can consider removing the condition for the iRODS cache hit condition. + if: steps.irods.outputs.cache-hit != 'true' || steps.icommands.outputs.cache-hit != 'true' + run: | + apt-get install -qq ./_build_irods/irods-{dev,runtime}*.deb + mkdir _build_icommands + cmake \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_BUILD_TYPE=Debug \ + -GNinja \ + -B _build_icommands \ + -S irods_client_icommands + cmake --build _build_icommands --target package