chore: build image which can be reused to speed up CI

timcadman · timcadman · commit 84290899aa75 · 2026-01-22T16:45:41.000+01:00
diff --git a/.github/workflows/build-env.yml b/.github/workflows/build-env.yml
@@ -0,0 +1,33 @@
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: 'Image tag (e.g., v7.0.0-dev)'
+        required: true
+      danger_version:
+        description: 'dsDangerClient version/branch'
+        required: true
+        default: 'v7.0.0-dev'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and Push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          build-args: |
+            DANGER_VERSION=${{ github.event.inputs.danger_version }}
+          tags: ghcr.io/${{ github.repository_owner }}/ds-base-env:${{ github.event.inputs.release_tag }}
diff --git a/.github/workflows/dsBaseClient_test_suite.yaml b/.github/workflows/dsBaseClient_test_suite.yaml
@@ -3,9 +3,9 @@
 # Adapted from `armadillo_azure-pipelines.yml` by Roberto Villegas-Diaz
 #
 # Inside the root directory $(Pipeline.Workspace) will be a file tree like:
-# /dsBaseClient               <- Checked out version of datashield/dsBaseClient
-# /dsBaseClient/logs          <- Where results of tests and logs are collated
-# /testStatus                 <- Checked out version of datashield/testStatus
+# /dsBaseClient                <- Checked out version of datashield/dsBaseClient
+# /dsBaseClient/logs           <- Where results of tests and logs are collated
+# /testStatus                  <- Checked out version of datashield/testStatus
 #
 # As of Sept. 2025 this takes ~ 95 mins to run.
 ################################################################################
@@ -24,11 +24,16 @@ jobs:
     permissions:
       contents: read
 
-    # These should all be constant, except TEST_FILTER. This can be used to test 
-    # subsets of test files in the testthat directory. Options are like:
-    # '*'               <- Run all tests.
-    # 'asNumericDS*'    <- Run all asNumericDS tests, i.e. all the arg, etc. tests.
-    # '*_smk_*'         <- Run all the smoke tests for all functions.
+    # This logic automatically selects the image tag
+    container:
+      image: >-
+        ghcr.io/${{ github.repository_owner }}/ds-base-env:${{
+          startsWith(github.ref_name, 'dev-') &&
+          replace(github.ref_name, 'dev-', 'v') ||
+          'latest'
+        }}-dev
+      options: --user root -v /var/run/docker.sock:/var/run/docker.sock
+
     env:
       TEST_FILTER: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-'
       _r_check_system_clock_: 0
@@ -55,57 +60,8 @@ jobs:
           persist-credentials: false
           token: ${{ env.GITHUB_TOKEN }}
 
-      - name: Uninstall default MySQL
-        run: |
-          curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
-          sudo service mysql stop || true
-          sudo apt-get update
-          sudo apt-get remove --purge mysql-client mysql-server mysql-common -y
-          sudo apt-get autoremove -y
-          sudo apt-get autoclean -y
-          sudo rm -rf /var/lib/mysql/
-
-      - uses: r-lib/actions/setup-pandoc@v2
-
-      - uses: r-lib/actions/setup-r@v2
-        with:
-          r-version: release
-          http-user-agent: release
-          use-public-rspm: true
-
-      - name: Install R and dependencies
-        run: |
-          sudo apt-get install --no-install-recommends software-properties-common dirmngr -y
-          wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
-          sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
-          sudo apt-get update -qq
-          sudo apt-get upgrade -y
-          sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y
-          sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools -y
-          sudo R -q -e "install.packages(c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), repos='https://cloud.r-project.org')"
-          sudo R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))"
-
-      - uses: r-lib/actions/setup-r-dependencies@v2
-        with:
-          dependencies: 'c("Imports")'
-          extra-packages: |
-            any::rcmdcheck
-            cran::devtools
-            cran::git2r
-            cran::RCurl
-            cran::readr
-            cran::magrittr
-            cran::xml2
-            cran::purrr
-            cran::dplyr
-            cran::stringr
-            cran::tidyr
-            cran::quarto
-            cran::knitr
-            cran::kableExtra
-            cran::rmarkdown
-            cran::downlit
-          needs: check
+      - name: Install PR version of dsBaseClient
+        run: R CMD INSTALL dsBaseClient
 
       - name: Check manual updated
         run: |
@@ -169,7 +125,6 @@ jobs:
               ),
               "coveragelist.csv"
             )'
-
           mv coveragelist.csv logs/
           mv test_* logs/
         working-directory: dsBaseClient
@@ -180,7 +135,6 @@ jobs:
           echo "Number of testsuites with issues: $issue_count"
           sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true
           cat issues.log || true
-          # continue with workflow even when some tests fail
           exit 0
         working-directory: dsBaseClient/logs
 
@@ -193,52 +147,14 @@ jobs:
 
       - name: Parse results from testthat and covr
         run: |
-          Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/
+          Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ logs/ https://github.com/datashield/${{ env.PROJECT_NAME }}/blob/${{ env.BRANCH_NAME }} '([^:]+)' '(?<=::)[^:]+(?=::)'
         working-directory: dsBaseClient
+        env:
+          PROJECT_NAME: ${{ env.PROJECT_NAME }}
+          BRANCH_NAME: ${{ env.BRANCH_NAME }}
 
       - name: Render report
         run: |
           cd testStatus
-
           mkdir -p new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
           mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
-          mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/
-
-          # Copy logs to new logs directory location
-          cp -rv ../dsBaseClient/logs/* new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
-          cp -rv ../dsBaseClient/logs/${{ env.WORKFLOW_ID }}.txt new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/
-
-          R -e 'input_dir <- file.path("../new/logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))'
-          mv source/test_report.html new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html
-          cp -r new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest
-
-        env:
-          PROJECT_NAME: ${{ env.PROJECT_NAME }}
-          BRANCH_NAME: ${{ env.BRANCH_NAME }}
-          WORKFLOW_ID: ${{ env.WORKFLOW_ID }}
-
-      - name: Upload test logs
-        uses: actions/upload-artifact@v4
-        with:
-          name: dsbaseclient-logs
-          path: testStatus/new
-
-      - name: Dump environment info
-        run: |
-          echo -e "\n#############################"
-          echo -e "ls /: ######################"
-          ls -al .
-          echo -e "\n#############################"
-          echo -e "lscpu: ######################"
-          lscpu
-          echo -e "\n#############################"
-          echo -e "memory: #####################"
-          free -m
-          echo -e "\n#############################"
-          echo -e "env: ########################"
-          env
-          echo -e "\n#############################"
-          echo -e "R sessionInfo(): ############"
-          R -e 'sessionInfo()'
-          sudo apt install tree -y
-          tree .
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,18 @@
+FROM rocker/r-ver:4.3.0
+
+# 1. System Libraries
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    sudo software-properties-common dirmngr wget curl \
+    libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev \
+    libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools \
+    docker.io docker-compose-v2 \
+    && rm -rf /var/lib/apt/lists/*
+
+# 2. R Packages (Stable dependencies)
+RUN R -e "options(repos = c(RSPM = 'https://packagemanager.posit.co/cran/__linux__/jammy/latest', CRAN = 'https://cloud.r-project.org')); \
+    install.packages(c('devtools','remotes','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071', \
+    'rcmdcheck','git2r','RCurl','readr','magrittr','xml2','purrr','dplyr','stringr','tidyr','knitr','kableExtra','rmarkdown','downlit'))"
+
+# 3. dsDangerClient (Pass the version as a build argument)
+ARG DANGER_VERSION=master
+RUN R -e "remotes::install_github('datashield/dsDangerClient', ref='${DANGER_VERSION}')"
