Refactoring molecule scenarios and GitHub workflows

.github/workflows/kics.yml

@@ -13,14 +13,17 @@ on:
           - warning
           - debug
   pull_request:
-  push:
-    branches:
-      - 'main'
-  merge_group:
-  schedule:
-    - cron: '15 6 * * 4'
+
+# Cancel outdated pipeline runs when a new push occurs in the pull request and a new pipeline starts.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  kics:
+  security_scan:
+    # For non-PR events (workflow_dispatch, schedule, etc.), github.event.pull_request.draft is null.
+    # GitHub Actions coerces null == false to true (null → 0, false → 0), so the job runs for all non-draft events.
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/test_linting.yml

@@ -17,12 +17,32 @@ on:
       rolename:
         required: true
         type: string
-  pull_request:
-    branches:
-      - '*'
+
+# DO NOT add a concurrency block to this reusable workflow. Keep it disabled.
+#
+# Inside a reusable workflow (called via "uses:"), github.workflow resolves to the
+# CALLER's workflow name, not "Test Linting". So the group below would evaluate to the
+# exact same value as the caller's own concurrency group (e.g.
+# "Test Collection Roles-refs/pull/<n>/merge"). With cancel-in-progress: true the called
+# workflow would then try to cancel the in-progress run of its own parent, which GitHub
+# refuses with:
+#   "Canceling since a deadlock was detected for concurrency group:
+#    'Test Collection Roles-refs/pull/<n>/merge' between a top level workflow and 'lint_collection'"
+# That deadlock cancels the lint job, which in turn skips the molecule jobs (needs: lint_*),
+# and the whole run ends as failure (see PR #430).
+#
+# Concurrency is already handled by every calling workflow (test_roles_pr.yml,
+# test_role_*.yml). Leave it to the callers and never re-enable it here.
+#
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 jobs:
   lint:
+    # For non-PR events (workflow_dispatch, workflow_call, etc.), github.event.pull_request.draft is null.
+    # GitHub Actions coerces null == false to true (null → 0, false → 0), so the job runs for all non-draft events.
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
@@ -32,12 +52,18 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.11"
+          cache: 'pip'
+          cache-dependency-path: requirements-test.txt
 
       - name: Install test dependencies.
         run: |
           python3 -m pip install --upgrade pip
           python3 -m pip install -r requirements-test.txt
 
+      - name: Install Ansible collections
+        run: |
+          ansible-galaxy collection install community.crypto community.general
+
       - name: Lint code (yamllint).
         run: |
           yamllint .

.github/workflows/test_plugins.yml

@@ -13,16 +13,22 @@ on:
           - warning
           - debug
   pull_request:
-    branches:
-      - 'main'
     paths:
       - 'plugins/**'
       - 'tests/**'
       - 'molecule/plugins/**'
       - '.github/workflows/test_plugins.yml'
 
+# Cancel outdated pipeline runs when a new push occurs in the pull request and a new pipeline starts.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  sanity_ansible_18_19:
+  sanity_core_2_18_2_19:
+    # For non-PR events (workflow_dispatch, merge_group, schedule, push, etc.), github.event.pull_request.draft is null.
+    # GitHub Actions coerces null == false to true (null → 0, false → 0), so the job runs for all non-draft events.
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     env:
       COLLECTION_NAMESPACE: netways
@@ -64,7 +70,10 @@ jobs:
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
 
-  sanity_ansible_20:
+  sanity_core_2_20:
+    # For non-PR events (workflow_dispatch, merge_group, schedule, push, etc.), github.event.pull_request.draft is null.
+    # GitHub Actions coerces null == false to true (null → 0, false → 0), so the job runs for all non-draft events.
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     env:
       COLLECTION_NAMESPACE: netways
@@ -105,10 +114,10 @@ jobs:
           PY_COLORS: '1'
           ANSIBLE_FORCE_COLOR: '1'
 
-  unit-test:
+  unit_tests:
     needs:
-      - sanity_ansible_18_19
-      - sanity_ansible_20
+      - sanity_core_2_18_2_19
+      - sanity_core_2_20
     runs-on: ubuntu-latest
     env:
       COLLECTION_NAMESPACE: netways
@@ -149,7 +158,7 @@ jobs:
           ANSIBLE_FORCE_COLOR: '1'
 
   molecule_plugins:
-    needs: unit-test
+    needs: unit_tests
     runs-on: ubuntu-latest
     env:
       COLLECTION_NAMESPACE: netways
@@ -158,7 +167,7 @@ jobs:
       fail-fast: false
       matrix:
         distro:
-          - ubuntu2204
+          - debian13
         scenario:
           - plugins
         release:
@@ -172,13 +181,53 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: "3.11"
+          cache: 'pip'
+          cache-dependency-path: requirements-test.txt
 
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip
           python3 -m pip install "ansible-core>=2.19,<2.20"
           python3 -m pip install -r requirements-test.txt
 
+      - name: Get latest Elasticsearch release
+        id: elastic-version
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        run: |
+          curl -fsSL \
+            "https://artifacts.elastic.co/packages/${{ matrix.release }}.x/apt/dists/stable/main/binary-amd64/Packages.gz" \
+            -o /tmp/Packages.gz
+          VERSION=$(zcat /tmp/Packages.gz \
+            | awk '$1=="Package:" && $2=="elasticsearch"{p=1} p && $1=="Version:"{print $2; p=0}' \
+            | sort -V \
+            | tail -n 1)
+          rm -f /tmp/Packages.gz
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Debug - latest Elasticsearch version
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        run: echo "Latest Elasticsearch ${{ matrix.release }}.x = ${{ steps.elastic-version.outputs.version }}"
+
+      - name: Restore Elastic apt cache
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        uses: actions/cache/restore@v5
+        with:
+          path: /tmp/elastic-apt-cache
+          key: ${{ runner.os }}-apt-elastic-${{ steps.elastic-version.outputs.version }}
+          restore-keys: |
+            ${{ runner.os }}-apt-elastic-${{ matrix.release }}.
+
+      - name: Debug - apt cache contents
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        run: |
+          echo "=== /tmp/elastic-apt-cache/ ==="
+          ls -lh /tmp/elastic-apt-cache/ 2>/dev/null || echo "(empty or does not exist)"
+          echo "=== Total size ==="
+          du -sh /tmp/elastic-apt-cache/ 2>/dev/null || echo "(n/a)"
+
+      - name: Ensure apt cache directory exists
+        run: mkdir -p /tmp/elastic-apt-cache
+
       - name: Install collection
         run: |
           mkdir -p ~/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE
@@ -195,8 +244,8 @@ jobs:
           ANSIBLE_FORCE_COLOR: '1'
           ELASTIC_RELEASE: ${{ matrix.release }}
 
-  python-cryptography:
-    needs: unit-test
+  cryptography_compatibility:
+    needs: unit_tests
     runs-on: ubuntu-latest
     env:
       COLLECTION_NAMESPACE: netways

.github/workflows/test_role_beats.yml

@@ -13,17 +13,21 @@ on:
           - warning
           - debug
   pull_request:
-    branches:
-      - 'feature/**'
-      - 'fix/**'
-      - '!doc/**'
     paths:
       - 'roles/beats/**'
       - '.github/workflows/test_role_beats.yml'
       - 'molecule/beats_**'
 
+# Cancel outdated pipeline runs when a new push occurs in the pull request and a new pipeline starts.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   lint_beats:
+    # For non-PR events (workflow_dispatch, merge_group, schedule, push, etc.), github.event.pull_request.draft is null.
+    # GitHub Actions coerces null == false to true (null → 0, false → 0), so the job runs for all non-draft events.
+    if: github.event.pull_request.draft == false
     uses: ./.github/workflows/test_linting.yml
     with:
       rolename: beats
@@ -40,15 +44,14 @@ jobs:
       fail-fast: false
       matrix:
         distro:
-          - ubuntu2204
+          - debian13
         scenario:
           - beats_default
-          - beats_peculiar
+          - beats_extended
         release:
-          - 7
           - 8
         ansible_version:
-          - "ansible>=9.0,<10.0" #Correspond ansible-core>=2.16,<2.17
+          - "ansible-core>=2.19,<2.20" #Correspond ansible>=12.0,<13.0
         python_version:
           - "3.11"
 
@@ -60,13 +63,53 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python_version }}
+          cache: 'pip'
+          cache-dependency-path: requirements-test.txt
 
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip
           python3 -m pip install "${{ matrix.ansible_version }}"
           python3 -m pip install -r requirements-test.txt
 
+      - name: Get latest Elasticsearch release
+        id: elastic-version
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        run: |
+          curl -fsSL \
+            "https://artifacts.elastic.co/packages/${{ matrix.release }}.x/apt/dists/stable/main/binary-amd64/Packages.gz" \
+            -o /tmp/Packages.gz
+          VERSION=$(zcat /tmp/Packages.gz \
+            | awk '$1=="Package:" && $2=="elasticsearch"{p=1} p && $1=="Version:"{print $2; p=0}' \
+            | sort -V \
+            | tail -n 1)
+          rm -f /tmp/Packages.gz
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Debug - latest Elasticsearch version
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        run: echo "Latest Elasticsearch ${{ matrix.release }}.x = ${{ steps.elastic-version.outputs.version }}"
+
+      - name: Restore Elastic apt cache
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        uses: actions/cache/restore@v5
+        with:
+          path: /tmp/elastic-apt-cache
+          key: ${{ runner.os }}-apt-elastic-${{ steps.elastic-version.outputs.version }}
+          restore-keys: |
+            ${{ runner.os }}-apt-elastic-${{ matrix.release }}.
+
+      - name: Debug - apt cache contents
+        if: contains(matrix.distro, 'ubuntu') || contains(matrix.distro, 'debian')
+        run: |
+          echo "=== /tmp/elastic-apt-cache/ ==="
+          ls -lh /tmp/elastic-apt-cache/ 2>/dev/null || echo "(empty or does not exist)"
+          echo "=== Total size ==="
+          du -sh /tmp/elastic-apt-cache/ 2>/dev/null || echo "(n/a)"
+
+      - name: Ensure apt cache directory exists
+        run: mkdir -p /tmp/elastic-apt-cache
+
       - name: Install collection
         run: |
           mkdir -p ~/.ansible/collections/ansible_collections/$COLLECTION_NAMESPACE