docker-image.yml

name: Fetchit Build and test

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

env:
  CGO_ENABLED: 0
  PODMAN_VER: v5.7.0

jobs:

  build-podman-v5:
    env:
      CGO_ENABLED: 1 # CGO is required for podman
    runs-on: ubuntu-latest
    steps:
      - name: Cache podman bin
        id: cache-podman-bin
        uses: actions/cache@v4
        with:
          path: |
            ./bin
          key: ${{ runner.os }}-podman-${{ env.PODMAN_VER }}-v2
          restore-keys: |
            ${{ runner.os }}-podman-${{ env.PODMAN_VER }}-v2
      - uses: actions/checkout@v4
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        with:
          repository: containers/podman
          ref: v5.7.0

      - uses: actions/setup-go@v4
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        with:
          go-version: ${{ env.GOVER }}

      - name: Cache go modules
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        uses: actions/cache@v4
        with:
          # In order:
          # * Module download cache
          # * Build cache (Linux)
          path: |
            ~/go/pkg/mod
            ~/.cache/go-build
          key: ${{ runner.os }}-go-podman-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-podman
      - name: Update apt cache
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        run: sudo apt-get update

      - name: Add build packages
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        run: sudo apt-get install -y libsystemd-dev libseccomp-dev pkg-config golang-github-proglottis-gpgme-dev

      - name: Install compatible crun for Podman v5
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        run: |
          CRUN_VERSION=1.18
          wget https://github.com/containers/crun/releases/download/${CRUN_VERSION}/crun-${CRUN_VERSION}-linux-amd64
          sudo install -m 755 crun-${CRUN_VERSION}-linux-amd64 /usr/bin/crun
          crun --version

      - name: Build podman v5
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        run: make binaries

      - name: Copy crun to bin directory for caching
        if: steps.cache-podman-bin.outputs.cache-hit != 'true'
        run: |
          mkdir -p bin
          cp /usr/bin/crun bin/crun

      # store podman binary as artifact
      - uses: actions/upload-artifact@v4
        with:
          name: podman-bins
          path: bin

  build:

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Install required packages
        run: sudo apt install curl

      - name: Build the image
        run: go mod tidy -compat=1.21 && go mod vendor && docker build . --file Dockerfile --tag quay.io/fetchit/fetchit-amd:latest

      - name: Export image
        run: docker save -o /tmp/fetchit.tar quay.io/fetchit/fetchit-amd:latest

      - name: Save container as artifact
        uses: actions/upload-artifact@v4
        with:
          name: fetchit-image
          path: /tmp/fetchit.tar

  build-ansible-amd:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Build the image
        run: go mod tidy -compat=1.21 && go mod vendor && make build-ansible-cross-build-linux-amd64

      - name: docker list images
        run: docker images

      - name: Export image
        run: podman save -o /tmp/ansible-amd.tar quay.io/fetchit/fetchit-ansible-amd:latest

      - name: Save container as artifact
        uses: actions/upload-artifact@v4
        with:
          name: ansible-image-amd
          path: /tmp/ansible-amd.tar

  build-ansible-arm:
    runs-on: ubuntu-latest
    needs: [ build, raw-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, quadlet-validate, quadlet-user-validate, quadlet-volume-network-validate, quadlet-kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, ansible-validate, loader-validate, disconnected-validate ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - uses: actions/checkout@v4

      - name: update system
        run: sudo apt-get update

      - name: Install required packages
        run: sudo apt install podman build-essential qemu-user qemu-user-static

      - name: Build the image
        run: go mod tidy -compat=1.21 && go mod vendor && CTR_CMD=podman make build-ansible-cross-build-linux-arm64

      - name: Export image
        run: podman save -o /tmp/ansible-arm.tar quay.io/fetchit/fetchit-ansible-arm:latest

      - name: Save container as artifact
        uses: actions/upload-artifact@v4
        with:
          name: ansible-image-arm
          path: /tmp/ansible-arm.tar

  pull-and-archive:
    runs-on: ubuntu-latest
    steps:
      - name: pull the colors image
        run: podman pull docker.io/mmumshad/simple-webapp-color:latest

      - name: archive the image
        run: podman save -o /tmp/colors.tar docker.io/mmumshad/simple-webapp-color:latest

      - name: save the container as an artifact
        uses: actions/upload-artifact@v4
        with:
          name: colors
          path: /tmp/colors.tar

  build-systemd-amd:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Build the image
        run: go mod tidy -compat=1.21 && go mod vendor && make build-systemd-cross-build-linux-amd64

      - name: docker list images
        run: docker images

      - name: Export image
        run: podman save -o /tmp/systemd-amd.tar quay.io/fetchit/fetchit-systemd-amd:latest

      - name: Save container as artifact
        uses: actions/upload-artifact@v4
        with:
          name: systemd-image-amd
          path: /tmp/systemd-amd.tar

  build-systemd-arm:
    runs-on: ubuntu-latest
    needs: [ build, raw-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, quadlet-validate, quadlet-user-validate, quadlet-volume-network-validate, quadlet-kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, ansible-validate, loader-validate, disconnected-validate ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - uses: actions/checkout@v4

      - name: update system
        run: sudo apt-get update

      - name: Install required packages
        run: sudo apt install podman build-essential qemu-user qemu-user-static

      - name: Build the image
        run: go mod tidy -compat=1.21 && go mod vendor && CTR_CMD=podman make build-systemd-cross-build-linux-arm64

      - name: Export image
        run: podman save -o /tmp/systemd-arm.tar quay.io/fetchit/fetchit-systemd-arm:latest

      - name: Save container as artifact
        uses: actions/upload-artifact@v4
        with:
          name: systemd-image-arm
          path: /tmp/systemd-arm.tar

  fetchit-config-target-no-config-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v4

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: create fetchit config directory
        run: sudo mkdir /root/.fetchit

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /root/.fetchit:/opt/mount -e FETCHIT_CONFIG_URL=https://raw.githubusercontent.com/josephsawaya/fetchit/fix-target-path/examples/raw-config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

  fetchit-config-reload-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v4

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: move config file to default location
        run: sudo mkdir "${HOME}"/.fetchit && sudo cp ./examples/config-url.yaml "${HOME}"/.fetchit/config.yaml

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v "${HOME}"/.fetchit/config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

  fetchit-config-PAT-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: swap in the GITHUB PAT token
        run: sudo sed -i "s/CHANGEME/${{ secrets.GITHUB_TOKEN }}/g" ./examples/pat-testing-config.yaml

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/pat-testing-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

  fetchit-PAT-kube-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: swap in the GITHUB PAT token
        run: sudo sed -i "s/CHANGEME/${{ secrets.GITHUB_TOKEN }}/g" ./examples/pat-testing-kube.yaml

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/pat-testing-kube.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for pod to be ready
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman pod ps | grep -v CON= | grep colors_pod | wc -l); sleep 2; done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman pod ps | grep -v CON= | grep colors_pod | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

  raw-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/raw-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

      - name: Check the capabilities of cap1
        run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Check that no capabilities exist for cap2
        run: if [[ $(sudo podman container inspect cap2 --format {{.EffectiveCaps}} | jq length) = "0" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: 'Check that the "owned-by:fetchit" label is applied'
        run: |
          declare -i numContainers=$(sudo podman ps --filter label=owned-by=fetchit | wc -l)
          if (( numContainers > 1 )); then
            echo 'Successfully applied labels'
          else
            exit 1
          fi

          # check against a garbage label
          declare -i checkAgainst=$(sudo podman ps --filter label=owned-by=abcdefghijklmnopqrstuvwxyz | wc -l)
          if (( checkAgainst == numContainers )); then
            echo 'container amount with label cannot equal container amount with nonexistent label'
            exit 1
          else
            echo 'Label was applied correctly'
          fi

  podman-secret-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: generate podman secret
        run: |
         export GH_PAT=${{ secrets.GITHUB_TOKEN }}
         podman secret create --env GH_PAT GH_PAT

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/podman-secret-raw.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

      - name: Check the capabilities of cap1
        run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Check that no capabilities exist for cap2
        run: if [[ $(sudo podman container inspect cap2 --format {{.EffectiveCaps}} | jq length) = "0" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: 'Check that the "owned-by:fetchit" label is applied'
        run: |
          declare -i numContainers=$(sudo podman ps --filter label=owned-by=fetchit | wc -l)
          if (( numContainers > 1 )); then
            echo 'Successfully applied labels'
          else
            exit 1
          fi

          # check against a garbage label
          declare -i checkAgainst=$(sudo podman ps --filter label=owned-by=abcdefghijklmnopqrstuvwxyz | wc -l)
          if (( checkAgainst == numContainers )); then
            echo 'container amount with label cannot equal container amount with nonexistent label'
            exit 1
          else
            echo 'Label was applied correctly'
          fi

  podman-config-secret-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: generate podman secret
        run: |
         export GH_PAT=${{ secrets.GITHUB_TOKEN }}
         podman secret create --env GH_PAT GH_PAT

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/podman-secret-raw-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

      - name: Check the capabilities of cap1
        run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Check that no capabilities exist for cap2
        run: if [[ $(sudo podman container inspect cap2 --format {{.EffectiveCaps}} | jq length) = "0" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: 'Check that the "owned-by:fetchit" label is applied'
        run: |
          declare -i numContainers=$(sudo podman ps --filter label=owned-by=fetchit | wc -l)
          if (( numContainers > 1 )); then
            echo 'Successfully applied labels'
          else
            exit 1
          fi

          # check against a garbage label
          declare -i checkAgainst=$(sudo podman ps --filter label=owned-by=abcdefghijklmnopqrstuvwxyz | wc -l)
          if (( checkAgainst == numContainers )); then
            echo 'container amount with label cannot equal container amount with nonexistent label'
            exit 1
          else
            echo 'Label was applied correctly'
          fi

  config-env-raw-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit with environment variable config
        run: |
          sudo podman run -d --name fetchit -v fetchit-volume:/opt -e FETCHIT_CONFIG="$(cat ./examples/raw-config.yaml)" -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

      - name: Check the capabilities of cap1
        run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Check that no capabilities exist for cap2
        run: if [[ $(sudo podman container inspect cap2 --format {{.EffectiveCaps}} | jq length) = "0" ]] ; then echo "Container successfully launched"; else exit 1; fi
  filetransfer-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: Create destination directory
        run: sudo mkdir /tmp/ft

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/filetransfer-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /tmp/ft/anotherfile.txt ]; do sleep 2; done"

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /tmp/ft/hello.txt ]; do sleep 2; done"

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: ensure container is gone
        run: sudo podman ps -a

  filetransfer-validate-exact-file:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: Create destination directory
        run: sudo mkdir -p /tmp/ft/single

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/filetransfer-config-single-file.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /tmp/ft/single/hello.txt ]; do sleep 2; done"

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: ensure container is gone
        run: sudo podman ps -a

  systemd-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, build-systemd-amd ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/systemd-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /etc/systemd/system/httpd.service ]; do sleep 2; done"

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: ensure systemd container is gone
        run: sudo podman ps -a

  systemd-autoupdate-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, build-systemd-amd ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the autoupdate timer & service
        run: sudo cp examples/systemd/podman-auto-update.service /usr/lib/systemd/system/ && sudo cp examples/systemd/podman-auto-update.timer /usr/lib/systemd/system/ && sudo systemctl daemon-reload

      - name: Load the autoupdate timer drop-in file
        run: sudo mkdir /etc/systemd/system/podman-auto-update.timer.d && sudo cp examples/systemd/podman-auto-update.timer.d-override.conf /etc/systemd/system/podman-auto-update.timer.d/override.conf && sudo systemctl daemon-reload

      - name: Enable and start the auto-update timer
        run: sudo systemctl enable --now podman-auto-update.timer

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: pull systemd amd artifact
        uses: actions/download-artifact@v4
        with:
          name: systemd-image-amd
          path: /tmp

      - name: Load the systemd image
        run: sudo podman load -i /tmp/systemd-amd.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-systemd-amd:latest quay.io/fetchit/fetchit-systemd:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/systemd-autoupdate.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: ensure httpd.service is running
        run: timeout 150 bash -c -- 'sysd=inactive ; until [ $sysd = "active" ]; do sysd=$(sudo systemctl is-active httpd.service); done'

      - name: Pull test httpd image
        run: sudo podman pull quay.io/sallyom/httpd-ex:latest

      - name: tag the image
        run: sudo podman tag quay.io/sallyom/httpd-ex:latest registry.access.redhat.com/ubi8/httpd-24:latest

      - name: Wait for podman auto-update
        run: timeout 150 bash -c -- 'msg=0 ; until [ $msg -ge 1 ]; do msg=$( curl -s localhost:8080/index.html | grep "Welcome to your static httpd application" | wc -l ); done'

      - name: check the service
        if: always()
        run: sudo systemctl status podman-auto-update.timer

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: list podman pods showing when they were started
        if: always()
        run: sudo podman ps -a

  systemd-enable-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, build-systemd-amd ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: pull systemd amd artifact
        uses: actions/download-artifact@v4
        with:
          name: systemd-image-amd
          path: /tmp

      - name: Load the systemd image
        run: sudo podman load -i /tmp/systemd-amd.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-systemd-amd:latest quay.io/fetchit/fetchit-systemd:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/systemd-enable.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /etc/systemd/system/httpd.service ]; do sleep 2; done"

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: ensure systemd containers are gone
        run: sudo podman ps -a

      - name: ensure httpd.service is running
        run: timeout 150 bash -c -- 'sysd=inactive ; until [ $sysd = "active" ]; do sysd=$(sudo systemctl is-active httpd.service); done'

  systemd-user-enable-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, build-systemd-amd ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: enable podman.socket
        run: |
          set -x
          loginctl enable-linger runner
          sleep 1
          ls -al /run/user/$UID
          XDG_RUNTIME_DIR=/run/user/$UID systemctl --user enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: pull systemd amd artifact
        uses: actions/download-artifact@v4
        with:
          name: systemd-image-amd
          path: /tmp

      - name: Load the systemd image
        run: podman load -i /tmp/systemd-amd.tar

      - name: tag the image
        run: podman tag quay.io/fetchit/fetchit-systemd-amd:latest quay.io/fetchit/fetchit-systemd:latest

      - name: Start fetchit
        run: podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/systemd-enable-user.yaml:/opt/mount/config.yaml -v /run/user/"${UID}"/podman/podman.sock:/run/podman/podman.sock -e XDG_RUNTIME_DIR="/run/user/${UID}" -e HOME="${HOME}" --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f ~/.config/systemd/user/httpd.service ]; do sleep 2; done"

      - name: Logs
        run: podman logs fetchit

      - name: list files
        run: ls ~/.config/systemd/user/httpd.service

      - name: ensure systemd containers are gone
        run: podman ps -a

      - name: ensure httpd.service is running
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(podman ps | grep httpd | wc -l); done'

  systemd-restart-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, build-systemd-amd ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: pull systemd amd artifact
        uses: actions/download-artifact@v4
        with:
          name: systemd-image-amd
          path: /tmp

      - name: Load the systemd image
        run: sudo podman load -i /tmp/systemd-amd.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-systemd-amd:latest quay.io/fetchit/fetchit-systemd:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/systemd-restart.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /etc/systemd/system/httpd.service ]; do sleep 2; done"

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: list files
        run: sudo ls /etc/systemd/system/httpd.service

      - name: ensure systemd containers are gone
        run: sudo podman ps -a

      - name: ensure httpd.service is running
        run: timeout 150 bash -c -- 'sysd=inactive ; until [ $sysd = "active" ]; do sysd=$(sudo systemctl is-active httpd.service); done'

  ansible-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, build-ansible-amd ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: ssh-keygen
        run: sudo ssh-keygen -t rsa -b 4096 -f /root/.ssh/id_rsa -N ""

      - name: cp pub into authorized
        run: sudo cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: Pull ansible artifact
        uses: actions/download-artifact@v4
        with:
          name: ansible-image-amd
          path: /tmp

      - name: Load the ansible image
        run: |
          sudo podman load -i /tmp/ansible-amd.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-ansible-amd:latest quay.io/fetchit/fetchit-ansible:latest

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/ansible.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: list packages
        run: timeout 150 bash -c -- 'z=0 ; until [ $z -ge 1 ]; do z=$(sudo apt list --installed | grep zsh | wc -l); done'
        continue-on-error: true

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

  systemd-validate-exact-file:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/systemd-config-single-file.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /etc/systemd/system/httpd.service ]; do sleep 2; done"

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: list files
        run: sudo ls /etc/systemd/system/httpd.service

      - name: ensure systemd container is gone
        run: sudo podman ps -a

  kube-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/kube-play-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for pod to be ready
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman pod ps | grep -v CON= | grep colors_pod | wc -l); sleep 2; done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman pod ps | grep -v CON= | grep colors_pod | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

  quadlet-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5 ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Prepare quadlet config for PR testing
        run: |
          cp ./examples/quadlet-config.yaml /tmp/quadlet-config.yaml
          sed -i 's|branch: main|branch: ${{ github.head_ref || github.ref_name }}|g' /tmp/quadlet-config.yaml
          sed -i 's|schedule: ".*/5 \* \* \* \*"|schedule: "*/1 * * * *"|g' /tmp/quadlet-config.yaml

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /tmp/quadlet-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for Quadlet directory to be created
        run: timeout 150 bash -c "until [ -d /etc/containers/systemd ]; do sleep 2; done"

      - name: Wait for Quadlet file to be placed
        run: timeout 150 bash -c "until [ -f /etc/containers/systemd/simple.container ]; do sleep 2; done"

      - name: Trigger daemon-reload manually
        run: sudo systemctl daemon-reload

      - name: Wait for service to be generated
        run: timeout 150 bash -c "until systemctl list-units --all | grep -q simple.service; do sleep 2; done"

      - name: Check service status
        run: sudo systemctl status simple.service || true

      - name: Wait for service to be active
        run: timeout 150 bash -c -- 'sysd=inactive ; until [ $sysd = "active" ]; do sysd=$(sudo systemctl is-active simple.service); sleep 2; done'

      - name: Verify container is running
        run: sudo podman ps | grep systemd-simple

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: List Quadlet files
        if: always()
        run: sudo ls -la /etc/containers/systemd/

      - name: Check all running containers
        if: always()
        run: sudo podman ps -a

  quadlet-user-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5 ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: enable podman.socket
        run: |
          set -x
          loginctl enable-linger runner
          sleep 1
          ls -al /run/user/$UID
          XDG_RUNTIME_DIR=/run/user/$UID systemctl --user enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Prepare quadlet rootless config for PR testing
        run: |
          cp ./examples/quadlet-rootless.yaml /tmp/quadlet-rootless.yaml
          sed -i 's|branch: main|branch: ${{ github.head_ref || github.ref_name }}|g' /tmp/quadlet-rootless.yaml
          sed -i 's|schedule: ".*/2 \* \* \* \*"|schedule: "*/1 * * * *"|g' /tmp/quadlet-rootless.yaml
          echo "=== Modified config ==="
          cat /tmp/quadlet-rootless.yaml

      - name: Start fetchit
        run: podman run -d --name fetchit -v fetchit-volume:/opt -v /tmp/quadlet-rootless.yaml:/opt/mount/config.yaml -v /run/user/"${UID}"/podman/podman.sock:/run/podman/podman.sock -e XDG_RUNTIME_DIR="/run/user/${UID}" -e HOME="${HOME}" --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for Quadlet directory to be created
        run: timeout 150 bash -c "until [ -d ~/.config/containers/systemd ]; do sleep 2; done"

      - name: DEBUG - Show what files exist while waiting
        if: always()
        run: |
          echo "=== Files in Quadlet directory ==="
          ls -la ~/.config/containers/systemd/ 2>/dev/null || echo "Directory doesn't exist yet"
          echo "=== Waiting for httpd.container... ==="

      - name: Wait for httpd Quadlet file to be placed
        run: timeout 150 bash -c "until [ -f ~/.config/containers/systemd/httpd.container ]; do sleep 2; done"


      - name: DEBUG - List all user services before daemon-reload
        run: systemctl --user list-units --all || true

      - name: DEBUG - Check for quadlet-systemctl containers
        if: always()
        run: podman ps -a | grep quadlet-systemctl || echo "No quadlet-systemctl containers found"

      - name: DEBUG - Show Quadlet files content
        if: always()
        run: cat ~/.config/containers/systemd/*.container || true

      - name: DEBUG - Check generator BEFORE manual daemon-reload
        if: always()
        run: |
          echo "=== Checking for generated service files BEFORE manual daemon-reload ==="
          find /run/user/$UID -name "*httpd*" 2>/dev/null || echo "No httpd service files found in /run/user/$UID"
          systemctl --user list-unit-files | grep httpd || echo "No httpd services in unit files"
          systemctl --user list-units --all | grep httpd || echo "No httpd services in units"

      - name: Trigger daemon-reload manually
        run: systemctl --user daemon-reload

      - name: DEBUG - Check if Quadlet generated service files
        if: always()
        run: ls -la ~/.config/systemd/user.control/ || ls -la /run/user/$UID/systemd/generator/ || echo "No generator directory found"

      - name: DEBUG - List all systemd generator locations
        if: always()
        run: find /run/user/$UID -name "*httpd*" 2>/dev/null || echo "No httpd service files found"

      - name: Wait for httpd service to be generated
        run: timeout 150 bash -c "until systemctl --user list-units --all | grep -q httpd.service; do sleep 2; done"

      - name: Check httpd service status
        run: systemctl --user status httpd.service || true

      - name: DEBUG - List all generated services
        if: always()
        run: systemctl --user list-unit-files | grep -E 'httpd' || true

      - name: DEBUG - Show podman containers state
        if: always()
        run: podman ps -a

      - name: Wait for httpd service to be active
        run: timeout 150 bash -c -- 'sysd=inactive ; until [ $sysd = "active" ]; do sysd=$(systemctl --user is-active httpd.service); sleep 2; done'

      - name: Verify httpd container is running
        run: podman ps | grep systemd-httpd

      - name: Logs
        if: always()
        run: podman logs fetchit

      - name: Service journal logs
        if: always()
        run: journalctl --user -u httpd.service || true

      - name: List Quadlet files
        if: always()
        run: ls -la ~/.config/containers/systemd/

      - name: Check all running containers
        if: always()
        run: podman ps -a

  quadlet-volume-network-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5 ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Prepare quadlet config for multi-resource testing
        run: |
          cp ./examples/quadlet-config.yaml /tmp/quadlet-multi.yaml
          sed -i 's|branch: main|branch: ${{ github.head_ref || github.ref_name }}|g' /tmp/quadlet-multi.yaml
          sed -i 's|schedule: ".*/5 \* \* \* \*"|schedule: "*/1 * * * *"|g' /tmp/quadlet-multi.yaml

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /tmp/quadlet-multi.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for network Quadlet file
        run: timeout 150 bash -c "until [ -f /etc/containers/systemd/httpd.network ]; do sleep 2; done"

      - name: Wait for volume Quadlet file
        run: timeout 150 bash -c "until [ -f /etc/containers/systemd/httpd.volume ]; do sleep 2; done"

      - name: Wait for container Quadlet file
        run: timeout 150 bash -c "until [ -f /etc/containers/systemd/httpd.container ]; do sleep 2; done"

      - name: Trigger daemon-reload manually
        run: sudo systemctl daemon-reload

      - name: Wait for network service to be generated
        run: timeout 150 bash -c "until systemctl list-units --all | grep -q httpd-network.service; do sleep 2; done"

      - name: Wait for volume service to be generated
        run: timeout 150 bash -c "until systemctl list-units --all | grep -q httpd-volume.service; do sleep 2; done"

      - name: Wait for container service to be generated
        run: timeout 150 bash -c "until systemctl list-units --all | grep -q httpd.service; do sleep 2; done"

      - name: Wait for httpd service to be active
        run: timeout 150 bash -c -- 'sysd=inactive ; until [ $sysd = "active" ]; do sysd=$(sudo systemctl is-active httpd.service); sleep 2; done'

      - name: Verify container is running
        run: sudo podman ps | grep systemd-httpd

      - name: Verify network exists
        run: sudo podman network ls | grep systemd-httpd || true

      - name: Verify volume exists
        run: sudo podman volume ls | grep httpd || true

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: Network service journal
        if: always()
        run: journalctl -u httpd-network.service || true

      - name: Volume service journal
        if: always()
        run: journalctl -u httpd-volume.service || true

      - name: Container service journal
        if: always()
        run: journalctl -u httpd.service || true

      - name: List Quadlet files
        if: always()
        run: sudo ls -la /etc/containers/systemd/

      - name: Check all running containers
        if: always()
        run: sudo podman ps -a

  quadlet-kube-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5 ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Create kube config
        run: |
          cat > /tmp/quadlet-kube.yaml <<EOF
          targetConfigs:
            - name: quadlet-kube
              url: https://github.com/containers/fetchit
              branch: ${{ github.head_ref || github.ref_name }}
              quadlet:
                - name: colors-kube
                  targetPath: examples/quadlet
                  schedule: "*/1 * * * *"
                  glob: "colors.kube"
                  root: true
                  enable: true
                  restart: false
          EOF

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /tmp/quadlet-kube.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for Quadlet kube file
        run: timeout 150 bash -c "until [ -f /etc/containers/systemd/colors.kube ]; do sleep 2; done"

      - name: Trigger daemon-reload manually
        run: sudo systemctl daemon-reload

      - name: Wait for service to be generated
        run: timeout 150 bash -c "until systemctl list-units --all | grep -q colors.service; do sleep 2; done"

      - name: Wait for service to be active
        run: timeout 150 bash -c -- 'sysd=inactive ; until [ $sysd = "active" ]; do sysd=$(sudo systemctl is-active colors.service); sleep 2; done'

      - name: Verify pod is running
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman pod ps | grep -v CON= | grep systemd-colors | wc -l); sleep 2; done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: Kube service journal
        if: always()
        run: journalctl -u colors.service || true

      - name: List Quadlet files
        if: always()
        run: sudo ls -la /etc/containers/systemd/

      - name: Check all running pods
        if: always()
        run: sudo podman pod ps

      - name: Check all running containers
        if: always()
        run: sudo podman ps -a

  clean-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Create unused volume
        run: sudo podman volume create test

      - name: Pull an image
        run: sudo podman image pull alpine:latest

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/clean-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify fetchit container
        run: timeout 150 bash -c -- 'f=0 ; until [ $f -eq 1 ]; do f=$(sudo podman ps | grep fetchit | wc -l); done'

      - name: Print the current images
        run: sudo podman image ls

      - name: wait until image is removed
        run: timeout 150 bash -c -- 'i=0 ; until [ $i -eq 0 ]; do i=$(sudo podman image ls alpine | grep -v REPOSITORY | wc -l); sleep 2; done'

      - name: Wait for volume to be cleaned up
        run: timeout 150 bash -c -- 'v=1 ; until [ $v -eq 0 ]; do v=$(sudo podman volume ls | grep test | wc -l); sleep 2; done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify volume is deleted
        run: if [[ $(sudo podman volume ls | grep test) ]] ; then exit 1; else echo "Volume correctly removed" 1; fi

      - name: verify image is deleted
        run: if [[ $(sudo podman image ls | grep alpine) ]] ; then exit 1; else echo "Image correctly removed" 1; fi

  glob-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/glob-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: verify container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

      - name: Print the current running container
        run: sudo podman ps

      - name: Check the capabilities of cap1
        run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi

  multi-engine-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: Create destination directory
        run: sudo mkdir /tmp/ft

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/full-suite.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /etc/systemd/system/httpd.service ]; do sleep 2; done"

      - name: check for file
        run: timeout 150 bash -c "until [ -f /tmp/ft/hello.txt ]; do sleep 2; done"

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: ensure only required containers are running (cap colors and fetchit)
        run: if [[ $(sudo podman ps -a | grep -v CON | grep -v fetchit | grep -v cap | grep -v colors | wc -l) > "0" ]] ; then exit 1; else echo "Containers properly cleaned"; fi

      - name: verify raw container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep colors | grep -v cap | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

  multi-engine-skew-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: Create destination directory
        run: sudo mkdir /tmp/ft

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v ./examples/full-suite-with-skew.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /etc/systemd/system/httpd.service ]; do sleep 2; done"

      - name: check for file
        run: timeout 150 bash -c "until [ -f /tmp/ft/hello.txt ]; do sleep 2; done"

      - name: check for file
        run: timeout 150 bash -c "until [ -f /tmp/ft/anotherfile.txt ]; do sleep 2; done"

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: ensure only required containers are running (cap colors and fetchit)
        run: if [[ $(sudo podman ps -a | grep -v CON | grep -v fetchit | grep -v cap | grep -v colors | wc -l) > "0" ]] ; then exit 1; else echo "Containers properly cleaned"; fi

      - name: verify raw container is running
        run: if [[ $(sudo podman ps | grep -v CON= | grep -v cap | grep colors | wc -l) = "2" ]] ; then echo "Container successfully launched"; else exit 1; fi

  make-change-to-repo:
    runs-on: ubuntu-latest
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v3
        with:
          path: main

      - name: checkout with token
        uses: actions/checkout@v3
        with:
          path: ci
          ref: ci

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: ensure value and commit
        run: |
          if [[ $( grep pink ./ci/examples/raw/color1.json | wc -l ) = "1" ]] ; then
            cat <<< $(jq '.Env.APP_COLOR |= "blue"' ./ci/examples/raw/color1.json) > ./ci/examples/raw/color1.json
            cd ci
            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
            git config --local user.name "github-actions[bot]"
            git commit -m "color changes" -a
            git push -f
          fi

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: set values relating to the current env
        run: |
          sed -i 's|  url: https://github.com/containers/fetchit|  url: https://github.com/${{ github.repository }}|g' /home/runner/work/fetchit/fetchit/main/examples/ci-config.yaml
          sed -i 's|   branch: ci|  branch: "{{ github.ref }}"|g' /home/runner/work/fetchit/fetchit/main/examples/ci-config.yaml

      - name: Start fetchit
        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /home/runner/work/fetchit/fetchit/main/examples/ci-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest

      - name: identify colors container
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do c=$(sudo podman ps | grep colors | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: get environment variable
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman exec colors1 env | grep blue| wc -l); done'

      - name: update variable
        run: |
          cat <<< $(jq '.Env.APP_COLOR |= "pink"' ./ci/examples/raw/color1.json) > ./ci/examples/raw/color1.json

      - name: Commit files
        run: |
          cd ci
            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
            git config --local user.name "github-actions[bot]"
            git commit -m "color changes" -a
            git push -f

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

      - name: get environment variable
        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman exec colors1 env | grep pink| wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

  loader-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - uses: actions/checkout@v2

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: Tag with a new name
        run: sudo podman tag docker.io/mmumshad/simple-webapp-color:latest quay.io/notreal/httpd:latest

      - name: archive the image
        run: |
          sudo mkdir -p /tmp/image
          sudo podman save -o /tmp/image/httpd.tar quay.io/notreal/httpd:latest
          sudo podman image rm quay.io/notreal/httpd:latest

      - name: start httpd and serve the httpd.tar image
        run: sudo podman run -d --name httpd -p 8080:8080 -v /tmp/image:/var/www/html registry.access.redhat.com/ubi8/httpd-24

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit --network=host -v ./examples/imageLoad-config.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock quay.io/fetchit/fetchit-amd:latest

      - name: wait until image exists
        run: timeout 150 bash -c -- 'i=1 ; until [ $i -eq 1 ]; do i=$(sudo podman image ls quay.io/notreal/httpd:latest | grep -v REPOSITORY | wc -l); done'

      - name: identify local container
        run: timeout 150 bash -c -- 'f=0 ; until [ $f -eq 1 ]; do f=$(sudo podman ps | grep notreal | wc -l); done'

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

  disconnected-validate:
    runs-on: ubuntu-latest
    needs: [ build, build-podman-v5, pull-and-archive ]
    steps:
      - name: checkout with token
        uses: actions/checkout@v3
        with:
          path: ci

      - name: pull in podman
        uses: actions/download-artifact@v4
        with:
          name: podman-bins
          path: bin

      - name: Install Podman and crun
        run: |
          chmod +x bin/podman bin/crun
          sudo mv bin/podman /usr/bin/podman
          sudo mv bin/crun /usr/bin/crun

      - name: Enable the podman socket
        run: sudo systemctl enable --now podman.socket

      - name: ssh-keygen
        run: sudo ssh-keygen -t rsa -b 4096 -f /root/.ssh/id_rsa -N ""

      - name: cp pub into authorized
        run: sudo cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: Pull ansible artifact
        uses: actions/download-artifact@v4
        with:
          name: ansible-image-amd
          path: /tmp

      - name: Load the ansible image
        run: |
         sudo podman load -i /tmp/ansible-amd.tar
         sudo podman tag quay.io/fetchit/fetchit-ansible-amd:latest quay.io/fetchit/fetchit-ansible:latest

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: colors
          path: /tmp

      - name: Load the image
        run: sudo podman load -i /tmp/colors.tar

      - name: Make directory
        run: mkdir -p /tmp/ft

      - name: Create an archive of the fetchit repo
        run: |
          sudo git clone https://github.com/containers/fetchit.git
          sudo mkdir -p /tmp/disco
          cd fetchit && sudo zip -r /tmp/disco/fetchit.zip ../fetchit

      - name: pull the web container
        run: podman pull registry.access.redhat.com/ubi8/httpd-24

      - name: start httpd and serve the fetchit.zip image
        run: sudo podman run -d --name httpd -p 9000:8080 -v /tmp/disco:/var/www/html registry.access.redhat.com/ubi8/httpd-24

      - name: Load the image
        run: sudo podman load -i /tmp/fetchit.tar

      - name: tag the image
        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest

      - name: Start fetchit
        run: sudo podman run -d --name fetchit --network=host -v fetchit-volume:/opt -v ./ci/examples/full-suite-disconnected.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock quay.io/fetchit/fetchit-amd:latest

      - name: Wait for fetchit to deploy
        run: timeout 150 bash -c "until [ -f /etc/systemd/system/httpd.service ]; do sleep 2; done"

      - name: check for file
        run: timeout 150 bash -c "until [ -f /tmp/ft/hello.txt ]; do sleep 2; done"

      - name: check for file
        run: timeout 150 bash -c "until [ -f /tmp/ft/anotherfile.txt ]; do sleep 2; done"

      - name: ensure only required containers are running (cap colors and fetchit)
        run: if [[ $(sudo podman ps -a | grep -v CON | grep -v httpd | grep -v fetchit | grep -v cap | grep -v colors | wc -l) > "0" ]] ; then exit 1; else echo "Containers properly cleaned"; fi

      - name: Logs
        if: always()
        run: sudo podman logs fetchit

  push-amd-image-to-registry:
    runs-on: ubuntu-latest
    needs: [ build, raw-validate, podman-secret-validate, podman-config-secret-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, quadlet-validate, quadlet-user-validate, quadlet-volume-network-validate, quadlet-kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, ansible-validate, loader-validate, disconnected-validate ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - name: Login to Quay
        uses: docker/login-action@v3
        with:
          registry: quay.io
          username: ${{ secrets.REGISTRY_USER }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the image
        run: docker load -i /tmp/fetchit.tar

      - name: push image
        run: docker push quay.io/fetchit/fetchit-amd:latest


  build-arm-and-manifest-list:
    runs-on: ubuntu-latest
    needs: [ build, raw-validate, podman-secret-validate, podman-config-secret-validate, fetchit-config-target-no-config-validate, fetchit-config-reload-validate, clean-validate, kube-validate, quadlet-validate, quadlet-user-validate, quadlet-volume-network-validate, quadlet-kube-validate, systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, systemd-restart-validate, systemd-validate-exact-file, multi-engine-validate, make-change-to-repo, filetransfer-validate, filetransfer-validate-exact-file, ansible-validate, loader-validate, disconnected-validate ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: update system
        run: sudo apt-get update

      - name: Install required packages
        run: sudo apt install podman build-essential qemu-user qemu-user-static

      - name: Build the arm image
        run: go mod tidy -compat=1.21 && go mod vendor && CTR_CMD=podman make build-containerized-cross-build-linux-arm64

      - name: Export image
        run: podman save -o /tmp/fetchit-arm.tar quay.io/fetchit/fetchit-arm:latest

      - name: Save container as artifact
        uses: actions/upload-artifact@v4
        with:
          name: fetchit-arm-image
          path: /tmp/fetchit-arm.tar

  push-manifest-list-to-registry:
    runs-on: ubuntu-latest
    needs: [ build-arm-and-manifest-list ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the amd image
        run: podman load -i /tmp/fetchit.tar

      - name: pull arm artifact
        uses: actions/download-artifact@v4
        with:
          name: fetchit-arm-image
          path: /tmp

      - name: Load the arm image
        run: podman load -i /tmp/fetchit-arm.tar

      - name: create manifest
        run: podman manifest create quay.io/fetchit/fetchit:latest

      - name: Login to Quay
        uses: docker/login-action@v3
        with:
          registry: quay.io
          username: ${{ secrets.REGISTRY_USER }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: podman login to quay.io
        run: podman login --authfile ~/.docker/config.json quay.io

      - name: add images and push manifest list
        run: |
          podman manifest add --arch arm64 quay.io/fetchit/fetchit:latest docker://quay.io/fetchit/fetchit-arm:latest
          podman manifest add quay.io/fetchit/fetchit:latest docker://quay.io/fetchit/fetchit-amd:latest
          podman manifest push --authfile ~/.docker/config.json quay.io/fetchit/fetchit:latest docker://quay.io/fetchit/fetchit:latest --format=docker

  publish-binary-to-github:
    runs-on: ubuntu-latest
    needs: [ build-arm-and-manifest-list ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - name: pull arm image
        uses: actions/download-artifact@v4
        with:
          name: fetchit-arm-image
          path: /tmp

      - name: Load the arm image
        run: podman load -i /tmp/fetchit-arm.tar

      - name: Pull amd image
        uses: actions/download-artifact@v4
        with:
          name: fetchit-image
          path: /tmp

      - name: Load the amd image
        run: podman load -i /tmp/fetchit.tar

      - name: move binary
        run: |
          arm=$(podman images | grep arm | awk '{print $3}')
          amd=$(podman images | grep amd | awk '{print $3}')
          podman cp $(podman create --rm $arm):/usr/local/bin/fetchit ./fetchit-arm
          podman cp $(podman create --rm $amd):/usr/local/bin/fetchit ./fetchit-amd

      - name: push binary
        uses: "marvinpinto/action-automatic-releases@latest"
        with:
          repo_token: "${{ secrets.GITHUB_TOKEN }}"
          automatic_release_tag: "latest"
          prerelease: true
          title: "Latest Build"
          files: |
            fetchit-arm
            fetchit-amd

  create-ansible-manifest:
    runs-on: ubuntu-latest
    needs: [ ansible-validate, build-ansible-arm ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: ansible-image-arm
          path: /tmp

      - name: Load the amd image
        run: podman load -i /tmp/ansible-arm.tar

      - name: pull amd artifact
        uses: actions/download-artifact@v4
        with:
          name: ansible-image-amd
          path: /tmp

      - name: Load the arm image
        run: podman load -i /tmp/ansible-amd.tar

      - name: Login to Quay
        uses: docker/login-action@v3
        with:
          registry: quay.io
          username: ${{ secrets.REGISTRY_USER }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: podman login to quay.io
        run: podman login --authfile ~/.docker/config.json quay.io

      - name: create manifest
        run: podman manifest create quay.io/fetchit/fetchit-ansible:latest

      - name: add images to manifest list
        run: |
          podman manifest add --arch arm64 quay.io/fetchit/fetchit-ansible:latest docker://quay.io/fetchit/fetchit-ansible-arm:latest
          podman manifest add quay.io/fetchit/fetchit-ansible:latest docker://quay.io/fetchit/fetchit-ansible-amd:latest
          podman manifest push --authfile ~/.docker/config.json quay.io/fetchit/fetchit-ansible:latest docker://quay.io/fetchit/fetchit-ansible:latest --format=docker

  create-systemd-manifest:
    runs-on: ubuntu-latest
    needs: [ systemd-validate, systemd-enable-validate, systemd-user-enable-validate, systemd-autoupdate-validate, build-systemd-arm ]
    if: >
      (github.event_name == 'push' || github.event_name == 'schedule') &&
      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
    steps:
      - name: pull artifact
        uses: actions/download-artifact@v4
        with:
          name: systemd-image-arm
          path: /tmp

      - name: Load the arm image
        run: podman load -i /tmp/systemd-arm.tar

      - name: pull amd artifact
        uses: actions/download-artifact@v4
        with:
          name: systemd-image-amd
          path: /tmp

      - name: Load the amd image
        run: podman load -i /tmp/systemd-amd.tar

      - name: Login to Quay
        uses: docker/login-action@v3
        with:
          registry: quay.io
          username: ${{ secrets.REGISTRY_USER }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: podman login to quay.io
        run: podman login --authfile ~/.docker/config.json quay.io

      - name: create manifest
        run: podman manifest create quay.io/fetchit/fetchit-systemd:latest

      - name: add images to manifest list
        run: |
          podman manifest add --arch arm64 quay.io/fetchit/fetchit-systemd:latest docker://quay.io/fetchit/fetchit-systemd-arm:latest
          podman manifest add quay.io/fetchit/fetchit-systemd:latest docker://quay.io/fetchit/fetchit-systemd-amd:latest
          podman manifest push --authfile ~/.docker/config.json quay.io/fetchit/fetchit-systemd:latest docker://quay.io/fetchit/fetchit-systemd:latest --format=docker