feat: k3s exp

.github/workflows/deploy-k8s.yml

@@ -0,0 +1,203 @@
+name: Deploy to K3s
+
+on:
+    workflow_call:
+        inputs:
+            action:
+                required: true
+                type: string
+                description: "'deploy' or 'teardown'"
+            image_tag:
+                required: false
+                type: string
+                description: "image tag to deploy"
+            namespace:
+                required: true
+                type: string
+                description: "k8s namespace (e.g. pubstar-sandbox, preview-pr-42)"
+            hostname:
+                required: true
+                type: string
+                description: "public hostname for this deployment"
+            env_file:
+                required: true
+                type: string
+                description: "sops-encrypted env file to decrypt (relative to infra/)"
+            overlay:
+                required: true
+                type: string
+                description: "kustomize overlay to use (sandbox or preview)"
+            pr_number:
+                required: false
+                type: string
+                description: "PR number (only for preview deployments)"
+        secrets:
+            SSH_PRIVATE_KEY:
+                required: true
+            SSH_USER:
+                required: true
+            SSH_HOST:
+                required: true
+            GHCR_USER:
+                required: true
+            GHCR_TOKEN:
+                required: true
+
+permissions:
+    contents: read
+
+jobs:
+    deploy:
+        if: inputs.action == 'deploy'
+        runs-on: ubuntu-latest
+        timeout-minutes: 15
+        steps:
+            - uses: actions/checkout@v6
+
+            - name: Start SSH agent
+              uses: webfactory/ssh-agent@v0.9.0
+              with:
+                  ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+            - name: Add known hosts
+              run: |
+                  mkdir -p ~/.ssh
+                  ssh-keyscan -H "${{ secrets.SSH_HOST }}" >> ~/.ssh/known_hosts
+
+            - name: Deploy to K3s
+              env:
+                  SSH_USER: ${{ secrets.SSH_USER }}
+                  SSH_HOST: ${{ secrets.SSH_HOST }}
+                  REPO: ${{ github.repository }}
+                  BRANCH: ${{ github.head_ref || github.ref_name }}
+                  GHCR_USER: ${{ secrets.GHCR_USER }}
+                  GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+                  IMAGE_TAG: ${{ inputs.image_tag }}
+                  NAMESPACE: ${{ inputs.namespace }}
+                  DEPLOY_HOST: ${{ inputs.hostname }}
+                  ENV_FILE: ${{ inputs.env_file }}
+                  OVERLAY: ${{ inputs.overlay }}
+                  PR_NUMBER: ${{ inputs.pr_number }}
+              run: |
+                  ssh "${SSH_USER}@${SSH_HOST}" \
+                    "env GHCR_USER='${GHCR_USER}' GHCR_TOKEN='${GHCR_TOKEN}' IMAGE_TAG='${IMAGE_TAG}' NAMESPACE='${NAMESPACE}' ENV_FILE='${ENV_FILE}' OVERLAY='${OVERLAY}' PR_NUMBER='${PR_NUMBER}' bash -s -- '${REPO}' '${BRANCH}'" <<'EOS'
+                    set -euo pipefail
+
+                    REPO="${1:?missing repo}"
+                    BRANCH="${2:-main}"
+
+                    : "${IMAGE_TAG:?missing IMAGE_TAG}"
+                    : "${GHCR_USER:?missing GHCR_USER}"
+                    : "${GHCR_TOKEN:?missing GHCR_TOKEN}"
+                    : "${NAMESPACE:?missing NAMESPACE}"
+                    : "${ENV_FILE:?missing ENV_FILE}"
+                    : "${OVERLAY:?missing OVERLAY}"
+
+                    REPO_NAME="${REPO##*/}"
+                    APP_DIR="/srv/k3s/${REPO_NAME}"
+                    REPO_SSH="git@github.com:${REPO}.git"
+
+                    ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null
+                    chmod 600 ~/.ssh/known_hosts
+
+                    if [[ ! -d "${APP_DIR}/.git" ]]; then
+                      mkdir -p "${APP_DIR}"
+                      git clone --branch "${BRANCH}" "${REPO_SSH}" "${APP_DIR}"
+                    fi
+
+                    cd "${APP_DIR}"
+                    git fetch --prune --tags origin
+                    git checkout --detach "origin/${BRANCH}"
+
+                    cd infra
+                    umask 077
+                    sops -d --input-type dotenv --output-type dotenv "$ENV_FILE" > ".env.k8s"
+
+                    # create namespace
+                    kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
+
+                    # label previews for cleanup tracking
+                    if [[ -n "${PR_NUMBER:-}" ]]; then
+                      kubectl label namespace "$NAMESPACE" \
+                        pubstar.io/preview=true \
+                        pubstar.io/pr-number="$PR_NUMBER" \
+                        --overwrite
+                    fi
+
+                    # GHCR pull secret
+                    kubectl create secret docker-registry ghcr \
+                      --docker-server=ghcr.io \
+                      --docker-username="$GHCR_USER" \
+                      --docker-password="$GHCR_TOKEN" \
+                      --namespace="$NAMESPACE" \
+                      --dry-run=client -o yaml | kubectl apply -f -
+
+                    # app secrets from decrypted env
+                    kubectl create secret generic pubstar-env \
+                      --from-env-file=".env.k8s" \
+                      --namespace="$NAMESPACE" \
+                      --dry-run=client -o yaml | kubectl apply -f -
+
+                    rm -f ".env.k8s"
+
+                    # build and apply manifests
+                    cd k8s/overlays/"$OVERLAY"
+
+                    if [[ "$OVERLAY" == "preview" ]]; then
+                      kustomize build . \
+                        | sed "s/__PR_NUMBER__/${PR_NUMBER}/g" \
+                        | sed "s/newTag: IMAGE_TAG/newTag: ${IMAGE_TAG}/g" \
+                        | sed "s/image: \(ghcr\.io\/knowledgefutures\/[^:]*\):IMAGE_TAG/image: \1:${IMAGE_TAG}/g" \
+                        | kubectl apply -n "$NAMESPACE" -f -
+                    else
+                      kustomize edit set image \
+                        "ghcr.io/knowledgefutures/platform:${IMAGE_TAG}" \
+                        "ghcr.io/knowledgefutures/platform-jobs:${IMAGE_TAG}" \
+                        "ghcr.io/knowledgefutures/platform-site-builder:${IMAGE_TAG}" \
+                        "ghcr.io/knowledgefutures/mock-coar-notify-server:${IMAGE_TAG}"
+                      kustomize build . | kubectl apply -n "$NAMESPACE" -f -
+                      git checkout -- kustomization.yaml 2>/dev/null || true
+                    fi
+
+                    echo "waiting for core deployment rollout..."
+                    kubectl rollout status deployment/core -n "$NAMESPACE" --timeout=600s
+
+                    echo ""
+                    kubectl get all -n "$NAMESPACE"
+                  EOS
+
+    teardown:
+        if: inputs.action == 'teardown'
+        runs-on: ubuntu-latest
+        timeout-minutes: 10
+        steps:
+            - name: Start SSH agent
+              uses: webfactory/ssh-agent@v0.9.0
+              with:
+                  ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+
+            - name: Add known hosts
+              run: |
+                  mkdir -p ~/.ssh
+                  ssh-keyscan -H "${{ secrets.SSH_HOST }}" >> ~/.ssh/known_hosts
+
+            - name: Teardown namespace
+              env:
+                  SSH_USER: ${{ secrets.SSH_USER }}
+                  SSH_HOST: ${{ secrets.SSH_HOST }}
+                  NAMESPACE: ${{ inputs.namespace }}
+              run: |
+                  ssh "${SSH_USER}@${SSH_HOST}" \
+                    "env NAMESPACE='${NAMESPACE}' bash -s" <<'EOS'
+                    set -euo pipefail
+                    : "${NAMESPACE:?missing NAMESPACE}"
+
+                    echo "tearing down namespace $NAMESPACE"
+
+                    if kubectl get namespace "$NAMESPACE" &>/dev/null; then
+                      kubectl delete namespace "$NAMESPACE"
+                      echo "namespace $NAMESPACE deleted"
+                    else
+                      echo "namespace $NAMESPACE not found, nothing to tear down"
+                    fi
+                  EOS

.github/workflows/on_main_2.yml

@@ -59,6 +59,25 @@ jobs:
             GHCR_USER: ${{ secrets.GHCR_USER }}
             GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
 
+    deploy-sandbox-k3s:
+        needs: build-all
+        permissions:
+            contents: read
+        uses: ./.github/workflows/deploy-k8s.yml
+        with:
+            action: deploy
+            image_tag: ${{ github.sha }}
+            namespace: pubstar-sandbox
+            hostname: sandbox.k3s.pubstar.org
+            env_file: .env.sandbox.enc
+            overlay: sandbox
+        secrets:
+            SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+            SSH_USER: ${{ secrets.SSH_USER }}
+            SSH_HOST: ${{ secrets.SSH_HOST_K3S }}
+            GHCR_USER: ${{ secrets.GHCR_USER }}
+            GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
     deploy-docs:
         permissions:
             contents: write

.github/workflows/on_pr.yml

@@ -165,6 +165,55 @@ jobs:
             GHCR_USER: ${{ secrets.GHCR_USER }}
             GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
 
+    deploy-preview-k3s:
+        if: >-
+            always() &&
+            contains(github.event.pull_request.labels.*.name, 'preview') &&
+            github.event.action != 'closed' &&
+            github.event.action != 'unlabeled' &&
+            needs.skip_build_sha.result == 'success' &&
+            (needs.build-all.result == 'success' ||
+              (needs.build-all.result == 'skipped' && needs.skip_build_sha.outputs.last-successful-build-sha != ''))
+        uses: ./.github/workflows/deploy-k8s.yml
+        needs:
+            - skip_build_sha
+            - build-all
+        permissions:
+            contents: read
+        with:
+            action: deploy
+            image_tag: ${{ needs.skip_build_sha.outputs.last-successful-build-sha || github.event.pull_request.head.sha }}
+            namespace: preview-pr-${{ github.event.pull_request.number }}
+            hostname: pr-${{ github.event.pull_request.number }}.k3s.pubstar.org
+            env_file: .env.preview.enc
+            overlay: preview
+            pr_number: ${{ github.event.pull_request.number }}
+        secrets:
+            SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+            SSH_USER: ${{ secrets.SSH_USER }}
+            SSH_HOST: ${{ secrets.SSH_HOST_K3S }}
+            GHCR_USER: ${{ secrets.GHCR_USER }}
+            GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
+    close-preview-k3s:
+        if: (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'preview')) || (github.event.action == 'unlabeled' && github.event.label.name == 'preview')
+        uses: ./.github/workflows/deploy-k8s.yml
+        permissions:
+            contents: read
+        with:
+            action: teardown
+            namespace: preview-pr-${{ github.event.pull_request.number }}
+            hostname: pr-${{ github.event.pull_request.number }}.k3s.pubstar.org
+            env_file: .env.preview.enc
+            overlay: preview
+            pr_number: ${{ github.event.pull_request.number }}
+        secrets:
+            SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+            SSH_USER: ${{ secrets.SSH_USER }}
+            SSH_HOST: ${{ secrets.SSH_HOST_K3S }}
+            GHCR_USER: ${{ secrets.GHCR_USER }}
+            GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
+
     deploy-docs-preview:
         permissions:
             contents: write

infra/.sops.yaml

@@ -16,3 +16,4 @@ creation_rules:
       - age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy
       - age15fsv503cl9ajfg9fphw45sfq0ytlptnr8wtlt48asytz4ev6rfks2pwt5s
       - age14dssfufn8n0n7lcvyxxp4rq7zuf5pv0ek4zl9h7atsy85ceynfvqa0qz59
+      - age19pjnf2dqg0sd3j4v8sfvlpytk0jg4v0230j5np8qku55y67r6a5qjs6rmn # k3s-exp

infra/k8s/base/configmaps/caddy-config.yaml

@@ -0,0 +1,67 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: caddy-config
+  labels:
+    app.kubernetes.io/name: pubstar
+    app.kubernetes.io/component: caddy
+data:
+  Caddyfile: |
+    {
+        admin off
+    }
+
+    :80 {
+        encode gzip
+
+        @assetsHost header_regexp assets Host ^(?:pr-\d+-assets|sandbox-assets)\.
+        handle @assetsHost {
+            reverse_proxy minio:9000
+        }
+
+        handle_path /assets* {
+            reverse_proxy minio:9000
+        }
+
+        handle_path /assets-ui* {
+            reverse_proxy minio:9001
+        }
+
+        handle_path /site-builder* {
+            reverse_proxy site-builder:4000
+        }
+
+        handle_path /sites/* {
+            @pathWithSlash path_regexp dir (.+)/$
+            handle @pathWithSlash {
+                redir {re.dir.1} permanent
+            }
+
+            rewrite * /pubstar-assets/sites{uri}
+
+            reverse_proxy minio:9000 {
+                @error status 403 404
+                handle_response @error {
+                    rewrite * {uri}/index.html
+                    reverse_proxy minio:9000 {
+                        @nestedError status 404
+                        handle_response @nestedError {
+                            respond "Not found" 404
+                        }
+                    }
+                }
+            }
+        }
+
+        handle /emails* {
+            reverse_proxy inbucket:9000
+        }
+
+        handle /mock-notify* {
+            reverse_proxy mock-notify:3000
+        }
+
+        handle {
+            reverse_proxy core:3000
+        }
+    }