feat: upgrading the always-on infra to use multiple individual stacks

Author: InputObject2

infrastructure/always-on/README.md

@@ -5,9 +5,9 @@ This directory contains the complete implementation spec and automation for the
 ## Overview
 
 Three machines (`srv-26`, `srv-27`, `srv-28`) form the backbone of the homelab infrastructure, providing:
-- **srv-26**: DNS, VPN gateway, WLAN controller, monitoring clients, GitHub Actions runner, Portainer server
-- **srv-27**: Secondary DNS, Grafana, Prometheus, Loki, InfluxDB (observability stack)
-- **srv-28**: Home Assistant, Matterbridge, FreePBX + VOIP, Portainer agent
+- **srv-26**: Primary DNS (Technitium), network fabric (Tailscale, Gatus, ntfy), WLAN controller (Omada), GitHub Actions runner
+- **srv-27**: Secondary DNS (Technitium, zone transfer from srv-26), observability stack (Prometheus, Grafana, Loki, InfluxDB, Promtail)
+- **srv-28**: Home automation (Home Assistant, Matterbridge, python-matter-server, matter-hub, ESPHome, Zigbee2MQTT, Mosquitto), VoIP (FreePBX, TFTP, MariaDB)
 
 ## Quick Start
 
@@ -109,9 +109,17 @@ infrastructure/always-on/
 │       └── portainer/          # Deploy Portainer server/agent
 │
 ├── stacks/                     # Docker Compose workloads (managed by Portainer)
-│   ├── srv-26/docker-compose.yml
-│   ├── srv-27/docker-compose.yml
-│   └── srv-28/docker-compose.yml
+│   ├── srv-26/
+│   │   ├── dns/docker-compose.yml       # Technitium DNS
+│   │   ├── infra/docker-compose.yml     # Tailscale, Gatus, ntfy
+│   │   ├── wlan/docker-compose.yml      # Omada controller
+│   │   └── cicd/docker-compose.yml      # GitHub Actions runner
+│   ├── srv-27/
+│   │   ├── dns/docker-compose.yml       # Technitium DNS (secondary)
+│   │   └── observability/docker-compose.yml  # Prometheus, Grafana, Loki, InfluxDB, Promtail
+│   └── srv-28/
+│       ├── ha/docker-compose.yml        # Home Assistant, Matterbridge, Matter, ESPHome, Zigbee2MQTT, Mosquitto
+│       └── voip/docker-compose.yml      # FreePBX, TFTP, MariaDB
 │
 ├── vault/                      # Vault configuration
 │   ├── policies/               # Vault policies (shared, srv-26, srv-27, srv-28)
@@ -139,8 +147,10 @@ infrastructure/always-on/
 - Or manually trigger: `ssh ansible@srv-host sudo systemctl start ansible-pull.service`
 
 **For Docker/application changes:**
-- Edit `stacks/<hostname>/docker-compose.yml`
-- Add as a Git-backed stack in Portainer, or push and let Portainer auto-sync
+- Edit the relevant `stacks/<hostname>/<stack>/docker-compose.yml`
+- Each stack is registered independently in Portainer as a Git-backed stack
+- Portainer stack path format: `infrastructure/always-on/stacks/<hostname>/<stack>`
+- Push to main branch and trigger a pull in Portainer, or let auto-sync run
 
 **For new secrets:**
 - Add to Vault: `vault kv put secrets/infra/srv-xx/app-name key=value`

infrastructure/always-on/ansible/bootstrap.yml

@@ -2,7 +2,7 @@
 # Bootstrap playbook to be run once from operator laptop
 # Usage: ansible-playbook infrastructure/always-on/ansible/bootstrap.yml \
 #  --inventory infrastructure/always-on/ansible/inventory/hosts.yml \
-#  --extra-vars "vault_token=hvs.xxxxx" \
+#  --extra-vars "vault_token=hvs.xxxxx vault_addr=https://vault.your-domain.com" \
 #  --ask-become-pass
 #
 # This playbook Sets up Vault policies and AppRoles, then applies common roles to all machines

infrastructure/always-on/ansible/roles/common/handlers/main.yml

@@ -13,3 +13,8 @@
 - name: reload systemd
   ansible.builtin.systemd:
     daemon_reload: yes
+
+- name: restart systemd-resolved
+  ansible.builtin.systemd:
+    name: systemd-resolved
+    state: restarted

infrastructure/always-on/ansible/roles/common/tasks/main.yml

@@ -121,3 +121,22 @@
   ansible.builtin.systemd:
     name: ansible-pull.timer
     state: started
+
+- name: Ensure [Resolve] section exists in resolved.conf for Technitium hosts
+  ansible.builtin.lineinfile:
+    path: /etc/systemd/resolved.conf
+    regexp: '^\[Resolve\]$'
+    line: '[Resolve]'
+    insertafter: EOF
+    state: present
+  when: "'technitium' in host_stacks"
+
+- name: Disable systemd-resolved DNS stub listener for Technitium hosts
+  ansible.builtin.lineinfile:
+    path: /etc/systemd/resolved.conf
+    regexp: '^DNSStubListener='
+    line: DNSStubListener=no
+    insertafter: '^\[Resolve\]$'
+    state: present
+  when: "'technitium' in host_stacks"
+  notify: restart systemd-resolved

infrastructure/always-on/ansible/roles/portainer/tasks/main.yml

@@ -19,6 +19,7 @@
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - portainer_data:/data
+      - /opt/secrets:/opt/secrets:ro
   when: portainer_role == "server"
 
 - name: Deploy Portainer agent

infrastructure/always-on/stacks/srv-26/cicd/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  github-runner:
+    image: myoung34/github-runner:latest
+    container_name: github-runner
+    restart: unless-stopped
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - runner_data:/home/runner
+    env_file:
+      - /opt/secrets/github-runner/github-runner.env
+
+volumes:
+  runner_data:

…-on/stacks/srv-27/docker-compose-dns.yml …-on/stacks/srv-26/dns/docker-compose.ymlinfrastructure/always-on/stacks/srv-27/docker-compose-dns.yml renamed to infrastructure/always-on/stacks/srv-26/dns/docker-compose.yml infrastructure/always-on/stacks/srv-27/docker-compose-dns.yml renamed to infrastructure/always-on/stacks/srv-26/dns/docker-compose.yml

@@ -15,4 +15,4 @@ services:
       - /opt/secrets/technitium/technitium.env
 
 volumes:
-  technitium_data:
+  technitium_data:

infrastructure/always-on/stacks/srv-26/docker-compose.yml

@@ -0,0 +1,40 @@
+version: '3.8'
+
+services:
+  tailscale:
+    image: tailscale/tailscale:latest
+    container_name: tailscale
+    restart: unless-stopped
+    network_mode: host
+    cap_add:
+      - NET_ADMIN
+    volumes:
+      - tailscale_data:/var/lib/tailscale
+    env_file:
+      - /opt/secrets/tailscale/tailscale.env
+
+  gatus:
+    image: twinproduction/gatus:v5.5.1
+    container_name: gatus
+    restart: unless-stopped
+    ports:
+      - "8080:8080/tcp"
+    volumes:
+      - ./gatus-config.yaml:/etc/gatus/config.yaml
+    env_file:
+      - /opt/secrets/gatus/gatus.env
+
+  ntfy:
+    image: binwiederhier/ntfy:latest
+    container_name: ntfy
+    restart: unless-stopped
+    ports:
+      - "8090:80/tcp"
+    volumes:
+      - ntfy_data:/var/lib/ntfy
+    env_file:
+      - /opt/secrets/ntfy/ntfy.env
+
+volumes:
+  tailscale_data:
+  ntfy_data:

infrastructure/always-on/stacks/srv-26/infra/docker-compose.yml

@@ -0,0 +1,20 @@
+version: '3.8'
+
+services:
+  omada-controller:
+    image: mbentley/omada-controller:latest
+    container_name: omada
+    restart: unless-stopped
+    ports:
+      - "8088:8088/tcp"
+      - "8043:8043/tcp"
+      - "29810:29810/udp"
+      - "29811:29811/tcp"
+      - "29812:29812/tcp"
+    volumes:
+      - omada_data:/opt/tplink/EAPController/data
+    env_file:
+      - /opt/secrets/omada/omada.env
+
+volumes:
+  omada_data: