Back to socat forwarding, proper upstream tracking

.env.example

@@ -27,18 +27,36 @@ MACHINE_CPUS=4
 # online (xfs_growfs). Shrinking is unsupported. Default shown.
 # PG_DATA_DISK_SIZE=140G
 
-# Optional: host loopback ports the forwarder listens on — what local clients
-# connect to. Offset from 5432 so a local PostgreSQL isn't shadowed. Defaults
-# shown. PG_BACKEND_PORT is the port each backend is exposed on (machine eth0).
+# Optional: the client-facing loopback ports — what local clients connect to
+# (psql/.pgpass/status print these). Offset from 5432 so a local PostgreSQL isn't
+# shadowed. The socat client proxy (`make proxy.install`) binds THESE.
+# PG_BACKEND_PORT is the port each backend is exposed on (machine eth0).
 # PG_CLIENT_ACTIVE_PORT=5442
 # PG_CLIENT_STAGING_PORT=5443
 # PG_BACKEND_PORT=5432
 
-# Optional: address the host forwarder binds its client listeners to. Default
-# 127.0.0.1 (loopback only). Set 0.0.0.0 ONLY if a sibling container/k3d can't
-# reach the Mac's loopback — it exposes the dev-credentialed backend on every
-# interface (LAN/Wi-Fi), so widen deliberately.
-# PG_FORWARD_BIND=127.0.0.1
+# Optional: verbose (debug-level) slog output for the SQLite tracking DB and the
+# socat proxy. ON by default (promote/reconcile are chatty on purpose); set 0 to
+# quiet it to info-level.
+# PG_PROXY_DEBUG=1
+
+# Optional: ANSI color for that structured output (rendered by lmittmann/tint).
+# "auto" (default) colors it only when stderr is a terminal — and honors
+# NO_COLOR / TERM=dumb — so piping or redirecting a run yields clean,
+# fully-dated lines. "always" forces color through a pipe (e.g. `|& less -R`),
+# "never" disables it outright.
+# PG_LOG_COLOR=auto
+
+# Optional: structured-log format. "text" (default) is the colored, human-first
+# rendering above; "json" swaps in the stdlib slog JSON handler for a run that
+# gets captured and machine-parsed rather than watched.
+# PG_LOG_FORMAT=text
+
+# Optional: address the client proxy binds its listeners to. Default 127.0.0.1
+# (loopback only). Set 0.0.0.0 ONLY if a sibling container/k3d can't reach the
+# Mac's loopback — it exposes the dev-credentialed backend on every interface
+# (LAN/Wi-Fi), so widen deliberately.
+# PG_CLIENT_BIND=127.0.0.1
 
 # Optional: minimum free GiB on the macOS volume before `make pg.up` / restore /
 # import will run. The VM disks are sparse images that grow and never shrink, so

.gitignore

@@ -1,5 +1,8 @@
 /.zed
 .env
+# Leftover local code-signing material from the removed forwarder/codesign flow.
+# Nothing builds or reads it any more — kept ignored so a stray private key can
+# never be committed; delete etc/keys yourself when you're done with it.
 /etc/keys/*
 
 # Everything under var/ is machine-local runtime or generated state — nothing

Makefile

@@ -142,7 +142,10 @@ deploy: machine pgdevd
 
 # Cheap guard for targets that exec into already-running machines: fail fast
 # with a clear message instead of a raw Apple CLI 'notFound' error when a
-# machine has never been created.
+# machine has never been created. Deliberately NOT used by the read-only
+# status targets: since `pg.staging.purge` a deleted machine is an expected
+# steady state, so status must still report (it prints ABSENT for that slot)
+# instead of refusing to run.
 .PHONY: machine.exists
 machine.exists:
 	@for slot in a b; do \
@@ -227,53 +230,61 @@ start: deploy
 	$(MAKE) status
 
 .PHONY: status/incus
-status/incus: machine.exists
+status/incus:
 	@for slot in a b; do \
 		name="$(MACHINE_PREFIX)-$$slot"; \
 		echo "── $$name ──"; \
-		container machine run --name "$$name" --root -- incus list 2>/dev/null || echo "  (Incus not up)"; \
+		if container machine inspect "$$name" >/dev/null 2>&1; then \
+			container machine run --name "$$name" --root -- incus list 2>/dev/null || echo "  (Incus not up)"; \
+		else \
+			echo "  (machine does not exist)"; \
+		fi; \
 	done
 
 # The one status command: active/staging machine roles, per-machine
 # state/endpoints, snapshot counts and snapshot timelines. Served by pgdevd
 # (one per machine) over the HTTP API; the active/staging split is a host-side
 # pointer (var/active-machine), not part of the daemon contract.
 .PHONY: status
-status: machine.exists pgdevd
+status: pgdevd
 	@$(PGDEV) status
 
-# ----- stable macOS client endpoints --------------------------------------
-# Each Apple machine's IP drifts and cannot be pinned, so a host-side in-process
-# Go forwarder (internal/forward, run by a per-user LaunchAgent) publishes
+# ----- stable macOS client endpoints (doc/issues/0004) ---------------------
+# Each Apple machine's IP drifts and cannot be pinned, so clients never talk to
+# a machine IP: a socat-based client proxy under per-user LaunchAgents publishes
 # permanent 127.0.0.1:5442 (active) / :5443 (staging) endpoints and relays each
-# to whichever machine currently holds that role (on its own eth0:5432). It owns
-# the listeners for their whole lifetime and re-points itself from the pointer
-# file — `pgdev promote` is just a pointer write. `start` (via `pgdev refresh`)
-# validates the LaunchAgent every run and self-heals a missing, unloaded, stale
-# (e.g. a plist left pointing at a deleted binary after the repo moved), or
-# crashed agent (PG_ENDPOINT_AUTOINSTALL=0 opts out).
-
-.PHONY: endpoint.install
-endpoint.install: machine.exists
-	@$(PGDEV) forward install
-
-# Restart the running forwarder so a macOS Local Network permission granted
-# AFTER it started actually takes effect. TCC caches its allow/deny decision at
-# process start, so ticking the Local Network box (System Settings → Privacy &
-# Security → Local Network) does nothing for an already-running agent — it keeps
-# failing to reach the VM subnet with EHOSTUNREACH ("server closed the
-# connection unexpectedly") until it is restarted. Run this once after granting.
-.PHONY: endpoint.restart
-endpoint.restart:
-	@$(PGDEV) forward restart
-
-.PHONY: endpoint.uninstall
-endpoint.uninstall:
-	@$(PGDEV) forward uninstall
-
-.PHONY: endpoint.status
-endpoint.status:
-	@$(PGDEV) forward status
+# to whichever machine currently holds that role (on its own eth0:5432).
+# macOS Local Network Privacy still has to be granted ONCE (System Settings →
+# Privacy & Security → Local Network, entry `socat`; restart the agents after
+# granting — proxy.uninstall + proxy.install — since the decision is cached at
+# process start). But only once: the grant is keyed to the signing identity, and
+# socat is one stable Homebrew binary we never rebuild. The removed Go forwarder
+# was re-signed on nearly every `make`, so it re-prompted per build.
+#
+# All machine tracking (active pointer, machine IPs,
+# reconciled targets) lives in var/pgdev.db (SQLite) — each reconcile is a
+# flock-guarded, DB-driven rewrite+reload of the socat LaunchAgents with an
+# explicit port-free gate + post-verify. Install is explicit: nothing here runs,
+# and promote/refresh stay hands-off, until `proxy.install`.
+
+# All four depend on `pgdevd` (the build target) so `make` rebuilds bin/pgdev
+# before invoking it — without this a stale CLI predating the `proxy` command
+# fails with `unknown command "proxy"`.
+.PHONY: proxy.install
+proxy.install: pgdevd machine.exists
+	@$(PGDEV) proxy install
+
+.PHONY: proxy.reconcile
+proxy.reconcile: pgdevd
+	@$(PGDEV) proxy reconcile
+
+.PHONY: proxy.status
+proxy.status: pgdevd
+	@$(PGDEV) proxy status
+
+.PHONY: proxy.uninstall
+proxy.uninstall: pgdevd
+	@$(PGDEV) proxy uninstall
 
 .PHONY: stop
 stop:
@@ -299,7 +310,7 @@ pg.down: deploy
 	$(PGDEV) down
 
 .PHONY: pg.status
-pg.status: machine.exists pgdevd
+pg.status: pgdevd
 	$(PGDEV) status
 	@$(PGDEV) endpoint
 
@@ -318,7 +329,7 @@ pg.shell: machine.exists
 	@$(call PG_DEV_IN,$(ACTIVE_SLOT),shell)
 
 .PHONY: pg.ip
-pg.ip: machine.exists pgdevd
+pg.ip: pgdevd
 	@$(PGDEV) ip
 
 .PHONY: pg.logs
@@ -338,7 +349,7 @@ pg.restore-last: machine.exists pgdevd
 	$(PGDEV) restore-last $(if $(force),--force,)
 
 .PHONY: pg.snapshots
-pg.snapshots: machine.exists pgdevd
+pg.snapshots: pgdevd
 	$(PGDEV) snapshots
 
 # ----- staging backend ----------------------------------------------------
@@ -383,10 +394,32 @@ pg.staging.start: machine.exists pgdevd
 # fresh backend on it. The active machine — and its data — is never touched.
 # Unlike the soft resets above this is slow (machine boot + provision) but
 # actually returns space to macOS; see 'make disk'/§1 of issues/0002.
+#
+# Deliberately NO disk.check: this IS the reclaim command (it deletes the machine
+# first, freeing the whole sparse image, then provisions a fresh EMPTY backend —
+# no restore, so no headroom needed). Gating it on free space is self-defeating —
+# you run it precisely when you're low, and it is what disk.check itself points
+# you to. Do not re-add disk.check here.
+# No machine.exists guard: rebuild is a delete+create path (applecli.Recreate's
+# delete is a no-op when the machine is gone), and it is the documented way back
+# from `pg.staging.purge` — which leaves vpg-b deleted. Guarding it would block
+# exactly the recovery it advertises.
 .PHONY: pg.staging.rebuild
-pg.staging.rebuild: disk.check machine.exists pgdevd
+pg.staging.rebuild: pgdevd
 	$(PGDEV) staging rebuild $(if $(force),--force,)
 
+# Reclaim WITHOUT rebuild: delete the staging machine (freeing its whole sparse
+# macOS disk) and LEAVE IT DOWN — no recreate/provision. Use this to reclaim
+# space (or just shut staging down) when you don't want a fresh backend yet;
+# bring it back later with pg.staging.rebuild or start. Like rebuild, deliberately
+# NO disk.check (it frees space). The active machine is never touched.
+# No machine.exists guard either: purge is idempotent (the delete no-ops when
+# the machine is already gone) and re-running it still forgets a stale IP and
+# drops the socat listener, which is the useful self-heal.
+.PHONY: pg.staging.purge
+pg.staging.purge: pgdevd
+	$(PGDEV) staging purge $(if $(force),--force,)
+
 # ----- destructive outer-machine lifecycle -------------------------------
 
 # Apple 1.1 frequently returns an XPC timeout when deleting a RUNNING machine