From a0a988b2900c3e64db861418993ded349286dd52 Mon Sep 17 00:00:00 2001 From: mesutoezdil Date: Wed, 26 Aug 2026 17:39:33 +0200 Subject: [PATCH] ateapi: backfill unset worker state during pod sync Follow-up to #517, which added a scheduler filter that skips any worker whose state is not STATE_ACTIVE. The syncer only writes State when it creates a worker record; the update path in syncWorkerToStore diffs Ip, SandboxClass and Labels and never touches State, and no other code path writes it onto an existing record. The store is Redis backed, so worker records outlive an ate-api-server rollout, and worker pods are not recreated by that rollout either. Every worker record written before the State field existed therefore stays at STATE_UNSPECIFIED and is skipped by the scheduler for the whole life of its pod. On an existing cluster upgrading past #517, actor placement fails with ErrNoCapacity until each worker pod is deleted and recreated. The startup sync does not help, since the field diff computes changed == false and never writes. --- cmd/ateapi/internal/controlapi/syncer.go | 6 +++--- cmd/ateapi/internal/controlapi/syncer_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/syncer.go b/cmd/ateapi/internal/controlapi/syncer.go index be87fe75f..a68ebf471 100644 --- a/cmd/ateapi/internal/controlapi/syncer.go +++ b/cmd/ateapi/internal/controlapi/syncer.go @@ -160,7 +160,7 @@ func (s *WorkerPoolSyncer) Start(ctx context.Context) { // Reconcile the other direction: enqueue every stored worker so records // whose pods no longer exist are cleaned up. This recovers delete events - // missed while ate-api-server was down — neither the watch relist nor + // missed while ate-api-server was down, neither the watch relist nor // the resync period can replay a delete across a process restart, // because the informer cache starts empty. Runs after the cache sync so // the indexer is an authoritative snapshot of live pods. @@ -221,7 +221,7 @@ func (s *WorkerPoolSyncer) reconcile(ctx context.Context, key workerKey) error { if pod.DeletionTimestamp != nil { // The pod has entered Terminating: mark the worker DRAINING so the // scheduler stops routing new actors to it. We deliberately do NOT touch - // the bound actor here — inside the pod ateom has received SIGTERM and is + // the bound actor here, inside the pod ateom has received SIGTERM and is // gracefully shutting the actor down. Actor cleanup happens on the Pod // Deleted event. return s.markWorkerDraining(ctx, key) @@ -348,7 +348,7 @@ func workerCapacity(pod *corev1.Pod) *ateapipb.WorkerCapacity { // markWorkerDraining transitions a worker to STATE_DRAINING so the scheduler // stops routing new actors to it while its pod is Terminating. If the worker is -// already gone or already draining there is nothing more to do — the Pod +// already gone or already draining there is nothing more to do, the Pod // Deleted event will clean up the record. A version conflict is returned so the // caller requeues and retries against the updated record. func (s *WorkerPoolSyncer) markWorkerDraining(ctx context.Context, key workerKey) error { diff --git a/cmd/ateapi/internal/controlapi/syncer_test.go b/cmd/ateapi/internal/controlapi/syncer_test.go index 67eeb9528..f59ec1fe1 100644 --- a/cmd/ateapi/internal/controlapi/syncer_test.go +++ b/cmd/ateapi/internal/controlapi/syncer_test.go @@ -421,7 +421,7 @@ func setupReconcileTest(t *testing.T, persistence store.Interface, initPools ... // TestSyncer_SoftDelete_MarksDraining verifies that a pod entering Terminating // (DeletionTimestamp set) flips its worker to STATE_DRAINING without deleting the -// worker record or touching the bound actor — the actor is still gracefully +// worker record or touching the bound actor, the actor is still gracefully // shutting down inside the pod. func TestSyncer_SoftDelete_MarksDraining(t *testing.T) { ctx := context.Background()