atenet-router: take parking-lot slots at the park transition, not at admission

cmd/atenet/internal/router/ingress/errors.go

@@ -51,10 +51,16 @@ func statusDescription(err error) string {
 	return status.Convert(err).Message()
 }
 
+// errParkingLotFull is returned by ActorResumer.ResumeActor when a request
+// reached its park transition but the lot had no free slot. mapResumeError
+// turns it into the client-facing 503 "router at capacity" denial.
+var errParkingLotFull = errors.New("parking lot full")
+
 // parkingFullErr returns a 503 denial signaling that the router's parking lot
-// is at capacity, so the request was shed without waiting. Clients should retry.
-func parkingFullErr(actorID string) error {
-	return extproc.NewReqError(envoy_type.StatusCode_ServiceUnavailable,
+// is at capacity, so the request was shed rather than parked. Clients should
+// retry. The cause is preserved for log inspection via Unwrap.
+func parkingFullErr(actorID string, cause error) error {
+	return extproc.WrapReqError(envoy_type.StatusCode_ServiceUnavailable, cause,
 		"actor %q unavailable: router at capacity", actorID)
 }
 
@@ -70,6 +76,12 @@ func mapResumeError(actorRef resources.ActorRef, err error) error {
 		return nil
 	}
 
+	// A caller shed at its park transition because the lot was full: the
+	// resume itself did not fail, the router declined to hold the request.
+	if errors.Is(err, errParkingLotFull) {
+		return parkingFullErr(actorRef.String(), err)
+	}
+
 	re := &extproc.ReqError{Cause: err}
 
 	// Bare context sentinels reach here when the request's own context ends

cmd/atenet/internal/router/ingress/flight.go

@@ -0,0 +1,103 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ingress
+
+import (
+	"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// resumeFlight is one in-flight, per-actor resume that concurrent requests
+// share. It replaces a singleflight.Group entry so callers can observe the
+// flight mid-run: parked exposes the park transition, which singleflight's
+// result-only channel cannot. Its lifecycle is park (at most once, flight
+// goroutine only) followed by ActorResumer.publish (exactly once).
+type resumeFlight struct {
+	// parked is closed at most once, by park, at the flight's first retryable
+	// error — the moment the flight stops resolving and starts waiting. A
+	// caller woken by it (or attaching after it) must hold a parking-lot slot
+	// to keep waiting. Never closed on the fast path.
+	parked chan struct{}
+	// parkedSignaled guards the parked close. Only the flight goroutine calls
+	// park, so a plain bool suffices.
+	parkedSignaled bool
+	// done is closed exactly once, by publish, after result is written and the
+	// flight is deleted from the registry. The write-result-then-close order
+	// is what lets every caller read result without further synchronization;
+	// the delete-then-close order is what keeps a completed flight unjoinable
+	// (the next request for the actor starts a fresh flight).
+	done chan struct{}
+	// result is the shared outcome, written exactly once before done closes.
+	result *resumeCallResult
+}
+
+func newResumeFlight() *resumeFlight {
+	return &resumeFlight{parked: make(chan struct{}), done: make(chan struct{})}
+}
+
+// park signals the flight's park transition: it stopped resolving and started
+// waiting, so from here on callers must hold parking-lot slots to keep
+// waiting. Idempotent; only the flight goroutine may call it.
+func (f *resumeFlight) park() {
+	if f.parkedSignaled {
+		return
+	}
+	f.parkedSignaled = true
+	close(f.parked)
+}
+
+// callerResult classifies f's completed outcome for one caller. It must only
+// be called after f.done is closed.
+func (f *resumeFlight) callerResult(reqID uint64) (*ateapipb.Actor, ResumeOutcome, error) {
+	res := f.result
+	if res == nil {
+		return nil, ResumeOutcomeNone, status.Error(codes.Internal, "resume call returned nil result")
+	}
+
+	// On error, return ResumeOutcomeNone ("none") so the failure is tagged
+	// under the 'outcome' label rather than misreported as an activation.
+	if res.err != nil {
+		return nil, ResumeOutcomeNone, res.err
+	}
+
+	// Disambiguate the shared-flight resume outcome:
+	// - ResumeOutcomeNone ("none"): resumed == false, actor was already active/running.
+	// - ResumeOutcomeTriggered ("triggered"): Cold activation leader (resumed == true, caller's reqID == leaderID).
+	// - ResumeOutcomeJoined ("joined"): Cold activation joiner (resumed == true, caller's reqID != leaderID).
+	outcome := ResumeOutcomeNone
+	if res.resumed {
+		if res.leaderID == reqID {
+			outcome = ResumeOutcomeTriggered
+		} else {
+			outcome = ResumeOutcomeJoined
+		}
+	}
+
+	return res.actor, outcome, nil
+}
+
+// publish completes f with result and makes it unjoinable. The order is
+// load-bearing: result before done (the channel close is what makes the write
+// visible to callers), and registry delete before done (so no caller can
+// attach to a completed flight — the next request for the actor starts a
+// fresh one, preserving forget-on-completion semantics).
+func (r *ActorResumer) publish(f *resumeFlight, key string, result *resumeCallResult) {
+	f.result = result
+	r.mu.Lock()
+	delete(r.flights, key)
+	r.mu.Unlock()
+	close(f.done)
+}

cmd/atenet/internal/router/ingress/ingress.go

@@ -72,9 +72,12 @@ type Handler struct {
 }
 
 func New(apiClient ateapipb.ControlClient, parkCfg ParkedRequestConfig, parkMetrics *ParkingMetrics) *Handler {
+	// The lot is shared: the resumer charges it at each caller's park
+	// transition; the handler keeps a reference only for the status page.
+	lot := newParkingLot(parkCfg, parkMetrics)
 	return &Handler{
-		resumer: NewActorResumer(apiClient, withParking(parkCfg)),
-		parking: newParkingLot(parkCfg, parkMetrics),
+		resumer: NewActorResumer(apiClient, withParking(parkCfg), withParkingLot(lot)),
+		parking: lot,
 	}
 }
 
@@ -116,19 +119,13 @@ func (h *Handler) HandleRequestHeaders(ctx context.Context, md *extproc.RequestM
 		}
 	}
 
-	// Admit the request to the parking lot before resuming. While resume is
-	// in-flight the request occupies a slot; if the actor's worker pool is
-	// momentarily saturated the resumer parks (retries) here rather than failing
-	// fast. A full lot sheds the request immediately so the router applies
-	// backpressure instead of queueing without bound.
-	release, ok := h.parking.enter(ctx)
-	if !ok {
-		return extproc.Result{}, parkingFullErr(actorRef.String())
-	}
-
+	// The resumer parks the request if the actor's worker pool is momentarily
+	// saturated, retrying rather than failing fast. Parking-lot admission
+	// happens inside, at the park transition: a request resolved on the first
+	// attempt never occupies a slot, so a full lot sheds only requests that
+	// would actually wait — never traffic to already-running actors.
 	slog.InfoContext(ctx, "ResumeActor", slog.Any("actor", actorRef))
 	actor, resumeOutcome, err := h.resumer.ResumeActor(ctx, actorRef)
-	release(parkOutcomeFor(err))
 	if err != nil {
 		return extproc.Result{Resume: string(resumeOutcome)}, mapResumeError(actorRef, err)
 	}

cmd/atenet/internal/router/ingress/ingress_test.go

@@ -21,7 +21,9 @@ import (
 	"errors"
 	"log/slog"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"testing/synctest"
 	"time"
 
 	corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
@@ -338,17 +340,31 @@ func TestHandleRequestHeadersHandlesConnectMethod(t *testing.T) {
 	}
 }
 
-// TestHandleRequestHeaders_ParkingLotFull verifies that when the parking lot is at capacity
-// the request is shed with a 503 before any resume is attempted.
-func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) {
+// TestHandleRequestHeaders_FullLotServesRunningActor pins the design guarantee
+// from docs/request-parking.md: a saturated parking lot cannot starve requests
+// to already-running actors, at any lot size. A request whose resume resolves
+// on the first attempt never occupies a slot, so it is served even with the
+// lot at capacity (issue #1081).
+func TestHandleRequestHeaders_FullLotServesRunningActor(t *testing.T) {
 	const testUUID = "123e4567-e89b-12d3-a456-426614174000"
 	authority := testUUID + ".team-a.actors.resources.substrate.ate.dev"
 
 	var resumeCalled bool
 	clientMock := &mockClient{
-		resumeFn: func(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) {
+		resumeFn: func(
+			ctx context.Context,
+			in *ateapipb.ResumeActorRequest,
+			opts ...grpc.CallOption,
+		) (*ateapipb.ResumeActorResponse, error) {
 			resumeCalled = true
-			return &ateapipb.ResumeActorResponse{Actor: &ateapipb.Actor{Status: &ateapipb.ActorStatus{WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.1"}}}}, nil
+			return &ateapipb.ResumeActorResponse{
+				Actor: &ateapipb.Actor{
+					Status: &ateapipb.ActorStatus{
+						State:            ateapipb.ActorState_ACTOR_STATE_RUNNING,
+						WorkerAssignment: &ateapipb.WorkerAssignment{WorkerPodIp: "10.0.0.1"},
+					},
+				},
+			}, nil
 		},
 	}
 
@@ -365,21 +381,77 @@ func TestHandleRequestHeaders_ParkingLotFull(t *testing.T) {
 		&corev3.HeaderValue{Key: ":authority", Value: authority},
 	)
 
-	_, err := h.HandleRequestHeaders(context.Background(), md)
-	if err == nil {
-		t.Fatal("expected error when parking lot is full")
-	}
-	var reqErr *extproc.ReqError
-	if !errors.As(err, &reqErr) {
-		t.Fatalf("expected *extproc.ReqError, got %T (%v)", err, err)
+	res, err := h.HandleRequestHeaders(context.Background(), md)
+	if err != nil {
+		t.Fatalf("a running actor must be served despite a full lot, got: %v", err)
 	}
-	if reqErr.StatusCode != int(envoy_type.StatusCode_ServiceUnavailable) {
-		t.Errorf("status code = %d, want %d (503)", reqErr.StatusCode, envoy_type.StatusCode_ServiceUnavailable)
+	if !resumeCalled {
+		t.Error("the resume lookup must still run when the lot is full")
 	}
-	if !strings.Contains(reqErr.Error(), "router at capacity") {
-		t.Errorf("error body = %q, want it to mention capacity", reqErr.Error())
+	const wantTarget = "10.0.0.1:443"
+	if res.Target != wantTarget {
+		t.Errorf("target = %q, want %q", res.Target, wantTarget)
 	}
-	if resumeCalled {
-		t.Error("resume must not be attempted for a shed request")
+	if got := h.parking.activeCount(); got != 1 {
+		t.Errorf("a first-attempt resolution must not occupy a slot; active = %d, want 1 (the priming entry)", got)
 	}
 }
+
+// TestHandleRequestHeaders_FullLotShedsParkedRequest verifies the other half
+// of lot admission: when the lot is full, a request whose resume actually
+// parks (first retryable failure) is shed with 503 "router at capacity" — at
+// the park transition, after its single initial attempt, not before any.
+func TestHandleRequestHeaders_FullLotShedsParkedRequest(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		const testUUID = "123e4567-e89b-12d3-a456-426614174000"
+		authority := testUUID + ".team-a.actors.resources.substrate.ate.dev"
+
+		var resumeCalls atomic.Int32
+		clientMock := &mockClient{
+			resumeFn: func(
+				ctx context.Context,
+				in *ateapipb.ResumeActorRequest,
+				opts ...grpc.CallOption,
+			) (*ateapipb.ResumeActorResponse, error) {
+				resumeCalls.Add(1)
+				return nil, status.Error(codes.ResourceExhausted, "no free workers available")
+			},
+		}
+
+		h := New(clientMock, ParkedRequestConfig{Budget: 500 * time.Millisecond, Max: 1}, nil)
+		release, ok := h.parking.enter(context.Background())
+		if !ok {
+			t.Fatal("priming enter should be admitted")
+		}
+		defer release(parkOutcomeServed)
+
+		md := requestMetadata(t, authority,
+			&corev3.HeaderValue{Key: ":authority", Value: authority},
+		)
+
+		_, err := h.HandleRequestHeaders(context.Background(), md)
+		if err == nil {
+			t.Fatal("expected error when a parked request finds the lot full")
+		}
+		var reqErr *extproc.ReqError
+		if !errors.As(err, &reqErr) {
+			t.Fatalf("expected *extproc.ReqError, got %T (%v)", err, err)
+		}
+		if reqErr.StatusCode != int(envoy_type.StatusCode_ServiceUnavailable) {
+			t.Errorf("status code = %d, want %d (503)", reqErr.StatusCode, envoy_type.StatusCode_ServiceUnavailable)
+		}
+		if !strings.Contains(reqErr.Error(), "router at capacity") {
+			t.Errorf("error body = %q, want it to mention capacity", reqErr.Error())
+		}
+		// Shedding happens at the park transition: exactly one attempt has run
+		// when the caller is turned away.
+		if got := resumeCalls.Load(); got != 1 {
+			t.Errorf("expected the caller shed after exactly 1 attempt, got %d", got)
+		}
+
+		// The abandoned flight retries on until its budget, like any flight
+		// whose callers left; sleep (fake time) past the budget so it exits
+		// before the bubble does.
+		time.Sleep(600 * time.Millisecond)
+	})
+}

cmd/atenet/internal/router/ingress/parking.go

@@ -135,9 +135,12 @@ func DefaultParkedRequestConfig() ParkedRequestConfig {
 	}
 }
 
-// parkingLot is a bounded, non-blocking admission gate for resume-gated
-// requests. Each admitted request holds a slot for the duration of its resume
-// attempt; when the lot is full further requests are shed immediately so the
+// parkingLot is a bounded, non-blocking admission gate for parked requests.
+// A caller enters at its resume flight's park transition — the first
+// retryable failure — and holds the slot for the rest of its wait; requests
+// resolved on the flight's first attempt never enter, so a saturated lot
+// cannot starve traffic to already-running actors (issue #1081). When the lot
+// is full, callers reaching their park transition are shed immediately so the
 // router applies backpressure instead of accumulating waiters without bound.
 //
 // With parking disabled (Max <= 0) enter always admits and performs no
@@ -154,12 +157,12 @@ func newParkingLot(cfg ParkedRequestConfig, m *ParkingMetrics) *parkingLot {
 	return &parkingLot{cfg: cfg, metrics: m}
 }
 
-// enter attempts to reserve a parking slot. On success it returns a release
-// func and ok=true; the caller MUST invoke release exactly once (passing the
-// request outcome, e.g. parkOutcomeServed) when the resume attempt completes.
-// ok=false means the lot is full and the request should be shed without
-// waiting. When parking is disabled every request is admitted and no slot
-// accounting or metrics are recorded.
+// enter reserves a parking slot for a caller whose resume flight just parked.
+// On success it returns a release func and ok=true; the caller MUST invoke
+// release exactly once (passing the request outcome, e.g. parkOutcomeServed)
+// when its wait ends. ok=false means the lot is full and the request should
+// be shed without waiting further. When parking is disabled every request is
+// admitted and no slot accounting or metrics are recorded.
 func (l *parkingLot) enter(ctx context.Context) (release func(outcome parkOutcome), ok bool) {
 	if !l.cfg.Enabled() {
 		return func(parkOutcome) {}, true