From 8100337d3fc56f071756c20cfbab2bd8450ae0c5 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Thu, 7 May 2026 17:54:01 -0400 Subject: [PATCH 1/4] fix(cobuilds): add socket + connect timeout Signed-off-by: Aramis Sennyey --- .../fix-redis-reconnect_2026-05-07-00-00.json | 10 ++++++++++ .../src/RedisCobuildLockProvider.ts | 13 +++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json diff --git a/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json b/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json new file mode 100644 index 00000000000..9dea113151a --- /dev/null +++ b/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-redis-cobuild-plugin", + "comment": "Detect half-dead Redis connections via `socketTimeout` and `keepAlive`, and replace the `count * 1000` reconnect backoff (0ms on first attempt) with exponential backoff plus jitter, capped at 10s.", + "type": "patch" + } + ], + "packageName": "@rushstack/rush-redis-cobuild-plugin" +} diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 3c2398baca8..3e4057d5d19 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -50,11 +50,20 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); - // Provide a default reconnect strategy that prevents more than 5 reconnect attempts. + // Provide socket defaults that detect half-dead connections and reconnect with + // backoff. this._options.socket = { + connectTimeout: 10_000, + socketTimeout: 30_000, reconnectStrategy: (count: number) => { this._terminal.writeErrorLine(`Redis client reconnecting attempt #${count}`); - return count < 5 ? count * 1000 : false; + if (count >= 5) { + return false; + } + // Exponential backoff with jitter, capped at 10s. Avoids the previous + // `count * 1000` formula which produced a 0ms delay on the first attempt. + const jitter: number = Math.floor(Math.random() * 200); + return Math.min(500 * 2 ** count, 10_000) + jitter; }, ...this._options.socket }; From ebd1496f5ccbe37e9f0890a8441437664daafd10 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Thu, 7 May 2026 17:59:29 -0400 Subject: [PATCH 2/4] scope to just timeout additions Signed-off-by: Aramis Sennyey --- .../fix-redis-reconnect_2026-05-07-00-00.json | 2 +- .../src/RedisCobuildLockProvider.ts | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json b/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json index 9dea113151a..5285b0c0e32 100644 --- a/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json +++ b/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@rushstack/rush-redis-cobuild-plugin", - "comment": "Detect half-dead Redis connections via `socketTimeout` and `keepAlive`, and replace the `count * 1000` reconnect backoff (0ms on first attempt) with exponential backoff plus jitter, capped at 10s.", + "comment": "Set Redis `connectTimeout` and `socketTimeout` so half-dead TCP connections (NAT/firewall) surface in seconds instead of stalling in-flight commands for many minutes while the kernel waits to give up.", "type": "patch" } ], diff --git a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts index 3e4057d5d19..defc6908b35 100644 --- a/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts +++ b/rush-plugins/rush-redis-cobuild-plugin/src/RedisCobuildLockProvider.ts @@ -50,20 +50,15 @@ export class RedisCobuildLockProvider implements ICobuildLockProvider { public constructor(options: IRedisCobuildLockProviderOptions, rushSession: RushSession) { this._options = RedisCobuildLockProvider.expandOptionsWithEnvironmentVariables(options); - // Provide socket defaults that detect half-dead connections and reconnect with - // backoff. + // Detect half-dead connections quickly. Without `socketTimeout`, a silently-dropped + // TCP connection (NAT/firewall) can stall in-flight commands for many minutes while + // the kernel waits to surface the failure. this._options.socket = { connectTimeout: 10_000, socketTimeout: 30_000, reconnectStrategy: (count: number) => { this._terminal.writeErrorLine(`Redis client reconnecting attempt #${count}`); - if (count >= 5) { - return false; - } - // Exponential backoff with jitter, capped at 10s. Avoids the previous - // `count * 1000` formula which produced a 0ms delay on the first attempt. - const jitter: number = Math.floor(Math.random() * 200); - return Math.min(500 * 2 ** count, 10_000) + jitter; + return count < 5 ? count * 1000 : false; }, ...this._options.socket }; From 48e41d5bb2309d1224f6a387226910d33e81661a Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Thu, 7 May 2026 18:12:49 -0400 Subject: [PATCH 3/4] rename changefile package Signed-off-by: Aramis Sennyey --- .../fix-redis-reconnect_2026-05-07-00-00.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json b/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json index 5285b0c0e32..43844f89bcf 100644 --- a/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json +++ b/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json @@ -1,10 +1,10 @@ { "changes": [ { - "packageName": "@rushstack/rush-redis-cobuild-plugin", + "packageName": "@microsoft/rush", "comment": "Set Redis `connectTimeout` and `socketTimeout` so half-dead TCP connections (NAT/firewall) surface in seconds instead of stalling in-flight commands for many minutes while the kernel waits to give up.", "type": "patch" } ], - "packageName": "@rushstack/rush-redis-cobuild-plugin" + "packageName": "@microsoft/rush" } From f665bf930f74685ea238aaca5921e224730e2147 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey Date: Thu, 7 May 2026 18:13:33 -0400 Subject: [PATCH 4/4] rename changefile package Signed-off-by: Aramis Sennyey --- .../rush}/fix-redis-reconnect_2026-05-07-00-00.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename common/changes/{@rushstack/rush-redis-cobuild-plugin => @microsoft/rush}/fix-redis-reconnect_2026-05-07-00-00.json (100%) diff --git a/common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json b/common/changes/@microsoft/rush/fix-redis-reconnect_2026-05-07-00-00.json similarity index 100% rename from common/changes/@rushstack/rush-redis-cobuild-plugin/fix-redis-reconnect_2026-05-07-00-00.json rename to common/changes/@microsoft/rush/fix-redis-reconnect_2026-05-07-00-00.json