diff --git a/src/Runner.Common/BrokerServer.cs b/src/Runner.Common/BrokerServer.cs
index 751ae1eee32..c642b4af1cf 100644
--- a/src/Runner.Common/BrokerServer.cs
+++ b/src/Runner.Common/BrokerServer.cs
@@ -108,7 +108,7 @@ public Task ForceRefreshConnection(VssCredentials credentials)
 
         public bool ShouldRetryException(Exception ex)
         {
-            if (ex is AccessDeniedException || ex is RunnerNotFoundException || ex is HostedRunnerDeprovisionedException)
+            if (ex is AccessDeniedException || ex is VssUnauthorizedException || ex is RunnerNotFoundException || ex is HostedRunnerDeprovisionedException)
             {
                 return false;
             }