Improve connection pool failure diagnostics

doc/snippets/Microsoft.Data.SqlClient/SqlException.xml

@@ -307,6 +307,41 @@
       <seealso cref="T:Microsoft.Data.SqlClient.SqlErrorCollection" />
       <seealso cref="T:Microsoft.Data.SqlClient.SqlError" />
     </Errors>
+    <ConnectionOpenRetryFailures>
+      <summary>
+        Gets the transient failures that caused retries before this connection open operation ultimately failed.
+      </summary>
+      <value>
+        A read-only list in retry order. The list is empty when the exception was not produced by a retried connection open operation.
+      </value>
+      <remarks>
+        <para>
+          The terminal failure is represented by this <see cref="T:Microsoft.Data.SqlClient.SqlException" />. This property contains only earlier failures that caused the driver to retry the same connection open operation.
+        </para>
+        <para>
+          The list is scoped to one call to <see cref="M:Microsoft.Data.SqlClient.SqlConnection.Open" /> or <see cref="M:Microsoft.Data.SqlClient.SqlConnection.OpenAsync(System.Threading.CancellationToken)" /> and is discarded when the connection opens successfully.
+        </para>
+        <para>
+          Formatter-based serialization does not preserve this diagnostic list.
+        </para>
+      </remarks>
+      <example>
+        <code language="c#">
+          try
+          {
+              connection.Open();
+          }
+          catch (SqlException ex)
+          {
+              Console.WriteLine($"Final failure: {ex.Message}");
+              foreach (SqlException retryFailure in ex.ConnectionOpenRetryFailures)
+              {
+                  Console.WriteLine($"Earlier retry failure: {retryFailure.Message}");
+              }
+          }
+        </code>
+      </example>
+    </ConnectionOpenRetryFailures>
     <GetObjectData>
       <summary>To be added</summary>
     </GetObjectData>

src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs

@@ -1860,6 +1860,8 @@ private SqlException(System.Runtime.Serialization.SerializationInfo info, System
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlException.xml' path='docs/members[@name="SqlException"]/Errors/*'/>
     [System.ComponentModel.DesignerSerializationVisibilityAttribute(System.ComponentModel.DesignerSerializationVisibility.Content)]
     public Microsoft.Data.SqlClient.SqlErrorCollection Errors { get { throw null; } }
+    /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlException.xml' path='docs/members[@name="SqlException"]/ConnectionOpenRetryFailures/*'/>
+    public System.Collections.Generic.IReadOnlyList<Microsoft.Data.SqlClient.SqlException> ConnectionOpenRetryFailures { get { throw null; } }
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlException.xml' path='docs/members[@name="SqlException"]/LineNumber/*'/>
     public int LineNumber { get { throw null; } }
     /// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlException.xml' path='docs/members[@name="SqlException"]/Number/*'/>

src/Microsoft.Data.SqlClient/src/Microsoft/Data/Common/AdapterUtil.cs

@@ -21,6 +21,7 @@
 using Microsoft.Data.Common.ConnectionString;
 using Microsoft.Data.SqlClient;
 using Microsoft.Data.SqlClient.Connection;
+using Microsoft.Data.SqlClient.ConnectionPool;
 using Microsoft.SqlServer.Server;
 using Microsoft.Win32;
 using IsolationLevel = System.Data.IsolationLevel;
@@ -1337,6 +1338,17 @@ internal static Exception UndefinedPopulationMechanism(string populationMechanis
         internal static Exception PooledOpenTimeout()
             => ADP.InvalidOperation(StringsHelper.GetString(Strings.ADP_PooledOpenTimeout));
 
+        internal static Exception PooledOpenTimeout(PoolAcquisitionDiagnostics diagnostics)
+        {
+            Exception exception = ADP.InvalidOperation(
+                string.Concat(
+                    StringsHelper.GetString(Strings.ADP_PooledOpenTimeout),
+                    " ",
+                    diagnostics.GetMessage()));
+            diagnostics.AddTo(exception.Data);
+            return exception;
+        }
+
         internal static Exception NonPooledOpenTimeout()
             => ADP.TimeoutException(StringsHelper.GetString(Strings.ADP_NonPooledOpenTimeout));
 #endregion

src/Microsoft.Data.SqlClient/src/Microsoft/Data/ProviderBase/DbConnectionInternal.cs

@@ -37,6 +37,12 @@ internal abstract class DbConnectionInternal
 
         private readonly int _objectId = Interlocked.Increment(ref _objectTypeCount);
 
+        /// <summary>
+        /// UTC time at which this internal connection was most recently handed to an owning
+        /// <see cref="DbConnection"/>. Cleared when it returns to the pool.
+        /// </summary>
+        private DateTime _checkoutTime;
+
         /// <summary>
         /// [usage must be thread safe] the owning object, when not in the pool. (both Pooled and Non-Pooled connections)
         /// </summary>
@@ -119,6 +125,17 @@ internal DbConnectionInternal(ConnectionState state, bool hidePassword, bool all
         /// </summary>
         internal DateTime ReturnedTime { get; set; }
 
+        /// <summary>
+        /// UTC timestamp of the current checkout, or <see cref="DateTime.MinValue"/> while the
+        /// connection is not owned by an application connection. The internal setter supports
+        /// deterministic timeout diagnostics tests.
+        /// </summary>
+        internal DateTime CheckoutTime
+        {
+            get => _checkoutTime;
+            set => _checkoutTime = value;
+        }
+
         /// <summary>
         /// The pool generation at the time this connection was created or added to the pool.
         /// Used by <see cref="ChannelDbConnectionPool"/> to detect stale connections after a pool clear.
@@ -730,8 +747,10 @@ internal void MakePooledConnection(IDbConnectionPool connectionPool)
             Pool = connectionPool;
         }
 
-        internal void PostPop(DbConnection newOwner)
+        internal void PostPop(DbConnection newOwner, DateTime checkoutTime)
         {
+            Debug.Assert(checkoutTime.Kind == DateTimeKind.Utc);
+
             // Called by IDbConnectionPool right after it pulls this from its pool, we take this
             // opportunity to ensure ownership and pool counts are legit.
             Debug.Assert(!IsEmancipated, "pooled object not in pool");
@@ -746,6 +765,7 @@ internal void PostPop(DbConnection newOwner)
 
             _owningObject.SetTarget(newOwner);
             _pooledCount--;
+            _checkoutTime = checkoutTime;
 
             SqlClientEventSource.Log.TryPoolerTraceEvent("<prov.DbConnectionInternal.PostPop|RES|CPOOL> {0}, Preparing to pop from pool,  owning connection {1}, pooledCount={2}", ObjectID, 0, _pooledCount);
 
@@ -819,12 +839,60 @@ internal void PrePush(DbConnection expectedOwner)
             SqlClientEventSource.Log.TryPoolerTraceEvent("<prov.DbConnectionInternal.PrePush|RES|CPOOL> {0}, Preparing to push into pool, owning connection {1}, pooledCount={2}", ObjectID, 0, _pooledCount);
 
             _pooledCount++;
+            _checkoutTime = DateTime.MinValue;
 
             // NOTE: doing this and checking for InternalError.PooledObjectHasOwner degrades the
             //    close by 2%
             _owningObject.SetTarget(null);
         }
 
+        /// <summary>
+        /// Classifies the connection for a timeout-only pool diagnostics snapshot.
+        /// The caller must hold this connection's monitor.
+        /// </summary>
+        /// <param name="utcNow">Current UTC time used to calculate checkout duration.</param>
+        /// <param name="checkoutDuration">How long the current or abandoned checkout has lasted.</param>
+        /// <returns>The connection's current pool usage state.</returns>
+        internal PoolConnectionUsageState GetPoolUsageState(
+            DateTime utcNow,
+            out TimeSpan checkoutDuration)
+        {
+            Debug.Assert(Monitor.IsEntered(this));
+            Debug.Assert(utcNow.Kind == DateTimeKind.Utc);
+
+            checkoutDuration = TimeSpan.Zero;
+
+            if (IsTxRootWaitingForTxEnd ||
+                (IsInPool && EnlistedTransaction is not null))
+            {
+                return PoolConnectionUsageState.TransactionHeld;
+            }
+
+            if (IsInPool)
+            {
+                return PoolConnectionUsageState.Idle;
+            }
+
+            if (_owningObject.TryGetTarget(out _))
+            {
+                checkoutDuration = GetCheckoutDuration(utcNow);
+                return PoolConnectionUsageState.CheckedOut;
+            }
+
+            if (_checkoutTime != DateTime.MinValue && IsEmancipated)
+            {
+                checkoutDuration = GetCheckoutDuration(utcNow);
+                return PoolConnectionUsageState.Abandoned;
+            }
+
+            return PoolConnectionUsageState.Unclassified;
+        }
+
+        private TimeSpan GetCheckoutDuration(DateTime utcNow) =>
+            utcNow > _checkoutTime
+                ? utcNow - _checkoutTime
+                : TimeSpan.Zero;
+
         internal void RemoveWeakReference(object value) =>
             ReferenceCollection?.Remove(value);
 

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Connection/SqlConnectionInternal.cs

@@ -313,6 +313,12 @@ internal class SqlConnectionInternal : DbConnectionInternal, IDisposable
 
         private readonly SqlConnectionTimeoutErrorInternal _timeoutErrorInternal;
 
+        /// <summary>
+        /// Transient failures that caused this physical connection open to retry. The list exists
+        /// only while the constructor's open operation is in progress and is cleared on success.
+        /// </summary>
+        private List<SqlException> _connectionOpenRetryFailures;
+
         /// <summary>
         /// Cache the whereabouts (DTC Address) for exporting.
         /// </summary>
@@ -452,9 +458,17 @@ internal SqlConnectionInternal(
                             && _timeout.MillisecondsRemaining >= transientRetryIntervalInMilliSeconds
                             && IsTransientError(sqlex))
                     {
+                        RecordConnectionOpenRetryFailure(sqlex);
                         Thread.Sleep(transientRetryIntervalInMilliSeconds);
                     }
+                    catch (SqlException sqlex)
+                    {
+                        AttachConnectionOpenRetryFailures(sqlex);
+                        throw;
+                    }
                 }
+
+                _connectionOpenRetryFailures = null;
             }
             // @TODO: CER Exception Handling was removed here (see GH#3581)
             finally
@@ -3407,6 +3421,7 @@ private void LoginNoFailover(
                 {
                     if (AttemptRetryADAuthWithTimeoutError(sqlex, timeout))
                     {
+                        RecordConnectionOpenRetryFailure(sqlex);
                         continue;
                     }
 
@@ -3429,6 +3444,8 @@ private void LoginNoFailover(
                     {
                         throw;
                     }
+
+                    RecordConnectionOpenRetryFailure(sqlex);
                 }
 
                 // We only get here when we failed to connect, but are going to re-try
@@ -3731,6 +3748,7 @@ private void LoginWithFailover(
                 {
                     if (AttemptRetryADAuthWithTimeoutError(sqlex, timeout))
                     {
+                        RecordConnectionOpenRetryFailure(sqlex);
                         continue;
                     }
 
@@ -3768,6 +3786,8 @@ private void LoginWithFailover(
                             throw;
                         }
                     }
+
+                    RecordConnectionOpenRetryFailure(sqlex);
                 }
 
                 // We only get here when we failed to connect, but are going to re-try
@@ -3838,6 +3858,26 @@ private bool IsDoNotRetryConnectError(SqlException exc)
             return errorNumberMatch || exc._doNotReconnect;
         }
 
+        /// <summary>
+        /// Records a transient failure immediately before the same connection open is retried.
+        /// </summary>
+        private void RecordConnectionOpenRetryFailure(SqlException exception) =>
+            (_connectionOpenRetryFailures ??= new List<SqlException>()).Add(exception);
+
+        /// <summary>
+        /// Attaches the retry ledger to the terminal failure without changing its errors or inner
+        /// exception.
+        /// </summary>
+        private void AttachConnectionOpenRetryFailures(
+            SqlException terminalException)
+        {
+            if (_connectionOpenRetryFailures is { Count: > 0 })
+            {
+                terminalException.SetConnectionOpenRetryFailures(
+                    _connectionOpenRetryFailures);
+            }
+        }
+
         /// <summary>
         /// Returns <c>true</c> if the SQL error is transient, as per <see cref="s_transientErrors"/>.
         /// </summary>

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/BlockingPeriodErrorState.cs

@@ -122,7 +122,10 @@ internal void Enter(Exception ex)
             lock (_lock)
             {
                 _inElevatedState = true;
-                _cachedException = ex;
+                _cachedException = ex is SqlException sqlException
+                    ? sqlException.InternalClone(
+                        includeConnectionOpenRetryFailures: false)
+                    : ex;
                 wait = _nextWait;
 
                 ITimer newTimer = ADP.UnsafeCreateTimer(