Fix flaky ConnectTimeout_MultipleCalls_AttemptReconnect test (#2460)

Author: JamesNK

src/Grpc.Net.Client/Balancer/Internal/ConnectionManager.cs

@@ -312,12 +312,20 @@ public void UpdateState(BalancerState state)
             {
                 case PickResultType.Complete:
                     var subchannel = result.Subchannel!;
-                    var address = subchannel.CurrentAddress;
+                    var (address, state) = subchannel.GetAddressAndState();
 
                     if (address != null)
                     {
-                        ConnectionManagerLog.PickResultSuccessful(Logger, subchannel.Id, address, subchannel.Transport.TransportStatus);
-                        return (subchannel, address, result.SubchannelCallTracker);
+                        if (state == ConnectivityState.Ready)
+                        {
+                            ConnectionManagerLog.PickResultSuccessful(Logger, subchannel.Id, address, subchannel.Transport.TransportStatus);
+                            return (subchannel, address, result.SubchannelCallTracker);
+                        }
+                        else
+                        {
+                            ConnectionManagerLog.PickResultSubchannelNotReady(Logger, subchannel.Id, address, state);
+                            previousPicker = currentPicker;
+                        }
                     }
                     else
                     {
@@ -499,6 +507,9 @@ internal static class ConnectionManagerLog
     private static readonly Action<ILogger, Status, Exception?> _resolverServiceConfigFallback =
         LoggerMessage.Define<Status>(LogLevel.Debug, new EventId(12, "ResolverServiceConfigFallback"), "Falling back to previously loaded service config. Resolver failure when retreiving or parsing service config with status: {Status}");
 
+    private static readonly Action<ILogger, string, BalancerAddress, ConnectivityState, Exception?> _pickResultSubchannelNotReady =
+        LoggerMessage.Define<string, BalancerAddress, ConnectivityState>(LogLevel.Debug, new EventId(13, "PickResultSubchannelNotReady"), "Picked subchannel id '{SubchannelId}' with address {CurrentAddress} doesn't have a ready state. Subchannel state: {State}");
+
     public static void ResolverUnsupportedLoadBalancingConfig(ILogger logger, IList<LoadBalancingConfig> loadBalancingConfigs)
     {
         if (logger.IsEnabled(LogLevel.Warning))
@@ -562,5 +573,10 @@ public static void ResolverServiceConfigFallback(ILogger logger, Status status)
     {
         _resolverServiceConfigFallback(logger, status, null);
     }
+
+    public static void PickResultSubchannelNotReady(ILogger logger, string subchannelId, BalancerAddress currentAddress, ConnectivityState state)
+    {
+        _pickResultSubchannelNotReady(logger, subchannelId, currentAddress, state, null);
+    }
 }
 #endif

src/Grpc.Net.Client/Balancer/Subchannel.cs

@@ -86,6 +86,14 @@ public BalancerAddress? CurrentAddress
     /// </summary>
     public BalancerAttributes Attributes { get; }
 
+    internal (BalancerAddress? Address, ConnectivityState State) GetAddressAndState()
+    {
+        lock (Lock)
+        {
+            return (CurrentAddress, State);
+        }
+    }
+
     internal Subchannel(ConnectionManager manager, IReadOnlyList<BalancerAddress> addresses)
     {
         Lock = new object();