Merge pull request #17391 from markdroth/reset_connection_backoff

Fix bug in subchannel backoff reset code.

src/core/ext/filters/client_channel/subchannel.cc

@@ -153,7 +153,7 @@ struct grpc_subchannel {
   /** have we started the backoff loop */
   bool backoff_begun;
   // reset_backoff() was called while alarm was pending
-  bool deferred_reset_backoff;
+  bool retry_immediately;
   /** our alarm */
   grpc_timer alarm;
 
@@ -709,8 +709,8 @@ static void on_alarm(void* arg, grpc_error* error) {
   if (c->disconnected) {
     error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Disconnected",
                                                              &error, 1);
-  } else if (c->deferred_reset_backoff) {
-    c->deferred_reset_backoff = false;
+  } else if (c->retry_immediately) {
+    c->retry_immediately = false;
     error = GRPC_ERROR_NONE;
   } else {
     GRPC_ERROR_REF(error);
@@ -887,12 +887,12 @@ static void on_subchannel_connected(void* arg, grpc_error* error) {
 
 void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel) {
   gpr_mu_lock(&subchannel->mu);
+  subchannel->backoff->Reset();
   if (subchannel->have_alarm) {
-    subchannel->deferred_reset_backoff = true;
+    subchannel->retry_immediately = true;
     grpc_timer_cancel(&subchannel->alarm);
   } else {
     subchannel->backoff_begun = false;
-    subchannel->backoff->Reset();
     maybe_start_connecting_locked(subchannel);
   }
   gpr_mu_unlock(&subchannel->mu);

test/cpp/end2end/client_lb_end2end_test.cc

@@ -537,6 +537,51 @@ TEST_F(ClientLbEnd2endTest, PickFirstResetConnectionBackoff) {
   EXPECT_LT(waited_ms, kInitialBackOffMs);
 }
 
+TEST_F(ClientLbEnd2endTest,
+       PickFirstResetConnectionBackoffNextAttemptStartsImmediately) {
+  ChannelArguments args;
+  constexpr int kInitialBackOffMs = 1000;
+  args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs);
+  const std::vector<int> ports = {grpc_pick_unused_port_or_die()};
+  auto channel = BuildChannel("pick_first", args);
+  auto stub = BuildStub(channel);
+  SetNextResolution(ports);
+  // Wait for connect, which should fail ~immediately, because the server
+  // is not up.
+  gpr_log(GPR_INFO, "=== INITIAL CONNECTION ATTEMPT");
+  EXPECT_FALSE(
+      channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10)));
+  // Reset connection backoff.
+  // Note that the time at which the third attempt will be started is
+  // actually computed at this point, so we record the start time here.
+  gpr_log(GPR_INFO, "=== RESETTING BACKOFF");
+  const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC);
+  experimental::ChannelResetConnectionBackoff(channel.get());
+  // Trigger a second connection attempt.  This should also fail
+  // ~immediately, but the retry should be scheduled for
+  // kInitialBackOffMs instead of applying the multiplier.
+  gpr_log(GPR_INFO, "=== POLLING FOR SECOND CONNECTION ATTEMPT");
+  EXPECT_FALSE(
+      channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10)));
+  // Bring up a server on the chosen port.
+  gpr_log(GPR_INFO, "=== STARTING BACKEND");
+  StartServers(1, ports);
+  // Wait for connect.  Should happen within kInitialBackOffMs.
+  // Give an extra 100ms to account for the time spent in the second and
+  // third connection attempts themselves (since what we really want to
+  // measure is the time between the two).  As long as this is less than
+  // the 1.6x increase we would see if the backoff state was not reset
+  // properly, the test is still proving that the backoff was reset.
+  constexpr int kWaitMs = kInitialBackOffMs + 100;
+  gpr_log(GPR_INFO, "=== POLLING FOR THIRD CONNECTION ATTEMPT");
+  EXPECT_TRUE(channel->WaitForConnected(
+      grpc_timeout_milliseconds_to_deadline(kWaitMs)));
+  const gpr_timespec t1 = gpr_now(GPR_CLOCK_MONOTONIC);
+  const grpc_millis waited_ms = gpr_time_to_millis(gpr_time_sub(t1, t0));
+  gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited_ms);
+  EXPECT_LT(waited_ms, kWaitMs);
+}
+
 TEST_F(ClientLbEnd2endTest, PickFirstUpdates) {
   // Start servers and send one RPC per server.
   const int kNumServers = 3;