fix race condition in grpc_tsi_alts_shutdown()

src/core/tsi/alts_transport_security.cc

@@ -26,17 +26,36 @@ alts_shared_resource* alts_get_shared_resource(void) {
   return &g_alts_resource;
 }
 
+static void grpc_tsi_alts_wait_for_cq_drain() {
+  gpr_mu_lock(&g_alts_resource.mu);
+  while (!g_alts_resource.is_cq_drained) {
+    gpr_cv_wait(&g_alts_resource.cv, &g_alts_resource.mu,
+                gpr_inf_future(GPR_CLOCK_REALTIME));
+  }
+  gpr_mu_unlock(&g_alts_resource.mu);
+}
+
+void grpc_tsi_alts_signal_for_cq_destroy() {
+  gpr_mu_lock(&g_alts_resource.mu);
+  g_alts_resource.is_cq_drained = true;
+  gpr_cv_signal(&g_alts_resource.cv);
+  gpr_mu_unlock(&g_alts_resource.mu);
+}
+
 void grpc_tsi_alts_init() {
   memset(&g_alts_resource, 0, sizeof(alts_shared_resource));
   gpr_mu_init(&g_alts_resource.mu);
+  gpr_cv_init(&g_alts_resource.cv);
 }
 
 void grpc_tsi_alts_shutdown() {
-  gpr_mu_destroy(&g_alts_resource.mu);
-  if (g_alts_resource.cq == nullptr) {
-    return;
+  if (g_alts_resource.cq != nullptr) {
+    grpc_completion_queue_shutdown(g_alts_resource.cq);
+    grpc_tsi_alts_wait_for_cq_drain();
+    grpc_completion_queue_destroy(g_alts_resource.cq);
+    grpc_channel_destroy(g_alts_resource.channel);
+    gpr_thd_join(g_alts_resource.thread_id);
   }
-  grpc_completion_queue_destroy(g_alts_resource.cq);
-  grpc_channel_destroy(g_alts_resource.channel);
-  gpr_thd_join(g_alts_resource.thread_id);
+  gpr_cv_destroy(&g_alts_resource.cv);
+  gpr_mu_destroy(&g_alts_resource.mu);
 }

src/core/tsi/alts_transport_security.h

@@ -28,10 +28,17 @@ typedef struct alts_shared_resource {
   grpc_channel* channel;
   grpc_completion_queue* cq;
   gpr_mu mu;
+  gpr_cv cv;
+  bool is_cq_drained;
 } alts_shared_resource;
 
 /* This method returns the address of alts_shared_resource object shared by all
  *    TSI handshakes. */
 alts_shared_resource* alts_get_shared_resource(void);
 
+/* This method signals the thread that invokes grpc_tsi_alts_shutdown() to
+ * continue with destroying the cq as a part of shutdown process. */
+
+void grpc_tsi_alts_signal_for_cq_destroy(void);
+
 #endif /* GRPC_CORE_TSI_ALTS_TRANSPORT_SECURITY_H */