Changed security handshakers to use generic handshaker API.

src/core/ext/transport/chttp2/client/secure/secure_channel_create.c

@@ -77,9 +77,6 @@ typedef struct {
   grpc_closure connected_closure;
 
   grpc_handshake_manager *handshake_mgr;
-
-  // TODO(roth): Remove once we eliminate on_secure_handshake_done().
-  grpc_channel_args *tmp_args;
 } connector;
 
 static void connector_ref(grpc_connector *con) {
@@ -91,62 +88,38 @@ static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
   connector *c = (connector *)con;
   if (gpr_unref(&c->refs)) {
     /* c->initial_string_buffer does not need to be destroyed */
-    grpc_channel_args_destroy(c->tmp_args);
     grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
     gpr_free(c);
   }
 }
 
-static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
-                                     grpc_security_status status,
-                                     grpc_endpoint *secure_endpoint,
-                                     grpc_auth_context *auth_context) {
-  connector *c = arg;
-  gpr_mu_lock(&c->mu);
-  grpc_error *error = GRPC_ERROR_NONE;
-  if (c->connecting_endpoint == NULL) {
-    memset(c->result, 0, sizeof(*c->result));
-    gpr_mu_unlock(&c->mu);
-  } else if (status != GRPC_SECURITY_OK) {
-    error = grpc_error_set_int(GRPC_ERROR_CREATE("Secure handshake failed"),
-                               GRPC_ERROR_INT_SECURITY_STATUS, status);
-    memset(c->result, 0, sizeof(*c->result));
-    c->connecting_endpoint = NULL;
-    gpr_mu_unlock(&c->mu);
-  } else {
-    grpc_arg auth_context_arg;
-    c->connecting_endpoint = NULL;
-    gpr_mu_unlock(&c->mu);
-    c->result->transport = grpc_create_chttp2_transport(
-        exec_ctx, c->args.channel_args, secure_endpoint, 1);
-    grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL);
-    auth_context_arg = grpc_auth_context_to_arg(auth_context);
-    c->result->channel_args =
-        grpc_channel_args_copy_and_add(c->tmp_args, &auth_context_arg, 1);
-  }
-  grpc_closure *notify = c->notify;
-  c->notify = NULL;
-  grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
-}
-
 static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
                               grpc_error *error) {
   grpc_handshaker_args *args = arg;
   connector *c = args->user_data;
-  c->tmp_args = args->args;
+  gpr_mu_lock(&c->mu);
   if (error != GRPC_ERROR_NONE) {
+    c->connecting_endpoint = NULL;
+    gpr_mu_unlock(&c->mu);
+    grpc_channel_args_destroy(args->args);
     gpr_free(args->read_buffer);
-    grpc_closure *notify = c->notify;
-    c->notify = NULL;
-    grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
   } else {
-    // TODO(roth, jboeuf): Convert security connector handshaking to use new
-    // handshake API, and then move the code from on_secure_handshake_done()
-    // into this function.
-    grpc_channel_security_connector_do_handshake(
-        exec_ctx, c->security_connector, args->endpoint, args->read_buffer,
-        c->args.deadline, on_secure_handshake_done, c);
+    if (c->connecting_endpoint == NULL) {
+      memset(c->result, 0, sizeof(*c->result));
+      gpr_mu_unlock(&c->mu);
+    } else {
+      c->connecting_endpoint = NULL;
+      gpr_mu_unlock(&c->mu);
+      c->result->transport = grpc_create_chttp2_transport(
+          exec_ctx, args->args, args->endpoint, 1);
+      grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport,
+                                          args->read_buffer);
+    }
+    c->result->channel_args = args->args;
   }
+  grpc_closure *notify = c->notify;
+  c->notify = NULL;
+  grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
   gpr_free(args);
 }
 
@@ -262,6 +235,8 @@ static grpc_subchannel *client_channel_factory_create_subchannel(
         grpc_http_connect_handshaker_create(proxy_name, args->server_name));
     gpr_free(proxy_name);
   }
+  grpc_channel_security_connector_create_handshakers(
+      exec_ctx, c->security_connector, c->handshake_mgr);
   gpr_mu_init(&c->mu);
   gpr_ref_init(&c->refs, 1);
   grpc_subchannel *s = grpc_subchannel_create(exec_ctx, &c->base, args);

src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c

@@ -54,6 +54,11 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
+typedef struct pending_handshake_manager_node {
+  grpc_handshake_manager* handshake_mgr;
+  struct pending_handshake_manager_node *next;
+} pending_handshake_manager_node;
+
 typedef struct server_secure_state {
   grpc_server *server;
   grpc_tcp_server *tcp;
@@ -63,6 +68,7 @@ typedef struct server_secure_state {
   gpr_mu mu;
   grpc_closure tcp_server_shutdown_complete;
   grpc_closure *server_destroy_listener_done;
+  pending_handshake_manager_node *pending_handshake_mgrs;
 } server_secure_state;
 
 typedef struct server_secure_connect {
@@ -70,49 +76,41 @@ typedef struct server_secure_connect {
   grpc_pollset *accepting_pollset;
   grpc_tcp_server_acceptor *acceptor;
   grpc_handshake_manager *handshake_mgr;
-  // TODO(roth): Remove the following two fields when we eliminate
-  // grpc_server_security_connector_do_handshake().
-  gpr_timespec deadline;
-  grpc_channel_args *args;
 } server_secure_connect;
 
-static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
-                                     grpc_security_status status,
-                                     grpc_endpoint *secure_endpoint,
-                                     grpc_auth_context *auth_context) {
-  server_secure_connect *connection_state = statep;
-  if (status == GRPC_SECURITY_OK) {
-    if (secure_endpoint) {
-      gpr_mu_lock(&connection_state->server_state->mu);
-      if (!connection_state->server_state->is_shutdown) {
-        grpc_transport *transport = grpc_create_chttp2_transport(
-            exec_ctx, grpc_server_get_channel_args(
-                          connection_state->server_state->server),
-            secure_endpoint, 0);
-        grpc_arg args_to_add[2];
-        args_to_add[0] = grpc_server_credentials_to_arg(
-            connection_state->server_state->creds);
-        args_to_add[1] = grpc_auth_context_to_arg(auth_context);
-        grpc_channel_args *args_copy = grpc_channel_args_copy_and_add(
-            connection_state->args, args_to_add, GPR_ARRAY_SIZE(args_to_add));
-        grpc_server_setup_transport(
-            exec_ctx, connection_state->server_state->server, transport,
-            connection_state->accepting_pollset, args_copy);
-        grpc_channel_args_destroy(args_copy);
-        grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL);
-      } else {
-        /* We need to consume this here, because the server may already have
-         * gone away. */
-        grpc_endpoint_destroy(exec_ctx, secure_endpoint);
-      }
-      gpr_mu_unlock(&connection_state->server_state->mu);
+static void pending_handshake_manager_add_locked(
+    server_secure_state* state, grpc_handshake_manager* handshake_mgr) {
+  pending_handshake_manager_node* node = gpr_malloc(sizeof(*node));
+  node->handshake_mgr = handshake_mgr;
+  node->next = state->pending_handshake_mgrs;
+  state->pending_handshake_mgrs = node;
+}
+
+static void pending_handshake_manager_remove_locked(
+    server_secure_state* state, grpc_handshake_manager* handshake_mgr) {
+  pending_handshake_manager_node** prev_node = &state->pending_handshake_mgrs;
+  for (pending_handshake_manager_node* node = state->pending_handshake_mgrs;
+       node != NULL; node = node->next) {
+    if (node->handshake_mgr == handshake_mgr) {
+      *prev_node = node->next;
+      gpr_free(node);
+      break;
     }
-  } else {
-    gpr_log(GPR_ERROR, "Secure transport failed with error %d", status);
+    prev_node = &node->next;
   }
-  grpc_channel_args_destroy(connection_state->args);
-  grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp);
-  gpr_free(connection_state);
+}
+
+static void pending_handshake_manager_shutdown_locked(
+    grpc_exec_ctx* exec_ctx, server_secure_state* state) {
+  pending_handshake_manager_node* prev_node = NULL;
+  for (pending_handshake_manager_node* node = state->pending_handshake_mgrs;
+       node != NULL; node = node->next) {
+    grpc_handshake_manager_shutdown(exec_ctx, node->handshake_mgr);
+    gpr_free(prev_node);
+    prev_node = node;
+  }
+  gpr_free(prev_node);
+  state->pending_handshake_mgrs = NULL;
 }
 
 static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
@@ -123,25 +121,39 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
     const char *error_str = grpc_error_string(error);
     gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
     grpc_error_free_string(error_str);
-    grpc_channel_args_destroy(args->args);
     gpr_free(args->read_buffer);
-    gpr_free(args);
-    grpc_handshake_manager_shutdown(exec_ctx, connection_state->handshake_mgr);
-    grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
-    grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp);
-    gpr_free(connection_state);
-    return;
+    if (args->endpoint != NULL) {
+      grpc_endpoint_destroy(exec_ctx, args->endpoint);
+    }
+    gpr_mu_lock(&connection_state->server_state->mu);
+  } else {
+    gpr_mu_lock(&connection_state->server_state->mu);
+    if (!connection_state->server_state->is_shutdown) {
+      grpc_arg channel_arg = grpc_server_credentials_to_arg(
+          connection_state->server_state->creds);
+      grpc_channel_args *args_copy =
+          grpc_channel_args_copy_and_add(args->args, &channel_arg, 1);
+      grpc_transport *transport = grpc_create_chttp2_transport(
+          exec_ctx, args_copy, args->endpoint, 0);
+      grpc_server_setup_transport(
+          exec_ctx, connection_state->server_state->server, transport,
+          connection_state->accepting_pollset, args_copy);
+      grpc_channel_args_destroy(args_copy);
+      grpc_chttp2_transport_start_reading(exec_ctx, transport,
+                                          args->read_buffer);
+    } else {
+      /* We need to consume this here, because the server may already have
+       * gone away. */
+      grpc_endpoint_destroy(exec_ctx, args->endpoint);
+    }
   }
+  pending_handshake_manager_remove_locked(
+      connection_state->server_state, connection_state->handshake_mgr);
+  gpr_mu_unlock(&connection_state->server_state->mu);
   grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
-  connection_state->handshake_mgr = NULL;
-  // TODO(roth, jboeuf): Convert security connector handshaking to use new
-  // handshake API, and then move the code from on_secure_handshake_done()
-  // into this function.
-  connection_state->args = args->args;
-  grpc_server_security_connector_do_handshake(
-      exec_ctx, connection_state->server_state->sc, connection_state->acceptor,
-      args->endpoint, args->read_buffer, connection_state->deadline,
-      on_secure_handshake_done, connection_state);
+  grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp);
+  gpr_free(connection_state);
+  grpc_channel_args_destroy(args->args);
   gpr_free(args);
 }
 
@@ -149,29 +161,32 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
                       grpc_pollset *accepting_pollset,
                       grpc_tcp_server_acceptor *acceptor) {
   server_secure_state *server_state = statep;
-  server_secure_connect *connection_state = NULL;
   gpr_mu_lock(&server_state->mu);
   if (server_state->is_shutdown) {
     gpr_mu_unlock(&server_state->mu);
     grpc_endpoint_destroy(exec_ctx, tcp);
     return;
   }
+  grpc_handshake_manager* handshake_mgr = grpc_handshake_manager_create();
+  pending_handshake_manager_add_locked(server_state, handshake_mgr);
   gpr_mu_unlock(&server_state->mu);
   grpc_tcp_server_ref(server_state->tcp);
-  connection_state = gpr_malloc(sizeof(*connection_state));
+  server_secure_connect *connection_state =
+      gpr_malloc(sizeof(*connection_state));
   connection_state->server_state = server_state;
   connection_state->accepting_pollset = accepting_pollset;
   connection_state->acceptor = acceptor;
-  connection_state->handshake_mgr = grpc_handshake_manager_create();
+  connection_state->handshake_mgr = handshake_mgr;
+  grpc_server_security_connector_create_handshakers(
+      exec_ctx, server_state->sc, connection_state->handshake_mgr);
   // TODO(roth): We should really get this timeout value from channel
   // args instead of hard-coding it.
-  connection_state->deadline = gpr_time_add(
+  gpr_timespec deadline = gpr_time_add(
       gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(120, GPR_TIMESPAN));
   grpc_handshake_manager_do_handshake(
       exec_ctx, connection_state->handshake_mgr, tcp,
       grpc_server_get_channel_args(connection_state->server_state->server),
-      connection_state->deadline, acceptor, on_handshake_done,
-      connection_state);
+      deadline, acceptor, on_handshake_done, connection_state);
 }
 
 /* Server callback: start listening on our ports */
@@ -193,9 +208,8 @@ static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *statep,
   gpr_mu_lock(&server_state->mu);
   grpc_closure *destroy_done = server_state->server_destroy_listener_done;
   GPR_ASSERT(server_state->is_shutdown);
+  pending_handshake_manager_shutdown_locked(exec_ctx, server_state);
   gpr_mu_unlock(&server_state->mu);
-  /* clean up */
-  grpc_server_security_connector_shutdown(exec_ctx, server_state->sc);
 
   /* Flush queued work before a synchronous unref. */
   grpc_exec_ctx_flush(exec_ctx);
@@ -260,7 +274,6 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
     gpr_free(msg);
     goto error;
   }
-  sc->channel_args = grpc_server_get_channel_args(server);
 
   /* resolve address */
   err = grpc_blocking_resolve_address(addr, "https", &resolved);

src/core/lib/http/httpcli_security_connector.c

@@ -38,6 +38,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
+
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/security/transport/handshake.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
@@ -45,47 +47,41 @@
 typedef struct {
   grpc_channel_security_connector base;
   tsi_ssl_handshaker_factory *handshaker_factory;
+  grpc_handshake_manager *handshake_mgr;
   char *secure_peer_name;
 } grpc_httpcli_ssl_channel_security_connector;
 
 static void httpcli_ssl_destroy(grpc_security_connector *sc) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;  // FIXME
   grpc_httpcli_ssl_channel_security_connector *c =
       (grpc_httpcli_ssl_channel_security_connector *)sc;
   if (c->handshaker_factory != NULL) {
     tsi_ssl_handshaker_factory_destroy(c->handshaker_factory);
   }
+  if (c->handshake_mgr != NULL) {
+    grpc_handshake_manager_destroy(&exec_ctx, c->handshake_mgr);
+  }
   if (c->secure_peer_name != NULL) gpr_free(c->secure_peer_name);
   gpr_free(sc);
+  grpc_exec_ctx_finish(&exec_ctx);
 }
 
-static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
-                                     grpc_channel_security_connector *sc,
-                                     grpc_endpoint *nonsecure_endpoint,
-                                     grpc_slice_buffer *read_buffer,
-                                     gpr_timespec deadline,
-                                     grpc_security_handshake_done_cb cb,
-                                     void *user_data) {
+static void httpcli_ssl_create_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_handshake_manager *handshake_mgr) {
   grpc_httpcli_ssl_channel_security_connector *c =
       (grpc_httpcli_ssl_channel_security_connector *)sc;
-  tsi_result result = TSI_OK;
-  tsi_handshaker *handshaker;
-  if (c->handshaker_factory == NULL) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-    return;
-  }
-  result = tsi_ssl_handshaker_factory_create_handshaker(
-      c->handshaker_factory, c->secure_peer_name, &handshaker);
-  if (result != TSI_OK) {
-    gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.",
-            tsi_result_to_string(result));
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-  } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, read_buffer, deadline, cb,
-                               user_data);
+  tsi_handshaker *handshaker = NULL;
+  if (c->handshaker_factory != NULL) {
+    tsi_result result = tsi_ssl_handshaker_factory_create_handshaker(
+        c->handshaker_factory, c->secure_peer_name, &handshaker);
+    if (result != TSI_OK) {
+      gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.",
+              tsi_result_to_string(result));
+    }
   }
+  grpc_security_create_handshakers(exec_ctx, handshaker, &sc->base,
+                                   handshake_mgr);
 }
 
 static void httpcli_ssl_check_peer(grpc_exec_ctx *exec_ctx,
@@ -140,7 +136,8 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create(
     *sc = NULL;
     return GRPC_SECURITY_ERROR;
   }
-  c->base.do_handshake = httpcli_ssl_do_handshake;
+  c->handshake_mgr = grpc_handshake_manager_create();
+  c->base.create_handshakers = httpcli_ssl_create_handshakers;
   *sc = &c->base;
   return GRPC_SECURITY_OK;
 }
@@ -152,18 +149,22 @@ typedef struct {
   void *arg;
 } on_done_closure;
 
-static void on_secure_transport_setup_done(grpc_exec_ctx *exec_ctx, void *rp,
-                                           grpc_security_status status,
-                                           grpc_endpoint *secure_endpoint,
-                                           grpc_auth_context *auth_context) {
-  on_done_closure *c = rp;
-  if (status != GRPC_SECURITY_OK) {
-    gpr_log(GPR_ERROR, "Secure transport setup failed with error %d.", status);
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error) {
+  grpc_handshaker_args *args = arg;
+  on_done_closure *c = args->user_data;
+  if (error != GRPC_ERROR_NONE) {
+    const char* msg = grpc_error_string(error);
+    gpr_log(GPR_ERROR, "Secure transport setup failed: %s", msg);
+    grpc_error_free_string(msg);
     c->func(exec_ctx, c->arg, NULL);
   } else {
-    c->func(exec_ctx, c->arg, secure_endpoint);
+    c->func(exec_ctx, c->arg, args->endpoint);
   }
   gpr_free(c);
+  grpc_channel_args_destroy(args->args);
+  gpr_free(args->read_buffer);
+  gpr_free(args);
 }
 
 static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
@@ -186,8 +187,13 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
   GPR_ASSERT(httpcli_ssl_channel_security_connector_create(
                  pem_root_certs, pem_root_certs_size, host, &sc) ==
              GRPC_SECURITY_OK);
-  grpc_channel_security_connector_do_handshake(
-      exec_ctx, sc, tcp, NULL, deadline, on_secure_transport_setup_done, c);
+  grpc_httpcli_ssl_channel_security_connector* httpcli_connector =
+      (grpc_httpcli_ssl_channel_security_connector*)sc;
+  grpc_channel_security_connector_create_handshakers(
+      exec_ctx, sc, httpcli_connector->handshake_mgr);
+  grpc_handshake_manager_do_handshake(
+      exec_ctx, httpcli_connector->handshake_mgr, tcp, NULL /* channel_args */,
+      deadline, NULL /* acceptor */, on_handshake_done, c /* user_data */);
   GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli");
 }
 

src/core/lib/security/transport/handshake.c

@@ -39,7 +39,9 @@
 #include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include "src/core/lib/iomgr/timer.h"
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/security/transport/tsi_error.h"
@@ -47,24 +49,22 @@
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
 typedef struct {
+  grpc_handshaker base;
+  grpc_handshaker_args* args;
+  grpc_closure* on_handshake_done;
   grpc_security_connector *connector;
   tsi_handshaker *handshaker;
-  bool is_client_side;
   unsigned char *handshake_buffer;
   size_t handshake_buffer_size;
   grpc_endpoint *wrapped_endpoint;
   grpc_endpoint *secure_endpoint;
   grpc_slice_buffer left_overs;
-  grpc_slice_buffer incoming;
   grpc_slice_buffer outgoing;
-  grpc_security_handshake_done_cb cb;
-  void *user_data;
   grpc_closure on_handshake_data_sent_to_peer;
   grpc_closure on_handshake_data_received_from_peer;
   grpc_auth_context *auth_context;
-  grpc_timer timer;
   gpr_refcount refs;
-} grpc_security_handshake;
+} security_handshaker;
 
 static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
                                                  void *setup,
@@ -73,40 +73,12 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
 static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, void *setup,
                                            grpc_error *error);
 
-static void security_connector_remove_handshake(grpc_security_handshake *h) {
-  GPR_ASSERT(!h->is_client_side);
-  grpc_security_connector_handshake_list *node;
-  grpc_security_connector_handshake_list *tmp;
-  grpc_server_security_connector *sc =
-      (grpc_server_security_connector *)h->connector;
-  gpr_mu_lock(&sc->mu);
-  node = sc->handshaking_handshakes;
-  if (node && node->handshake == h) {
-    sc->handshaking_handshakes = node->next;
-    gpr_free(node);
-    gpr_mu_unlock(&sc->mu);
-    return;
-  }
-  while (node) {
-    if (node->next->handshake == h) {
-      tmp = node->next;
-      node->next = node->next->next;
-      gpr_free(tmp);
-      gpr_mu_unlock(&sc->mu);
-      return;
-    }
-    node = node->next;
-  }
-  gpr_mu_unlock(&sc->mu);
-}
-
-static void unref_handshake(grpc_security_handshake *h) {
+static void unref_handshake(security_handshaker *h) {
   if (gpr_unref(&h->refs)) {
     if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker);
     if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer);
     grpc_slice_buffer_destroy(&h->left_overs);
     grpc_slice_buffer_destroy(&h->outgoing);
-    grpc_slice_buffer_destroy(&h->incoming);
     GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake");
     GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake");
     gpr_free(h);
@@ -114,36 +86,38 @@ static void unref_handshake(grpc_security_handshake *h) {
 }
 
 static void security_handshake_done(grpc_exec_ctx *exec_ctx,
-                                    grpc_security_handshake *h,
+                                    security_handshaker *h,
                                     grpc_error *error) {
-  grpc_timer_cancel(exec_ctx, &h->timer);
-  if (!h->is_client_side) {
-    security_connector_remove_handshake(h);
-  }
   if (error == GRPC_ERROR_NONE) {
-    h->cb(exec_ctx, h->user_data, GRPC_SECURITY_OK, h->secure_endpoint,
-          h->auth_context);
+    h->args->endpoint = h->secure_endpoint;
+    grpc_arg auth_context_arg = grpc_auth_context_to_arg(h->auth_context);
+    grpc_channel_args* tmp_args = h->args->args;
+    h->args->args =
+        grpc_channel_args_copy_and_add(tmp_args, &auth_context_arg, 1);
+    grpc_channel_args_destroy(tmp_args);
   } else {
     const char *msg = grpc_error_string(error);
     gpr_log(GPR_DEBUG, "Security handshake failed: %s", msg);
     grpc_error_free_string(msg);
-
     if (h->secure_endpoint != NULL) {
       grpc_endpoint_shutdown(exec_ctx, h->secure_endpoint);
       grpc_endpoint_destroy(exec_ctx, h->secure_endpoint);
     } else {
       grpc_endpoint_destroy(exec_ctx, h->wrapped_endpoint);
     }
-    h->cb(exec_ctx, h->user_data, GRPC_SECURITY_ERROR, NULL, NULL);
   }
+  // Clear out the read buffer before it gets passed to the transport,
+  // since any excess bytes were already moved to h->left_overs.
+  grpc_slice_buffer_reset_and_unref(h->args->read_buffer);
+  h->args = NULL;
+  grpc_exec_ctx_sched(exec_ctx, h->on_handshake_done, error, NULL);
   unref_handshake(h);
-  GRPC_ERROR_UNREF(error);
 }
 
 static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data,
                             grpc_security_status status,
                             grpc_auth_context *auth_context) {
-  grpc_security_handshake *h = user_data;
+  security_handshaker *h = user_data;
   tsi_frame_protector *protector;
   tsi_result result;
   if (status != GRPC_SECURITY_OK) {
@@ -172,7 +146,7 @@ static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data,
   return;
 }
 
-static void check_peer(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h) {
+static void check_peer(grpc_exec_ctx *exec_ctx, security_handshaker *h) {
   tsi_peer peer;
   tsi_result result = tsi_handshaker_extract_peer(h->handshaker, &peer);
 
@@ -187,7 +161,7 @@ static void check_peer(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h) {
 }
 
 static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx,
-                                         grpc_security_handshake *h) {
+                                         security_handshaker *h) {
   size_t offset = 0;
   tsi_result result = TSI_OK;
   grpc_slice to_send;
@@ -215,8 +189,6 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx,
       grpc_slice_from_copied_buffer((const char *)h->handshake_buffer, offset);
   grpc_slice_buffer_reset_and_unref(&h->outgoing);
   grpc_slice_buffer_add(&h->outgoing, to_send);
-  /* TODO(klempner,jboeuf): This should probably use the client setup
-     deadline */
   grpc_endpoint_write(exec_ctx, h->wrapped_endpoint, &h->outgoing,
                       &h->on_handshake_data_sent_to_peer);
 }
@@ -224,7 +196,7 @@ static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx,
 static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
                                                  void *handshake,
                                                  grpc_error *error) {
-  grpc_security_handshake *h = handshake;
+  security_handshaker *h = handshake;
   size_t consumed_slice_size = 0;
   tsi_result result = TSI_OK;
   size_t i;
@@ -238,10 +210,10 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
     return;
   }
 
-  for (i = 0; i < h->incoming.count; i++) {
-    consumed_slice_size = GRPC_SLICE_LENGTH(h->incoming.slices[i]);
+  for (i = 0; i < h->args->read_buffer->count; i++) {
+    consumed_slice_size = GRPC_SLICE_LENGTH(h->args->read_buffer->slices[i]);
     result = tsi_handshaker_process_bytes_from_peer(
-        h->handshaker, GRPC_SLICE_START_PTR(h->incoming.slices[i]),
+        h->handshaker, GRPC_SLICE_START_PTR(h->args->read_buffer->slices[i]),
         &consumed_slice_size);
     if (!tsi_handshaker_is_in_progress(h->handshaker)) break;
   }
@@ -249,7 +221,7 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
   if (tsi_handshaker_is_in_progress(h->handshaker)) {
     /* We may need more data. */
     if (result == TSI_INCOMPLETE_DATA) {
-      grpc_endpoint_read(exec_ctx, h->wrapped_endpoint, &h->incoming,
+      grpc_endpoint_read(exec_ctx, h->wrapped_endpoint, h->args->read_buffer,
                          &h->on_handshake_data_received_from_peer);
       return;
     } else {
@@ -267,32 +239,33 @@ static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
 
   /* Handshake is done and successful this point. */
   has_left_overs_in_current_slice =
-      (consumed_slice_size < GRPC_SLICE_LENGTH(h->incoming.slices[i]));
+      (consumed_slice_size <
+       GRPC_SLICE_LENGTH(h->args->read_buffer->slices[i]));
   num_left_overs =
-      (has_left_overs_in_current_slice ? 1 : 0) + h->incoming.count - i - 1;
-  if (num_left_overs == 0) {
-    check_peer(exec_ctx, h);
-    return;
+      (has_left_overs_in_current_slice ? 1 : 0)
+      + h->args->read_buffer->count - i - 1;
+  if (num_left_overs > 0) {
+    /* Put the leftovers in our buffer (ownership transfered). */
+    if (has_left_overs_in_current_slice) {
+      grpc_slice_buffer_add(
+          &h->left_overs,
+          grpc_slice_split_tail(&h->args->read_buffer->slices[i],
+          consumed_slice_size));
+      /* split_tail above increments refcount. */
+      grpc_slice_unref(h->args->read_buffer->slices[i]);
+    }
+    grpc_slice_buffer_addn(
+        &h->left_overs, &h->args->read_buffer->slices[i + 1],
+        num_left_overs - (size_t)has_left_overs_in_current_slice);
   }
 
-  /* Put the leftovers in our buffer (ownership transfered). */
-  if (has_left_overs_in_current_slice) {
-    grpc_slice_buffer_add(
-        &h->left_overs,
-        grpc_slice_split_tail(&h->incoming.slices[i], consumed_slice_size));
-    grpc_slice_unref(
-        h->incoming.slices[i]); /* split_tail above increments refcount. */
-  }
-  grpc_slice_buffer_addn(
-      &h->left_overs, &h->incoming.slices[i + 1],
-      num_left_overs - (size_t)has_left_overs_in_current_slice);
   check_peer(exec_ctx, h);
 }
 
 /* If handshake is NULL, the handshake is done. */
 static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx,
                                            void *handshake, grpc_error *error) {
-  grpc_security_handshake *h = handshake;
+  security_handshaker *h = handshake;
 
   /* Make sure that write is OK. */
   if (error != GRPC_ERROR_NONE) {
@@ -305,70 +278,110 @@ static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx,
 
   /* We may be done. */
   if (tsi_handshaker_is_in_progress(h->handshaker)) {
-    /* TODO(klempner,jboeuf): This should probably use the client setup
-       deadline */
-    grpc_endpoint_read(exec_ctx, h->wrapped_endpoint, &h->incoming,
+    grpc_endpoint_read(exec_ctx, h->wrapped_endpoint, h->args->read_buffer,
                        &h->on_handshake_data_received_from_peer);
   } else {
     check_peer(exec_ctx, h);
   }
 }
 
-static void on_timeout(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
-  grpc_security_handshake *h = arg;
-  if (error == GRPC_ERROR_NONE) {
-    grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint);
-  }
+//
+// public handshaker API
+//
+
+static void security_handshaker_destroy(grpc_exec_ctx* exec_ctx,
+                                            grpc_handshaker* handshaker) {
+  security_handshaker* h = (security_handshaker*)handshaker;
   unref_handshake(h);
 }
 
-void grpc_do_security_handshake(
+static void security_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
+                                             grpc_handshaker* handshaker) {
+  security_handshaker *h = (security_handshaker*)handshaker;
+  grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint);
+}
+
+static void security_handshaker_do_handshake(
+    grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker,
+    grpc_tcp_server_acceptor* acceptor, grpc_closure* on_handshake_done,
+    grpc_handshaker_args* args) {
+  security_handshaker* h = (security_handshaker*)handshaker;
+  h->args = args;
+  h->on_handshake_done = on_handshake_done;
+  h->wrapped_endpoint = args->endpoint;  // FIXME: remove?
+  gpr_ref(&h->refs);
+  send_handshake_bytes_to_peer(exec_ctx, h);
+}
+
+static const grpc_handshaker_vtable security_handshaker_vtable = {
+  security_handshaker_destroy, security_handshaker_shutdown,
+  security_handshaker_do_handshake};
+
+static grpc_handshaker* security_handshaker_create(
     grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
-    grpc_security_connector *connector, bool is_client_side,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb,
-    void *user_data) {
-  grpc_security_connector_handshake_list *handshake_node;
-  grpc_security_handshake *h = gpr_malloc(sizeof(grpc_security_handshake));
-  memset(h, 0, sizeof(grpc_security_handshake));
+    grpc_security_connector *connector) {
+  security_handshaker *h = gpr_malloc(sizeof(security_handshaker));
+  memset(h, 0, sizeof(security_handshaker));
+  grpc_handshaker_init(&security_handshaker_vtable, &h->base);
   h->handshaker = handshaker;
   h->connector = GRPC_SECURITY_CONNECTOR_REF(connector, "handshake");
-  h->is_client_side = is_client_side;
   h->handshake_buffer_size = GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE;
   h->handshake_buffer = gpr_malloc(h->handshake_buffer_size);
-  h->wrapped_endpoint = nonsecure_endpoint;
-  h->user_data = user_data;
-  h->cb = cb;
-  gpr_ref_init(&h->refs, 2); /* timer and handshake proper each get a ref */
+  gpr_ref_init(&h->refs, 1);
   grpc_closure_init(&h->on_handshake_data_sent_to_peer,
                     on_handshake_data_sent_to_peer, h);
   grpc_closure_init(&h->on_handshake_data_received_from_peer,
                     on_handshake_data_received_from_peer, h);
   grpc_slice_buffer_init(&h->left_overs);
   grpc_slice_buffer_init(&h->outgoing);
-  grpc_slice_buffer_init(&h->incoming);
-  if (read_buffer != NULL) {
-    grpc_slice_buffer_move_into(read_buffer, &h->incoming);
-    gpr_free(read_buffer);
-  }
-  if (!is_client_side) {
-    grpc_server_security_connector *server_connector =
-        (grpc_server_security_connector *)connector;
-    handshake_node = gpr_malloc(sizeof(grpc_security_connector_handshake_list));
-    handshake_node->handshake = h;
-    gpr_mu_lock(&server_connector->mu);
-    handshake_node->next = server_connector->handshaking_handshakes;
-    server_connector->handshaking_handshakes = handshake_node;
-    gpr_mu_unlock(&server_connector->mu);
-  }
-  send_handshake_bytes_to_peer(exec_ctx, h);
-  grpc_timer_init(exec_ctx, &h->timer,
-                  gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC),
-                  on_timeout, h, gpr_now(GPR_CLOCK_MONOTONIC));
+  return &h->base;
 }
 
-void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx,
-                                      void *handshake) {
-  grpc_security_handshake *h = handshake;
-  grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint);
+//
+// fail_handshaker
+//
+
+static void fail_handshaker_destroy(grpc_exec_ctx* exec_ctx,
+                                    grpc_handshaker* handshaker) {
+  gpr_free(handshaker);
+}
+
+static void fail_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
+                                     grpc_handshaker* handshaker) {}
+
+static void fail_handshaker_do_handshake(
+    grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker,
+    grpc_tcp_server_acceptor* acceptor, grpc_closure* on_handshake_done,
+    grpc_handshaker_args* args) {
+  grpc_exec_ctx_sched(
+      exec_ctx, on_handshake_done,
+      GRPC_ERROR_CREATE("Failed to create security handshaker"), NULL);
+}
+
+static const grpc_handshaker_vtable fail_handshaker_vtable = {
+  fail_handshaker_destroy, fail_handshaker_shutdown,
+  fail_handshaker_do_handshake};
+
+static grpc_handshaker* fail_handshaker_create() {
+  grpc_handshaker* h = gpr_malloc(sizeof(*h));
+  grpc_handshaker_init(&fail_handshaker_vtable, h);
+  return h;
+}
+
+//
+// exported functions
+//
+
+void grpc_security_create_handshakers(
+    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
+    grpc_security_connector *connector, grpc_handshake_manager *handshake_mgr) {
+  // If no TSI handshaker was created, add a handshaker that always fails.
+  // Otherwise, add a real security handshaker.
+  if (handshaker == NULL) {
+    grpc_handshake_manager_add(handshake_mgr, fail_handshaker_create());
+  } else {
+    grpc_handshake_manager_add(
+        handshake_mgr,
+        security_handshaker_create(exec_ctx, handshaker, connector));
+  }
 }

src/core/lib/security/transport/handshake.h

@@ -37,14 +37,10 @@
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/security/transport/security_connector.h"
 
-/* Calls the callback upon completion. Takes owership of handshaker and
- * read_buffer. */
-void grpc_do_security_handshake(
+/// Creates any necessary security handshakers and adds them to
+/// \a handshake_mgr.
+void grpc_security_create_handshakers(
     grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
-    grpc_security_connector *connector, bool is_client_side,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data);
-
-void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
+    grpc_security_connector *connector, grpc_handshake_manager *handshake_mgr);
 
 #endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H */

src/core/lib/security/transport/security_connector.c

@@ -111,45 +111,19 @@ const tsi_peer_property *tsi_peer_get_property_by_name(const tsi_peer *peer,
   return NULL;
 }
 
-void grpc_server_security_connector_shutdown(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector) {
-  grpc_security_connector_handshake_list *tmp;
-  gpr_mu_lock(&connector->mu);
-  while (connector->handshaking_handshakes) {
-    tmp = connector->handshaking_handshakes;
-    grpc_security_handshake_shutdown(
-        exec_ctx, connector->handshaking_handshakes->handshake);
-    connector->handshaking_handshakes = tmp->next;
-    gpr_free(tmp);
+void grpc_channel_security_connector_create_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector,
+    grpc_handshake_manager *handshake_mgr) {
+  if (connector != NULL) {
+    connector->create_handshakers(exec_ctx, connector, handshake_mgr);
   }
-  gpr_mu_unlock(&connector->mu);
 }
 
-void grpc_channel_security_connector_do_handshake(
-    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb,
-    void *user_data) {
-  if (sc == NULL || nonsecure_endpoint == NULL) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-  } else {
-    sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, read_buffer, deadline,
-                     cb, user_data);
-  }
-}
-
-void grpc_server_security_connector_do_handshake(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data) {
-  if (sc == NULL || nonsecure_endpoint == NULL) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-  } else {
-    sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, read_buffer,
-                     deadline, cb, user_data);
+void grpc_server_security_connector_create_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector,
+    grpc_handshake_manager *handshake_mgr) {
+  if (connector != NULL) {
+    connector->create_handshakers(exec_ctx, connector, handshake_mgr);
   }
 }
 
@@ -263,8 +237,6 @@ static void fake_channel_destroy(grpc_security_connector *sc) {
 }
 
 static void fake_server_destroy(grpc_security_connector *sc) {
-  grpc_server_security_connector *c = (grpc_server_security_connector *)sc;
-  gpr_mu_destroy(&c->mu);
   gpr_free(sc);
 }
 
@@ -313,26 +285,20 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
   cb(exec_ctx, user_data, GRPC_SECURITY_OK);
 }
 
-static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx,
-                                      grpc_channel_security_connector *sc,
-                                      grpc_endpoint *nonsecure_endpoint,
-                                      grpc_slice_buffer *read_buffer,
-                                      gpr_timespec deadline,
-                                      grpc_security_handshake_done_cb cb,
-                                      void *user_data) {
-  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base,
-                             true, nonsecure_endpoint, read_buffer, deadline,
-                             cb, user_data);
+static void fake_channel_create_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_handshake_manager *handshake_mgr) {
+  grpc_security_create_handshakers(
+      exec_ctx, tsi_create_fake_handshaker(true /* is_client */), &sc->base,
+      handshake_mgr);
 }
 
-static void fake_server_do_handshake(
+static void fake_server_create_handshakers(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data) {
-  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base,
-                             false, nonsecure_endpoint, read_buffer, deadline,
-                             cb, user_data);
+    grpc_handshake_manager *handshake_mgr) {
+  grpc_security_create_handshakers(
+      exec_ctx, tsi_create_fake_handshaker(false /* is_client */), &sc->base,
+      handshake_mgr);
 }
 
 static grpc_security_connector_vtable fake_channel_vtable = {
@@ -350,7 +316,7 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create(
   c->base.vtable = &fake_channel_vtable;
   c->request_metadata_creds = grpc_call_credentials_ref(request_metadata_creds);
   c->check_call_host = fake_channel_check_call_host;
-  c->do_handshake = fake_channel_do_handshake;
+  c->create_handshakers = fake_channel_create_handshakers;
   return c;
 }
 
@@ -362,8 +328,7 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create(
   gpr_ref_init(&c->base.refcount, 1);
   c->base.vtable = &fake_server_vtable;
   c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME;
-  c->do_handshake = fake_server_do_handshake;
-  gpr_mu_init(&c->mu);
+  c->create_handshakers = fake_server_create_handshakers;
   return c;
 }
 
@@ -396,11 +361,9 @@ static void ssl_channel_destroy(grpc_security_connector *sc) {
 static void ssl_server_destroy(grpc_security_connector *sc) {
   grpc_ssl_server_security_connector *c =
       (grpc_ssl_server_security_connector *)sc;
-
   if (c->handshaker_factory != NULL) {
     tsi_ssl_handshaker_factory_destroy(c->handshaker_factory);
   }
-  gpr_mu_destroy(&c->base.mu);
   gpr_free(sc);
 }
 
@@ -419,49 +382,34 @@ static grpc_security_status ssl_create_handshaker(
   return GRPC_SECURITY_OK;
 }
 
-static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
-                                     grpc_channel_security_connector *sc,
-                                     grpc_endpoint *nonsecure_endpoint,
-                                     grpc_slice_buffer *read_buffer,
-                                     gpr_timespec deadline,
-                                     grpc_security_handshake_done_cb cb,
-                                     void *user_data) {
-  grpc_ssl_channel_security_connector *c =
-      (grpc_ssl_channel_security_connector *)sc;
-  tsi_handshaker *handshaker;
-  grpc_security_status status = ssl_create_handshaker(
-      c->handshaker_factory, true,
+static void ssl_channel_create_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_handshake_manager *handshake_mgr) {
+  grpc_ssl_channel_security_connector* c =
+      (grpc_ssl_channel_security_connector*)sc;
+  // Instantiate TSI handshaker.
+  tsi_handshaker *tsi_hs = NULL;
+  ssl_create_handshaker(
+      c->handshaker_factory, true /* is_client */,
       c->overridden_target_name != NULL ? c->overridden_target_name
                                         : c->target_name,
-      &handshaker);
-  if (status != GRPC_SECURITY_OK) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, status, NULL, NULL);
-  } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, read_buffer, deadline, cb,
-                               user_data);
-  }
+      &tsi_hs);
+  // Create handshakers.
+  grpc_security_create_handshakers(exec_ctx, tsi_hs, &sc->base, handshake_mgr);
 }
 
-static void ssl_server_do_handshake(
+static void ssl_server_create_handshakers(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data) {
-  grpc_ssl_server_security_connector *c =
-      (grpc_ssl_server_security_connector *)sc;
-  tsi_handshaker *handshaker;
-  grpc_security_status status =
-      ssl_create_handshaker(c->handshaker_factory, false, NULL, &handshaker);
-  if (status != GRPC_SECURITY_OK) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, status, NULL, NULL);
-  } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false,
-                               nonsecure_endpoint, read_buffer, deadline, cb,
-                               user_data);
-  }
+    grpc_handshake_manager *handshake_mgr) {
+  grpc_ssl_server_security_connector* c =
+      (grpc_ssl_server_security_connector*)sc;
+  // Instantiate TSI handshaker.
+  tsi_handshaker *tsi_hs = NULL;
+  ssl_create_handshaker(
+      c->handshaker_factory, false /* is_client */, NULL /* peer_name */,
+      &tsi_hs);
+  // Create handshakers.
+  grpc_security_create_handshakers(exec_ctx, tsi_hs, &sc->base, handshake_mgr);
 }
 
 static int ssl_host_matches_name(const tsi_peer *peer, const char *peer_name) {
@@ -765,7 +713,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = ssl_channel_check_call_host;
-  c->base.do_handshake = ssl_channel_do_handshake;
+  c->base.create_handshakers = ssl_channel_create_handshakers;
   gpr_split_host_port(target_name, &c->target_name, &port);
   gpr_free(port);
   if (overridden_target_name != NULL) {
@@ -840,8 +788,7 @@ grpc_security_status grpc_ssl_server_security_connector_create(
     *sc = NULL;
     goto error;
   }
-  gpr_mu_init(&c->base.mu);
-  c->base.do_handshake = ssl_server_do_handshake;
+  c->base.create_handshakers = ssl_server_create_handshakers;
   *sc = &c->base;
   gpr_free((void *)alpn_protocol_strings);
   gpr_free(alpn_protocol_string_lengths);

src/core/lib/security/transport/security_connector.h

@@ -35,6 +35,8 @@
 #define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
 
 #include <grpc/grpc_security.h>
+
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/tcp_server.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
@@ -141,11 +143,9 @@ struct grpc_channel_security_connector {
                           grpc_channel_security_connector *sc, const char *host,
                           grpc_auth_context *auth_context,
                           grpc_security_call_host_check_cb cb, void *user_data);
-  void (*do_handshake)(grpc_exec_ctx *exec_ctx,
-                       grpc_channel_security_connector *sc,
-                       grpc_endpoint *nonsecure_endpoint,
-                       grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-                       grpc_security_handshake_done_cb cb, void *user_data);
+  void (*create_handshakers)(
+      grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+      grpc_handshake_manager *handshake_mgr);
 };
 
 /* Checks that the host that will be set for a call is acceptable. */
@@ -154,11 +154,10 @@ void grpc_channel_security_connector_check_call_host(
     const char *host, grpc_auth_context *auth_context,
     grpc_security_call_host_check_cb cb, void *user_data);
 
-/* Handshake. */
-void grpc_channel_security_connector_do_handshake(
+/* Registers handshakers with \a handshake_mgr. */
+void grpc_channel_security_connector_create_handshakers(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data);
+    grpc_handshake_manager *handshake_mgr);
 
 /* --- server_security_connector object. ---
 
@@ -169,25 +168,14 @@ typedef struct grpc_server_security_connector grpc_server_security_connector;
 
 struct grpc_server_security_connector {
   grpc_security_connector base;
-  gpr_mu mu;
-  grpc_security_connector_handshake_list *handshaking_handshakes;
-  const grpc_channel_args *channel_args;
-  void (*do_handshake)(grpc_exec_ctx *exec_ctx,
-                       grpc_server_security_connector *sc,
-                       grpc_tcp_server_acceptor *acceptor,
-                       grpc_endpoint *nonsecure_endpoint,
-                       grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-                       grpc_security_handshake_done_cb cb, void *user_data);
+  void (*create_handshakers)(
+      grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
+      grpc_handshake_manager *handshake_mgr);
 };
 
-void grpc_server_security_connector_do_handshake(
+void grpc_server_security_connector_create_handshakers(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data);
-
-void grpc_server_security_connector_shutdown(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector);
+    grpc_handshake_manager *handshake_mgr);
 
 /* --- Creation security connectors. --- */
 

test/core/security/ssl_server_fuzzer.c

@@ -58,17 +58,14 @@ struct handshake_state {
   bool done_callback_called;
 };
 
-static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
-                                     grpc_security_status status,
-                                     grpc_endpoint *secure_endpoint,
-                                     grpc_auth_context *auth_context) {
-  struct handshake_state *state = (struct handshake_state *)statep;
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error) {
+  grpc_handshaker_args *args = arg;
+  struct handshake_state *state = args->user_data;
   GPR_ASSERT(state->done_callback_called == false);
   state->done_callback_called = true;
   // The fuzzer should not pass the handshake.
-  GPR_ASSERT(status != GRPC_SECURITY_OK);
-  GPR_ASSERT(secure_endpoint == NULL);
-  GPR_ASSERT(auth_context == NULL);
+  GPR_ASSERT(error != GRPC_ERROR_NONE);
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
@@ -108,15 +105,18 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   grpc_security_status status =
       grpc_server_credentials_create_security_connector(creds, &sc);
   GPR_ASSERT(status == GRPC_SECURITY_OK);
-  sc->channel_args = NULL;
   gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_MONOTONIC),
                                        gpr_time_from_seconds(1, GPR_TIMESPAN));
 
   struct handshake_state state;
   state.done_callback_called = false;
-  grpc_server_security_connector_do_handshake(&exec_ctx, sc, NULL,
-                                              mock_endpoint, NULL, deadline,
-                                              on_secure_handshake_done, &state);
+  grpc_handshake_manager *handshake_mgr = grpc_handshake_manager_create();
+  grpc_server_security_connector_create_handshakers(&exec_ctx, sc,
+                                                    handshake_mgr);
+  grpc_handshake_manager_do_handshake(&exec_ctx, handshake_mgr, mock_endpoint,
+                                      NULL /* channel_args */, deadline,
+                                      NULL /* acceptor */, on_handshake_done,
+                                      &state);
   grpc_exec_ctx_flush(&exec_ctx);
 
   // If the given string happens to be part of the correct client hello, the
@@ -129,6 +129,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
 
   GPR_ASSERT(state.done_callback_called);
 
+  grpc_handshake_manager_destroy(&exec_ctx, handshake_mgr);
   GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "test");
   grpc_server_credentials_release(creds);
   grpc_slice_unref(cert_slice);