Merge branch 'master' into change_cronet_interface

.gitignore

@@ -17,7 +17,7 @@ py27/
 py34/
 
 # Node installation output
-^node_modules
+node_modules
 src/node/extension_binary/
 
 # gcov coverage data

.gitmodules

@@ -17,6 +17,6 @@
 [submodule "third_party/thrift"]
 	path = third_party/thrift
 	url = https://github.com/apache/thrift.git
-[submodule "third_party/google_benchmark"]
-	path = third_party/google_benchmark
+[submodule "third_party/benchmark"]
+	path = third_party/benchmark
 	url = https://github.com/google/benchmark

BUILD

@@ -287,9 +287,9 @@ cc_library(
     "src/core/lib/security/credentials/plugin/plugin_credentials.h",
     "src/core/lib/security/credentials/ssl/ssl_credentials.h",
     "src/core/lib/security/transport/auth_filters.h",
-    "src/core/lib/security/transport/handshake.h",
     "src/core/lib/security/transport/secure_endpoint.h",
     "src/core/lib/security/transport/security_connector.h",
+    "src/core/lib/security/transport/security_handshaker.h",
     "src/core/lib/security/transport/tsi_error.h",
     "src/core/lib/security/util/b64.h",
     "src/core/lib/security/util/json_util.h",
@@ -298,6 +298,7 @@ cc_library(
     "src/core/lib/tsi/ssl_types.h",
     "src/core/lib/tsi/transport_security.h",
     "src/core/lib/tsi/transport_security_interface.h",
+    "src/core/ext/transport/chttp2/server/chttp2_server.h",
     "src/core/ext/client_channel/client_channel.h",
     "src/core/ext/client_channel/client_channel_factory.h",
     "src/core/ext/client_channel/connector.h",
@@ -313,6 +314,7 @@ cc_library(
     "src/core/ext/client_channel/subchannel.h",
     "src/core/ext/client_channel/subchannel_index.h",
     "src/core/ext/client_channel/uri_parser.h",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.h",
     "src/core/ext/lb_policy/grpclb/grpclb.h",
     "src/core/ext/lb_policy/grpclb/load_balancer_api.h",
     "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h",
@@ -481,9 +483,9 @@ cc_library(
     "src/core/lib/security/credentials/plugin/plugin_credentials.c",
     "src/core/lib/security/credentials/ssl/ssl_credentials.c",
     "src/core/lib/security/transport/client_auth_filter.c",
-    "src/core/lib/security/transport/handshake.c",
     "src/core/lib/security/transport/secure_endpoint.c",
     "src/core/lib/security/transport/security_connector.c",
+    "src/core/lib/security/transport/security_handshaker.c",
     "src/core/lib/security/transport/server_auth_filter.c",
     "src/core/lib/security/transport/tsi_error.c",
     "src/core/lib/security/util/b64.c",
@@ -492,6 +494,7 @@ cc_library(
     "src/core/lib/tsi/fake_transport_security.c",
     "src/core/lib/tsi/ssl_transport_security.c",
     "src/core/lib/tsi/transport_security.c",
+    "src/core/ext/transport/chttp2/server/chttp2_server.c",
     "src/core/ext/transport/chttp2/client/secure/secure_channel_create.c",
     "src/core/ext/client_channel/channel_connectivity.c",
     "src/core/ext/client_channel/client_channel.c",
@@ -511,6 +514,7 @@ cc_library(
     "src/core/ext/client_channel/subchannel.c",
     "src/core/ext/client_channel/subchannel_index.c",
     "src/core/ext/client_channel/uri_parser.c",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.c",
     "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c",
     "src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c",
     "src/core/ext/transport/chttp2/client/insecure/channel_create.c",
@@ -737,9 +741,9 @@ cc_library(
     "src/core/lib/security/credentials/plugin/plugin_credentials.h",
     "src/core/lib/security/credentials/ssl/ssl_credentials.h",
     "src/core/lib/security/transport/auth_filters.h",
-    "src/core/lib/security/transport/handshake.h",
     "src/core/lib/security/transport/secure_endpoint.h",
     "src/core/lib/security/transport/security_connector.h",
+    "src/core/lib/security/transport/security_handshaker.h",
     "src/core/lib/security/transport/tsi_error.h",
     "src/core/lib/security/util/b64.h",
     "src/core/lib/security/util/json_util.h",
@@ -748,6 +752,7 @@ cc_library(
     "src/core/lib/tsi/ssl_types.h",
     "src/core/lib/tsi/transport_security.h",
     "src/core/lib/tsi/transport_security_interface.h",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.h",
     "src/core/lib/surface/init.c",
     "src/core/lib/channel/channel_args.c",
     "src/core/lib/channel/channel_stack.c",
@@ -921,9 +926,9 @@ cc_library(
     "src/core/lib/security/credentials/plugin/plugin_credentials.c",
     "src/core/lib/security/credentials/ssl/ssl_credentials.c",
     "src/core/lib/security/transport/client_auth_filter.c",
-    "src/core/lib/security/transport/handshake.c",
     "src/core/lib/security/transport/secure_endpoint.c",
     "src/core/lib/security/transport/security_connector.c",
+    "src/core/lib/security/transport/security_handshaker.c",
     "src/core/lib/security/transport/server_auth_filter.c",
     "src/core/lib/security/transport/tsi_error.c",
     "src/core/lib/security/util/b64.c",
@@ -932,6 +937,7 @@ cc_library(
     "src/core/lib/tsi/fake_transport_security.c",
     "src/core/lib/tsi/ssl_transport_security.c",
     "src/core/lib/tsi/transport_security.c",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.c",
     "src/core/plugin_registry/grpc_cronet_plugin_registry.c",
   ],
   hdrs = [
@@ -1097,6 +1103,8 @@ cc_library(
     "src/core/ext/transport/chttp2/transport/stream_map.h",
     "src/core/ext/transport/chttp2/transport/varint.h",
     "src/core/ext/transport/chttp2/alpn/alpn.h",
+    "src/core/ext/transport/chttp2/server/chttp2_server.h",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.h",
     "src/core/ext/client_channel/client_channel.h",
     "src/core/ext/client_channel/client_channel_factory.h",
     "src/core/ext/client_channel/connector.h",
@@ -1266,8 +1274,10 @@ cc_library(
     "src/core/ext/transport/chttp2/transport/varint.c",
     "src/core/ext/transport/chttp2/transport/writing.c",
     "src/core/ext/transport/chttp2/alpn/alpn.c",
+    "src/core/ext/transport/chttp2/server/chttp2_server.c",
     "src/core/ext/transport/chttp2/client/insecure/channel_create.c",
     "src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.c",
     "src/core/ext/client_channel/channel_connectivity.c",
     "src/core/ext/client_channel/client_channel.c",
     "src/core/ext/client_channel/client_channel_factory.c",
@@ -1515,6 +1525,152 @@ cc_library(
     "src/cpp/server/dynamic_thread_pool.h",
     "src/cpp/server/thread_pool_interface.h",
     "src/cpp/thread_manager/thread_manager.h",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.h",
+    "src/core/ext/transport/chttp2/transport/bin_decoder.h",
+    "src/core/ext/transport/chttp2/transport/bin_encoder.h",
+    "src/core/ext/transport/chttp2/transport/chttp2_transport.h",
+    "src/core/ext/transport/chttp2/transport/frame.h",
+    "src/core/ext/transport/chttp2/transport/frame_data.h",
+    "src/core/ext/transport/chttp2/transport/frame_goaway.h",
+    "src/core/ext/transport/chttp2/transport/frame_ping.h",
+    "src/core/ext/transport/chttp2/transport/frame_rst_stream.h",
+    "src/core/ext/transport/chttp2/transport/frame_settings.h",
+    "src/core/ext/transport/chttp2/transport/frame_window_update.h",
+    "src/core/ext/transport/chttp2/transport/hpack_encoder.h",
+    "src/core/ext/transport/chttp2/transport/hpack_parser.h",
+    "src/core/ext/transport/chttp2/transport/hpack_table.h",
+    "src/core/ext/transport/chttp2/transport/http2_errors.h",
+    "src/core/ext/transport/chttp2/transport/huffsyms.h",
+    "src/core/ext/transport/chttp2/transport/incoming_metadata.h",
+    "src/core/ext/transport/chttp2/transport/internal.h",
+    "src/core/ext/transport/chttp2/transport/status_conversion.h",
+    "src/core/ext/transport/chttp2/transport/stream_map.h",
+    "src/core/ext/transport/chttp2/transport/varint.h",
+    "src/core/lib/channel/channel_args.h",
+    "src/core/lib/channel/channel_stack.h",
+    "src/core/lib/channel/channel_stack_builder.h",
+    "src/core/lib/channel/compress_filter.h",
+    "src/core/lib/channel/connected_channel.h",
+    "src/core/lib/channel/context.h",
+    "src/core/lib/channel/deadline_filter.h",
+    "src/core/lib/channel/handshaker.h",
+    "src/core/lib/channel/http_client_filter.h",
+    "src/core/lib/channel/http_server_filter.h",
+    "src/core/lib/channel/message_size_filter.h",
+    "src/core/lib/compression/algorithm_metadata.h",
+    "src/core/lib/compression/message_compress.h",
+    "src/core/lib/debug/trace.h",
+    "src/core/lib/http/format_request.h",
+    "src/core/lib/http/httpcli.h",
+    "src/core/lib/http/parser.h",
+    "src/core/lib/iomgr/closure.h",
+    "src/core/lib/iomgr/combiner.h",
+    "src/core/lib/iomgr/endpoint.h",
+    "src/core/lib/iomgr/endpoint_pair.h",
+    "src/core/lib/iomgr/error.h",
+    "src/core/lib/iomgr/ev_epoll_linux.h",
+    "src/core/lib/iomgr/ev_poll_posix.h",
+    "src/core/lib/iomgr/ev_posix.h",
+    "src/core/lib/iomgr/exec_ctx.h",
+    "src/core/lib/iomgr/executor.h",
+    "src/core/lib/iomgr/iocp_windows.h",
+    "src/core/lib/iomgr/iomgr.h",
+    "src/core/lib/iomgr/iomgr_internal.h",
+    "src/core/lib/iomgr/iomgr_posix.h",
+    "src/core/lib/iomgr/load_file.h",
+    "src/core/lib/iomgr/network_status_tracker.h",
+    "src/core/lib/iomgr/polling_entity.h",
+    "src/core/lib/iomgr/pollset.h",
+    "src/core/lib/iomgr/pollset_set.h",
+    "src/core/lib/iomgr/pollset_set_windows.h",
+    "src/core/lib/iomgr/pollset_uv.h",
+    "src/core/lib/iomgr/pollset_windows.h",
+    "src/core/lib/iomgr/port.h",
+    "src/core/lib/iomgr/resolve_address.h",
+    "src/core/lib/iomgr/resource_quota.h",
+    "src/core/lib/iomgr/sockaddr.h",
+    "src/core/lib/iomgr/sockaddr_posix.h",
+    "src/core/lib/iomgr/sockaddr_utils.h",
+    "src/core/lib/iomgr/sockaddr_windows.h",
+    "src/core/lib/iomgr/socket_mutator.h",
+    "src/core/lib/iomgr/socket_utils.h",
+    "src/core/lib/iomgr/socket_utils_posix.h",
+    "src/core/lib/iomgr/socket_windows.h",
+    "src/core/lib/iomgr/tcp_client.h",
+    "src/core/lib/iomgr/tcp_client_posix.h",
+    "src/core/lib/iomgr/tcp_posix.h",
+    "src/core/lib/iomgr/tcp_server.h",
+    "src/core/lib/iomgr/tcp_uv.h",
+    "src/core/lib/iomgr/tcp_windows.h",
+    "src/core/lib/iomgr/time_averaged_stats.h",
+    "src/core/lib/iomgr/timer.h",
+    "src/core/lib/iomgr/timer_generic.h",
+    "src/core/lib/iomgr/timer_heap.h",
+    "src/core/lib/iomgr/timer_uv.h",
+    "src/core/lib/iomgr/udp_server.h",
+    "src/core/lib/iomgr/unix_sockets_posix.h",
+    "src/core/lib/iomgr/wakeup_fd_cv.h",
+    "src/core/lib/iomgr/wakeup_fd_pipe.h",
+    "src/core/lib/iomgr/wakeup_fd_posix.h",
+    "src/core/lib/iomgr/workqueue.h",
+    "src/core/lib/iomgr/workqueue_uv.h",
+    "src/core/lib/iomgr/workqueue_windows.h",
+    "src/core/lib/json/json.h",
+    "src/core/lib/json/json_common.h",
+    "src/core/lib/json/json_reader.h",
+    "src/core/lib/json/json_writer.h",
+    "src/core/lib/slice/percent_encoding.h",
+    "src/core/lib/slice/slice_string_helpers.h",
+    "src/core/lib/surface/api_trace.h",
+    "src/core/lib/surface/call.h",
+    "src/core/lib/surface/call_test_only.h",
+    "src/core/lib/surface/channel.h",
+    "src/core/lib/surface/channel_init.h",
+    "src/core/lib/surface/channel_stack_type.h",
+    "src/core/lib/surface/completion_queue.h",
+    "src/core/lib/surface/event_string.h",
+    "src/core/lib/surface/init.h",
+    "src/core/lib/surface/lame_client.h",
+    "src/core/lib/surface/server.h",
+    "src/core/lib/transport/byte_stream.h",
+    "src/core/lib/transport/connectivity_state.h",
+    "src/core/lib/transport/mdstr_hash_table.h",
+    "src/core/lib/transport/metadata.h",
+    "src/core/lib/transport/metadata_batch.h",
+    "src/core/lib/transport/pid_controller.h",
+    "src/core/lib/transport/service_config.h",
+    "src/core/lib/transport/static_metadata.h",
+    "src/core/lib/transport/timeout_encoding.h",
+    "src/core/lib/transport/transport.h",
+    "src/core/lib/transport/transport_impl.h",
+    "src/core/ext/transport/chttp2/alpn/alpn.h",
+    "src/core/ext/client_channel/client_channel.h",
+    "src/core/ext/client_channel/client_channel_factory.h",
+    "src/core/ext/client_channel/connector.h",
+    "src/core/ext/client_channel/http_connect_handshaker.h",
+    "src/core/ext/client_channel/initial_connect_string.h",
+    "src/core/ext/client_channel/lb_policy.h",
+    "src/core/ext/client_channel/lb_policy_factory.h",
+    "src/core/ext/client_channel/lb_policy_registry.h",
+    "src/core/ext/client_channel/parse_address.h",
+    "src/core/ext/client_channel/resolver.h",
+    "src/core/ext/client_channel/resolver_factory.h",
+    "src/core/ext/client_channel/resolver_registry.h",
+    "src/core/ext/client_channel/subchannel.h",
+    "src/core/ext/client_channel/subchannel_index.h",
+    "src/core/ext/client_channel/uri_parser.h",
+    "src/core/ext/transport/chttp2/server/chttp2_server.h",
+    "src/core/ext/census/aggregation.h",
+    "src/core/ext/census/base_resources.h",
+    "src/core/ext/census/census_interface.h",
+    "src/core/ext/census/census_rpc_stats.h",
+    "src/core/ext/census/gen/census.pb.h",
+    "src/core/ext/census/gen/trace_context.pb.h",
+    "src/core/ext/census/grpc_filter.h",
+    "src/core/ext/census/mlog.h",
+    "src/core/ext/census/resource.h",
+    "src/core/ext/census/rpc_metric_id.h",
+    "src/core/ext/census/trace_context.h",
     "src/cpp/client/cronet_credentials.cc",
     "src/cpp/client/insecure_credentials.cc",
     "src/cpp/common/insecure_create_auth_context.cc",
@@ -1548,6 +1704,178 @@ cc_library(
     "src/cpp/util/string_ref.cc",
     "src/cpp/util/time_cc.cc",
     "src/cpp/codegen/codegen_init.cc",
+    "src/core/ext/transport/chttp2/client/insecure/channel_create.c",
+    "src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.c",
+    "src/core/ext/transport/chttp2/transport/bin_decoder.c",
+    "src/core/ext/transport/chttp2/transport/bin_encoder.c",
+    "src/core/ext/transport/chttp2/transport/chttp2_plugin.c",
+    "src/core/ext/transport/chttp2/transport/chttp2_transport.c",
+    "src/core/ext/transport/chttp2/transport/frame_data.c",
+    "src/core/ext/transport/chttp2/transport/frame_goaway.c",
+    "src/core/ext/transport/chttp2/transport/frame_ping.c",
+    "src/core/ext/transport/chttp2/transport/frame_rst_stream.c",
+    "src/core/ext/transport/chttp2/transport/frame_settings.c",
+    "src/core/ext/transport/chttp2/transport/frame_window_update.c",
+    "src/core/ext/transport/chttp2/transport/hpack_encoder.c",
+    "src/core/ext/transport/chttp2/transport/hpack_parser.c",
+    "src/core/ext/transport/chttp2/transport/hpack_table.c",
+    "src/core/ext/transport/chttp2/transport/huffsyms.c",
+    "src/core/ext/transport/chttp2/transport/incoming_metadata.c",
+    "src/core/ext/transport/chttp2/transport/parsing.c",
+    "src/core/ext/transport/chttp2/transport/status_conversion.c",
+    "src/core/ext/transport/chttp2/transport/stream_lists.c",
+    "src/core/ext/transport/chttp2/transport/stream_map.c",
+    "src/core/ext/transport/chttp2/transport/varint.c",
+    "src/core/ext/transport/chttp2/transport/writing.c",
+    "src/core/lib/channel/channel_args.c",
+    "src/core/lib/channel/channel_stack.c",
+    "src/core/lib/channel/channel_stack_builder.c",
+    "src/core/lib/channel/compress_filter.c",
+    "src/core/lib/channel/connected_channel.c",
+    "src/core/lib/channel/deadline_filter.c",
+    "src/core/lib/channel/handshaker.c",
+    "src/core/lib/channel/http_client_filter.c",
+    "src/core/lib/channel/http_server_filter.c",
+    "src/core/lib/channel/message_size_filter.c",
+    "src/core/lib/compression/compression.c",
+    "src/core/lib/compression/message_compress.c",
+    "src/core/lib/debug/trace.c",
+    "src/core/lib/http/format_request.c",
+    "src/core/lib/http/httpcli.c",
+    "src/core/lib/http/parser.c",
+    "src/core/lib/iomgr/closure.c",
+    "src/core/lib/iomgr/combiner.c",
+    "src/core/lib/iomgr/endpoint.c",
+    "src/core/lib/iomgr/endpoint_pair_posix.c",
+    "src/core/lib/iomgr/endpoint_pair_uv.c",
+    "src/core/lib/iomgr/endpoint_pair_windows.c",
+    "src/core/lib/iomgr/error.c",
+    "src/core/lib/iomgr/ev_epoll_linux.c",
+    "src/core/lib/iomgr/ev_poll_posix.c",
+    "src/core/lib/iomgr/ev_posix.c",
+    "src/core/lib/iomgr/exec_ctx.c",
+    "src/core/lib/iomgr/executor.c",
+    "src/core/lib/iomgr/iocp_windows.c",
+    "src/core/lib/iomgr/iomgr.c",
+    "src/core/lib/iomgr/iomgr_posix.c",
+    "src/core/lib/iomgr/iomgr_uv.c",
+    "src/core/lib/iomgr/iomgr_windows.c",
+    "src/core/lib/iomgr/load_file.c",
+    "src/core/lib/iomgr/network_status_tracker.c",
+    "src/core/lib/iomgr/polling_entity.c",
+    "src/core/lib/iomgr/pollset_set_uv.c",
+    "src/core/lib/iomgr/pollset_set_windows.c",
+    "src/core/lib/iomgr/pollset_uv.c",
+    "src/core/lib/iomgr/pollset_windows.c",
+    "src/core/lib/iomgr/resolve_address_posix.c",
+    "src/core/lib/iomgr/resolve_address_uv.c",
+    "src/core/lib/iomgr/resolve_address_windows.c",
+    "src/core/lib/iomgr/resource_quota.c",
+    "src/core/lib/iomgr/sockaddr_utils.c",
+    "src/core/lib/iomgr/socket_mutator.c",
+    "src/core/lib/iomgr/socket_utils_common_posix.c",
+    "src/core/lib/iomgr/socket_utils_linux.c",
+    "src/core/lib/iomgr/socket_utils_posix.c",
+    "src/core/lib/iomgr/socket_utils_uv.c",
+    "src/core/lib/iomgr/socket_utils_windows.c",
+    "src/core/lib/iomgr/socket_windows.c",
+    "src/core/lib/iomgr/tcp_client_posix.c",
+    "src/core/lib/iomgr/tcp_client_uv.c",
+    "src/core/lib/iomgr/tcp_client_windows.c",
+    "src/core/lib/iomgr/tcp_posix.c",
+    "src/core/lib/iomgr/tcp_server_posix.c",
+    "src/core/lib/iomgr/tcp_server_uv.c",
+    "src/core/lib/iomgr/tcp_server_windows.c",
+    "src/core/lib/iomgr/tcp_uv.c",
+    "src/core/lib/iomgr/tcp_windows.c",
+    "src/core/lib/iomgr/time_averaged_stats.c",
+    "src/core/lib/iomgr/timer_generic.c",
+    "src/core/lib/iomgr/timer_heap.c",
+    "src/core/lib/iomgr/timer_uv.c",
+    "src/core/lib/iomgr/udp_server.c",
+    "src/core/lib/iomgr/unix_sockets_posix.c",
+    "src/core/lib/iomgr/unix_sockets_posix_noop.c",
+    "src/core/lib/iomgr/wakeup_fd_cv.c",
+    "src/core/lib/iomgr/wakeup_fd_eventfd.c",
+    "src/core/lib/iomgr/wakeup_fd_nospecial.c",
+    "src/core/lib/iomgr/wakeup_fd_pipe.c",
+    "src/core/lib/iomgr/wakeup_fd_posix.c",
+    "src/core/lib/iomgr/workqueue_uv.c",
+    "src/core/lib/iomgr/workqueue_windows.c",
+    "src/core/lib/json/json.c",
+    "src/core/lib/json/json_reader.c",
+    "src/core/lib/json/json_string.c",
+    "src/core/lib/json/json_writer.c",
+    "src/core/lib/slice/percent_encoding.c",
+    "src/core/lib/slice/slice.c",
+    "src/core/lib/slice/slice_buffer.c",
+    "src/core/lib/slice/slice_string_helpers.c",
+    "src/core/lib/surface/alarm.c",
+    "src/core/lib/surface/api_trace.c",
+    "src/core/lib/surface/byte_buffer.c",
+    "src/core/lib/surface/byte_buffer_reader.c",
+    "src/core/lib/surface/call.c",
+    "src/core/lib/surface/call_details.c",
+    "src/core/lib/surface/call_log_batch.c",
+    "src/core/lib/surface/channel.c",
+    "src/core/lib/surface/channel_init.c",
+    "src/core/lib/surface/channel_ping.c",
+    "src/core/lib/surface/channel_stack_type.c",
+    "src/core/lib/surface/completion_queue.c",
+    "src/core/lib/surface/event_string.c",
+    "src/core/lib/surface/lame_client.c",
+    "src/core/lib/surface/metadata_array.c",
+    "src/core/lib/surface/server.c",
+    "src/core/lib/surface/validate_metadata.c",
+    "src/core/lib/surface/version.c",
+    "src/core/lib/transport/byte_stream.c",
+    "src/core/lib/transport/connectivity_state.c",
+    "src/core/lib/transport/mdstr_hash_table.c",
+    "src/core/lib/transport/metadata.c",
+    "src/core/lib/transport/metadata_batch.c",
+    "src/core/lib/transport/pid_controller.c",
+    "src/core/lib/transport/service_config.c",
+    "src/core/lib/transport/static_metadata.c",
+    "src/core/lib/transport/timeout_encoding.c",
+    "src/core/lib/transport/transport.c",
+    "src/core/lib/transport/transport_op_string.c",
+    "src/core/ext/transport/chttp2/alpn/alpn.c",
+    "src/core/ext/client_channel/channel_connectivity.c",
+    "src/core/ext/client_channel/client_channel.c",
+    "src/core/ext/client_channel/client_channel_factory.c",
+    "src/core/ext/client_channel/client_channel_plugin.c",
+    "src/core/ext/client_channel/connector.c",
+    "src/core/ext/client_channel/default_initial_connect_string.c",
+    "src/core/ext/client_channel/http_connect_handshaker.c",
+    "src/core/ext/client_channel/initial_connect_string.c",
+    "src/core/ext/client_channel/lb_policy.c",
+    "src/core/ext/client_channel/lb_policy_factory.c",
+    "src/core/ext/client_channel/lb_policy_registry.c",
+    "src/core/ext/client_channel/parse_address.c",
+    "src/core/ext/client_channel/resolver.c",
+    "src/core/ext/client_channel/resolver_factory.c",
+    "src/core/ext/client_channel/resolver_registry.c",
+    "src/core/ext/client_channel/subchannel.c",
+    "src/core/ext/client_channel/subchannel_index.c",
+    "src/core/ext/client_channel/uri_parser.c",
+    "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c",
+    "src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c",
+    "src/core/ext/transport/chttp2/server/chttp2_server.c",
+    "src/core/ext/census/base_resources.c",
+    "src/core/ext/census/context.c",
+    "src/core/ext/census/gen/census.pb.c",
+    "src/core/ext/census/gen/trace_context.pb.c",
+    "src/core/ext/census/grpc_context.c",
+    "src/core/ext/census/grpc_filter.c",
+    "src/core/ext/census/grpc_plugin.c",
+    "src/core/ext/census/initialize.c",
+    "src/core/ext/census/mlog.c",
+    "src/core/ext/census/operation.c",
+    "src/core/ext/census/placeholders.c",
+    "src/core/ext/census/resource.c",
+    "src/core/ext/census/trace_context.c",
+    "src/core/ext/census/tracing.c",
   ],
   hdrs = [
     "include/grpc++/alarm.h",
@@ -1637,6 +1965,16 @@ cc_library(
     "include/grpc/impl/codegen/sync_generic.h",
     "include/grpc/impl/codegen/sync_posix.h",
     "include/grpc/impl/codegen/sync_windows.h",
+    "include/grpc/byte_buffer.h",
+    "include/grpc/byte_buffer_reader.h",
+    "include/grpc/compression.h",
+    "include/grpc/grpc.h",
+    "include/grpc/grpc_posix.h",
+    "include/grpc/grpc_security_constants.h",
+    "include/grpc/slice.h",
+    "include/grpc/slice_buffer.h",
+    "include/grpc/status.h",
+    "include/grpc/census.h",
   ],
   includes = [
     "include",
@@ -1646,6 +1984,7 @@ cc_library(
     "//external:libssl",
     ":gpr",
     ":grpc_cronet",
+    "//external:nanopb",
   ],
 )
 
@@ -2142,9 +2481,9 @@ objc_library(
     "src/core/lib/security/credentials/plugin/plugin_credentials.c",
     "src/core/lib/security/credentials/ssl/ssl_credentials.c",
     "src/core/lib/security/transport/client_auth_filter.c",
-    "src/core/lib/security/transport/handshake.c",
     "src/core/lib/security/transport/secure_endpoint.c",
     "src/core/lib/security/transport/security_connector.c",
+    "src/core/lib/security/transport/security_handshaker.c",
     "src/core/lib/security/transport/server_auth_filter.c",
     "src/core/lib/security/transport/tsi_error.c",
     "src/core/lib/security/util/b64.c",
@@ -2153,6 +2492,7 @@ objc_library(
     "src/core/lib/tsi/fake_transport_security.c",
     "src/core/lib/tsi/ssl_transport_security.c",
     "src/core/lib/tsi/transport_security.c",
+    "src/core/ext/transport/chttp2/server/chttp2_server.c",
     "src/core/ext/transport/chttp2/client/secure/secure_channel_create.c",
     "src/core/ext/client_channel/channel_connectivity.c",
     "src/core/ext/client_channel/client_channel.c",
@@ -2172,6 +2512,7 @@ objc_library(
     "src/core/ext/client_channel/subchannel.c",
     "src/core/ext/client_channel/subchannel_index.c",
     "src/core/ext/client_channel/uri_parser.c",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.c",
     "src/core/ext/transport/chttp2/server/insecure/server_chttp2.c",
     "src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c",
     "src/core/ext/transport/chttp2/client/insecure/channel_create.c",
@@ -2361,9 +2702,9 @@ objc_library(
     "src/core/lib/security/credentials/plugin/plugin_credentials.h",
     "src/core/lib/security/credentials/ssl/ssl_credentials.h",
     "src/core/lib/security/transport/auth_filters.h",
-    "src/core/lib/security/transport/handshake.h",
     "src/core/lib/security/transport/secure_endpoint.h",
     "src/core/lib/security/transport/security_connector.h",
+    "src/core/lib/security/transport/security_handshaker.h",
     "src/core/lib/security/transport/tsi_error.h",
     "src/core/lib/security/util/b64.h",
     "src/core/lib/security/util/json_util.h",
@@ -2372,6 +2713,7 @@ objc_library(
     "src/core/lib/tsi/ssl_types.h",
     "src/core/lib/tsi/transport_security.h",
     "src/core/lib/tsi/transport_security_interface.h",
+    "src/core/ext/transport/chttp2/server/chttp2_server.h",
     "src/core/ext/client_channel/client_channel.h",
     "src/core/ext/client_channel/client_channel_factory.h",
     "src/core/ext/client_channel/connector.h",
@@ -2387,6 +2729,7 @@ objc_library(
     "src/core/ext/client_channel/subchannel.h",
     "src/core/ext/client_channel/subchannel_index.h",
     "src/core/ext/client_channel/uri_parser.h",
+    "src/core/ext/transport/chttp2/client/chttp2_connector.h",
     "src/core/ext/lb_policy/grpclb/grpclb.h",
     "src/core/ext/lb_policy/grpclb/load_balancer_api.h",
     "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h",

CMakeLists.txt

@@ -436,9 +436,9 @@ add_library(grpc
   src/core/lib/security/credentials/plugin/plugin_credentials.c
   src/core/lib/security/credentials/ssl/ssl_credentials.c
   src/core/lib/security/transport/client_auth_filter.c
-  src/core/lib/security/transport/handshake.c
   src/core/lib/security/transport/secure_endpoint.c
   src/core/lib/security/transport/security_connector.c
+  src/core/lib/security/transport/security_handshaker.c
   src/core/lib/security/transport/server_auth_filter.c
   src/core/lib/security/transport/tsi_error.c
   src/core/lib/security/util/b64.c
@@ -447,6 +447,7 @@ add_library(grpc
   src/core/lib/tsi/fake_transport_security.c
   src/core/lib/tsi/ssl_transport_security.c
   src/core/lib/tsi/transport_security.c
+  src/core/ext/transport/chttp2/server/chttp2_server.c
   src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
   src/core/ext/client_channel/channel_connectivity.c
   src/core/ext/client_channel/client_channel.c
@@ -466,6 +467,7 @@ add_library(grpc
   src/core/ext/client_channel/subchannel.c
   src/core/ext/client_channel/subchannel_index.c
   src/core/ext/client_channel/uri_parser.c
+  src/core/ext/transport/chttp2/client/chttp2_connector.c
   src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
   src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c
   src/core/ext/transport/chttp2/client/insecure/channel_create.c
@@ -736,9 +738,9 @@ add_library(grpc_cronet
   src/core/lib/security/credentials/plugin/plugin_credentials.c
   src/core/lib/security/credentials/ssl/ssl_credentials.c
   src/core/lib/security/transport/client_auth_filter.c
-  src/core/lib/security/transport/handshake.c
   src/core/lib/security/transport/secure_endpoint.c
   src/core/lib/security/transport/security_connector.c
+  src/core/lib/security/transport/security_handshaker.c
   src/core/lib/security/transport/server_auth_filter.c
   src/core/lib/security/transport/tsi_error.c
   src/core/lib/security/util/b64.c
@@ -747,6 +749,7 @@ add_library(grpc_cronet
   src/core/lib/tsi/fake_transport_security.c
   src/core/lib/tsi/ssl_transport_security.c
   src/core/lib/tsi/transport_security.c
+  src/core/ext/transport/chttp2/client/chttp2_connector.c
   src/core/plugin_registry/grpc_cronet_plugin_registry.c
 )
 
@@ -951,8 +954,10 @@ add_library(grpc_unsecure
   src/core/ext/transport/chttp2/transport/varint.c
   src/core/ext/transport/chttp2/transport/writing.c
   src/core/ext/transport/chttp2/alpn/alpn.c
+  src/core/ext/transport/chttp2/server/chttp2_server.c
   src/core/ext/transport/chttp2/client/insecure/channel_create.c
   src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
+  src/core/ext/transport/chttp2/client/chttp2_connector.c
   src/core/ext/client_channel/channel_connectivity.c
   src/core/ext/client_channel/client_channel.c
   src/core/ext/client_channel/client_channel_factory.c
@@ -1256,6 +1261,181 @@ add_library(grpc++_cronet
   src/cpp/util/string_ref.cc
   src/cpp/util/time_cc.cc
   src/cpp/codegen/codegen_init.cc
+  src/core/ext/transport/chttp2/client/insecure/channel_create.c
+  src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
+  src/core/ext/transport/chttp2/client/chttp2_connector.c
+  src/core/ext/transport/chttp2/transport/bin_decoder.c
+  src/core/ext/transport/chttp2/transport/bin_encoder.c
+  src/core/ext/transport/chttp2/transport/chttp2_plugin.c
+  src/core/ext/transport/chttp2/transport/chttp2_transport.c
+  src/core/ext/transport/chttp2/transport/frame_data.c
+  src/core/ext/transport/chttp2/transport/frame_goaway.c
+  src/core/ext/transport/chttp2/transport/frame_ping.c
+  src/core/ext/transport/chttp2/transport/frame_rst_stream.c
+  src/core/ext/transport/chttp2/transport/frame_settings.c
+  src/core/ext/transport/chttp2/transport/frame_window_update.c
+  src/core/ext/transport/chttp2/transport/hpack_encoder.c
+  src/core/ext/transport/chttp2/transport/hpack_parser.c
+  src/core/ext/transport/chttp2/transport/hpack_table.c
+  src/core/ext/transport/chttp2/transport/huffsyms.c
+  src/core/ext/transport/chttp2/transport/incoming_metadata.c
+  src/core/ext/transport/chttp2/transport/parsing.c
+  src/core/ext/transport/chttp2/transport/status_conversion.c
+  src/core/ext/transport/chttp2/transport/stream_lists.c
+  src/core/ext/transport/chttp2/transport/stream_map.c
+  src/core/ext/transport/chttp2/transport/varint.c
+  src/core/ext/transport/chttp2/transport/writing.c
+  src/core/lib/channel/channel_args.c
+  src/core/lib/channel/channel_stack.c
+  src/core/lib/channel/channel_stack_builder.c
+  src/core/lib/channel/compress_filter.c
+  src/core/lib/channel/connected_channel.c
+  src/core/lib/channel/deadline_filter.c
+  src/core/lib/channel/handshaker.c
+  src/core/lib/channel/http_client_filter.c
+  src/core/lib/channel/http_server_filter.c
+  src/core/lib/channel/message_size_filter.c
+  src/core/lib/compression/compression.c
+  src/core/lib/compression/message_compress.c
+  src/core/lib/debug/trace.c
+  src/core/lib/http/format_request.c
+  src/core/lib/http/httpcli.c
+  src/core/lib/http/parser.c
+  src/core/lib/iomgr/closure.c
+  src/core/lib/iomgr/combiner.c
+  src/core/lib/iomgr/endpoint.c
+  src/core/lib/iomgr/endpoint_pair_posix.c
+  src/core/lib/iomgr/endpoint_pair_uv.c
+  src/core/lib/iomgr/endpoint_pair_windows.c
+  src/core/lib/iomgr/error.c
+  src/core/lib/iomgr/ev_epoll_linux.c
+  src/core/lib/iomgr/ev_poll_posix.c
+  src/core/lib/iomgr/ev_posix.c
+  src/core/lib/iomgr/exec_ctx.c
+  src/core/lib/iomgr/executor.c
+  src/core/lib/iomgr/iocp_windows.c
+  src/core/lib/iomgr/iomgr.c
+  src/core/lib/iomgr/iomgr_posix.c
+  src/core/lib/iomgr/iomgr_uv.c
+  src/core/lib/iomgr/iomgr_windows.c
+  src/core/lib/iomgr/load_file.c
+  src/core/lib/iomgr/network_status_tracker.c
+  src/core/lib/iomgr/polling_entity.c
+  src/core/lib/iomgr/pollset_set_uv.c
+  src/core/lib/iomgr/pollset_set_windows.c
+  src/core/lib/iomgr/pollset_uv.c
+  src/core/lib/iomgr/pollset_windows.c
+  src/core/lib/iomgr/resolve_address_posix.c
+  src/core/lib/iomgr/resolve_address_uv.c
+  src/core/lib/iomgr/resolve_address_windows.c
+  src/core/lib/iomgr/resource_quota.c
+  src/core/lib/iomgr/sockaddr_utils.c
+  src/core/lib/iomgr/socket_mutator.c
+  src/core/lib/iomgr/socket_utils_common_posix.c
+  src/core/lib/iomgr/socket_utils_linux.c
+  src/core/lib/iomgr/socket_utils_posix.c
+  src/core/lib/iomgr/socket_utils_uv.c
+  src/core/lib/iomgr/socket_utils_windows.c
+  src/core/lib/iomgr/socket_windows.c
+  src/core/lib/iomgr/tcp_client_posix.c
+  src/core/lib/iomgr/tcp_client_uv.c
+  src/core/lib/iomgr/tcp_client_windows.c
+  src/core/lib/iomgr/tcp_posix.c
+  src/core/lib/iomgr/tcp_server_posix.c
+  src/core/lib/iomgr/tcp_server_uv.c
+  src/core/lib/iomgr/tcp_server_windows.c
+  src/core/lib/iomgr/tcp_uv.c
+  src/core/lib/iomgr/tcp_windows.c
+  src/core/lib/iomgr/time_averaged_stats.c
+  src/core/lib/iomgr/timer_generic.c
+  src/core/lib/iomgr/timer_heap.c
+  src/core/lib/iomgr/timer_uv.c
+  src/core/lib/iomgr/udp_server.c
+  src/core/lib/iomgr/unix_sockets_posix.c
+  src/core/lib/iomgr/unix_sockets_posix_noop.c
+  src/core/lib/iomgr/wakeup_fd_cv.c
+  src/core/lib/iomgr/wakeup_fd_eventfd.c
+  src/core/lib/iomgr/wakeup_fd_nospecial.c
+  src/core/lib/iomgr/wakeup_fd_pipe.c
+  src/core/lib/iomgr/wakeup_fd_posix.c
+  src/core/lib/iomgr/workqueue_uv.c
+  src/core/lib/iomgr/workqueue_windows.c
+  src/core/lib/json/json.c
+  src/core/lib/json/json_reader.c
+  src/core/lib/json/json_string.c
+  src/core/lib/json/json_writer.c
+  src/core/lib/slice/percent_encoding.c
+  src/core/lib/slice/slice.c
+  src/core/lib/slice/slice_buffer.c
+  src/core/lib/slice/slice_string_helpers.c
+  src/core/lib/surface/alarm.c
+  src/core/lib/surface/api_trace.c
+  src/core/lib/surface/byte_buffer.c
+  src/core/lib/surface/byte_buffer_reader.c
+  src/core/lib/surface/call.c
+  src/core/lib/surface/call_details.c
+  src/core/lib/surface/call_log_batch.c
+  src/core/lib/surface/channel.c
+  src/core/lib/surface/channel_init.c
+  src/core/lib/surface/channel_ping.c
+  src/core/lib/surface/channel_stack_type.c
+  src/core/lib/surface/completion_queue.c
+  src/core/lib/surface/event_string.c
+  src/core/lib/surface/lame_client.c
+  src/core/lib/surface/metadata_array.c
+  src/core/lib/surface/server.c
+  src/core/lib/surface/validate_metadata.c
+  src/core/lib/surface/version.c
+  src/core/lib/transport/byte_stream.c
+  src/core/lib/transport/connectivity_state.c
+  src/core/lib/transport/mdstr_hash_table.c
+  src/core/lib/transport/metadata.c
+  src/core/lib/transport/metadata_batch.c
+  src/core/lib/transport/pid_controller.c
+  src/core/lib/transport/service_config.c
+  src/core/lib/transport/static_metadata.c
+  src/core/lib/transport/timeout_encoding.c
+  src/core/lib/transport/transport.c
+  src/core/lib/transport/transport_op_string.c
+  src/core/ext/transport/chttp2/alpn/alpn.c
+  src/core/ext/client_channel/channel_connectivity.c
+  src/core/ext/client_channel/client_channel.c
+  src/core/ext/client_channel/client_channel_factory.c
+  src/core/ext/client_channel/client_channel_plugin.c
+  src/core/ext/client_channel/connector.c
+  src/core/ext/client_channel/default_initial_connect_string.c
+  src/core/ext/client_channel/http_connect_handshaker.c
+  src/core/ext/client_channel/initial_connect_string.c
+  src/core/ext/client_channel/lb_policy.c
+  src/core/ext/client_channel/lb_policy_factory.c
+  src/core/ext/client_channel/lb_policy_registry.c
+  src/core/ext/client_channel/parse_address.c
+  src/core/ext/client_channel/resolver.c
+  src/core/ext/client_channel/resolver_factory.c
+  src/core/ext/client_channel/resolver_registry.c
+  src/core/ext/client_channel/subchannel.c
+  src/core/ext/client_channel/subchannel_index.c
+  src/core/ext/client_channel/uri_parser.c
+  src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
+  src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c
+  src/core/ext/transport/chttp2/server/chttp2_server.c
+  src/core/ext/census/base_resources.c
+  src/core/ext/census/context.c
+  src/core/ext/census/gen/census.pb.c
+  src/core/ext/census/gen/trace_context.pb.c
+  src/core/ext/census/grpc_context.c
+  src/core/ext/census/grpc_filter.c
+  src/core/ext/census/grpc_plugin.c
+  src/core/ext/census/initialize.c
+  src/core/ext/census/mlog.c
+  src/core/ext/census/operation.c
+  src/core/ext/census/placeholders.c
+  src/core/ext/census/resource.c
+  src/core/ext/census/trace_context.c
+  src/core/ext/census/tracing.c
+  third_party/nanopb/pb_common.c
+  third_party/nanopb/pb_decode.c
+  third_party/nanopb/pb_encode.c
 )
 
 target_include_directories(grpc++_cronet
@@ -1362,6 +1542,16 @@ foreach(_hdr
   include/grpc/impl/codegen/sync_generic.h
   include/grpc/impl/codegen/sync_posix.h
   include/grpc/impl/codegen/sync_windows.h
+  include/grpc/byte_buffer.h
+  include/grpc/byte_buffer_reader.h
+  include/grpc/compression.h
+  include/grpc/grpc.h
+  include/grpc/grpc_posix.h
+  include/grpc/grpc_security_constants.h
+  include/grpc/slice.h
+  include/grpc/slice_buffer.h
+  include/grpc/status.h
+  include/grpc/census.h
 )
   string(REPLACE "include/" "" _path ${_hdr})
   get_filename_component(_path ${_path} PATH)

Makefile

@@ -1260,9 +1260,9 @@ pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc
 pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc
 
 ifeq ($(EMBED_OPENSSL),true)
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 else
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 endif
 
 
@@ -2775,9 +2775,9 @@ LIBGRPC_SRC = \
     src/core/lib/security/credentials/plugin/plugin_credentials.c \
     src/core/lib/security/credentials/ssl/ssl_credentials.c \
     src/core/lib/security/transport/client_auth_filter.c \
-    src/core/lib/security/transport/handshake.c \
     src/core/lib/security/transport/secure_endpoint.c \
     src/core/lib/security/transport/security_connector.c \
+    src/core/lib/security/transport/security_handshaker.c \
     src/core/lib/security/transport/server_auth_filter.c \
     src/core/lib/security/transport/tsi_error.c \
     src/core/lib/security/util/b64.c \
@@ -2786,6 +2786,7 @@ LIBGRPC_SRC = \
     src/core/lib/tsi/fake_transport_security.c \
     src/core/lib/tsi/ssl_transport_security.c \
     src/core/lib/tsi/transport_security.c \
+    src/core/ext/transport/chttp2/server/chttp2_server.c \
     src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \
     src/core/ext/client_channel/channel_connectivity.c \
     src/core/ext/client_channel/client_channel.c \
@@ -2805,6 +2806,7 @@ LIBGRPC_SRC = \
     src/core/ext/client_channel/subchannel.c \
     src/core/ext/client_channel/subchannel_index.c \
     src/core/ext/client_channel/uri_parser.c \
+    src/core/ext/transport/chttp2/client/chttp2_connector.c \
     src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \
     src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c \
     src/core/ext/transport/chttp2/client/insecure/channel_create.c \
@@ -3093,9 +3095,9 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/security/credentials/plugin/plugin_credentials.c \
     src/core/lib/security/credentials/ssl/ssl_credentials.c \
     src/core/lib/security/transport/client_auth_filter.c \
-    src/core/lib/security/transport/handshake.c \
     src/core/lib/security/transport/secure_endpoint.c \
     src/core/lib/security/transport/security_connector.c \
+    src/core/lib/security/transport/security_handshaker.c \
     src/core/lib/security/transport/server_auth_filter.c \
     src/core/lib/security/transport/tsi_error.c \
     src/core/lib/security/util/b64.c \
@@ -3104,6 +3106,7 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/tsi/fake_transport_security.c \
     src/core/lib/tsi/ssl_transport_security.c \
     src/core/lib/tsi/transport_security.c \
+    src/core/ext/transport/chttp2/client/chttp2_connector.c \
     src/core/plugin_registry/grpc_cronet_plugin_registry.c \
 
 PUBLIC_HEADERS_C += \
@@ -3562,8 +3565,10 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/ext/transport/chttp2/transport/varint.c \
     src/core/ext/transport/chttp2/transport/writing.c \
     src/core/ext/transport/chttp2/alpn/alpn.c \
+    src/core/ext/transport/chttp2/server/chttp2_server.c \
     src/core/ext/transport/chttp2/client/insecure/channel_create.c \
     src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c \
+    src/core/ext/transport/chttp2/client/chttp2_connector.c \
     src/core/ext/client_channel/channel_connectivity.c \
     src/core/ext/client_channel/client_channel.c \
     src/core/ext/client_channel/client_channel_factory.c \
@@ -3979,6 +3984,181 @@ LIBGRPC++_CRONET_SRC = \
     src/cpp/util/string_ref.cc \
     src/cpp/util/time_cc.cc \
     src/cpp/codegen/codegen_init.cc \
+    src/core/ext/transport/chttp2/client/insecure/channel_create.c \
+    src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c \
+    src/core/ext/transport/chttp2/client/chttp2_connector.c \
+    src/core/ext/transport/chttp2/transport/bin_decoder.c \
+    src/core/ext/transport/chttp2/transport/bin_encoder.c \
+    src/core/ext/transport/chttp2/transport/chttp2_plugin.c \
+    src/core/ext/transport/chttp2/transport/chttp2_transport.c \
+    src/core/ext/transport/chttp2/transport/frame_data.c \
+    src/core/ext/transport/chttp2/transport/frame_goaway.c \
+    src/core/ext/transport/chttp2/transport/frame_ping.c \
+    src/core/ext/transport/chttp2/transport/frame_rst_stream.c \
+    src/core/ext/transport/chttp2/transport/frame_settings.c \
+    src/core/ext/transport/chttp2/transport/frame_window_update.c \
+    src/core/ext/transport/chttp2/transport/hpack_encoder.c \
+    src/core/ext/transport/chttp2/transport/hpack_parser.c \
+    src/core/ext/transport/chttp2/transport/hpack_table.c \
+    src/core/ext/transport/chttp2/transport/huffsyms.c \
+    src/core/ext/transport/chttp2/transport/incoming_metadata.c \
+    src/core/ext/transport/chttp2/transport/parsing.c \
+    src/core/ext/transport/chttp2/transport/status_conversion.c \
+    src/core/ext/transport/chttp2/transport/stream_lists.c \
+    src/core/ext/transport/chttp2/transport/stream_map.c \
+    src/core/ext/transport/chttp2/transport/varint.c \
+    src/core/ext/transport/chttp2/transport/writing.c \
+    src/core/lib/channel/channel_args.c \
+    src/core/lib/channel/channel_stack.c \
+    src/core/lib/channel/channel_stack_builder.c \
+    src/core/lib/channel/compress_filter.c \
+    src/core/lib/channel/connected_channel.c \
+    src/core/lib/channel/deadline_filter.c \
+    src/core/lib/channel/handshaker.c \
+    src/core/lib/channel/http_client_filter.c \
+    src/core/lib/channel/http_server_filter.c \
+    src/core/lib/channel/message_size_filter.c \
+    src/core/lib/compression/compression.c \
+    src/core/lib/compression/message_compress.c \
+    src/core/lib/debug/trace.c \
+    src/core/lib/http/format_request.c \
+    src/core/lib/http/httpcli.c \
+    src/core/lib/http/parser.c \
+    src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
+    src/core/lib/iomgr/endpoint.c \
+    src/core/lib/iomgr/endpoint_pair_posix.c \
+    src/core/lib/iomgr/endpoint_pair_uv.c \
+    src/core/lib/iomgr/endpoint_pair_windows.c \
+    src/core/lib/iomgr/error.c \
+    src/core/lib/iomgr/ev_epoll_linux.c \
+    src/core/lib/iomgr/ev_poll_posix.c \
+    src/core/lib/iomgr/ev_posix.c \
+    src/core/lib/iomgr/exec_ctx.c \
+    src/core/lib/iomgr/executor.c \
+    src/core/lib/iomgr/iocp_windows.c \
+    src/core/lib/iomgr/iomgr.c \
+    src/core/lib/iomgr/iomgr_posix.c \
+    src/core/lib/iomgr/iomgr_uv.c \
+    src/core/lib/iomgr/iomgr_windows.c \
+    src/core/lib/iomgr/load_file.c \
+    src/core/lib/iomgr/network_status_tracker.c \
+    src/core/lib/iomgr/polling_entity.c \
+    src/core/lib/iomgr/pollset_set_uv.c \
+    src/core/lib/iomgr/pollset_set_windows.c \
+    src/core/lib/iomgr/pollset_uv.c \
+    src/core/lib/iomgr/pollset_windows.c \
+    src/core/lib/iomgr/resolve_address_posix.c \
+    src/core/lib/iomgr/resolve_address_uv.c \
+    src/core/lib/iomgr/resolve_address_windows.c \
+    src/core/lib/iomgr/resource_quota.c \
+    src/core/lib/iomgr/sockaddr_utils.c \
+    src/core/lib/iomgr/socket_mutator.c \
+    src/core/lib/iomgr/socket_utils_common_posix.c \
+    src/core/lib/iomgr/socket_utils_linux.c \
+    src/core/lib/iomgr/socket_utils_posix.c \
+    src/core/lib/iomgr/socket_utils_uv.c \
+    src/core/lib/iomgr/socket_utils_windows.c \
+    src/core/lib/iomgr/socket_windows.c \
+    src/core/lib/iomgr/tcp_client_posix.c \
+    src/core/lib/iomgr/tcp_client_uv.c \
+    src/core/lib/iomgr/tcp_client_windows.c \
+    src/core/lib/iomgr/tcp_posix.c \
+    src/core/lib/iomgr/tcp_server_posix.c \
+    src/core/lib/iomgr/tcp_server_uv.c \
+    src/core/lib/iomgr/tcp_server_windows.c \
+    src/core/lib/iomgr/tcp_uv.c \
+    src/core/lib/iomgr/tcp_windows.c \
+    src/core/lib/iomgr/time_averaged_stats.c \
+    src/core/lib/iomgr/timer_generic.c \
+    src/core/lib/iomgr/timer_heap.c \
+    src/core/lib/iomgr/timer_uv.c \
+    src/core/lib/iomgr/udp_server.c \
+    src/core/lib/iomgr/unix_sockets_posix.c \
+    src/core/lib/iomgr/unix_sockets_posix_noop.c \
+    src/core/lib/iomgr/wakeup_fd_cv.c \
+    src/core/lib/iomgr/wakeup_fd_eventfd.c \
+    src/core/lib/iomgr/wakeup_fd_nospecial.c \
+    src/core/lib/iomgr/wakeup_fd_pipe.c \
+    src/core/lib/iomgr/wakeup_fd_posix.c \
+    src/core/lib/iomgr/workqueue_uv.c \
+    src/core/lib/iomgr/workqueue_windows.c \
+    src/core/lib/json/json.c \
+    src/core/lib/json/json_reader.c \
+    src/core/lib/json/json_string.c \
+    src/core/lib/json/json_writer.c \
+    src/core/lib/slice/percent_encoding.c \
+    src/core/lib/slice/slice.c \
+    src/core/lib/slice/slice_buffer.c \
+    src/core/lib/slice/slice_string_helpers.c \
+    src/core/lib/surface/alarm.c \
+    src/core/lib/surface/api_trace.c \
+    src/core/lib/surface/byte_buffer.c \
+    src/core/lib/surface/byte_buffer_reader.c \
+    src/core/lib/surface/call.c \
+    src/core/lib/surface/call_details.c \
+    src/core/lib/surface/call_log_batch.c \
+    src/core/lib/surface/channel.c \
+    src/core/lib/surface/channel_init.c \
+    src/core/lib/surface/channel_ping.c \
+    src/core/lib/surface/channel_stack_type.c \
+    src/core/lib/surface/completion_queue.c \
+    src/core/lib/surface/event_string.c \
+    src/core/lib/surface/lame_client.c \
+    src/core/lib/surface/metadata_array.c \
+    src/core/lib/surface/server.c \
+    src/core/lib/surface/validate_metadata.c \
+    src/core/lib/surface/version.c \
+    src/core/lib/transport/byte_stream.c \
+    src/core/lib/transport/connectivity_state.c \
+    src/core/lib/transport/mdstr_hash_table.c \
+    src/core/lib/transport/metadata.c \
+    src/core/lib/transport/metadata_batch.c \
+    src/core/lib/transport/pid_controller.c \
+    src/core/lib/transport/service_config.c \
+    src/core/lib/transport/static_metadata.c \
+    src/core/lib/transport/timeout_encoding.c \
+    src/core/lib/transport/transport.c \
+    src/core/lib/transport/transport_op_string.c \
+    src/core/ext/transport/chttp2/alpn/alpn.c \
+    src/core/ext/client_channel/channel_connectivity.c \
+    src/core/ext/client_channel/client_channel.c \
+    src/core/ext/client_channel/client_channel_factory.c \
+    src/core/ext/client_channel/client_channel_plugin.c \
+    src/core/ext/client_channel/connector.c \
+    src/core/ext/client_channel/default_initial_connect_string.c \
+    src/core/ext/client_channel/http_connect_handshaker.c \
+    src/core/ext/client_channel/initial_connect_string.c \
+    src/core/ext/client_channel/lb_policy.c \
+    src/core/ext/client_channel/lb_policy_factory.c \
+    src/core/ext/client_channel/lb_policy_registry.c \
+    src/core/ext/client_channel/parse_address.c \
+    src/core/ext/client_channel/resolver.c \
+    src/core/ext/client_channel/resolver_factory.c \
+    src/core/ext/client_channel/resolver_registry.c \
+    src/core/ext/client_channel/subchannel.c \
+    src/core/ext/client_channel/subchannel_index.c \
+    src/core/ext/client_channel/uri_parser.c \
+    src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \
+    src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c \
+    src/core/ext/transport/chttp2/server/chttp2_server.c \
+    src/core/ext/census/base_resources.c \
+    src/core/ext/census/context.c \
+    src/core/ext/census/gen/census.pb.c \
+    src/core/ext/census/gen/trace_context.pb.c \
+    src/core/ext/census/grpc_context.c \
+    src/core/ext/census/grpc_filter.c \
+    src/core/ext/census/grpc_plugin.c \
+    src/core/ext/census/initialize.c \
+    src/core/ext/census/mlog.c \
+    src/core/ext/census/operation.c \
+    src/core/ext/census/placeholders.c \
+    src/core/ext/census/resource.c \
+    src/core/ext/census/trace_context.c \
+    src/core/ext/census/tracing.c \
+    third_party/nanopb/pb_common.c \
+    third_party/nanopb/pb_decode.c \
+    third_party/nanopb/pb_encode.c \
 
 PUBLIC_HEADERS_CXX += \
     include/grpc++/alarm.h \
@@ -4068,6 +4248,16 @@ PUBLIC_HEADERS_CXX += \
     include/grpc/impl/codegen/sync_generic.h \
     include/grpc/impl/codegen/sync_posix.h \
     include/grpc/impl/codegen/sync_windows.h \
+    include/grpc/byte_buffer.h \
+    include/grpc/byte_buffer_reader.h \
+    include/grpc/compression.h \
+    include/grpc/grpc.h \
+    include/grpc/grpc_posix.h \
+    include/grpc/grpc_security_constants.h \
+    include/grpc/slice.h \
+    include/grpc/slice_buffer.h \
+    include/grpc/status.h \
+    include/grpc/census.h \
 
 LIBGRPC++_CRONET_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_CRONET_SRC))))
 
@@ -6808,43 +6998,43 @@ ifneq ($(NO_DEPS),true)
 endif
 
 
-LIBGOOGLE_BENCHMARK_SRC = \
-    third_party/google_benchmark/src/benchmark.cc \
-    third_party/google_benchmark/src/benchmark_register.cc \
-    third_party/google_benchmark/src/colorprint.cc \
-    third_party/google_benchmark/src/commandlineflags.cc \
-    third_party/google_benchmark/src/complexity.cc \
-    third_party/google_benchmark/src/console_reporter.cc \
-    third_party/google_benchmark/src/csv_reporter.cc \
-    third_party/google_benchmark/src/json_reporter.cc \
-    third_party/google_benchmark/src/reporter.cc \
-    third_party/google_benchmark/src/sleep.cc \
-    third_party/google_benchmark/src/string_util.cc \
-    third_party/google_benchmark/src/sysinfo.cc \
-    third_party/google_benchmark/src/timers.cc \
+LIBBENCHMARK_SRC = \
+    third_party/benchmark/src/benchmark.cc \
+    third_party/benchmark/src/benchmark_register.cc \
+    third_party/benchmark/src/colorprint.cc \
+    third_party/benchmark/src/commandlineflags.cc \
+    third_party/benchmark/src/complexity.cc \
+    third_party/benchmark/src/console_reporter.cc \
+    third_party/benchmark/src/csv_reporter.cc \
+    third_party/benchmark/src/json_reporter.cc \
+    third_party/benchmark/src/reporter.cc \
+    third_party/benchmark/src/sleep.cc \
+    third_party/benchmark/src/string_util.cc \
+    third_party/benchmark/src/sysinfo.cc \
+    third_party/benchmark/src/timers.cc \
 
 PUBLIC_HEADERS_CXX += \
 
-LIBGOOGLE_BENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGOOGLE_BENCHMARK_SRC))))
+LIBBENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBBENCHMARK_SRC))))
 
-$(LIBGOOGLE_BENCHMARK_OBJS): CPPFLAGS += -Ithird_party/google_benchmark/include -DHAVE_POSIX_REGEX
+$(LIBBENCHMARK_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX
 
 ifeq ($(NO_PROTOBUF),true)
 
 # You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
 
-$(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a: protobuf_dep_error
+$(LIBDIR)/$(CONFIG)/libbenchmark.a: protobuf_dep_error
 
 
 else
 
-$(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(LIBGOOGLE_BENCHMARK_OBJS) 
+$(LIBDIR)/$(CONFIG)/libbenchmark.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(LIBBENCHMARK_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
-	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBGOOGLE_BENCHMARK_OBJS) 
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libbenchmark.a
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBBENCHMARK_OBJS) 
 ifeq ($(SYSTEM),Darwin)
-	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libbenchmark.a
 endif
 
 
@@ -6853,7 +7043,7 @@ endif
 endif
 
 ifneq ($(NO_DEPS),true)
--include $(LIBGOOGLE_BENCHMARK_OBJS:.o=.dep)
+-include $(LIBBENCHMARK_OBJS:.o=.dep)
 endif
 
 
@@ -11546,16 +11736,16 @@ $(BINDIR)/$(CONFIG)/bm_fullstack: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack
+	$(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o:  $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o:  $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_bm_fullstack: $(BM_FULLSTACK_OBJS:.o=.dep)
 
@@ -13002,16 +13192,16 @@ $(BINDIR)/$(CONFIG)/noop-benchmark: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/noop-benchmark: $(PROTOBUF_DEP) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+$(BINDIR)/$(CONFIG)/noop-benchmark: $(PROTOBUF_DEP) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/noop-benchmark
+	$(Q) $(LDXX) $(LDFLAGS) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/noop-benchmark
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/noop-benchmark.o:  $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/noop-benchmark.o:  $(LIBDIR)/$(CONFIG)/libbenchmark.a
 
 deps_noop-benchmark: $(NOOP-BENCHMARK_OBJS:.o=.dep)
 
@@ -16577,9 +16767,9 @@ src/core/lib/security/credentials/oauth2/oauth2_credentials.c: $(OPENSSL_DEP)
 src/core/lib/security/credentials/plugin/plugin_credentials.c: $(OPENSSL_DEP)
 src/core/lib/security/credentials/ssl/ssl_credentials.c: $(OPENSSL_DEP)
 src/core/lib/security/transport/client_auth_filter.c: $(OPENSSL_DEP)
-src/core/lib/security/transport/handshake.c: $(OPENSSL_DEP)
 src/core/lib/security/transport/secure_endpoint.c: $(OPENSSL_DEP)
 src/core/lib/security/transport/security_connector.c: $(OPENSSL_DEP)
+src/core/lib/security/transport/security_handshaker.c: $(OPENSSL_DEP)
 src/core/lib/security/transport/server_auth_filter.c: $(OPENSSL_DEP)
 src/core/lib/security/transport/tsi_error.c: $(OPENSSL_DEP)
 src/core/lib/security/util/b64.c: $(OPENSSL_DEP)

binding.gyp

@@ -716,9 +716,9 @@
         'src/core/lib/security/credentials/plugin/plugin_credentials.c',
         'src/core/lib/security/credentials/ssl/ssl_credentials.c',
         'src/core/lib/security/transport/client_auth_filter.c',
-        'src/core/lib/security/transport/handshake.c',
         'src/core/lib/security/transport/secure_endpoint.c',
         'src/core/lib/security/transport/security_connector.c',
+        'src/core/lib/security/transport/security_handshaker.c',
         'src/core/lib/security/transport/server_auth_filter.c',
         'src/core/lib/security/transport/tsi_error.c',
         'src/core/lib/security/util/b64.c',
@@ -727,6 +727,7 @@
         'src/core/lib/tsi/fake_transport_security.c',
         'src/core/lib/tsi/ssl_transport_security.c',
         'src/core/lib/tsi/transport_security.c',
+        'src/core/ext/transport/chttp2/server/chttp2_server.c',
         'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c',
         'src/core/ext/client_channel/channel_connectivity.c',
         'src/core/ext/client_channel/client_channel.c',
@@ -746,6 +747,7 @@
         'src/core/ext/client_channel/subchannel.c',
         'src/core/ext/client_channel/subchannel_index.c',
         'src/core/ext/client_channel/uri_parser.c',
+        'src/core/ext/transport/chttp2/client/chttp2_connector.c',
         'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
         'src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c',
         'src/core/ext/transport/chttp2/client/insecure/channel_create.c',

build.yaml

@@ -495,9 +495,9 @@ filegroups:
   - src/core/lib/security/credentials/plugin/plugin_credentials.h
   - src/core/lib/security/credentials/ssl/ssl_credentials.h
   - src/core/lib/security/transport/auth_filters.h
-  - src/core/lib/security/transport/handshake.h
   - src/core/lib/security/transport/secure_endpoint.h
   - src/core/lib/security/transport/security_connector.h
+  - src/core/lib/security/transport/security_handshaker.h
   - src/core/lib/security/transport/tsi_error.h
   - src/core/lib/security/util/b64.h
   - src/core/lib/security/util/json_util.h
@@ -518,9 +518,9 @@ filegroups:
   - src/core/lib/security/credentials/plugin/plugin_credentials.c
   - src/core/lib/security/credentials/ssl/ssl_credentials.c
   - src/core/lib/security/transport/client_auth_filter.c
-  - src/core/lib/security/transport/handshake.c
   - src/core/lib/security/transport/secure_endpoint.c
   - src/core/lib/security/transport/security_connector.c
+  - src/core/lib/security/transport/security_handshaker.c
   - src/core/lib/security/transport/server_auth_filter.c
   - src/core/lib/security/transport/tsi_error.c
   - src/core/lib/security/util/b64.c
@@ -621,11 +621,20 @@ filegroups:
   - src/core/ext/transport/chttp2/alpn/alpn.c
   deps:
   - gpr
+- name: grpc_transport_chttp2_client_connector
+  headers:
+  - src/core/ext/transport/chttp2/client/chttp2_connector.h
+  src:
+  - src/core/ext/transport/chttp2/client/chttp2_connector.c
+  uses:
+  - grpc_transport_chttp2
+  - grpc_base
 - name: grpc_transport_chttp2_client_insecure
   src:
   - src/core/ext/transport/chttp2/client/insecure/channel_create.c
   - src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c
   uses:
+  - grpc_transport_chttp2_client_connector
   - grpc_transport_chttp2
   - grpc_base
   - grpc_client_channel
@@ -637,6 +646,15 @@ filegroups:
   - grpc_base
   - grpc_client_channel
   - grpc_secure
+  - grpc_transport_chttp2_client_connector
+- name: grpc_transport_chttp2_server
+  headers:
+  - src/core/ext/transport/chttp2/server/chttp2_server.h
+  src:
+  - src/core/ext/transport/chttp2/server/chttp2_server.c
+  uses:
+  - grpc_transport_chttp2
+  - grpc_base
 - name: grpc_transport_chttp2_server_insecure
   src:
   - src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
@@ -644,6 +662,7 @@ filegroups:
   uses:
   - grpc_transport_chttp2
   - grpc_base
+  - grpc_transport_chttp2_server
 - name: grpc_transport_chttp2_server_secure
   src:
   - src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -651,6 +670,7 @@ filegroups:
   - grpc_transport_chttp2
   - grpc_base
   - grpc_secure
+  - grpc_transport_chttp2_server
 - name: grpc_transport_cronet_client_secure
   public_headers:
   - include/grpc/grpc_cronet.h
@@ -1039,6 +1059,9 @@ libs:
   - grpc++_base
   - grpc++_codegen_base
   - grpc++_codegen_base_src
+  - grpc_transport_chttp2_client_insecure
+  - grpc_transport_chttp2_server_insecure
+  - census
   platforms:
   - linux
   secure: true
@@ -2827,7 +2850,7 @@ targets:
   src:
   - test/cpp/microbenchmarks/bm_fullstack.cc
   deps:
-  - google_benchmark
+  - benchmark
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -3277,7 +3300,7 @@ targets:
   src:
   - test/cpp/microbenchmarks/noop-benchmark.cc
   deps:
-  - google_benchmark
+  - benchmark
 - name: proto_server_reflection_test
   gtest: true
   build: test
@@ -3763,6 +3786,8 @@ configs:
       UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
     timeout_multiplier: 1.5
 defaults:
+  benchmark:
+    CPPFLAGS: -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX
   boringssl:
     CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas
       -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI)
@@ -3771,8 +3796,6 @@ defaults:
   global:
     CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter
     LDFLAGS: -g
-  google_benchmark:
-    CPPFLAGS: -Ithird_party/google_benchmark/include -DHAVE_POSIX_REGEX
   zlib:
     CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration
       $(W_NO_SHIFT_NEGATIVE_VALUE) -fvisibility=hidden

config.m4

@@ -232,9 +232,9 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/security/credentials/plugin/plugin_credentials.c \
     src/core/lib/security/credentials/ssl/ssl_credentials.c \
     src/core/lib/security/transport/client_auth_filter.c \
-    src/core/lib/security/transport/handshake.c \
     src/core/lib/security/transport/secure_endpoint.c \
     src/core/lib/security/transport/security_connector.c \
+    src/core/lib/security/transport/security_handshaker.c \
     src/core/lib/security/transport/server_auth_filter.c \
     src/core/lib/security/transport/tsi_error.c \
     src/core/lib/security/util/b64.c \
@@ -243,6 +243,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/tsi/fake_transport_security.c \
     src/core/lib/tsi/ssl_transport_security.c \
     src/core/lib/tsi/transport_security.c \
+    src/core/ext/transport/chttp2/server/chttp2_server.c \
     src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \
     src/core/ext/client_channel/channel_connectivity.c \
     src/core/ext/client_channel/client_channel.c \
@@ -262,6 +263,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/client_channel/subchannel.c \
     src/core/ext/client_channel/subchannel_index.c \
     src/core/ext/client_channel/uri_parser.c \
+    src/core/ext/transport/chttp2/client/chttp2_connector.c \
     src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \
     src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c \
     src/core/ext/transport/chttp2/client/insecure/channel_create.c \
@@ -609,8 +611,10 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/resolver/dns/native)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/resolver/sockaddr)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/alpn)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/client)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/client/insecure)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/client/secure)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/insecure)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/server/secure)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/transport/chttp2/transport)

doc/connection-backoff.md

@@ -7,9 +7,10 @@ requests) and instead do some form of exponential backoff.
 
 We have several parameters:
  1. INITIAL_BACKOFF (how long to wait after the first failure before retrying)
- 2. MULTIPLIER (factor with which to multiply backoff after a failed retry)
- 3. MAX_BACKOFF (upper bound on backoff)
- 4. MIN_CONNECT_TIMEOUT (minimum time we're willing to give a connection to
+ 1. MULTIPLIER (factor with which to multiply backoff after a failed retry)
+ 1. JITTER (by how much to randomize backoffs).
+ 1. MAX_BACKOFF (upper bound on backoff)
+ 1. MIN_CONNECT_TIMEOUT (minimum time we're willing to give a connection to
     complete)
 
 ## Proposed Backoff Algorithm

doc/connectivity-semantics-and-api.md

@@ -16,7 +16,7 @@ reconnect, or in the case of HTTP/2 GO_AWAY, re-resolve the name and reconnect.
 
 To hide the details of all this activity from the user of the gRPC API (i.e.,
 application code) while exposing meaningful information about the state of a
-channel, we use a state machine with four states, defined below:
+channel, we use a state machine with five states, defined below:
 
 CONNECTING: The channel is trying to establish a connection and is waiting to
 make progress on one of the steps involved in name resolution, TCP connection
@@ -116,7 +116,7 @@ Channel State API
 
 All gRPC libraries will expose a channel-level API method to poll the current
 state of a channel. In C++, this method is called GetCurrentState and returns
-an enum for one of the four legal states.
+an enum for one of the five legal states.
 
 All libraries should also expose an API that enables the application (user of
 the gRPC API) to be notified when the channel state changes. Since state

examples/cpp/helloworld/greeter_client.cc

@@ -51,7 +51,7 @@ class GreeterClient {
   GreeterClient(std::shared_ptr<Channel> channel)
       : stub_(Greeter::NewStub(channel)) {}
 
-  // Assambles the client's payload, sends it and presents the response back
+  // Assembles the client's payload, sends it and presents the response back
   // from the server.
   std::string SayHello(const std::string& user) {
     // Data we are sending to the server.

examples/python/multiplex/run_codegen.py

@@ -29,7 +29,7 @@
 
 """Generates protocol messages and gRPC stubs."""
 
-from grpc.tools import protoc
+from grpc_tools import protoc
 
 protoc.main(
     (

examples/python/route_guide/run_codegen.py

@@ -29,7 +29,7 @@
 
 """Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
 
-from grpc.tools import protoc
+from grpc_tools import protoc
 
 protoc.main(
     (

gRPC-Core.podspec

@@ -376,9 +376,9 @@ Pod::Spec.new do |s|
                       'src/core/lib/security/credentials/plugin/plugin_credentials.h',
                       'src/core/lib/security/credentials/ssl/ssl_credentials.h',
                       'src/core/lib/security/transport/auth_filters.h',
-                      'src/core/lib/security/transport/handshake.h',
                       'src/core/lib/security/transport/secure_endpoint.h',
                       'src/core/lib/security/transport/security_connector.h',
+                      'src/core/lib/security/transport/security_handshaker.h',
                       'src/core/lib/security/transport/tsi_error.h',
                       'src/core/lib/security/util/b64.h',
                       'src/core/lib/security/util/json_util.h',
@@ -387,6 +387,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/tsi/ssl_types.h',
                       'src/core/lib/tsi/transport_security.h',
                       'src/core/lib/tsi/transport_security_interface.h',
+                      'src/core/ext/transport/chttp2/server/chttp2_server.h',
                       'src/core/ext/client_channel/client_channel.h',
                       'src/core/ext/client_channel/client_channel_factory.h',
                       'src/core/ext/client_channel/connector.h',
@@ -402,6 +403,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/client_channel/subchannel.h',
                       'src/core/ext/client_channel/subchannel_index.h',
                       'src/core/ext/client_channel/uri_parser.h',
+                      'src/core/ext/transport/chttp2/client/chttp2_connector.h',
                       'src/core/ext/lb_policy/grpclb/grpclb.h',
                       'src/core/ext/lb_policy/grpclb/load_balancer_api.h',
                       'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h',
@@ -574,9 +576,9 @@ Pod::Spec.new do |s|
                       'src/core/lib/security/credentials/plugin/plugin_credentials.c',
                       'src/core/lib/security/credentials/ssl/ssl_credentials.c',
                       'src/core/lib/security/transport/client_auth_filter.c',
-                      'src/core/lib/security/transport/handshake.c',
                       'src/core/lib/security/transport/secure_endpoint.c',
                       'src/core/lib/security/transport/security_connector.c',
+                      'src/core/lib/security/transport/security_handshaker.c',
                       'src/core/lib/security/transport/server_auth_filter.c',
                       'src/core/lib/security/transport/tsi_error.c',
                       'src/core/lib/security/util/b64.c',
@@ -585,6 +587,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/tsi/fake_transport_security.c',
                       'src/core/lib/tsi/ssl_transport_security.c',
                       'src/core/lib/tsi/transport_security.c',
+                      'src/core/ext/transport/chttp2/server/chttp2_server.c',
                       'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c',
                       'src/core/ext/client_channel/channel_connectivity.c',
                       'src/core/ext/client_channel/client_channel.c',
@@ -604,6 +607,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/client_channel/subchannel.c',
                       'src/core/ext/client_channel/subchannel_index.c',
                       'src/core/ext/client_channel/uri_parser.c',
+                      'src/core/ext/transport/chttp2/client/chttp2_connector.c',
                       'src/core/ext/transport/chttp2/server/insecure/server_chttp2.c',
                       'src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c',
                       'src/core/ext/transport/chttp2/client/insecure/channel_create.c',
@@ -779,9 +783,9 @@ Pod::Spec.new do |s|
                               'src/core/lib/security/credentials/plugin/plugin_credentials.h',
                               'src/core/lib/security/credentials/ssl/ssl_credentials.h',
                               'src/core/lib/security/transport/auth_filters.h',
-                              'src/core/lib/security/transport/handshake.h',
                               'src/core/lib/security/transport/secure_endpoint.h',
                               'src/core/lib/security/transport/security_connector.h',
+                              'src/core/lib/security/transport/security_handshaker.h',
                               'src/core/lib/security/transport/tsi_error.h',
                               'src/core/lib/security/util/b64.h',
                               'src/core/lib/security/util/json_util.h',
@@ -790,6 +794,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/tsi/ssl_types.h',
                               'src/core/lib/tsi/transport_security.h',
                               'src/core/lib/tsi/transport_security_interface.h',
+                              'src/core/ext/transport/chttp2/server/chttp2_server.h',
                               'src/core/ext/client_channel/client_channel.h',
                               'src/core/ext/client_channel/client_channel_factory.h',
                               'src/core/ext/client_channel/connector.h',
@@ -805,6 +810,7 @@ Pod::Spec.new do |s|
                               'src/core/ext/client_channel/subchannel.h',
                               'src/core/ext/client_channel/subchannel_index.h',
                               'src/core/ext/client_channel/uri_parser.h',
+                              'src/core/ext/transport/chttp2/client/chttp2_connector.h',
                               'src/core/ext/lb_policy/grpclb/grpclb.h',
                               'src/core/ext/lb_policy/grpclb/load_balancer_api.h',
                               'src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h',

grpc.gemspec

@@ -295,9 +295,9 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.h )
   s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.h )
   s.files += %w( src/core/lib/security/transport/auth_filters.h )
-  s.files += %w( src/core/lib/security/transport/handshake.h )
   s.files += %w( src/core/lib/security/transport/secure_endpoint.h )
   s.files += %w( src/core/lib/security/transport/security_connector.h )
+  s.files += %w( src/core/lib/security/transport/security_handshaker.h )
   s.files += %w( src/core/lib/security/transport/tsi_error.h )
   s.files += %w( src/core/lib/security/util/b64.h )
   s.files += %w( src/core/lib/security/util/json_util.h )
@@ -306,6 +306,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/tsi/ssl_types.h )
   s.files += %w( src/core/lib/tsi/transport_security.h )
   s.files += %w( src/core/lib/tsi/transport_security_interface.h )
+  s.files += %w( src/core/ext/transport/chttp2/server/chttp2_server.h )
   s.files += %w( src/core/ext/client_channel/client_channel.h )
   s.files += %w( src/core/ext/client_channel/client_channel_factory.h )
   s.files += %w( src/core/ext/client_channel/connector.h )
@@ -321,6 +322,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/client_channel/subchannel.h )
   s.files += %w( src/core/ext/client_channel/subchannel_index.h )
   s.files += %w( src/core/ext/client_channel/uri_parser.h )
+  s.files += %w( src/core/ext/transport/chttp2/client/chttp2_connector.h )
   s.files += %w( src/core/ext/lb_policy/grpclb/grpclb.h )
   s.files += %w( src/core/ext/lb_policy/grpclb/load_balancer_api.h )
   s.files += %w( src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h )
@@ -493,9 +495,9 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.c )
   s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.c )
   s.files += %w( src/core/lib/security/transport/client_auth_filter.c )
-  s.files += %w( src/core/lib/security/transport/handshake.c )
   s.files += %w( src/core/lib/security/transport/secure_endpoint.c )
   s.files += %w( src/core/lib/security/transport/security_connector.c )
+  s.files += %w( src/core/lib/security/transport/security_handshaker.c )
   s.files += %w( src/core/lib/security/transport/server_auth_filter.c )
   s.files += %w( src/core/lib/security/transport/tsi_error.c )
   s.files += %w( src/core/lib/security/util/b64.c )
@@ -504,6 +506,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/tsi/fake_transport_security.c )
   s.files += %w( src/core/lib/tsi/ssl_transport_security.c )
   s.files += %w( src/core/lib/tsi/transport_security.c )
+  s.files += %w( src/core/ext/transport/chttp2/server/chttp2_server.c )
   s.files += %w( src/core/ext/transport/chttp2/client/secure/secure_channel_create.c )
   s.files += %w( src/core/ext/client_channel/channel_connectivity.c )
   s.files += %w( src/core/ext/client_channel/client_channel.c )
@@ -523,6 +526,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/client_channel/subchannel.c )
   s.files += %w( src/core/ext/client_channel/subchannel_index.c )
   s.files += %w( src/core/ext/client_channel/uri_parser.c )
+  s.files += %w( src/core/ext/transport/chttp2/client/chttp2_connector.c )
   s.files += %w( src/core/ext/transport/chttp2/server/insecure/server_chttp2.c )
   s.files += %w( src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c )
   s.files += %w( src/core/ext/transport/chttp2/client/insecure/channel_create.c )

include/grpc++/impl/codegen/completion_queue.h

@@ -52,6 +52,7 @@
 #include <grpc++/impl/codegen/grpc_library.h>
 #include <grpc++/impl/codegen/status.h>
 #include <grpc++/impl/codegen/time.h>
+#include <grpc/impl/codegen/atm.h>
 
 struct grpc_completion_queue;
 
@@ -101,6 +102,7 @@ class CompletionQueue : private GrpcLibraryCodegen {
   /// instance.
   CompletionQueue() {
     cq_ = g_core_codegen_interface->grpc_completion_queue_create(nullptr);
+    InitialAvalanching();  // reserve this for the future shutdown
   }
 
   /// Wrap \a take, taking ownership of the instance.
@@ -151,7 +153,8 @@ class CompletionQueue : private GrpcLibraryCodegen {
 
   /// Request the shutdown of the queue.
   ///
-  /// \warning This method must be called at some point. Once invoked, \a Next
+  /// \warning This method must be called at some point if this completion queue
+  /// is accessed with Next or AsyncNext. Once invoked, \a Next
   /// will start to return false and \a AsyncNext will return \a
   /// NextStatus::SHUTDOWN. Only once either one of these methods does that
   /// (that is, once the queue has been \em drained) can an instance of this
@@ -165,6 +168,21 @@ class CompletionQueue : private GrpcLibraryCodegen {
   /// owership is performed.
   grpc_completion_queue* cq() { return cq_; }
 
+  /// Manage state of avalanching operations : completion queue tags that
+  /// trigger other completion queue operations. The underlying core completion
+  /// queue should not really shutdown until all avalanching operations have
+  /// been finalized. Note that we maintain the requirement that an avalanche
+  /// registration must take place before CQ shutdown (which must be maintained
+  /// elsehwere)
+  void InitialAvalanching() {
+    gpr_atm_rel_store(&avalanches_in_flight_, static_cast<gpr_atm>(1));
+  }
+  void RegisterAvalanching() {
+    gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_,
+                                 static_cast<gpr_atm>(1));
+  };
+  void CompleteAvalanching();
+
  private:
   // Friend synchronous wrappers so that they can access Pluck(), which is
   // a semi-private API geared towards the synchronous implementation.
@@ -229,6 +247,8 @@ class CompletionQueue : private GrpcLibraryCodegen {
   }
 
   grpc_completion_queue* cq_;  // owned
+
+  gpr_atm avalanches_in_flight_;
 };
 
 /// A specific type of completion queue used by the processing of notifications

include/grpc++/impl/codegen/server_interface.h

@@ -140,7 +140,7 @@ class ServerInterface : public CallHook {
                      ServerAsyncStreamingInterface* stream,
                      CompletionQueue* call_cq, void* tag,
                      bool delete_on_finalize);
-    virtual ~BaseAsyncRequest() {}
+    virtual ~BaseAsyncRequest();
 
     bool FinalizeResult(void** tag, bool* status) override;
 

package.xml

@@ -303,9 +303,9 @@
     <file baseinstalldir="/" name="src/core/lib/security/credentials/plugin/plugin_credentials.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/credentials/ssl/ssl_credentials.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/auth_filters.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/transport/handshake.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/secure_endpoint.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/security_connector.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/security_handshaker.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/tsi_error.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/util/b64.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/util/json_util.h" role="src" />
@@ -314,6 +314,7 @@
     <file baseinstalldir="/" name="src/core/lib/tsi/ssl_types.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/transport_security.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/transport_security_interface.h" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/transport/chttp2/server/chttp2_server.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/client_channel.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/client_channel_factory.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/connector.h" role="src" />
@@ -329,6 +330,7 @@
     <file baseinstalldir="/" name="src/core/ext/client_channel/subchannel.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/subchannel_index.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/uri_parser.h" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/chttp2_connector.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/lb_policy/grpclb/grpclb.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/lb_policy/grpclb/load_balancer_api.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" role="src" />
@@ -501,9 +503,9 @@
     <file baseinstalldir="/" name="src/core/lib/security/credentials/plugin/plugin_credentials.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/credentials/ssl/ssl_credentials.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/client_auth_filter.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/transport/handshake.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/secure_endpoint.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/security_connector.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/security_handshaker.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/server_auth_filter.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/transport/tsi_error.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/security/util/b64.c" role="src" />
@@ -512,6 +514,7 @@
     <file baseinstalldir="/" name="src/core/lib/tsi/fake_transport_security.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/ssl_transport_security.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/transport_security.c" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/transport/chttp2/server/chttp2_server.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/secure/secure_channel_create.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/channel_connectivity.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/client_channel.c" role="src" />
@@ -531,6 +534,7 @@
     <file baseinstalldir="/" name="src/core/ext/client_channel/subchannel.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/subchannel_index.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/client_channel/uri_parser.c" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/chttp2_connector.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/server/insecure/server_chttp2.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/client/insecure/channel_create.c" role="src" />

src/google_benchmark/gen_build_yaml.py → src/benchmark/gen_build_yaml.py

@@ -39,15 +39,15 @@ os.chdir(os.path.dirname(sys.argv[0])+'/../..')
 out = {}
 
 out['libs'] = [{
-    'name': 'google_benchmark',
+    'name': 'benchmark',
     'build': 'private',
     'language': 'c++',
     'secure': 'no',
-    'defaults': 'google_benchmark',
-    'src': sorted(glob.glob('third_party/google_benchmark/src/*.cc')),
+    'defaults': 'benchmark',
+    'src': sorted(glob.glob('third_party/benchmark/src/*.cc')),
     'headers': sorted(
-        glob.glob('third_party/google_benchmark/src/*.h') +
-        glob.glob('third_party/google_benchmark/include/benchmark/*.h')),
+        glob.glob('third_party/benchmark/src/*.h') +
+        glob.glob('third_party/benchmark/include/benchmark/*.h')),
 }]
 
 print yaml.dump(out)

src/compiler/csharp_generator.cc

@@ -68,13 +68,13 @@ namespace {
 // Currently, we cannot easily reuse the functionality as
 // google/protobuf/compiler/csharp/csharp_doc_comment.h is not a public header.
 // TODO(jtattermusch): reuse the functionality from google/protobuf.
-void GenerateDocCommentBodyImpl(grpc::protobuf::io::Printer *printer,
+bool GenerateDocCommentBodyImpl(grpc::protobuf::io::Printer *printer,
                                 grpc::protobuf::SourceLocation location) {
   grpc::string comments = location.leading_comments.empty()
                               ? location.trailing_comments
                               : location.leading_comments;
   if (comments.empty()) {
-    return;
+    return false;
   }
   // XML escaping... no need for apostrophes etc as the whole text is going to
   // be a child
@@ -107,18 +107,84 @@ void GenerateDocCommentBodyImpl(grpc::protobuf::io::Printer *printer,
         printer->Print("///\n");
       }
       last_was_empty = false;
-      printer->Print("/// $line$\n", "line", *it);
+      printer->Print("///$line$\n", "line", *it);
     }
   }
   printer->Print("/// </summary>\n");
+  return true;
 }
 
 template <typename DescriptorType>
-void GenerateDocCommentBody(grpc::protobuf::io::Printer *printer,
+bool GenerateDocCommentBody(grpc::protobuf::io::Printer *printer,
                             const DescriptorType *descriptor) {
   grpc::protobuf::SourceLocation location;
-  if (descriptor->GetSourceLocation(&location)) {
-    GenerateDocCommentBodyImpl(printer, location);
+  if (!descriptor->GetSourceLocation(&location)) {
+    return false;
+  }
+  return GenerateDocCommentBodyImpl(printer, location);
+}
+
+void GenerateDocCommentServerMethod(grpc::protobuf::io::Printer *printer,
+                                    const MethodDescriptor *method) {
+  if (GenerateDocCommentBody(printer, method)) {
+    if (method->client_streaming()) {
+      printer->Print(
+          "/// <param name=\"requestStream\">Used for reading requests from "
+          "the client.</param>\n");
+    } else {
+      printer->Print(
+          "/// <param name=\"request\">The request received from the "
+          "client.</param>\n");
+    }
+    if (method->server_streaming()) {
+      printer->Print(
+          "/// <param name=\"responseStream\">Used for sending responses back "
+          "to the client.</param>\n");
+    }
+    printer->Print(
+        "/// <param name=\"context\">The context of the server-side call "
+        "handler being invoked.</param>\n");
+    if (method->server_streaming()) {
+      printer->Print(
+          "/// <returns>A task indicating completion of the "
+          "handler.</returns>\n");
+    } else {
+      printer->Print(
+          "/// <returns>The response to send back to the client (wrapped by a "
+          "task).</returns>\n");
+    }
+  }
+}
+
+void GenerateDocCommentClientMethod(grpc::protobuf::io::Printer *printer,
+                                    const MethodDescriptor *method,
+                                    bool is_sync, bool use_call_options) {
+  if (GenerateDocCommentBody(printer, method)) {
+    if (!method->client_streaming()) {
+      printer->Print(
+          "/// <param name=\"request\">The request to send to the "
+          "server.</param>\n");
+    }
+    if (!use_call_options) {
+      printer->Print(
+          "/// <param name=\"headers\">The initial metadata to send with the "
+          "call. This parameter is optional.</param>\n");
+      printer->Print(
+          "/// <param name=\"deadline\">An optional deadline for the call. The "
+          "call will be cancelled if deadline is hit.</param>\n");
+      printer->Print(
+          "/// <param name=\"cancellationToken\">An optional token for "
+          "canceling the call.</param>\n");
+    } else {
+      printer->Print(
+          "/// <param name=\"options\">The options for the call.</param>\n");
+    }
+    if (is_sync) {
+      printer->Print(
+          "/// <returns>The response received from the server.</returns>\n");
+    } else {
+      printer->Print("/// <returns>The call object.</returns>\n");
+    }
   }
 }
 
@@ -319,7 +385,7 @@ void GenerateServerClass(Printer *out, const ServiceDescriptor *service) {
   out->Indent();
   for (int i = 0; i < service->method_count(); i++) {
     const MethodDescriptor *method = service->method(i);
-    GenerateDocCommentBody(out, method);
+    GenerateDocCommentServerMethod(out, method);
     out->Print(
         "public virtual $returntype$ "
         "$methodname$($request$$response_stream_maybe$, "
@@ -393,7 +459,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
 
     if (method_type == METHODTYPE_NO_STREAMING) {
       // unary calls have an extra synchronous stub method
-      GenerateDocCommentBody(out, method);
+      GenerateDocCommentClientMethod(out, method, true, false);
       out->Print(
           "public virtual $response$ $methodname$($request$ request, Metadata "
           "headers = null, DateTime? deadline = null, CancellationToken "
@@ -411,7 +477,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
       out->Print("}\n");
 
       // overload taking CallOptions as a param
-      GenerateDocCommentBody(out, method);
+      GenerateDocCommentClientMethod(out, method, true, true);
       out->Print(
           "public virtual $response$ $methodname$($request$ request, "
           "CallOptions options)\n",
@@ -432,7 +498,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
     if (method_type == METHODTYPE_NO_STREAMING) {
       method_name += "Async";  // prevent name clash with synchronous method.
     }
-    GenerateDocCommentBody(out, method);
+    GenerateDocCommentClientMethod(out, method, false, false);
     out->Print(
         "public virtual $returntype$ $methodname$($request_maybe$Metadata "
         "headers = null, DateTime? deadline = null, CancellationToken "
@@ -452,7 +518,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
     out->Print("}\n");
 
     // overload taking CallOptions as a param
-    GenerateDocCommentBody(out, method);
+    GenerateDocCommentClientMethod(out, method, false, true);
     out->Print(
         "public virtual $returntype$ $methodname$($request_maybe$CallOptions "
         "options)\n",
@@ -517,6 +583,9 @@ void GenerateBindServiceMethod(Printer *out, const ServiceDescriptor *service) {
   out->Print(
       "/// <summary>Creates service definition that can be registered with a "
       "server</summary>\n");
+  out->Print(
+      "/// <param name=\"serviceImpl\">An object implementing the server-side"
+      " handling logic.</param>\n");
   out->Print(
       "public static ServerServiceDefinition BindService($implclass$ "
       "serviceImpl)\n",

src/compiler/python_generator.cc

@@ -760,6 +760,32 @@ PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config)
 
 PythonGrpcGenerator::~PythonGrpcGenerator() {}
 
+static bool GenerateGrpc(GeneratorContext* context, PrivateGenerator& generator,
+                         grpc::string file_name, bool generate_in_pb2_grpc) {
+  bool success;
+  std::unique_ptr<ZeroCopyOutputStream> output;
+  std::unique_ptr<CodedOutputStream> coded_output;
+  grpc::string grpc_code;
+
+  if (generate_in_pb2_grpc) {
+    output.reset(context->Open(file_name));
+    generator.generate_in_pb2_grpc = true;
+  } else {
+    output.reset(context->OpenForInsert(file_name, "module_scope"));
+    generator.generate_in_pb2_grpc = false;
+  }
+
+  coded_output.reset(new CodedOutputStream(output.get()));
+  tie(success, grpc_code) = generator.GetGrpcServices();
+
+  if (success) {
+    coded_output->WriteRaw(grpc_code.data(), grpc_code.size());
+    return true;
+  } else {
+    return false;
+  }
+}
+
 bool PythonGrpcGenerator::Generate(const FileDescriptor* file,
                                    const grpc::string& parameter,
                                    GeneratorContext* context,
@@ -780,28 +806,15 @@ bool PythonGrpcGenerator::Generate(const FileDescriptor* file,
   }
 
   PrivateGenerator generator(config_, file);
-
-  std::unique_ptr<ZeroCopyOutputStream> pb2_output(
-      context->OpenForAppend(pb2_file_name));
-  std::unique_ptr<ZeroCopyOutputStream> grpc_output(
-      context->Open(pb2_grpc_file_name));
-  CodedOutputStream pb2_coded_out(pb2_output.get());
-  CodedOutputStream grpc_coded_out(grpc_output.get());
-  bool success = false;
-  grpc::string pb2_code;
-  grpc::string grpc_code;
-  generator.generate_in_pb2_grpc = false;
-  tie(success, pb2_code) = generator.GetGrpcServices();
-  if (success) {
-    generator.generate_in_pb2_grpc = true;
-    tie(success, grpc_code) = generator.GetGrpcServices();
-    if (success) {
-      pb2_coded_out.WriteRaw(pb2_code.data(), pb2_code.size());
-      grpc_coded_out.WriteRaw(grpc_code.data(), grpc_code.size());
-      return true;
-    }
+  if (parameter == "grpc_2_0") {
+    return GenerateGrpc(context, generator, pb2_grpc_file_name, true);
+  } else if (parameter == "") {
+    return GenerateGrpc(context, generator, pb2_grpc_file_name, true) &&
+           GenerateGrpc(context, generator, pb2_file_name, false);
+  } else {
+    *error = "Invalid parameter '" + parameter + "'.";
+    return false;
   }
-  return false;
 }
 
 }  // namespace grpc_python_generator

src/core/ext/client_channel/client_channel.c

@@ -683,9 +683,15 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
                                      "Failed to create subchannel", &error, 1));
   } else if (GET_CALL(calld) == CANCELLED_CALL) {
     /* already cancelled before subchannel became ready */
-    fail_locked(exec_ctx, calld,
-                GRPC_ERROR_CREATE_REFERENCING(
-                    "Cancelled before creating subchannel", &error, 1));
+    grpc_error *cancellation_error = GRPC_ERROR_CREATE_REFERENCING(
+        "Cancelled before creating subchannel", &error, 1);
+    /* if due to deadline, attach the deadline exceeded status to the error */
+    if (gpr_time_cmp(calld->deadline, gpr_now(GPR_CLOCK_MONOTONIC)) < 0) {
+      cancellation_error =
+          grpc_error_set_int(cancellation_error, GRPC_ERROR_INT_GRPC_STATUS,
+                             GRPC_STATUS_DEADLINE_EXCEEDED);
+    }
+    fail_locked(exec_ctx, calld, cancellation_error);
   } else {
     /* Create call on subchannel. */
     grpc_subchannel_call *subchannel_call = NULL;
@@ -809,7 +815,6 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
         initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY;
       }
     }
-    // TODO(dgq): make this deadline configurable somehow.
     const grpc_lb_policy_pick_args inputs = {
         initial_metadata, initial_metadata_flags, &calld->lb_token_mdelem,
         gpr_inf_future(GPR_CLOCK_MONOTONIC)};

src/core/ext/client_channel/http_connect_handshaker.c

@@ -41,9 +41,9 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/client_channel/uri_parser.h"
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/http/format_request.h"
 #include "src/core/lib/http/parser.h"
-#include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/support/env.h"
 
 typedef struct http_connect_handshaker {
@@ -53,27 +53,38 @@ typedef struct http_connect_handshaker {
   char* proxy_server;
   char* server_name;
 
+  gpr_refcount refcount;
+  gpr_mu mu;
+
+  bool shutdown;
+  // Endpoint and read buffer to destroy after a shutdown.
+  grpc_endpoint* endpoint_to_destroy;
+  grpc_slice_buffer* read_buffer_to_destroy;
+
   // State saved while performing the handshake.
-  grpc_endpoint* endpoint;
-  grpc_channel_args* args;
-  grpc_handshaker_done_cb cb;
-  void* user_data;
+  grpc_handshaker_args* args;
+  grpc_closure* on_handshake_done;
 
   // Objects for processing the HTTP CONNECT request and response.
   grpc_slice_buffer write_buffer;
-  grpc_slice_buffer* read_buffer;  // Ownership passes through this object.
   grpc_closure request_done_closure;
   grpc_closure response_read_closure;
   grpc_http_parser http_parser;
   grpc_http_response http_response;
-  grpc_timer timeout_timer;
-
-  gpr_refcount refcount;
 } http_connect_handshaker;
 
 // Unref and clean up handshaker.
-static void http_connect_handshaker_unref(http_connect_handshaker* handshaker) {
+static void http_connect_handshaker_unref(grpc_exec_ctx* exec_ctx,
+                                          http_connect_handshaker* handshaker) {
   if (gpr_unref(&handshaker->refcount)) {
+    gpr_mu_destroy(&handshaker->mu);
+    if (handshaker->endpoint_to_destroy != NULL) {
+      grpc_endpoint_destroy(exec_ctx, handshaker->endpoint_to_destroy);
+    }
+    if (handshaker->read_buffer_to_destroy != NULL) {
+      grpc_slice_buffer_destroy(handshaker->read_buffer_to_destroy);
+      gpr_free(handshaker->read_buffer_to_destroy);
+    }
     gpr_free(handshaker->proxy_server);
     gpr_free(handshaker->server_name);
     grpc_slice_buffer_destroy(&handshaker->write_buffer);
@@ -83,28 +94,64 @@ static void http_connect_handshaker_unref(http_connect_handshaker* handshaker) {
   }
 }
 
-// Callback invoked when deadline is exceeded.
-static void on_timeout(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
-  http_connect_handshaker* handshaker = arg;
-  if (error == GRPC_ERROR_NONE) {  // Timer fired, rather than being cancelled.
-    grpc_endpoint_shutdown(exec_ctx, handshaker->endpoint);
+// Set args fields to NULL, saving the endpoint and read buffer for
+// later destruction.
+static void cleanup_args_for_failure_locked(
+    http_connect_handshaker* handshaker) {
+  handshaker->endpoint_to_destroy = handshaker->args->endpoint;
+  handshaker->args->endpoint = NULL;
+  handshaker->read_buffer_to_destroy = handshaker->args->read_buffer;
+  handshaker->args->read_buffer = NULL;
+  grpc_channel_args_destroy(handshaker->args->args);
+  handshaker->args->args = NULL;
+}
+
+// If the handshake failed or we're shutting down, clean up and invoke the
+// callback with the error.
+static void handshake_failed_locked(grpc_exec_ctx* exec_ctx,
+                                    http_connect_handshaker* handshaker,
+                                    grpc_error* error) {
+  if (error == GRPC_ERROR_NONE) {
+    // If we were shut down after an endpoint operation succeeded but
+    // before the endpoint callback was invoked, we need to generate our
+    // own error.
+    error = GRPC_ERROR_CREATE("Handshaker shutdown");
   }
-  http_connect_handshaker_unref(handshaker);
+  if (!handshaker->shutdown) {
+    // TODO(ctiller): It is currently necessary to shutdown endpoints
+    // before destroying them, even if we know that there are no
+    // pending read/write callbacks.  This should be fixed, at which
+    // point this can be removed.
+    grpc_endpoint_shutdown(exec_ctx, handshaker->args->endpoint);
+    // Not shutting down, so the handshake failed.  Clean up before
+    // invoking the callback.
+    cleanup_args_for_failure_locked(handshaker);
+    // Set shutdown to true so that subsequent calls to
+    // http_connect_handshaker_shutdown() do nothing.
+    handshaker->shutdown = true;
+  }
+  // Invoke callback.
+  grpc_exec_ctx_sched(exec_ctx, handshaker->on_handshake_done, error, NULL);
 }
 
 // Callback invoked when finished writing HTTP CONNECT request.
 static void on_write_done(grpc_exec_ctx* exec_ctx, void* arg,
                           grpc_error* error) {
   http_connect_handshaker* handshaker = arg;
-  if (error != GRPC_ERROR_NONE) {
-    // If the write failed, invoke the callback immediately with the error.
-    handshaker->cb(exec_ctx, handshaker->endpoint, handshaker->args,
-                   handshaker->read_buffer, handshaker->user_data,
-                   GRPC_ERROR_REF(error));
+  gpr_mu_lock(&handshaker->mu);
+  if (error != GRPC_ERROR_NONE || handshaker->shutdown) {
+    // If the write failed or we're shutting down, clean up and invoke the
+    // callback with the error.
+    handshake_failed_locked(exec_ctx, handshaker, GRPC_ERROR_REF(error));
+    gpr_mu_unlock(&handshaker->mu);
+    http_connect_handshaker_unref(exec_ctx, handshaker);
   } else {
     // Otherwise, read the response.
-    grpc_endpoint_read(exec_ctx, handshaker->endpoint, handshaker->read_buffer,
+    // The read callback inherits our ref to the handshaker.
+    grpc_endpoint_read(exec_ctx, handshaker->args->endpoint,
+                       handshaker->args->read_buffer,
                        &handshaker->response_read_closure);
+    gpr_mu_unlock(&handshaker->mu);
   }
 }
 
@@ -112,36 +159,40 @@ static void on_write_done(grpc_exec_ctx* exec_ctx, void* arg,
 static void on_read_done(grpc_exec_ctx* exec_ctx, void* arg,
                          grpc_error* error) {
   http_connect_handshaker* handshaker = arg;
-  if (error != GRPC_ERROR_NONE) {
-    GRPC_ERROR_REF(error);  // Take ref to pass to the handshake-done callback.
+  gpr_mu_lock(&handshaker->mu);
+  if (error != GRPC_ERROR_NONE || handshaker->shutdown) {
+    // If the read failed or we're shutting down, clean up and invoke the
+    // callback with the error.
+    handshake_failed_locked(exec_ctx, handshaker, GRPC_ERROR_REF(error));
     goto done;
   }
   // Add buffer to parser.
-  for (size_t i = 0; i < handshaker->read_buffer->count; ++i) {
-    if (GRPC_SLICE_LENGTH(handshaker->read_buffer->slices[i]) > 0) {
+  for (size_t i = 0; i < handshaker->args->read_buffer->count; ++i) {
+    if (GRPC_SLICE_LENGTH(handshaker->args->read_buffer->slices[i]) > 0) {
       size_t body_start_offset = 0;
       error = grpc_http_parser_parse(&handshaker->http_parser,
-                                     handshaker->read_buffer->slices[i],
+                                     handshaker->args->read_buffer->slices[i],
                                      &body_start_offset);
-      if (error != GRPC_ERROR_NONE) goto done;
+      if (error != GRPC_ERROR_NONE) {
+        handshake_failed_locked(exec_ctx, handshaker, error);
+        goto done;
+      }
       if (handshaker->http_parser.state == GRPC_HTTP_BODY) {
-        // We've gotten back a successul response, so stop the timeout timer.
-        grpc_timer_cancel(exec_ctx, &handshaker->timeout_timer);
         // Remove the data we've already read from the read buffer,
         // leaving only the leftover bytes (if any).
         grpc_slice_buffer tmp_buffer;
         grpc_slice_buffer_init(&tmp_buffer);
         if (body_start_offset <
-            GRPC_SLICE_LENGTH(handshaker->read_buffer->slices[i])) {
+            GRPC_SLICE_LENGTH(handshaker->args->read_buffer->slices[i])) {
           grpc_slice_buffer_add(
               &tmp_buffer,
-              grpc_slice_split_tail(&handshaker->read_buffer->slices[i],
+              grpc_slice_split_tail(&handshaker->args->read_buffer->slices[i],
                                     body_start_offset));
         }
         grpc_slice_buffer_addn(&tmp_buffer,
-                               &handshaker->read_buffer->slices[i + 1],
-                               handshaker->read_buffer->count - i - 1);
-        grpc_slice_buffer_swap(handshaker->read_buffer, &tmp_buffer);
+                               &handshaker->args->read_buffer->slices[i + 1],
+                               handshaker->args->read_buffer->count - i - 1);
+        grpc_slice_buffer_swap(handshaker->args->read_buffer, &tmp_buffer);
         grpc_slice_buffer_destroy(&tmp_buffer);
         break;
       }
@@ -159,9 +210,11 @@ static void on_read_done(grpc_exec_ctx* exec_ctx, void* arg,
   // complete (e.g., handling chunked transfer encoding or looking
   // at the Content-Length: header).
   if (handshaker->http_parser.state != GRPC_HTTP_BODY) {
-    grpc_slice_buffer_reset_and_unref(handshaker->read_buffer);
-    grpc_endpoint_read(exec_ctx, handshaker->endpoint, handshaker->read_buffer,
+    grpc_slice_buffer_reset_and_unref(handshaker->args->read_buffer);
+    grpc_endpoint_read(exec_ctx, handshaker->args->endpoint,
+                       handshaker->args->read_buffer,
                        &handshaker->response_read_closure);
+    gpr_mu_unlock(&handshaker->mu);
     return;
   }
   // Make sure we got a 2xx response.
@@ -172,11 +225,17 @@ static void on_read_done(grpc_exec_ctx* exec_ctx, void* arg,
                  handshaker->http_response.status);
     error = GRPC_ERROR_CREATE(msg);
     gpr_free(msg);
+    handshake_failed_locked(exec_ctx, handshaker, error);
+    goto done;
   }
+  // Success.  Invoke handshake-done callback.
+  grpc_exec_ctx_sched(exec_ctx, handshaker->on_handshake_done, error, NULL);
 done:
-  // Invoke handshake-done callback.
-  handshaker->cb(exec_ctx, handshaker->endpoint, handshaker->args,
-                 handshaker->read_buffer, handshaker->user_data, error);
+  // Set shutdown to true so that subsequent calls to
+  // http_connect_handshaker_shutdown() do nothing.
+  handshaker->shutdown = true;
+  gpr_mu_unlock(&handshaker->mu);
+  http_connect_handshaker_unref(exec_ctx, handshaker);
 }
 
 //
@@ -186,25 +245,30 @@ done:
 static void http_connect_handshaker_destroy(grpc_exec_ctx* exec_ctx,
                                             grpc_handshaker* handshaker_in) {
   http_connect_handshaker* handshaker = (http_connect_handshaker*)handshaker_in;
-  http_connect_handshaker_unref(handshaker);
+  http_connect_handshaker_unref(exec_ctx, handshaker);
 }
 
 static void http_connect_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
-                                             grpc_handshaker* handshaker) {}
+                                             grpc_handshaker* handshaker_in) {
+  http_connect_handshaker* handshaker = (http_connect_handshaker*)handshaker_in;
+  gpr_mu_lock(&handshaker->mu);
+  if (!handshaker->shutdown) {
+    handshaker->shutdown = true;
+    grpc_endpoint_shutdown(exec_ctx, handshaker->args->endpoint);
+    cleanup_args_for_failure_locked(handshaker);
+  }
+  gpr_mu_unlock(&handshaker->mu);
+}
 
 static void http_connect_handshaker_do_handshake(
     grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker_in,
-    grpc_endpoint* endpoint, grpc_channel_args* args,
-    grpc_slice_buffer* read_buffer, gpr_timespec deadline,
-    grpc_tcp_server_acceptor* acceptor, grpc_handshaker_done_cb cb,
-    void* user_data) {
+    grpc_tcp_server_acceptor* acceptor, grpc_closure* on_handshake_done,
+    grpc_handshaker_args* args) {
   http_connect_handshaker* handshaker = (http_connect_handshaker*)handshaker_in;
+  gpr_mu_lock(&handshaker->mu);
   // Save state in the handshaker object.
-  handshaker->endpoint = endpoint;
   handshaker->args = args;
-  handshaker->cb = cb;
-  handshaker->user_data = user_data;
-  handshaker->read_buffer = read_buffer;
+  handshaker->on_handshake_done = on_handshake_done;
   // Send HTTP CONNECT request.
   gpr_log(GPR_INFO, "Connecting to server %s via HTTP proxy %s",
           handshaker->server_name, handshaker->proxy_server);
@@ -216,16 +280,14 @@ static void http_connect_handshaker_do_handshake(
   request.handshaker = &grpc_httpcli_plaintext;
   grpc_slice request_slice = grpc_httpcli_format_connect_request(&request);
   grpc_slice_buffer_add(&handshaker->write_buffer, request_slice);
-  grpc_endpoint_write(exec_ctx, endpoint, &handshaker->write_buffer,
-                      &handshaker->request_done_closure);
-  // Set timeout timer.  The timer gets a reference to the handshaker.
+  // Take a new ref to be held by the write callback.
   gpr_ref(&handshaker->refcount);
-  grpc_timer_init(exec_ctx, &handshaker->timeout_timer,
-                  gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC),
-                  on_timeout, handshaker, gpr_now(GPR_CLOCK_MONOTONIC));
+  grpc_endpoint_write(exec_ctx, args->endpoint, &handshaker->write_buffer,
+                      &handshaker->request_done_closure);
+  gpr_mu_unlock(&handshaker->mu);
 }
 
-static const struct grpc_handshaker_vtable http_connect_handshaker_vtable = {
+static const grpc_handshaker_vtable http_connect_handshaker_vtable = {
     http_connect_handshaker_destroy, http_connect_handshaker_shutdown,
     http_connect_handshaker_do_handshake};
 
@@ -233,10 +295,11 @@ grpc_handshaker* grpc_http_connect_handshaker_create(const char* proxy_server,
                                                      const char* server_name) {
   GPR_ASSERT(proxy_server != NULL);
   GPR_ASSERT(server_name != NULL);
-  http_connect_handshaker* handshaker =
-      gpr_malloc(sizeof(http_connect_handshaker));
+  http_connect_handshaker* handshaker = gpr_malloc(sizeof(*handshaker));
   memset(handshaker, 0, sizeof(*handshaker));
   grpc_handshaker_init(&http_connect_handshaker_vtable, &handshaker->base);
+  gpr_mu_init(&handshaker->mu);
+  gpr_ref_init(&handshaker->refcount, 1);
   handshaker->proxy_server = gpr_strdup(proxy_server);
   handshaker->server_name = gpr_strdup(server_name);
   grpc_slice_buffer_init(&handshaker->write_buffer);
@@ -246,7 +309,6 @@ grpc_handshaker* grpc_http_connect_handshaker_create(const char* proxy_server,
                     handshaker);
   grpc_http_parser_init(&handshaker->http_parser, GRPC_HTTP_RESPONSE,
                         &handshaker->http_response);
-  gpr_ref_init(&handshaker->refcount, 1);
   return &handshaker->base;
 }
 

src/core/ext/client_channel/subchannel.c

@@ -119,9 +119,9 @@ struct grpc_subchannel {
   gpr_mu mu;
 
   /** have we seen a disconnection? */
-  int disconnected;
+  bool disconnected;
   /** are we connecting */
-  int connecting;
+  bool connecting;
   /** connectivity state tracking */
   grpc_connectivity_state_tracker state_tracker;
 
@@ -132,7 +132,9 @@ struct grpc_subchannel {
   /** backoff state */
   gpr_backoff backoff_state;
   /** do we have an active alarm? */
-  int have_alarm;
+  bool have_alarm;
+  /** have we started the backoff loop */
+  bool backoff_begun;
   /** our alarm */
   grpc_timer alarm;
 };
@@ -264,7 +266,7 @@ static void disconnect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
   grpc_subchannel_index_unregister(exec_ctx, c->key, c);
   gpr_mu_lock(&c->mu);
   GPR_ASSERT(!c->disconnected);
-  c->disconnected = 1;
+  c->disconnected = true;
   grpc_connector_shutdown(exec_ctx, c->connector);
   con = GET_CONNECTED_SUBCHANNEL(c, no_barrier);
   if (con != NULL) {
@@ -334,16 +336,18 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
   int initial_backoff_ms =
       GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS * 1000;
   int max_backoff_ms = GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS * 1000;
+  int min_backoff_ms = GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS * 1000;
   bool fixed_reconnect_backoff = false;
   if (c->args) {
     for (size_t i = 0; i < c->args->num_args; i++) {
       if (0 == strcmp(c->args->args[i].key,
-                      "grpc.testing.fixed_reconnect_backoff")) {
+                      "grpc.testing.fixed_reconnect_backoff_ms")) {
         GPR_ASSERT(c->args->args[i].type == GRPC_ARG_INTEGER);
         fixed_reconnect_backoff = true;
-        initial_backoff_ms = max_backoff_ms = grpc_channel_arg_get_integer(
-            &c->args->args[i],
-            (grpc_integer_options){initial_backoff_ms, 100, INT_MAX});
+        initial_backoff_ms = min_backoff_ms = max_backoff_ms =
+            grpc_channel_arg_get_integer(
+                &c->args->args[i],
+                (grpc_integer_options){initial_backoff_ms, 100, INT_MAX});
       } else if (0 == strcmp(c->args->args[i].key,
                              GRPC_ARG_MAX_RECONNECT_BACKOFF_MS)) {
         fixed_reconnect_backoff = false;
@@ -360,17 +364,18 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
     }
   }
   gpr_backoff_init(
-      &c->backoff_state,
+      &c->backoff_state, initial_backoff_ms,
       fixed_reconnect_backoff ? 1.0
                               : GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER,
       fixed_reconnect_backoff ? 0.0 : GRPC_SUBCHANNEL_RECONNECT_JITTER,
-      initial_backoff_ms, max_backoff_ms);
+      min_backoff_ms, max_backoff_ms);
   gpr_mu_init(&c->mu);
 
   return grpc_subchannel_index_register(exec_ctx, key, c);
 }
 
-static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
+static void continue_connect_locked(grpc_exec_ctx *exec_ctx,
+                                    grpc_subchannel *c) {
   grpc_connect_in_args args;
 
   args.interested_parties = c->pollset_set;
@@ -386,12 +391,6 @@ static void continue_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
                          &c->connected);
 }
 
-static void start_connect(grpc_exec_ctx *exec_ctx, grpc_subchannel *c) {
-  c->next_attempt =
-      gpr_backoff_begin(&c->backoff_state, gpr_now(GPR_CLOCK_MONOTONIC));
-  continue_connect(exec_ctx, c);
-}
-
 grpc_connectivity_state grpc_subchannel_check_connectivity(grpc_subchannel *c,
                                                            grpc_error **error) {
   grpc_connectivity_state state;
@@ -418,6 +417,73 @@ static void on_external_state_watcher_done(grpc_exec_ctx *exec_ctx, void *arg,
   follow_up->cb(exec_ctx, follow_up->cb_arg, error);
 }
 
+static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  grpc_subchannel *c = arg;
+  gpr_mu_lock(&c->mu);
+  c->have_alarm = false;
+  if (c->disconnected) {
+    error = GRPC_ERROR_CREATE_REFERENCING("Disconnected", &error, 1);
+  } else {
+    GRPC_ERROR_REF(error);
+  }
+  if (error == GRPC_ERROR_NONE) {
+    gpr_log(GPR_INFO, "Failed to connect to channel, retrying");
+    c->next_attempt =
+        gpr_backoff_step(&c->backoff_state, gpr_now(GPR_CLOCK_MONOTONIC));
+    continue_connect_locked(exec_ctx, c);
+    gpr_mu_unlock(&c->mu);
+  } else {
+    gpr_mu_unlock(&c->mu);
+    GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
+  }
+  GRPC_ERROR_UNREF(error);
+}
+
+static void maybe_start_connecting_locked(grpc_exec_ctx *exec_ctx,
+                                          grpc_subchannel *c) {
+  if (c->disconnected) {
+    /* Don't try to connect if we're already disconnected */
+    return;
+  }
+
+  if (c->connecting) {
+    /* Already connecting: don't restart */
+    return;
+  }
+
+  if (GET_CONNECTED_SUBCHANNEL(c, no_barrier) != NULL) {
+    /* Already connected: don't restart */
+    return;
+  }
+
+  if (!grpc_connectivity_state_has_watchers(&c->state_tracker)) {
+    /* Nobody is interested in connecting: so don't just yet */
+    return;
+  }
+
+  c->connecting = true;
+  GRPC_SUBCHANNEL_WEAK_REF(c, "connecting");
+
+  gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
+  if (!c->backoff_begun) {
+    c->backoff_begun = true;
+    c->next_attempt = gpr_backoff_begin(&c->backoff_state, now);
+    continue_connect_locked(exec_ctx, c);
+  } else {
+    GPR_ASSERT(!c->have_alarm);
+    c->have_alarm = true;
+    gpr_timespec time_til_next = gpr_time_sub(c->next_attempt, now);
+    if (gpr_time_cmp(time_til_next, gpr_time_0(time_til_next.clock_type)) <=
+        0) {
+      gpr_log(GPR_INFO, "Retry immediately");
+    } else {
+      gpr_log(GPR_INFO, "Retry in %" PRId64 ".%09d seconds",
+              time_til_next.tv_sec, time_til_next.tv_nsec);
+    }
+    grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now);
+  }
+}
+
 void grpc_subchannel_notify_on_state_change(
     grpc_exec_ctx *exec_ctx, grpc_subchannel *c,
     grpc_pollset_set *interested_parties, grpc_connectivity_state *state,
@@ -449,13 +515,9 @@ void grpc_subchannel_notify_on_state_change(
     w->next = &c->root_external_state_watcher;
     w->prev = w->next->prev;
     w->next->prev = w->prev->next = w;
-    if (grpc_connectivity_state_notify_on_state_change(
-            exec_ctx, &c->state_tracker, state, &w->closure)) {
-      c->connecting = 1;
-      /* released by connection */
-      GRPC_SUBCHANNEL_WEAK_REF(c, "connecting");
-      start_connect(exec_ctx, c);
-    }
+    grpc_connectivity_state_notify_on_state_change(exec_ctx, &c->state_tracker,
+                                                   state, &w->closure);
+    maybe_start_connecting_locked(exec_ctx, c);
     gpr_mu_unlock(&c->mu);
   }
 }
@@ -575,12 +637,9 @@ static void publish_transport_locked(grpc_exec_ctx *exec_ctx,
                     Re-evaluate if we really need this. */
   gpr_atm_full_barrier();
   GPR_ASSERT(gpr_atm_rel_cas(&c->connected_subchannel, 0, (gpr_atm)con));
-  c->connecting = 0;
 
   /* setup subchannel watching connected subchannel for changes; subchannel
-     ref
-     for connecting is donated
-     to the state watcher */
+     ref for connecting is donated to the state watcher */
   GRPC_SUBCHANNEL_WEAK_REF(c, "state_watcher");
   GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   grpc_connected_subchannel_notify_on_state_change(
@@ -592,28 +651,6 @@ static void publish_transport_locked(grpc_exec_ctx *exec_ctx,
                               GRPC_ERROR_NONE, "connected");
 }
 
-static void on_alarm(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
-  grpc_subchannel *c = arg;
-  gpr_mu_lock(&c->mu);
-  c->have_alarm = 0;
-  if (c->disconnected) {
-    error = GRPC_ERROR_CREATE_REFERENCING("Disconnected", &error, 1);
-  } else {
-    GRPC_ERROR_REF(error);
-  }
-  if (error == GRPC_ERROR_NONE) {
-    gpr_log(GPR_INFO, "Failed to connect to channel, retrying");
-    c->next_attempt =
-        gpr_backoff_step(&c->backoff_state, gpr_now(GPR_CLOCK_MONOTONIC));
-    continue_connect(exec_ctx, c);
-    gpr_mu_unlock(&c->mu);
-  } else {
-    gpr_mu_unlock(&c->mu);
-    GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
 static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
                                  grpc_error *error) {
   grpc_subchannel *c = arg;
@@ -621,35 +658,28 @@ static void subchannel_connected(grpc_exec_ctx *exec_ctx, void *arg,
 
   GRPC_SUBCHANNEL_WEAK_REF(c, "connected");
   gpr_mu_lock(&c->mu);
+  c->connecting = false;
   if (c->connecting_result.transport != NULL) {
     publish_transport_locked(exec_ctx, c);
   } else if (c->disconnected) {
     GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   } else {
-    gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
-    GPR_ASSERT(!c->have_alarm);
-    c->have_alarm = 1;
     grpc_connectivity_state_set(
         exec_ctx, &c->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
         grpc_error_set_int(
             GRPC_ERROR_CREATE_REFERENCING("Connect Failed", &error, 1),
             GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
         "connect_failed");
-    gpr_timespec time_til_next = gpr_time_sub(c->next_attempt, now);
+
     const char *errmsg = grpc_error_string(error);
     gpr_log(GPR_INFO, "Connect failed: %s", errmsg);
-    if (gpr_time_cmp(time_til_next, gpr_time_0(time_til_next.clock_type)) <=
-        0) {
-      gpr_log(GPR_INFO, "Retry immediately");
-    } else {
-      gpr_log(GPR_INFO, "Retry in %" PRId64 ".%09d seconds",
-              time_til_next.tv_sec, time_til_next.tv_nsec);
-    }
-    grpc_timer_init(exec_ctx, &c->alarm, c->next_attempt, on_alarm, c, now);
     grpc_error_free_string(errmsg);
+
+    maybe_start_connecting_locked(exec_ctx, c);
+    GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
   }
   gpr_mu_unlock(&c->mu);
-  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connecting");
+  GRPC_SUBCHANNEL_WEAK_UNREF(exec_ctx, c, "connected");
   grpc_channel_args_destroy(delete_channel_args);
 }
 

src/core/ext/lb_policy/grpclb/grpclb.c

@@ -123,10 +123,11 @@
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/transport/static_metadata.h"
 
-#define BACKOFF_MULTIPLIER 1.6
-#define BACKOFF_JITTER 0.2
-#define BACKOFF_MIN_SECONDS 10
-#define BACKOFF_MAX_SECONDS 60
+#define GRPC_GRPCLB_MIN_CONNECT_TIMEOUT_SECONDS 20
+#define GRPC_GRPCLB_INITIAL_CONNECT_BACKOFF_SECONDS 1
+#define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6
+#define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120
+#define GRPC_GRPCLB_RECONNECT_JITTER 0.2
 
 int grpc_lb_glb_trace = 0;
 
@@ -1107,9 +1108,12 @@ static void lb_call_init_locked(glb_lb_policy *glb_policy) {
   grpc_closure_init(&glb_policy->lb_on_response_received,
                     lb_on_response_received, glb_policy);
 
-  gpr_backoff_init(&glb_policy->lb_call_backoff_state, BACKOFF_MULTIPLIER,
-                   BACKOFF_JITTER, BACKOFF_MIN_SECONDS * 1000,
-                   BACKOFF_MAX_SECONDS * 1000);
+  gpr_backoff_init(&glb_policy->lb_call_backoff_state,
+                   GRPC_GRPCLB_INITIAL_CONNECT_BACKOFF_SECONDS,
+                   GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER,
+                   GRPC_GRPCLB_RECONNECT_JITTER,
+                   GRPC_GRPCLB_MIN_CONNECT_TIMEOUT_SECONDS * 1000,
+                   GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS * 1000);
 }
 
 static void lb_call_destroy_locked(glb_lb_policy *glb_policy) {

src/core/ext/resolver/dns/native/dns_resolver.c

@@ -46,10 +46,11 @@
 #include "src/core/lib/support/backoff.h"
 #include "src/core/lib/support/string.h"
 
-#define BACKOFF_MULTIPLIER 1.6
-#define BACKOFF_JITTER 0.2
-#define BACKOFF_MIN_SECONDS 1
-#define BACKOFF_MAX_SECONDS 120
+#define GRPC_DNS_MIN_CONNECT_TIMEOUT_SECONDS 1
+#define GRPC_DNS_INITIAL_CONNECT_BACKOFF_SECONDS 1
+#define GRPC_DNS_RECONNECT_BACKOFF_MULTIPLIER 1.6
+#define GRPC_DNS_RECONNECT_MAX_BACKOFF_SECONDS 120
+#define GRPC_DNS_RECONNECT_JITTER 0.2
 
 typedef struct {
   /** base class: must be first */
@@ -269,8 +270,11 @@ static grpc_resolver *dns_create(grpc_resolver_args *args,
   server_name_arg.value.string = (char *)path;
   r->channel_args =
       grpc_channel_args_copy_and_add(args->args, &server_name_arg, 1);
-  gpr_backoff_init(&r->backoff_state, BACKOFF_MULTIPLIER, BACKOFF_JITTER,
-                   BACKOFF_MIN_SECONDS * 1000, BACKOFF_MAX_SECONDS * 1000);
+  gpr_backoff_init(&r->backoff_state, GRPC_DNS_INITIAL_CONNECT_BACKOFF_SECONDS,
+                   GRPC_DNS_RECONNECT_BACKOFF_MULTIPLIER,
+                   GRPC_DNS_RECONNECT_JITTER,
+                   GRPC_DNS_MIN_CONNECT_TIMEOUT_SECONDS * 1000,
+                   GRPC_DNS_RECONNECT_MAX_BACKOFF_SECONDS * 1000);
   return &r->base;
 }
 

src/core/ext/transport/chttp2/client/chttp2_connector.c

@@ -0,0 +1,269 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/ext/transport/chttp2/client/chttp2_connector.h"
+
+#include <grpc/grpc.h>
+
+#include <string.h>
+
+#include <grpc/slice_buffer.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/string_util.h>
+
+#include "src/core/ext/client_channel/connector.h"
+#include "src/core/ext/client_channel/http_connect_handshaker.h"
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/security/transport/security_connector.h"
+
+typedef struct {
+  grpc_connector base;
+
+  gpr_mu mu;
+  gpr_refcount refs;
+
+  bool shutdown;
+  bool connecting;
+
+  char *server_name;
+  grpc_chttp2_add_handshakers_func add_handshakers;
+  void *add_handshakers_user_data;
+
+  grpc_closure *notify;
+  grpc_connect_in_args args;
+  grpc_connect_out_args *result;
+  grpc_closure initial_string_sent;
+  grpc_slice_buffer initial_string_buffer;
+
+  grpc_endpoint *endpoint;  // Non-NULL until handshaking starts.
+
+  grpc_closure connected;
+
+  grpc_handshake_manager *handshake_mgr;
+} chttp2_connector;
+
+static void chttp2_connector_ref(grpc_connector *con) {
+  chttp2_connector *c = (chttp2_connector *)con;
+  gpr_ref(&c->refs);
+}
+
+static void chttp2_connector_unref(grpc_exec_ctx *exec_ctx,
+                                   grpc_connector *con) {
+  chttp2_connector *c = (chttp2_connector *)con;
+  if (gpr_unref(&c->refs)) {
+    /* c->initial_string_buffer does not need to be destroyed */
+    gpr_mu_destroy(&c->mu);
+    // If handshaking is not yet in progress, destroy the endpoint.
+    // Otherwise, the handshaker will do this for us.
+    if (c->endpoint != NULL) grpc_endpoint_destroy(exec_ctx, c->endpoint);
+    gpr_free(c->server_name);
+    gpr_free(c);
+  }
+}
+
+static void chttp2_connector_shutdown(grpc_exec_ctx *exec_ctx,
+                                      grpc_connector *con) {
+  chttp2_connector *c = (chttp2_connector *)con;
+  gpr_mu_lock(&c->mu);
+  c->shutdown = true;
+  if (c->handshake_mgr != NULL) {
+    grpc_handshake_manager_shutdown(exec_ctx, c->handshake_mgr);
+  }
+  // If handshaking is not yet in progress, shutdown the endpoint.
+  // Otherwise, the handshaker will do this for us.
+  if (!c->connecting && c->endpoint != NULL) {
+    grpc_endpoint_shutdown(exec_ctx, c->endpoint);
+  }
+  gpr_mu_unlock(&c->mu);
+}
+
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error) {
+  grpc_handshaker_args *args = arg;
+  chttp2_connector *c = args->user_data;
+  gpr_mu_lock(&c->mu);
+  if (error != GRPC_ERROR_NONE || c->shutdown) {
+    if (error == GRPC_ERROR_NONE) {
+      error = GRPC_ERROR_CREATE("connector shutdown");
+      // We were shut down after handshaking completed successfully, so
+      // destroy the endpoint here.
+      // TODO(ctiller): It is currently necessary to shutdown endpoints
+      // before destroying them, even if we know that there are no
+      // pending read/write callbacks.  This should be fixed, at which
+      // point this can be removed.
+      grpc_endpoint_shutdown(exec_ctx, args->endpoint);
+      grpc_endpoint_destroy(exec_ctx, args->endpoint);
+      grpc_channel_args_destroy(args->args);
+      grpc_slice_buffer_destroy(args->read_buffer);
+      gpr_free(args->read_buffer);
+    } else {
+      error = GRPC_ERROR_REF(error);
+    }
+    memset(c->result, 0, sizeof(*c->result));
+  } else {
+    c->result->transport =
+        grpc_create_chttp2_transport(exec_ctx, args->args, args->endpoint, 1);
+    GPR_ASSERT(c->result->transport);
+    grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport,
+                                        args->read_buffer);
+    c->result->channel_args = args->args;
+  }
+  grpc_closure *notify = c->notify;
+  c->notify = NULL;
+  grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+  grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
+  c->handshake_mgr = NULL;
+  gpr_mu_unlock(&c->mu);
+  chttp2_connector_unref(exec_ctx, (grpc_connector *)c);
+}
+
+static void start_handshake_locked(grpc_exec_ctx *exec_ctx,
+                                   chttp2_connector *c) {
+  c->handshake_mgr = grpc_handshake_manager_create();
+  char *proxy_name = grpc_get_http_proxy_server();
+  if (proxy_name != NULL) {
+    grpc_handshake_manager_add(
+        c->handshake_mgr,
+        grpc_http_connect_handshaker_create(proxy_name, c->server_name));
+    gpr_free(proxy_name);
+  }
+  if (c->add_handshakers != NULL) {
+    c->add_handshakers(exec_ctx, c->add_handshakers_user_data,
+                       c->handshake_mgr);
+  }
+  grpc_handshake_manager_do_handshake(
+      exec_ctx, c->handshake_mgr, c->endpoint, c->args.channel_args,
+      c->args.deadline, NULL /* acceptor */, on_handshake_done, c);
+  c->endpoint = NULL;  // Endpoint handed off to handshake manager.
+}
+
+static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
+                                           grpc_error *error) {
+  chttp2_connector *c = arg;
+  gpr_mu_lock(&c->mu);
+  if (error != GRPC_ERROR_NONE || c->shutdown) {
+    if (error == GRPC_ERROR_NONE) {
+      error = GRPC_ERROR_CREATE("connector shutdown");
+    } else {
+      error = GRPC_ERROR_REF(error);
+    }
+    memset(c->result, 0, sizeof(*c->result));
+    grpc_closure *notify = c->notify;
+    c->notify = NULL;
+    grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+    gpr_mu_unlock(&c->mu);
+    chttp2_connector_unref(exec_ctx, arg);
+  } else {
+    start_handshake_locked(exec_ctx, c);
+    gpr_mu_unlock(&c->mu);
+  }
+}
+
+static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  chttp2_connector *c = arg;
+  gpr_mu_lock(&c->mu);
+  GPR_ASSERT(c->connecting);
+  c->connecting = false;
+  if (error != GRPC_ERROR_NONE || c->shutdown) {
+    if (error == GRPC_ERROR_NONE) {
+      error = GRPC_ERROR_CREATE("connector shutdown");
+    } else {
+      error = GRPC_ERROR_REF(error);
+    }
+    memset(c->result, 0, sizeof(*c->result));
+    grpc_closure *notify = c->notify;
+    c->notify = NULL;
+    grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+    if (c->endpoint != NULL) grpc_endpoint_shutdown(exec_ctx, c->endpoint);
+    gpr_mu_unlock(&c->mu);
+    chttp2_connector_unref(exec_ctx, arg);
+  } else {
+    GPR_ASSERT(c->endpoint != NULL);
+    if (!GRPC_SLICE_IS_EMPTY(c->args.initial_connect_string)) {
+      grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent,
+                        c);
+      grpc_slice_buffer_init(&c->initial_string_buffer);
+      grpc_slice_buffer_add(&c->initial_string_buffer,
+                            c->args.initial_connect_string);
+      grpc_endpoint_write(exec_ctx, c->endpoint, &c->initial_string_buffer,
+                          &c->initial_string_sent);
+    } else {
+      start_handshake_locked(exec_ctx, c);
+    }
+    gpr_mu_unlock(&c->mu);
+  }
+}
+
+static void chttp2_connector_connect(grpc_exec_ctx *exec_ctx,
+                                     grpc_connector *con,
+                                     const grpc_connect_in_args *args,
+                                     grpc_connect_out_args *result,
+                                     grpc_closure *notify) {
+  chttp2_connector *c = (chttp2_connector *)con;
+  gpr_mu_lock(&c->mu);
+  GPR_ASSERT(c->notify == NULL);
+  c->notify = notify;
+  c->args = *args;
+  c->result = result;
+  GPR_ASSERT(c->endpoint == NULL);
+  chttp2_connector_ref(con);  // Ref taken for callback.
+  grpc_closure_init(&c->connected, connected, c);
+  GPR_ASSERT(!c->connecting);
+  c->connecting = true;
+  grpc_tcp_client_connect(exec_ctx, &c->connected, &c->endpoint,
+                          args->interested_parties, args->channel_args,
+                          args->addr, args->deadline);
+  gpr_mu_unlock(&c->mu);
+}
+
+static const grpc_connector_vtable chttp2_connector_vtable = {
+    chttp2_connector_ref, chttp2_connector_unref, chttp2_connector_shutdown,
+    chttp2_connector_connect};
+
+grpc_connector *grpc_chttp2_connector_create(
+    grpc_exec_ctx *exec_ctx, const char *server_name,
+    grpc_chttp2_add_handshakers_func add_handshakers,
+    void *add_handshakers_user_data) {
+  chttp2_connector *c = gpr_malloc(sizeof(*c));
+  memset(c, 0, sizeof(*c));
+  c->base.vtable = &chttp2_connector_vtable;
+  gpr_mu_init(&c->mu);
+  gpr_ref_init(&c->refs, 1);
+  c->server_name = gpr_strdup(server_name);
+  c->add_handshakers = add_handshakers;
+  c->add_handshakers_user_data = add_handshakers_user_data;
+  return &c->base;
+}

src/core/ext/transport/chttp2/client/chttp2_connector.h

@@ -0,0 +1,52 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_CLIENT_CHTTP2_CONNECTOR_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_CLIENT_CHTTP2_CONNECTOR_H
+
+#include "src/core/ext/client_channel/connector.h"
+#include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+
+typedef void (*grpc_chttp2_add_handshakers_func)(
+    grpc_exec_ctx* exec_ctx, void* user_data,
+    grpc_handshake_manager* handshake_mgr);
+
+/// If \a add_handshakers is non-NULL, it will be called with
+/// \a add_handshakers_user_data to add handshakers.
+grpc_connector* grpc_chttp2_connector_create(
+    grpc_exec_ctx* exec_ctx, const char* server_name,
+    grpc_chttp2_add_handshakers_func add_handshakers,
+    void* add_handshakers_user_data);
+
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_CLIENT_CHTTP2_CONNECTOR_H */

src/core/ext/transport/chttp2/client/insecure/channel_create.c

@@ -33,138 +33,17 @@
 
 #include <grpc/grpc.h>
 
-#include <stdlib.h>
 #include <string.h>
 
-#include <grpc/slice.h>
-#include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
+#include <grpc/support/string_util.h>
 
 #include "src/core/ext/client_channel/client_channel.h"
-#include "src/core/ext/client_channel/http_connect_handshaker.h"
 #include "src/core/ext/client_channel/resolver_registry.h"
-#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/compress_filter.h"
-#include "src/core/lib/channel/handshaker.h"
-#include "src/core/lib/channel/http_client_filter.h"
-#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/ext/transport/chttp2/client/chttp2_connector.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
 
-//
-// connector
-//
-
-typedef struct {
-  grpc_connector base;
-  gpr_refcount refs;
-
-  grpc_closure *notify;
-  grpc_connect_in_args args;
-  grpc_connect_out_args *result;
-  grpc_closure initial_string_sent;
-  grpc_slice_buffer initial_string_buffer;
-
-  grpc_endpoint *tcp;
-
-  grpc_closure connected;
-
-  grpc_handshake_manager *handshake_mgr;
-} connector;
-
-static void connector_ref(grpc_connector *con) {
-  connector *c = (connector *)con;
-  gpr_ref(&c->refs);
-}
-
-static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
-  connector *c = (connector *)con;
-  if (gpr_unref(&c->refs)) {
-    /* c->initial_string_buffer does not need to be destroyed */
-    grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
-    gpr_free(c);
-  }
-}
-
-static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
-                                           grpc_error *error) {
-  connector_unref(exec_ctx, arg);
-}
-
-static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
-                              grpc_channel_args *args,
-                              grpc_slice_buffer *read_buffer, void *user_data,
-                              grpc_error *error) {
-  connector *c = user_data;
-  if (error != GRPC_ERROR_NONE) {
-    grpc_channel_args_destroy(args);
-    gpr_free(read_buffer);
-  } else {
-    c->result->transport =
-        grpc_create_chttp2_transport(exec_ctx, args, endpoint, 1);
-    GPR_ASSERT(c->result->transport);
-    grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport,
-                                        read_buffer);
-    c->result->channel_args = args;
-  }
-  grpc_closure *notify = c->notify;
-  c->notify = NULL;
-  grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
-}
-
-static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
-  connector *c = arg;
-  grpc_endpoint *tcp = c->tcp;
-  if (tcp != NULL) {
-    if (!GRPC_SLICE_IS_EMPTY(c->args.initial_connect_string)) {
-      grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent,
-                        c);
-      grpc_slice_buffer_init(&c->initial_string_buffer);
-      grpc_slice_buffer_add(&c->initial_string_buffer,
-                            c->args.initial_connect_string);
-      connector_ref(arg);
-      grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer,
-                          &c->initial_string_sent);
-    } else {
-      grpc_handshake_manager_do_handshake(
-          exec_ctx, c->handshake_mgr, tcp, c->args.channel_args,
-          c->args.deadline, NULL /* acceptor */, on_handshake_done, c);
-    }
-  } else {
-    memset(c->result, 0, sizeof(*c->result));
-    grpc_closure *notify = c->notify;
-    c->notify = NULL;
-    grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
-  }
-}
-
-static void connector_shutdown(grpc_exec_ctx *exec_ctx, grpc_connector *con) {}
-
-static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con,
-                              const grpc_connect_in_args *args,
-                              grpc_connect_out_args *result,
-                              grpc_closure *notify) {
-  connector *c = (connector *)con;
-  GPR_ASSERT(c->notify == NULL);
-  GPR_ASSERT(notify->cb);
-  c->notify = notify;
-  c->args = *args;
-  c->result = result;
-  c->tcp = NULL;
-  grpc_closure_init(&c->connected, connected, c);
-  grpc_tcp_client_connect(exec_ctx, &c->connected, &c->tcp,
-                          args->interested_parties, args->channel_args,
-                          args->addr, args->deadline);
-}
-
-static const grpc_connector_vtable connector_vtable = {
-    connector_ref, connector_unref, connector_shutdown, connector_connect};
-
-//
-// client_channel_factory
-//
-
 static void client_channel_factory_ref(
     grpc_client_channel_factory *cc_factory) {}
 
@@ -174,20 +53,11 @@ static void client_channel_factory_unref(
 static grpc_subchannel *client_channel_factory_create_subchannel(
     grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory,
     const grpc_subchannel_args *args) {
-  connector *c = gpr_malloc(sizeof(*c));
-  memset(c, 0, sizeof(*c));
-  c->base.vtable = &connector_vtable;
-  gpr_ref_init(&c->refs, 1);
-  c->handshake_mgr = grpc_handshake_manager_create();
-  char *proxy_name = grpc_get_http_proxy_server();
-  if (proxy_name != NULL) {
-    grpc_handshake_manager_add(
-        c->handshake_mgr,
-        grpc_http_connect_handshaker_create(proxy_name, args->server_name));
-    gpr_free(proxy_name);
-  }
-  grpc_subchannel *s = grpc_subchannel_create(exec_ctx, &c->base, args);
-  grpc_connector_unref(exec_ctx, &c->base);
+  grpc_connector *connector = grpc_chttp2_connector_create(
+      exec_ctx, args->server_name, NULL /* add_handshakers */,
+      NULL /* user_data */);
+  grpc_subchannel *s = grpc_subchannel_create(exec_ctx, connector, args);
+  grpc_connector_unref(exec_ctx, connector);
   return s;
 }
 
@@ -198,16 +68,14 @@ static grpc_channel *client_channel_factory_create_channel(
   grpc_channel *channel =
       grpc_channel_create(exec_ctx, target, args, GRPC_CLIENT_CHANNEL, NULL);
   grpc_resolver *resolver = grpc_resolver_create(target, args);
-  if (!resolver) {
+  if (resolver == NULL) {
     GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel,
                                 "client_channel_factory_create_channel");
     return NULL;
   }
-
   grpc_client_channel_finish_initialization(
       exec_ctx, grpc_channel_get_channel_stack(channel), resolver, cc_factory);
   GRPC_RESOLVER_UNREF(exec_ctx, resolver, "create_channel");
-
   return channel;
 }
 
@@ -230,16 +98,13 @@ grpc_channel *grpc_insecure_channel_create(const char *target,
   GRPC_API_TRACE(
       "grpc_insecure_channel_create(target=%p, args=%p, reserved=%p)", 3,
       (target, args, reserved));
-  GPR_ASSERT(!reserved);
-
+  GPR_ASSERT(reserved == NULL);
   grpc_client_channel_factory *factory =
       (grpc_client_channel_factory *)&client_channel_factory;
   grpc_channel *channel = client_channel_factory_create_channel(
       &exec_ctx, factory, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, args);
-
   grpc_client_channel_factory_unref(&exec_ctx, factory);
   grpc_exec_ctx_finish(&exec_ctx);
-
   return channel != NULL ? channel : grpc_lame_client_channel_create(
                                          target, GRPC_STATUS_INTERNAL,
                                          "Failed to create client channel");

src/core/ext/transport/chttp2/client/secure/secure_channel_create.c

@@ -33,196 +33,19 @@
 
 #include <grpc/grpc.h>
 
-#include <stdlib.h>
 #include <string.h>
 
-#include <grpc/slice.h>
-#include <grpc/slice_buffer.h>
 #include <grpc/support/alloc.h>
+#include <grpc/support/string_util.h>
 
 #include "src/core/ext/client_channel/client_channel.h"
-#include "src/core/ext/client_channel/http_connect_handshaker.h"
 #include "src/core/ext/client_channel/resolver_registry.h"
-#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/client/chttp2_connector.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/handshaker.h"
-#include "src/core/lib/iomgr/tcp_client.h"
-#include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
-#include "src/core/lib/tsi/transport_security_interface.h"
-
-//
-// connector
-//
-
-typedef struct {
-  grpc_connector base;
-  gpr_refcount refs;
-
-  grpc_channel_security_connector *security_connector;
-
-  grpc_closure *notify;
-  grpc_connect_in_args args;
-  grpc_connect_out_args *result;
-  grpc_closure initial_string_sent;
-  grpc_slice_buffer initial_string_buffer;
-
-  gpr_mu mu;
-  grpc_endpoint *connecting_endpoint;
-  grpc_endpoint *newly_connecting_endpoint;
-
-  grpc_closure connected_closure;
-
-  grpc_handshake_manager *handshake_mgr;
-
-  // TODO(roth): Remove once we eliminate on_secure_handshake_done().
-  grpc_channel_args *tmp_args;
-} connector;
-
-static void connector_ref(grpc_connector *con) {
-  connector *c = (connector *)con;
-  gpr_ref(&c->refs);
-}
-
-static void connector_unref(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
-  connector *c = (connector *)con;
-  if (gpr_unref(&c->refs)) {
-    /* c->initial_string_buffer does not need to be destroyed */
-    grpc_channel_args_destroy(c->tmp_args);
-    grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
-    gpr_free(c);
-  }
-}
-
-static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
-                                     grpc_security_status status,
-                                     grpc_endpoint *secure_endpoint,
-                                     grpc_auth_context *auth_context) {
-  connector *c = arg;
-  gpr_mu_lock(&c->mu);
-  grpc_error *error = GRPC_ERROR_NONE;
-  if (c->connecting_endpoint == NULL) {
-    memset(c->result, 0, sizeof(*c->result));
-    gpr_mu_unlock(&c->mu);
-  } else if (status != GRPC_SECURITY_OK) {
-    error = grpc_error_set_int(GRPC_ERROR_CREATE("Secure handshake failed"),
-                               GRPC_ERROR_INT_SECURITY_STATUS, status);
-    memset(c->result, 0, sizeof(*c->result));
-    c->connecting_endpoint = NULL;
-    gpr_mu_unlock(&c->mu);
-  } else {
-    grpc_arg auth_context_arg;
-    c->connecting_endpoint = NULL;
-    gpr_mu_unlock(&c->mu);
-    c->result->transport = grpc_create_chttp2_transport(
-        exec_ctx, c->args.channel_args, secure_endpoint, 1);
-    grpc_chttp2_transport_start_reading(exec_ctx, c->result->transport, NULL);
-    auth_context_arg = grpc_auth_context_to_arg(auth_context);
-    c->result->channel_args =
-        grpc_channel_args_copy_and_add(c->tmp_args, &auth_context_arg, 1);
-  }
-  grpc_closure *notify = c->notify;
-  c->notify = NULL;
-  grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
-}
-
-static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
-                              grpc_channel_args *args,
-                              grpc_slice_buffer *read_buffer, void *user_data,
-                              grpc_error *error) {
-  connector *c = user_data;
-  c->tmp_args = args;
-  if (error != GRPC_ERROR_NONE) {
-    gpr_free(read_buffer);
-    grpc_closure *notify = c->notify;
-    c->notify = NULL;
-    grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
-  } else {
-    // TODO(roth, jboeuf): Convert security connector handshaking to use new
-    // handshake API, and then move the code from on_secure_handshake_done()
-    // into this function.
-    grpc_channel_security_connector_do_handshake(
-        exec_ctx, c->security_connector, endpoint, read_buffer,
-        c->args.deadline, on_secure_handshake_done, c);
-  }
-}
-
-static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
-                                           grpc_error *error) {
-  connector *c = arg;
-  grpc_handshake_manager_do_handshake(
-      exec_ctx, c->handshake_mgr, c->connecting_endpoint, c->args.channel_args,
-      c->args.deadline, NULL /* acceptor */, on_handshake_done, c);
-}
-
-static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
-  connector *c = arg;
-  grpc_endpoint *tcp = c->newly_connecting_endpoint;
-  if (tcp != NULL) {
-    gpr_mu_lock(&c->mu);
-    GPR_ASSERT(c->connecting_endpoint == NULL);
-    c->connecting_endpoint = tcp;
-    gpr_mu_unlock(&c->mu);
-    if (!GRPC_SLICE_IS_EMPTY(c->args.initial_connect_string)) {
-      grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent,
-                        c);
-      grpc_slice_buffer_init(&c->initial_string_buffer);
-      grpc_slice_buffer_add(&c->initial_string_buffer,
-                            c->args.initial_connect_string);
-      grpc_endpoint_write(exec_ctx, tcp, &c->initial_string_buffer,
-                          &c->initial_string_sent);
-    } else {
-      grpc_handshake_manager_do_handshake(
-          exec_ctx, c->handshake_mgr, tcp, c->args.channel_args,
-          c->args.deadline, NULL /* acceptor */, on_handshake_done, c);
-    }
-  } else {
-    memset(c->result, 0, sizeof(*c->result));
-    grpc_closure *notify = c->notify;
-    c->notify = NULL;
-    grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_REF(error), NULL);
-  }
-}
-
-static void connector_shutdown(grpc_exec_ctx *exec_ctx, grpc_connector *con) {
-  connector *c = (connector *)con;
-  grpc_endpoint *ep;
-  gpr_mu_lock(&c->mu);
-  ep = c->connecting_endpoint;
-  c->connecting_endpoint = NULL;
-  gpr_mu_unlock(&c->mu);
-  if (ep) {
-    grpc_endpoint_shutdown(exec_ctx, ep);
-  }
-}
-
-static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con,
-                              const grpc_connect_in_args *args,
-                              grpc_connect_out_args *result,
-                              grpc_closure *notify) {
-  connector *c = (connector *)con;
-  GPR_ASSERT(c->notify == NULL);
-  c->notify = notify;
-  c->args = *args;
-  c->result = result;
-  gpr_mu_lock(&c->mu);
-  GPR_ASSERT(c->connecting_endpoint == NULL);
-  gpr_mu_unlock(&c->mu);
-  grpc_closure_init(&c->connected_closure, connected, c);
-  grpc_tcp_client_connect(
-      exec_ctx, &c->connected_closure, &c->newly_connecting_endpoint,
-      args->interested_parties, args->channel_args, args->addr, args->deadline);
-}
-
-static const grpc_connector_vtable connector_vtable = {
-    connector_ref, connector_unref, connector_shutdown, connector_connect};
-
-//
-// client_channel_factory
-//
 
 typedef struct {
   grpc_client_channel_factory base;
@@ -246,26 +69,20 @@ static void client_channel_factory_unref(
   }
 }
 
+static void add_handshakers(grpc_exec_ctx *exec_ctx, void *security_connector,
+                            grpc_handshake_manager *handshake_mgr) {
+  grpc_channel_security_connector_add_handshakers(exec_ctx, security_connector,
+                                                  handshake_mgr);
+}
+
 static grpc_subchannel *client_channel_factory_create_subchannel(
     grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory,
     const grpc_subchannel_args *args) {
   client_channel_factory *f = (client_channel_factory *)cc_factory;
-  connector *c = gpr_malloc(sizeof(*c));
-  memset(c, 0, sizeof(*c));
-  c->base.vtable = &connector_vtable;
-  c->security_connector = f->security_connector;
-  c->handshake_mgr = grpc_handshake_manager_create();
-  char *proxy_name = grpc_get_http_proxy_server();
-  if (proxy_name != NULL) {
-    grpc_handshake_manager_add(
-        c->handshake_mgr,
-        grpc_http_connect_handshaker_create(proxy_name, args->server_name));
-    gpr_free(proxy_name);
-  }
-  gpr_mu_init(&c->mu);
-  gpr_ref_init(&c->refs, 1);
-  grpc_subchannel *s = grpc_subchannel_create(exec_ctx, &c->base, args);
-  grpc_connector_unref(exec_ctx, &c->base);
+  grpc_connector *connector = grpc_chttp2_connector_create(
+      exec_ctx, args->server_name, add_handshakers, f->security_connector);
+  grpc_subchannel *s = grpc_subchannel_create(exec_ctx, connector, args);
+  grpc_connector_unref(exec_ctx, connector);
   return s;
 }
 
@@ -277,15 +94,14 @@ static grpc_channel *client_channel_factory_create_channel(
   grpc_channel *channel =
       grpc_channel_create(exec_ctx, target, args, GRPC_CLIENT_CHANNEL, NULL);
   grpc_resolver *resolver = grpc_resolver_create(target, args);
-  if (resolver != NULL) {
-    grpc_client_channel_finish_initialization(
-        exec_ctx, grpc_channel_get_channel_stack(channel), resolver, &f->base);
-    GRPC_RESOLVER_UNREF(exec_ctx, resolver, "create");
-  } else {
+  if (resolver == NULL) {
     GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel,
                                 "client_channel_factory_create_channel");
-    channel = NULL;
+    return NULL;
   }
+  grpc_client_channel_finish_initialization(
+      exec_ctx, grpc_channel_get_channel_stack(channel), resolver, &f->base);
+  GRPC_RESOLVER_UNREF(exec_ctx, resolver, "create_channel");
   return channel;
 }
 

src/core/ext/transport/chttp2/server/chttp2_server.c

@@ -0,0 +1,362 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/ext/transport/chttp2/server/chttp2_server.h"
+
+#include <grpc/grpc.h>
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/channel/http_server_filter.h"
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/resolve_address.h"
+#include "src/core/lib/iomgr/tcp_server.h"
+#include "src/core/lib/surface/api_trace.h"
+#include "src/core/lib/surface/server.h"
+
+void grpc_chttp2_server_handshaker_factory_add_handshakers(
+    grpc_exec_ctx *exec_ctx,
+    grpc_chttp2_server_handshaker_factory *handshaker_factory,
+    grpc_handshake_manager *handshake_mgr) {
+  if (handshaker_factory != NULL) {
+    handshaker_factory->vtable->add_handshakers(exec_ctx, handshaker_factory,
+                                                handshake_mgr);
+  }
+}
+
+void grpc_chttp2_server_handshaker_factory_destroy(
+    grpc_exec_ctx *exec_ctx,
+    grpc_chttp2_server_handshaker_factory *handshaker_factory) {
+  if (handshaker_factory != NULL) {
+    handshaker_factory->vtable->destroy(exec_ctx, handshaker_factory);
+  }
+}
+
+typedef struct pending_handshake_manager_node {
+  grpc_handshake_manager *handshake_mgr;
+  struct pending_handshake_manager_node *next;
+} pending_handshake_manager_node;
+
+typedef struct {
+  grpc_server *server;
+  grpc_tcp_server *tcp_server;
+  grpc_channel_args *args;
+  grpc_chttp2_server_handshaker_factory *handshaker_factory;
+  gpr_mu mu;
+  bool shutdown;
+  grpc_closure tcp_server_shutdown_complete;
+  grpc_closure *server_destroy_listener_done;
+  pending_handshake_manager_node *pending_handshake_mgrs;
+} server_state;
+
+typedef struct {
+  server_state *server_state;
+  grpc_pollset *accepting_pollset;
+  grpc_tcp_server_acceptor *acceptor;
+  grpc_handshake_manager *handshake_mgr;
+} server_connection_state;
+
+static void pending_handshake_manager_add_locked(
+    server_state *state, grpc_handshake_manager *handshake_mgr) {
+  pending_handshake_manager_node *node = gpr_malloc(sizeof(*node));
+  node->handshake_mgr = handshake_mgr;
+  node->next = state->pending_handshake_mgrs;
+  state->pending_handshake_mgrs = node;
+}
+
+static void pending_handshake_manager_remove_locked(
+    server_state *state, grpc_handshake_manager *handshake_mgr) {
+  pending_handshake_manager_node **prev_node = &state->pending_handshake_mgrs;
+  for (pending_handshake_manager_node *node = state->pending_handshake_mgrs;
+       node != NULL; node = node->next) {
+    if (node->handshake_mgr == handshake_mgr) {
+      *prev_node = node->next;
+      gpr_free(node);
+      break;
+    }
+    prev_node = &node->next;
+  }
+}
+
+static void pending_handshake_manager_shutdown_locked(grpc_exec_ctx *exec_ctx,
+                                                      server_state *state) {
+  pending_handshake_manager_node *prev_node = NULL;
+  for (pending_handshake_manager_node *node = state->pending_handshake_mgrs;
+       node != NULL; node = node->next) {
+    grpc_handshake_manager_shutdown(exec_ctx, node->handshake_mgr);
+    gpr_free(prev_node);
+    prev_node = node;
+  }
+  gpr_free(prev_node);
+  state->pending_handshake_mgrs = NULL;
+}
+
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error) {
+  grpc_handshaker_args *args = arg;
+  server_connection_state *connection_state = args->user_data;
+  gpr_mu_lock(&connection_state->server_state->mu);
+  if (error != GRPC_ERROR_NONE || connection_state->server_state->shutdown) {
+    const char *error_str = grpc_error_string(error);
+    gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
+    grpc_error_free_string(error_str);
+    if (error == GRPC_ERROR_NONE && args->endpoint != NULL) {
+      // We were shut down after handshaking completed successfully, so
+      // destroy the endpoint here.
+      // TODO(ctiller): It is currently necessary to shutdown endpoints
+      // before destroying them, even if we know that there are no
+      // pending read/write callbacks.  This should be fixed, at which
+      // point this can be removed.
+      grpc_endpoint_shutdown(exec_ctx, args->endpoint);
+      grpc_endpoint_destroy(exec_ctx, args->endpoint);
+      grpc_channel_args_destroy(args->args);
+      grpc_slice_buffer_destroy(args->read_buffer);
+      gpr_free(args->read_buffer);
+    }
+  } else {
+    // If the handshaking succeeded but there is no endpoint, then the
+    // handshaker may have handed off the connection to some external
+    // code, so we can just clean up here without creating a transport.
+    if (args->endpoint != NULL) {
+      grpc_transport *transport =
+          grpc_create_chttp2_transport(exec_ctx, args->args, args->endpoint, 0);
+      grpc_server_setup_transport(
+          exec_ctx, connection_state->server_state->server, transport,
+          connection_state->accepting_pollset, args->args);
+      grpc_chttp2_transport_start_reading(exec_ctx, transport,
+                                          args->read_buffer);
+      grpc_channel_args_destroy(args->args);
+    }
+  }
+  pending_handshake_manager_remove_locked(connection_state->server_state,
+                                          connection_state->handshake_mgr);
+  gpr_mu_unlock(&connection_state->server_state->mu);
+  grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
+  grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp_server);
+  gpr_free(connection_state->acceptor);
+  gpr_free(connection_state);
+}
+
+static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
+                      grpc_pollset *accepting_pollset,
+                      grpc_tcp_server_acceptor *acceptor) {
+  server_state *state = arg;
+  gpr_mu_lock(&state->mu);
+  if (state->shutdown) {
+    gpr_mu_unlock(&state->mu);
+    grpc_endpoint_destroy(exec_ctx, tcp);
+    gpr_free(acceptor);
+    return;
+  }
+  grpc_handshake_manager *handshake_mgr = grpc_handshake_manager_create();
+  pending_handshake_manager_add_locked(state, handshake_mgr);
+  gpr_mu_unlock(&state->mu);
+  grpc_tcp_server_ref(state->tcp_server);
+  server_connection_state *connection_state =
+      gpr_malloc(sizeof(*connection_state));
+  connection_state->server_state = state;
+  connection_state->accepting_pollset = accepting_pollset;
+  connection_state->acceptor = acceptor;
+  connection_state->handshake_mgr = handshake_mgr;
+  grpc_chttp2_server_handshaker_factory_add_handshakers(
+      exec_ctx, state->handshaker_factory, connection_state->handshake_mgr);
+  // TODO(roth): We should really get this timeout value from channel
+  // args instead of hard-coding it.
+  const gpr_timespec deadline = gpr_time_add(
+      gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(120, GPR_TIMESPAN));
+  grpc_handshake_manager_do_handshake(exec_ctx, connection_state->handshake_mgr,
+                                      tcp, state->args, deadline, acceptor,
+                                      on_handshake_done, connection_state);
+}
+
+/* Server callback: start listening on our ports */
+static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server,
+                                  void *arg, grpc_pollset **pollsets,
+                                  size_t pollset_count) {
+  server_state *state = arg;
+  gpr_mu_lock(&state->mu);
+  state->shutdown = false;
+  gpr_mu_unlock(&state->mu);
+  grpc_tcp_server_start(exec_ctx, state->tcp_server, pollsets, pollset_count,
+                        on_accept, state);
+}
+
+static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *arg,
+                                         grpc_error *error) {
+  server_state *state = arg;
+  /* ensure all threads have unlocked */
+  gpr_mu_lock(&state->mu);
+  grpc_closure *destroy_done = state->server_destroy_listener_done;
+  GPR_ASSERT(state->shutdown);
+  pending_handshake_manager_shutdown_locked(exec_ctx, state);
+  gpr_mu_unlock(&state->mu);
+  // Flush queued work before destroying handshaker factory, since that
+  // may do a synchronous unref.
+  grpc_exec_ctx_flush(exec_ctx);
+  grpc_chttp2_server_handshaker_factory_destroy(exec_ctx,
+                                                state->handshaker_factory);
+  if (destroy_done != NULL) {
+    destroy_done->cb(exec_ctx, destroy_done->cb_arg, GRPC_ERROR_REF(error));
+    grpc_exec_ctx_flush(exec_ctx);
+  }
+  grpc_channel_args_destroy(state->args);
+  gpr_mu_destroy(&state->mu);
+  gpr_free(state);
+}
+
+/* Server callback: destroy the tcp listener (so we don't generate further
+   callbacks) */
+static void server_destroy_listener(grpc_exec_ctx *exec_ctx,
+                                    grpc_server *server, void *arg,
+                                    grpc_closure *destroy_done) {
+  server_state *state = arg;
+  gpr_mu_lock(&state->mu);
+  state->shutdown = true;
+  state->server_destroy_listener_done = destroy_done;
+  grpc_tcp_server *tcp_server = state->tcp_server;
+  gpr_mu_unlock(&state->mu);
+  grpc_tcp_server_shutdown_listeners(exec_ctx, tcp_server);
+  grpc_tcp_server_unref(exec_ctx, tcp_server);
+}
+
+grpc_error *grpc_chttp2_server_add_port(
+    grpc_exec_ctx *exec_ctx, grpc_server *server, const char *addr,
+    grpc_channel_args *args,
+    grpc_chttp2_server_handshaker_factory *handshaker_factory, int *port_num) {
+  grpc_resolved_addresses *resolved = NULL;
+  grpc_tcp_server *tcp_server = NULL;
+  size_t i;
+  size_t count = 0;
+  int port_temp;
+  grpc_error *err = GRPC_ERROR_NONE;
+  server_state *state = NULL;
+  grpc_error **errors = NULL;
+
+  *port_num = -1;
+
+  /* resolve address */
+  err = grpc_blocking_resolve_address(addr, "https", &resolved);
+  if (err != GRPC_ERROR_NONE) {
+    goto error;
+  }
+  state = gpr_malloc(sizeof(*state));
+  memset(state, 0, sizeof(*state));
+  grpc_closure_init(&state->tcp_server_shutdown_complete,
+                    tcp_server_shutdown_complete, state);
+  err = grpc_tcp_server_create(exec_ctx, &state->tcp_server_shutdown_complete,
+                               args, &tcp_server);
+  if (err != GRPC_ERROR_NONE) {
+    goto error;
+  }
+
+  state->server = server;
+  state->tcp_server = tcp_server;
+  state->args = args;
+  state->handshaker_factory = handshaker_factory;
+  state->shutdown = true;
+  gpr_mu_init(&state->mu);
+
+  const size_t naddrs = resolved->naddrs;
+  errors = gpr_malloc(sizeof(*errors) * naddrs);
+  for (i = 0; i < naddrs; i++) {
+    errors[i] =
+        grpc_tcp_server_add_port(tcp_server, &resolved->addrs[i], &port_temp);
+    if (errors[i] == GRPC_ERROR_NONE) {
+      if (*port_num == -1) {
+        *port_num = port_temp;
+      } else {
+        GPR_ASSERT(*port_num == port_temp);
+      }
+      count++;
+    }
+  }
+  if (count == 0) {
+    char *msg;
+    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
+                 naddrs);
+    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
+    gpr_free(msg);
+    goto error;
+  } else if (count != naddrs) {
+    char *msg;
+    gpr_asprintf(&msg, "Only %" PRIuPTR
+                       " addresses added out of total %" PRIuPTR " resolved",
+                 count, naddrs);
+    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
+    gpr_free(msg);
+
+    const char *warning_message = grpc_error_string(err);
+    gpr_log(GPR_INFO, "WARNING: %s", warning_message);
+    grpc_error_free_string(warning_message);
+    /* we managed to bind some addresses: continue */
+  }
+  grpc_resolved_addresses_destroy(resolved);
+
+  /* Register with the server only upon success */
+  grpc_server_add_listener(exec_ctx, server, state, server_start_listener,
+                           server_destroy_listener);
+  goto done;
+
+/* Error path: cleanup and return */
+error:
+  GPR_ASSERT(err != GRPC_ERROR_NONE);
+  if (resolved) {
+    grpc_resolved_addresses_destroy(resolved);
+  }
+  if (tcp_server) {
+    grpc_tcp_server_unref(exec_ctx, tcp_server);
+  } else {
+    grpc_channel_args_destroy(args);
+    grpc_chttp2_server_handshaker_factory_destroy(exec_ctx, handshaker_factory);
+    gpr_free(state);
+  }
+  *port_num = 0;
+
+done:
+  if (errors != NULL) {
+    for (i = 0; i < naddrs; i++) {
+      GRPC_ERROR_UNREF(errors[i]);
+    }
+    gpr_free(errors);
+  }
+  return err;
+}

src/core/ext/transport/chttp2/server/chttp2_server.h

@@ -0,0 +1,78 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_SERVER_CHTTP2_SERVER_H
+#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_SERVER_CHTTP2_SERVER_H
+
+#include <grpc/impl/codegen/grpc_types.h>
+
+#include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+
+/// A server handshaker factory is used to create handshakers for server
+/// connections.
+typedef struct grpc_chttp2_server_handshaker_factory
+    grpc_chttp2_server_handshaker_factory;
+
+typedef struct {
+  void (*add_handshakers)(
+      grpc_exec_ctx *exec_ctx,
+      grpc_chttp2_server_handshaker_factory *handshaker_factory,
+      grpc_handshake_manager *handshake_mgr);
+  void (*destroy)(grpc_exec_ctx *exec_ctx,
+                  grpc_chttp2_server_handshaker_factory *handshaker_factory);
+} grpc_chttp2_server_handshaker_factory_vtable;
+
+struct grpc_chttp2_server_handshaker_factory {
+  const grpc_chttp2_server_handshaker_factory_vtable *vtable;
+};
+
+void grpc_chttp2_server_handshaker_factory_add_handshakers(
+    grpc_exec_ctx *exec_ctx,
+    grpc_chttp2_server_handshaker_factory *handshaker_factory,
+    grpc_handshake_manager *handshake_mgr);
+
+void grpc_chttp2_server_handshaker_factory_destroy(
+    grpc_exec_ctx *exec_ctx,
+    grpc_chttp2_server_handshaker_factory *handshaker_factory);
+
+/// Adds a port to \a server.  Sets \a port_num to the port number.
+/// If \a handshaker_factory is not NULL, it will be used to create
+/// handshakers for the port.
+/// Takes ownership of \a args and \a handshaker_factory.
+grpc_error *grpc_chttp2_server_add_port(
+    grpc_exec_ctx *exec_ctx, grpc_server *server, const char *addr,
+    grpc_channel_args *args,
+    grpc_chttp2_server_handshaker_factory *handshaker_factory, int *port_num);
+
+#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_SERVER_CHTTP2_SERVER_H */

src/core/ext/transport/chttp2/server/insecure/server_chttp2.c

@@ -33,180 +33,28 @@
 
 #include <grpc/grpc.h>
 
-#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/useful.h>
 
-#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
+#include "src/core/ext/transport/chttp2/server/chttp2_server.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/handshaker.h"
-#include "src/core/lib/channel/http_server_filter.h"
-#include "src/core/lib/iomgr/resolve_address.h"
-#include "src/core/lib/iomgr/tcp_server.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-typedef struct server_connect_state {
-  grpc_server *server;
-  grpc_pollset *accepting_pollset;
-  grpc_tcp_server_acceptor *acceptor;
-  grpc_handshake_manager *handshake_mgr;
-} server_connect_state;
-
-static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
-                              grpc_channel_args *args,
-                              grpc_slice_buffer *read_buffer, void *user_data,
-                              grpc_error *error) {
-  server_connect_state *state = user_data;
-  if (error != GRPC_ERROR_NONE) {
-    const char *error_str = grpc_error_string(error);
-    gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
-    grpc_error_free_string(error_str);
-    GRPC_ERROR_UNREF(error);
-    grpc_handshake_manager_shutdown(exec_ctx, state->handshake_mgr);
-    gpr_free(read_buffer);
-  } else {
-    // Beware that the call to grpc_create_chttp2_transport() has to happen
-    // before grpc_tcp_server_destroy(). This is fine here, but similar code
-    // asynchronously doing a handshake instead of calling
-    // grpc_tcp_server_start() (as in server_secure_chttp2.c) needs to add
-    // synchronization to avoid this case.
-    grpc_transport *transport =
-        grpc_create_chttp2_transport(exec_ctx, args, endpoint, 0);
-    grpc_server_setup_transport(exec_ctx, state->server, transport,
-                                state->accepting_pollset,
-                                grpc_server_get_channel_args(state->server));
-    grpc_chttp2_transport_start_reading(exec_ctx, transport, read_buffer);
-  }
-  // Clean up.
-  grpc_channel_args_destroy(args);
-  grpc_handshake_manager_destroy(exec_ctx, state->handshake_mgr);
-  gpr_free(state);
-}
-
-static void on_accept(grpc_exec_ctx *exec_ctx, void *server, grpc_endpoint *tcp,
-                      grpc_pollset *accepting_pollset,
-                      grpc_tcp_server_acceptor *acceptor) {
-  server_connect_state *state = gpr_malloc(sizeof(server_connect_state));
-  state->server = server;
-  state->accepting_pollset = accepting_pollset;
-  state->acceptor = acceptor;
-  state->handshake_mgr = grpc_handshake_manager_create();
-  // TODO(roth): We should really get this timeout value from channel
-  // args instead of hard-coding it.
-  const gpr_timespec deadline = gpr_time_add(
-      gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(120, GPR_TIMESPAN));
-  grpc_handshake_manager_do_handshake(
-      exec_ctx, state->handshake_mgr, tcp, grpc_server_get_channel_args(server),
-      deadline, acceptor, on_handshake_done, state);
-}
-
-/* Server callback: start listening on our ports */
-static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp,
-                  grpc_pollset **pollsets, size_t pollset_count) {
-  grpc_tcp_server *tcp = tcpp;
-  grpc_tcp_server_start(exec_ctx, tcp, pollsets, pollset_count, on_accept,
-                        server);
-}
-
-/* Server callback: destroy the tcp listener (so we don't generate further
-   callbacks) */
-static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *tcpp,
-                    grpc_closure *destroy_done) {
-  grpc_tcp_server *tcp = tcpp;
-  grpc_tcp_server_shutdown_listeners(exec_ctx, tcp);
-  grpc_tcp_server_unref(exec_ctx, tcp);
-  grpc_exec_ctx_sched(exec_ctx, destroy_done, GRPC_ERROR_NONE, NULL);
-}
-
 int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
-  grpc_resolved_addresses *resolved = NULL;
-  grpc_tcp_server *tcp = NULL;
-  size_t i;
-  size_t count = 0;
-  int port_num = -1;
-  int port_temp;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_error *err = GRPC_ERROR_NONE;
-
+  int port_num = 0;
   GRPC_API_TRACE("grpc_server_add_insecure_http2_port(server=%p, addr=%s)", 2,
                  (server, addr));
-
-  grpc_error **errors = NULL;
-  err = grpc_blocking_resolve_address(addr, "https", &resolved);
-  if (err != GRPC_ERROR_NONE) {
-    goto error;
-  }
-
-  err = grpc_tcp_server_create(&exec_ctx, NULL,
-                               grpc_server_get_channel_args(server), &tcp);
+  grpc_error *err = grpc_chttp2_server_add_port(
+      &exec_ctx, server, addr,
+      grpc_channel_args_copy(grpc_server_get_channel_args(server)),
+      NULL /* handshaker_factory */, &port_num);
   if (err != GRPC_ERROR_NONE) {
-    goto error;
-  }
-
-  const size_t naddrs = resolved->naddrs;
-  errors = gpr_malloc(sizeof(*errors) * naddrs);
-  for (i = 0; i < naddrs; i++) {
-    errors[i] = grpc_tcp_server_add_port(tcp, &resolved->addrs[i], &port_temp);
-    if (errors[i] == GRPC_ERROR_NONE) {
-      if (port_num == -1) {
-        port_num = port_temp;
-      } else {
-        GPR_ASSERT(port_num == port_temp);
-      }
-      count++;
-    }
+    const char *msg = grpc_error_string(err);
+    gpr_log(GPR_ERROR, "%s", msg);
+    grpc_error_free_string(msg);
+    GRPC_ERROR_UNREF(err);
   }
-  if (count == 0) {
-    char *msg;
-    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
-                 naddrs);
-    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
-    gpr_free(msg);
-    goto error;
-  } else if (count != naddrs) {
-    char *msg;
-    gpr_asprintf(&msg, "Only %" PRIuPTR
-                       " addresses added out of total %" PRIuPTR " resolved",
-                 count, naddrs);
-    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, naddrs);
-    gpr_free(msg);
-
-    const char *warning_message = grpc_error_string(err);
-    gpr_log(GPR_INFO, "WARNING: %s", warning_message);
-    grpc_error_free_string(warning_message);
-    /* we managed to bind some addresses: continue */
-  }
-  grpc_resolved_addresses_destroy(resolved);
-
-  /* Register with the server only upon success */
-  grpc_server_add_listener(&exec_ctx, server, tcp, start, destroy);
-  goto done;
-
-/* Error path: cleanup and return */
-error:
-  GPR_ASSERT(err != GRPC_ERROR_NONE);
-  if (resolved) {
-    grpc_resolved_addresses_destroy(resolved);
-  }
-  if (tcp) {
-    grpc_tcp_server_unref(&exec_ctx, tcp);
-  }
-  port_num = 0;
-
-  const char *msg = grpc_error_string(err);
-  gpr_log(GPR_ERROR, "%s", msg);
-  grpc_error_free_string(msg);
-  GRPC_ERROR_UNREF(err);
-
-done:
   grpc_exec_ctx_finish(&exec_ctx);
-  if (errors != NULL) {
-    for (i = 0; i < naddrs; i++) {
-      GRPC_ERROR_UNREF(errors[i]);
-    }
-  }
-  gpr_free(errors);
   return port_num;
 }

src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c

@@ -38,218 +38,63 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include <grpc/support/sync.h>
-#include <grpc/support/useful.h>
+
+#include "src/core/ext/transport/chttp2/server/chttp2_server.h"
+
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker.h"
-#include "src/core/lib/channel/http_server_filter.h"
-#include "src/core/lib/iomgr/endpoint.h"
-#include "src/core/lib/iomgr/resolve_address.h"
-#include "src/core/lib/iomgr/tcp_server.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/security/transport/auth_filters.h"
-#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-typedef struct server_secure_state {
-  grpc_server *server;
-  grpc_tcp_server *tcp;
-  grpc_server_security_connector *sc;
-  grpc_server_credentials *creds;
-  bool is_shutdown;
-  gpr_mu mu;
-  grpc_closure tcp_server_shutdown_complete;
-  grpc_closure *server_destroy_listener_done;
-} server_secure_state;
-
-typedef struct server_secure_connect {
-  server_secure_state *server_state;
-  grpc_pollset *accepting_pollset;
-  grpc_tcp_server_acceptor *acceptor;
-  grpc_handshake_manager *handshake_mgr;
-  // TODO(roth): Remove the following two fields when we eliminate
-  // grpc_server_security_connector_do_handshake().
-  gpr_timespec deadline;
-  grpc_channel_args *args;
-} server_secure_connect;
-
-static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
-                                     grpc_security_status status,
-                                     grpc_endpoint *secure_endpoint,
-                                     grpc_auth_context *auth_context) {
-  server_secure_connect *connection_state = statep;
-  if (status == GRPC_SECURITY_OK) {
-    if (secure_endpoint) {
-      gpr_mu_lock(&connection_state->server_state->mu);
-      if (!connection_state->server_state->is_shutdown) {
-        grpc_transport *transport = grpc_create_chttp2_transport(
-            exec_ctx, grpc_server_get_channel_args(
-                          connection_state->server_state->server),
-            secure_endpoint, 0);
-        grpc_arg args_to_add[2];
-        args_to_add[0] = grpc_server_credentials_to_arg(
-            connection_state->server_state->creds);
-        args_to_add[1] = grpc_auth_context_to_arg(auth_context);
-        grpc_channel_args *args_copy = grpc_channel_args_copy_and_add(
-            connection_state->args, args_to_add, GPR_ARRAY_SIZE(args_to_add));
-        grpc_server_setup_transport(
-            exec_ctx, connection_state->server_state->server, transport,
-            connection_state->accepting_pollset, args_copy);
-        grpc_channel_args_destroy(args_copy);
-        grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL);
-      } else {
-        /* We need to consume this here, because the server may already have
-         * gone away. */
-        grpc_endpoint_destroy(exec_ctx, secure_endpoint);
-      }
-      gpr_mu_unlock(&connection_state->server_state->mu);
-    }
-  } else {
-    gpr_log(GPR_ERROR, "Secure transport failed with error %d", status);
-  }
-  grpc_channel_args_destroy(connection_state->args);
-  grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp);
-  gpr_free(connection_state);
-}
-
-static void on_handshake_done(grpc_exec_ctx *exec_ctx, grpc_endpoint *endpoint,
-                              grpc_channel_args *args,
-                              grpc_slice_buffer *read_buffer, void *user_data,
-                              grpc_error *error) {
-  server_secure_connect *connection_state = user_data;
-  if (error != GRPC_ERROR_NONE) {
-    const char *error_str = grpc_error_string(error);
-    gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
-    grpc_error_free_string(error_str);
-    GRPC_ERROR_UNREF(error);
-    grpc_channel_args_destroy(args);
-    gpr_free(read_buffer);
-    grpc_handshake_manager_shutdown(exec_ctx, connection_state->handshake_mgr);
-    grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
-    grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp);
-    gpr_free(connection_state);
-    return;
-  }
-  grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
-  connection_state->handshake_mgr = NULL;
-  // TODO(roth, jboeuf): Convert security connector handshaking to use new
-  // handshake API, and then move the code from on_secure_handshake_done()
-  // into this function.
-  connection_state->args = args;
-  grpc_server_security_connector_do_handshake(
-      exec_ctx, connection_state->server_state->sc, connection_state->acceptor,
-      endpoint, read_buffer, connection_state->deadline,
-      on_secure_handshake_done, connection_state);
-}
-
-static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
-                      grpc_pollset *accepting_pollset,
-                      grpc_tcp_server_acceptor *acceptor) {
-  server_secure_state *server_state = statep;
-  server_secure_connect *connection_state = NULL;
-  gpr_mu_lock(&server_state->mu);
-  if (server_state->is_shutdown) {
-    gpr_mu_unlock(&server_state->mu);
-    grpc_endpoint_destroy(exec_ctx, tcp);
-    return;
-  }
-  gpr_mu_unlock(&server_state->mu);
-  grpc_tcp_server_ref(server_state->tcp);
-  connection_state = gpr_malloc(sizeof(*connection_state));
-  connection_state->server_state = server_state;
-  connection_state->accepting_pollset = accepting_pollset;
-  connection_state->acceptor = acceptor;
-  connection_state->handshake_mgr = grpc_handshake_manager_create();
-  // TODO(roth): We should really get this timeout value from channel
-  // args instead of hard-coding it.
-  connection_state->deadline = gpr_time_add(
-      gpr_now(GPR_CLOCK_MONOTONIC), gpr_time_from_seconds(120, GPR_TIMESPAN));
-  grpc_handshake_manager_do_handshake(
-      exec_ctx, connection_state->handshake_mgr, tcp,
-      grpc_server_get_channel_args(connection_state->server_state->server),
-      connection_state->deadline, acceptor, on_handshake_done,
-      connection_state);
+typedef struct {
+  grpc_chttp2_server_handshaker_factory base;
+  grpc_server_security_connector *security_connector;
+} server_security_handshaker_factory;
+
+static void server_security_handshaker_factory_add_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_server_handshaker_factory *hf,
+    grpc_handshake_manager *handshake_mgr) {
+  server_security_handshaker_factory *handshaker_factory =
+      (server_security_handshaker_factory *)hf;
+  grpc_server_security_connector_add_handshakers(
+      exec_ctx, handshaker_factory->security_connector, handshake_mgr);
 }
 
-/* Server callback: start listening on our ports */
-static void server_start_listener(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                                  void *statep, grpc_pollset **pollsets,
-                                  size_t pollset_count) {
-  server_secure_state *server_state = statep;
-  gpr_mu_lock(&server_state->mu);
-  server_state->is_shutdown = false;
-  gpr_mu_unlock(&server_state->mu);
-  grpc_tcp_server_start(exec_ctx, server_state->tcp, pollsets, pollset_count,
-                        on_accept, server_state);
+static void server_security_handshaker_factory_destroy(
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_server_handshaker_factory *hf) {
+  server_security_handshaker_factory *handshaker_factory =
+      (server_security_handshaker_factory *)hf;
+  GRPC_SECURITY_CONNECTOR_UNREF(&handshaker_factory->security_connector->base,
+                                "server");
+  gpr_free(hf);
 }
 
-static void tcp_server_shutdown_complete(grpc_exec_ctx *exec_ctx, void *statep,
-                                         grpc_error *error) {
-  server_secure_state *server_state = statep;
-  /* ensure all threads have unlocked */
-  gpr_mu_lock(&server_state->mu);
-  grpc_closure *destroy_done = server_state->server_destroy_listener_done;
-  GPR_ASSERT(server_state->is_shutdown);
-  gpr_mu_unlock(&server_state->mu);
-  /* clean up */
-  grpc_server_security_connector_shutdown(exec_ctx, server_state->sc);
-
-  /* Flush queued work before a synchronous unref. */
-  grpc_exec_ctx_flush(exec_ctx);
-  GRPC_SECURITY_CONNECTOR_UNREF(&server_state->sc->base, "server");
-  grpc_server_credentials_unref(server_state->creds);
-
-  if (destroy_done != NULL) {
-    destroy_done->cb(exec_ctx, destroy_done->cb_arg, GRPC_ERROR_REF(error));
-    grpc_exec_ctx_flush(exec_ctx);
-  }
-  gpr_free(server_state);
-}
-
-static void server_destroy_listener(grpc_exec_ctx *exec_ctx,
-                                    grpc_server *server, void *statep,
-                                    grpc_closure *callback) {
-  server_secure_state *server_state = statep;
-  grpc_tcp_server *tcp;
-  gpr_mu_lock(&server_state->mu);
-  server_state->is_shutdown = true;
-  server_state->server_destroy_listener_done = callback;
-  tcp = server_state->tcp;
-  gpr_mu_unlock(&server_state->mu);
-  grpc_tcp_server_shutdown_listeners(exec_ctx, tcp);
-  grpc_tcp_server_unref(exec_ctx, server_state->tcp);
-}
+static const grpc_chttp2_server_handshaker_factory_vtable
+    server_security_handshaker_factory_vtable = {
+        server_security_handshaker_factory_add_handshakers,
+        server_security_handshaker_factory_destroy};
 
 int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
                                       grpc_server_credentials *creds) {
-  grpc_resolved_addresses *resolved = NULL;
-  grpc_tcp_server *tcp = NULL;
-  server_secure_state *server_state = NULL;
-  size_t i;
-  size_t count = 0;
-  int port_num = -1;
-  int port_temp;
-  grpc_security_status status = GRPC_SECURITY_ERROR;
-  grpc_server_security_connector *sc = NULL;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_error *err = GRPC_ERROR_NONE;
-  grpc_error **errors = NULL;
-
+  grpc_server_security_connector *sc = NULL;
+  int port_num = 0;
   GRPC_API_TRACE(
       "grpc_server_add_secure_http2_port("
       "server=%p, addr=%s, creds=%p)",
       3, (server, addr, creds));
-
-  /* create security context */
+  // Create security context.
   if (creds == NULL) {
     err = GRPC_ERROR_CREATE(
         "No credentials specified for secure server port (creds==NULL)");
-    goto error;
+    goto done;
   }
-  status = grpc_server_credentials_create_security_connector(creds, &sc);
+  grpc_security_status status =
+      grpc_server_credentials_create_security_connector(creds, &sc);
   if (status != GRPC_SECURITY_OK) {
     char *msg;
     gpr_asprintf(&msg,
@@ -258,107 +103,28 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
     err = grpc_error_set_int(GRPC_ERROR_CREATE(msg),
                              GRPC_ERROR_INT_SECURITY_STATUS, status);
     gpr_free(msg);
-    goto error;
+    goto done;
   }
-  sc->channel_args = grpc_server_get_channel_args(server);
-
-  /* resolve address */
-  err = grpc_blocking_resolve_address(addr, "https", &resolved);
-  if (err != GRPC_ERROR_NONE) {
-    goto error;
-  }
-  server_state = gpr_malloc(sizeof(*server_state));
-  memset(server_state, 0, sizeof(*server_state));
-  grpc_closure_init(&server_state->tcp_server_shutdown_complete,
-                    tcp_server_shutdown_complete, server_state);
-  err = grpc_tcp_server_create(&exec_ctx,
-                               &server_state->tcp_server_shutdown_complete,
-                               grpc_server_get_channel_args(server), &tcp);
+  // Create handshaker factory.
+  server_security_handshaker_factory *handshaker_factory =
+      gpr_malloc(sizeof(*handshaker_factory));
+  memset(handshaker_factory, 0, sizeof(*handshaker_factory));
+  handshaker_factory->base.vtable = &server_security_handshaker_factory_vtable;
+  handshaker_factory->security_connector = sc;
+  // Create channel args.
+  grpc_arg channel_arg = grpc_server_credentials_to_arg(creds);
+  grpc_channel_args *args = grpc_channel_args_copy_and_add(
+      grpc_server_get_channel_args(server), &channel_arg, 1);
+  // Add server port.
+  err = grpc_chttp2_server_add_port(&exec_ctx, server, addr, args,
+                                    &handshaker_factory->base, &port_num);
+done:
+  grpc_exec_ctx_finish(&exec_ctx);
   if (err != GRPC_ERROR_NONE) {
-    goto error;
+    const char *msg = grpc_error_string(err);
+    gpr_log(GPR_ERROR, "%s", msg);
+    grpc_error_free_string(msg);
+    GRPC_ERROR_UNREF(err);
   }
-
-  server_state->server = server;
-  server_state->tcp = tcp;
-  server_state->sc = sc;
-  server_state->creds = grpc_server_credentials_ref(creds);
-  server_state->is_shutdown = true;
-  gpr_mu_init(&server_state->mu);
-
-  errors = gpr_malloc(sizeof(*errors) * resolved->naddrs);
-  for (i = 0; i < resolved->naddrs; i++) {
-    errors[i] = grpc_tcp_server_add_port(tcp, &resolved->addrs[i], &port_temp);
-    if (errors[i] == GRPC_ERROR_NONE) {
-      if (port_num == -1) {
-        port_num = port_temp;
-      } else {
-        GPR_ASSERT(port_num == port_temp);
-      }
-      count++;
-    }
-  }
-  if (count == 0) {
-    char *msg;
-    gpr_asprintf(&msg, "No address added out of total %" PRIuPTR " resolved",
-                 resolved->naddrs);
-    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, resolved->naddrs);
-    gpr_free(msg);
-    goto error;
-  } else if (count != resolved->naddrs) {
-    char *msg;
-    gpr_asprintf(&msg, "Only %" PRIuPTR
-                       " addresses added out of total %" PRIuPTR " resolved",
-                 count, resolved->naddrs);
-    err = GRPC_ERROR_CREATE_REFERENCING(msg, errors, resolved->naddrs);
-    gpr_free(msg);
-
-    const char *warning_message = grpc_error_string(err);
-    gpr_log(GPR_INFO, "WARNING: %s", warning_message);
-    grpc_error_free_string(warning_message);
-    /* we managed to bind some addresses: continue */
-  } else {
-    for (i = 0; i < resolved->naddrs; i++) {
-      GRPC_ERROR_UNREF(errors[i]);
-    }
-  }
-  gpr_free(errors);
-  errors = NULL;
-  grpc_resolved_addresses_destroy(resolved);
-
-  /* Register with the server only upon success */
-  grpc_server_add_listener(&exec_ctx, server, server_state,
-                           server_start_listener, server_destroy_listener);
-
-  grpc_exec_ctx_finish(&exec_ctx);
   return port_num;
-
-/* Error path: cleanup and return */
-error:
-  GPR_ASSERT(err != GRPC_ERROR_NONE);
-  if (errors != NULL) {
-    for (i = 0; i < resolved->naddrs; i++) {
-      GRPC_ERROR_UNREF(errors[i]);
-    }
-    gpr_free(errors);
-  }
-  if (resolved) {
-    grpc_resolved_addresses_destroy(resolved);
-  }
-  if (tcp) {
-    grpc_tcp_server_unref(&exec_ctx, tcp);
-  } else {
-    if (sc) {
-      grpc_exec_ctx_flush(&exec_ctx);
-      GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "server");
-    }
-    if (server_state) {
-      gpr_free(server_state);
-    }
-  }
-  grpc_exec_ctx_finish(&exec_ctx);
-  const char *msg = grpc_error_string(err);
-  GRPC_ERROR_UNREF(err);
-  gpr_log(GPR_ERROR, "%s", msg);
-  grpc_error_free_string(msg);
-  return 0;
 }

src/core/ext/transport/chttp2/transport/chttp2_transport.c

@@ -111,9 +111,6 @@ static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx,
 static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx,
                                                 void *byte_stream,
                                                 grpc_error *error_ignored);
-static void fail_pending_writes(grpc_exec_ctx *exec_ctx,
-                                grpc_chttp2_transport *t, grpc_chttp2_stream *s,
-                                grpc_error *error);
 
 static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *t,
                              grpc_error *error);
@@ -602,11 +599,13 @@ static void set_write_state(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                                  write_state_name(t->write_state),
                                  write_state_name(st), reason));
   t->write_state = st;
-  if (st == GRPC_CHTTP2_WRITE_STATE_IDLE &&
-      t->close_transport_on_writes_finished != NULL) {
-    grpc_error *err = t->close_transport_on_writes_finished;
-    t->close_transport_on_writes_finished = NULL;
-    close_transport_locked(exec_ctx, t, err);
+  if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) {
+    grpc_exec_ctx_enqueue_list(exec_ctx, &t->run_after_write, NULL);
+    if (t->close_transport_on_writes_finished != NULL) {
+      grpc_error *err = t->close_transport_on_writes_finished;
+      t->close_transport_on_writes_finished = NULL;
+      close_transport_locked(exec_ctx, t, err);
+    }
   }
 }
 
@@ -704,8 +703,6 @@ static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *tp,
     }
   }
 
-  grpc_chttp2_end_write(exec_ctx, t, GRPC_ERROR_REF(error));
-
   switch (t->write_state) {
     case GRPC_CHTTP2_WRITE_STATE_IDLE:
       GPR_UNREACHABLE_CODE(break);
@@ -734,6 +731,8 @@ static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *tp,
       break;
   }
 
+  grpc_chttp2_end_write(exec_ctx, t, GRPC_ERROR_REF(error));
+
   GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "writing");
   GPR_TIMER_END("terminate_writing_with_lock", 0);
 }
@@ -823,7 +822,14 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx,
   }
 }
 
+/* Flag that this closure barrier wants stats to be updated before finishing */
 #define CLOSURE_BARRIER_STATS_BIT (1 << 0)
+/* Flag that this closure barrier may be covering a write in a pollset, and so
+   we should not complete this closure until we can prove that the write got
+   scheduled */
+#define CLOSURE_BARRIER_MAY_COVER_WRITE (1 << 1)
+/* First bit of the reference count, stored in the high order bits (with the low
+   bits being used for flags defined above) */
 #define CLOSURE_BARRIER_FIRST_REF_BIT (1 << 16)
 
 static grpc_closure *add_closure_barrier(grpc_closure *closure) {
@@ -850,6 +856,16 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
     return;
   }
   closure->next_data.scratch -= CLOSURE_BARRIER_FIRST_REF_BIT;
+  if (grpc_http_trace) {
+    const char *errstr = grpc_error_string(error);
+    gpr_log(GPR_DEBUG,
+            "complete_closure_step: %p refs=%d flags=0x%04x desc=%s err=%s",
+            closure,
+            (int)(closure->next_data.scratch / CLOSURE_BARRIER_FIRST_REF_BIT),
+            (int)(closure->next_data.scratch % CLOSURE_BARRIER_FIRST_REF_BIT),
+            desc, errstr);
+    grpc_error_free_string(errstr);
+  }
   if (error != GRPC_ERROR_NONE) {
     if (closure->error_data.error == GRPC_ERROR_NONE) {
       closure->error_data.error =
@@ -866,7 +882,13 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
       grpc_transport_move_stats(&s->stats, s->collecting_stats);
       s->collecting_stats = NULL;
     }
-    grpc_closure_run(exec_ctx, closure, closure->error_data.error);
+    if ((t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) ||
+        !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) {
+      grpc_closure_run(exec_ctx, closure, closure->error_data.error);
+    } else {
+      grpc_closure_list_append(&t->run_after_write, closure,
+                               closure->error_data.error);
+    }
   }
 }
 
@@ -1011,6 +1033,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
 
   if (op->send_initial_metadata != NULL) {
     GPR_ASSERT(s->send_initial_metadata_finished == NULL);
+    on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->send_initial_metadata_finished = add_closure_barrier(on_complete);
     s->send_initial_metadata = op->send_initial_metadata;
     const size_t metadata_size =
@@ -1064,6 +1087,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->send_message != NULL) {
+    on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->fetching_send_message_finished = add_closure_barrier(op->on_complete);
     if (s->write_closed) {
       grpc_chttp2_complete_closure_step(
@@ -1101,6 +1125,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
 
   if (op->send_trailing_metadata != NULL) {
     GPR_ASSERT(s->send_trailing_metadata_finished == NULL);
+    on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->send_trailing_metadata_finished = add_closure_barrier(on_complete);
     s->send_trailing_metadata = op->send_trailing_metadata;
     const size_t metadata_size =
@@ -1534,9 +1559,9 @@ static grpc_error *removal_error(grpc_error *extra_error, grpc_chttp2_stream *s,
   return error;
 }
 
-static void fail_pending_writes(grpc_exec_ctx *exec_ctx,
-                                grpc_chttp2_transport *t, grpc_chttp2_stream *s,
-                                grpc_error *error) {
+void grpc_chttp2_fail_pending_writes(grpc_exec_ctx *exec_ctx,
+                                     grpc_chttp2_transport *t,
+                                     grpc_chttp2_stream *s, grpc_error *error) {
   error =
       removal_error(error, s, "Pending writes failed due to stream closure");
   s->send_initial_metadata = NULL;
@@ -1590,13 +1615,16 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx,
   if (close_writes && !s->write_closed) {
     s->write_closed_error = GRPC_ERROR_REF(error);
     s->write_closed = true;
-    fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error));
+    grpc_chttp2_fail_pending_writes(exec_ctx, t, s, GRPC_ERROR_REF(error));
     grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s);
   }
   if (s->read_closed && s->write_closed) {
     if (s->id != 0) {
       remove_stream(exec_ctx, t, s->id,
                     removal_error(GRPC_ERROR_REF(error), s, "Stream removed"));
+    } else {
+      /* Purge streams waiting on concurrency still waiting for id assignment */
+      grpc_chttp2_list_remove_waiting_for_concurrency(t, s);
     }
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2");
   }

src/core/ext/transport/chttp2/transport/internal.h

@@ -327,6 +327,9 @@ struct grpc_chttp2_transport {
    */
   grpc_error *close_transport_on_writes_finished;
 
+  /* a list of closures to run after writes are finished */
+  grpc_closure_list run_after_write;
+
   /* buffer pool state */
   /** have we scheduled a benign cleanup? */
   bool benign_reclaimer_registered;
@@ -493,6 +496,8 @@ void grpc_chttp2_list_add_waiting_for_concurrency(grpc_chttp2_transport *t,
                                                   grpc_chttp2_stream *s);
 int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t,
                                                  grpc_chttp2_stream **s);
+void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t,
+                                                     grpc_chttp2_stream *s);
 
 void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream *s);
@@ -689,4 +694,8 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx,
                                                        grpc_chttp2_transport *t,
                                                        grpc_chttp2_stream *s);
 
+void grpc_chttp2_fail_pending_writes(grpc_exec_ctx *exec_ctx,
+                                     grpc_chttp2_transport *t,
+                                     grpc_chttp2_stream *s, grpc_error *error);
+
 #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_INTERNAL_H */

src/core/ext/transport/chttp2/transport/stream_lists.c

@@ -158,6 +158,11 @@ int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t,
   return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY);
 }
 
+void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t,
+                                                     grpc_chttp2_stream *s) {
+  stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY);
+}
+
 void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream *s) {
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);

src/core/ext/transport/cronet/transport/cronet_transport.c

@@ -148,6 +148,9 @@ struct write_state {
 struct op_state {
   bool state_op_done[OP_NUM_OPS];
   bool state_callback_received[OP_NUM_OPS];
+  bool fail_state;
+  bool flush_read;
+  grpc_error *cancel_error;
   /* data structure for storing data coming from server */
   struct read_state rs;
   /* data structure for storing data going to the server */
@@ -247,6 +250,12 @@ static void free_read_buffer(stream_obj *s) {
   }
 }
 
+static grpc_error *make_error_with_desc(int error_code, const char *desc) {
+  grpc_error *error = GRPC_ERROR_CREATE(desc);
+  error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, error_code);
+  return error;
+}
+
 /*
   Add a new stream op to op storage.
 */
@@ -432,6 +441,18 @@ static void on_response_headers_received(
             grpc_mdstr_from_string(headers->headers[i].value)));
   }
   s->state.state_callback_received[OP_RECV_INITIAL_METADATA] = true;
+  if (!(s->state.state_op_done[OP_CANCEL_ERROR] ||
+        s->state.state_callback_received[OP_FAILED])) {
+    /* Do an extra read to trigger on_succeeded() callback in case connection
+     is closed */
+    GPR_ASSERT(s->state.rs.length_field_received == false);
+    s->state.rs.read_buffer = s->state.rs.grpc_header_bytes;
+    s->state.rs.received_bytes = 0;
+    s->state.rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES;
+    CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs);
+    bidirectional_stream_read(s->cbs, s->state.rs.read_buffer,
+                              s->state.rs.remaining_bytes);
+  }
   gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
@@ -462,7 +483,11 @@ static void on_read_completed(bidirectional_stream *stream, char *data,
              count);
   gpr_mu_lock(&s->mu);
   s->state.state_callback_received[OP_RECV_MESSAGE] = true;
-  if (count > 0) {
+  if (count > 0 && s->state.flush_read) {
+    CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs);
+    bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, 4096);
+    gpr_mu_unlock(&s->mu);
+  } else if (count > 0) {
     s->state.rs.received_bytes += count;
     s->state.rs.remaining_bytes -= count;
     if (s->state.rs.remaining_bytes > 0) {
@@ -477,6 +502,10 @@ static void on_read_completed(bidirectional_stream *stream, char *data,
       execute_from_storage(s);
     }
   } else {
+    if (s->state.flush_read) {
+      gpr_free(s->state.rs.read_buffer);
+      s->state.rs.read_buffer = NULL;
+    }
     s->state.rs.read_stream_closed = true;
     gpr_mu_unlock(&s->mu);
     execute_from_storage(s);
@@ -506,10 +535,27 @@ static void on_response_trailers_received(
             grpc_mdstr_from_string(trailers->headers[i].key),
             grpc_mdstr_from_string(trailers->headers[i].value)));
     s->state.rs.trailing_metadata_valid = true;
+    if (0 == strcmp(trailers->headers[i].key, "grpc-status") &&
+        0 != strcmp(trailers->headers[i].value, "0")) {
+      s->state.fail_state = true;
+    }
   }
   s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true;
-  gpr_mu_unlock(&s->mu);
-  execute_from_storage(s);
+  /* Send a EOS when server terminates the stream (testServerFinishesRequest) to
+   * trigger on_succeeded */
+  if (!s->state.state_op_done[OP_SEND_TRAILING_METADATA] &&
+      !(s->state.state_op_done[OP_CANCEL_ERROR] ||
+        s->state.state_callback_received[OP_FAILED])) {
+    CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", s->cbs);
+    s->state.state_callback_received[OP_SEND_MESSAGE] = false;
+    bidirectional_stream_write(s->cbs, "", 0, true);
+    s->state.state_op_done[OP_SEND_TRAILING_METADATA] = true;
+
+    gpr_mu_unlock(&s->mu);
+  } else {
+    gpr_mu_unlock(&s->mu);
+    execute_from_storage(s);
+  }
 }
 
 /*
@@ -630,9 +676,9 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op,
   /* When call is canceled, every op can be run, except under following
   conditions
   */
-  bool is_canceled_of_failed = stream_state->state_op_done[OP_CANCEL_ERROR] ||
+  bool is_canceled_or_failed = stream_state->state_op_done[OP_CANCEL_ERROR] ||
                                stream_state->state_callback_received[OP_FAILED];
-  if (is_canceled_of_failed) {
+  if (is_canceled_or_failed) {
     if (op_id == OP_SEND_INITIAL_METADATA) result = false;
     if (op_id == OP_SEND_MESSAGE) result = false;
     if (op_id == OP_SEND_TRAILING_METADATA) result = false;
@@ -776,16 +822,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
       op_can_be_run(stream_op, stream_state, &oas->state,
                     OP_SEND_INITIAL_METADATA)) {
     CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas);
-    /* This OP is the beginning. Reset various states */
-    memset(&s->header_array, 0, sizeof(s->header_array));
-    memset(&stream_state->rs, 0, sizeof(stream_state->rs));
-    memset(&stream_state->ws, 0, sizeof(stream_state->ws));
-    memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done));
-    memset(stream_state->state_callback_received, 0,
-           sizeof(stream_state->state_callback_received));
     /* Start new cronet stream. It is destroyed in on_succeeded, on_canceled,
      * on_failed */
     GPR_ASSERT(s->cbs == NULL);
+    GPR_ASSERT(!stream_state->state_op_done[OP_SEND_INITIAL_METADATA]);
     s->cbs = bidirectional_stream_create(s->curr_ct.engine, s->curr_gs,
                                          &cronet_callbacks);
     CRONET_LOG(GPR_DEBUG, "%p = bidirectional_stream_create()", s->cbs);
@@ -804,10 +844,13 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_RECV_INITIAL_METADATA)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_RECV_INITIAL_METADATA", oas);
-    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
-        stream_state->state_callback_received[OP_FAILED]) {
+    if (stream_state->state_op_done[OP_CANCEL_ERROR]) {
       grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
                           GRPC_ERROR_CANCELLED, NULL);
+    } else if (stream_state->state_callback_received[OP_FAILED]) {
+      grpc_exec_ctx_sched(
+          exec_ctx, stream_op->recv_initial_metadata_ready,
+          make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."), NULL);
     } else {
       grpc_chttp2_incoming_metadata_buffer_publish(
           &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata);
@@ -861,12 +904,19 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_RECV_MESSAGE)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_RECV_MESSAGE", oas);
-    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
-        stream_state->state_callback_received[OP_FAILED]) {
-      CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed.");
+    if (stream_state->state_op_done[OP_CANCEL_ERROR]) {
+      CRONET_LOG(GPR_DEBUG, "Stream is cancelled.");
       grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready,
                           GRPC_ERROR_CANCELLED, NULL);
       stream_state->state_op_done[OP_RECV_MESSAGE] = true;
+      result = ACTION_TAKEN_NO_CALLBACK;
+    } else if (stream_state->state_callback_received[OP_FAILED]) {
+      CRONET_LOG(GPR_DEBUG, "Stream failed.");
+      grpc_exec_ctx_sched(
+          exec_ctx, stream_op->recv_message_ready,
+          make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."), NULL);
+      stream_state->state_op_done[OP_RECV_MESSAGE] = true;
+      result = ACTION_TAKEN_NO_CALLBACK;
     } else if (stream_state->rs.read_stream_closed == true) {
       /* No more data will be received */
       CRONET_LOG(GPR_DEBUG, "read stream closed");
@@ -874,6 +924,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
                           GRPC_ERROR_NONE, NULL);
       stream_state->state_op_done[OP_RECV_MESSAGE] = true;
       oas->state.state_op_done[OP_RECV_MESSAGE] = true;
+      result = ACTION_TAKEN_NO_CALLBACK;
     } else if (stream_state->rs.length_field_received == false) {
       if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES &&
           stream_state->rs.remaining_bytes == 0) {
@@ -942,10 +993,15 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
                           GRPC_ERROR_NONE, NULL);
       stream_state->state_op_done[OP_RECV_MESSAGE] = true;
       oas->state.state_op_done[OP_RECV_MESSAGE] = true;
-      /* Clear read state of the stream, so next read op (if it were to come)
-       * will work */
-      stream_state->rs.received_bytes = stream_state->rs.remaining_bytes =
-          stream_state->rs.length_field_received = 0;
+      /* Do an extra read to trigger on_succeeded() callback in case connection
+         is closed */
+      stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes;
+      stream_state->rs.received_bytes = 0;
+      stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES;
+      stream_state->rs.length_field_received = false;
+      CRONET_LOG(GPR_DEBUG, "bidirectional_stream_read(%p)", s->cbs);
+      bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer,
+                                       stream_state->rs.remaining_bytes);
       result = ACTION_TAKEN_NO_CALLBACK;
     }
   } else if (stream_op->recv_trailing_metadata &&
@@ -981,17 +1037,25 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
     CRONET_LOG(GPR_DEBUG, "W: bidirectional_stream_cancel(%p)", s->cbs);
     if (s->cbs) {
       bidirectional_stream_cancel(s->cbs);
+      result = ACTION_TAKEN_WITH_CALLBACK;
+    } else {
+      result = ACTION_TAKEN_NO_CALLBACK;
     }
     stream_state->state_op_done[OP_CANCEL_ERROR] = true;
-    result = ACTION_TAKEN_WITH_CALLBACK;
+    if (!stream_state->cancel_error) {
+      stream_state->cancel_error = GRPC_ERROR_REF(stream_op->cancel_error);
+    }
   } else if (stream_op->on_complete &&
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_ON_COMPLETE)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_ON_COMPLETE", oas);
-    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
-        stream_state->state_callback_received[OP_FAILED]) {
+    if (stream_state->state_op_done[OP_CANCEL_ERROR]) {
       grpc_exec_ctx_sched(exec_ctx, stream_op->on_complete,
-                          GRPC_ERROR_CANCELLED, NULL);
+                          GRPC_ERROR_REF(stream_state->cancel_error), NULL);
+    } else if (stream_state->state_callback_received[OP_FAILED]) {
+      grpc_exec_ctx_sched(
+          exec_ctx, stream_op->on_complete,
+          make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."), NULL);
     } else {
       /* All actions in this stream_op are complete. Call the on_complete
        * callback
@@ -1012,6 +1076,15 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
       make a note */
     if (stream_op->recv_message)
       stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true;
+  } else if (stream_state->fail_state && !stream_state->flush_read) {
+    CRONET_LOG(GPR_DEBUG, "running: %p  flush read", oas);
+    if (stream_state->rs.read_buffer &&
+        stream_state->rs.read_buffer != stream_state->rs.grpc_header_bytes) {
+      gpr_free(stream_state->rs.read_buffer);
+      stream_state->rs.read_buffer = NULL;
+    }
+    stream_state->rs.read_buffer = gpr_malloc(4096);
+    stream_state->flush_read = true;
   } else {
     result = NO_ACTION_POSSIBLE;
   }
@@ -1037,6 +1110,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done));
   memset(s->state.state_callback_received, 0,
          sizeof(s->state.state_callback_received));
+  s->state.fail_state = s->state.flush_read = false;
+  s->state.cancel_error = NULL;
   gpr_mu_init(&s->mu);
   return 0;
 }
@@ -1082,7 +1157,10 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
 }
 
 static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
-                           grpc_stream *gs, void *and_free_memory) {}
+                           grpc_stream *gs, void *and_free_memory) {
+  stream_obj *s = (stream_obj *)gs;
+  GRPC_ERROR_UNREF(s->state.cancel_error);
+}
 
 static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {}
 

src/core/lib/channel/handshaker.c

@@ -38,12 +38,13 @@
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/iomgr/timer.h"
 
 //
 // grpc_handshaker
 //
 
-void grpc_handshaker_init(const struct grpc_handshaker_vtable* vtable,
+void grpc_handshaker_init(const grpc_handshaker_vtable* vtable,
                           grpc_handshaker* handshaker) {
   handshaker->vtable = vtable;
 }
@@ -60,45 +61,43 @@ void grpc_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
 
 void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx,
                                   grpc_handshaker* handshaker,
-                                  grpc_endpoint* endpoint,
-                                  grpc_channel_args* args,
-                                  grpc_slice_buffer* read_buffer,
-                                  gpr_timespec deadline,
                                   grpc_tcp_server_acceptor* acceptor,
-                                  grpc_handshaker_done_cb cb, void* user_data) {
-  handshaker->vtable->do_handshake(exec_ctx, handshaker, endpoint, args,
-                                   read_buffer, deadline, acceptor, cb,
-                                   user_data);
+                                  grpc_closure* on_handshake_done,
+                                  grpc_handshaker_args* args) {
+  handshaker->vtable->do_handshake(exec_ctx, handshaker, acceptor,
+                                   on_handshake_done, args);
 }
 
 //
 // grpc_handshake_manager
 //
 
-// State used while chaining handshakers.
-struct grpc_handshaker_state {
-  // The index of the handshaker to invoke next.
-  size_t index;
-  // The deadline for all handshakers.
-  gpr_timespec deadline;
-  // The acceptor to call the handshakers with.
-  grpc_tcp_server_acceptor* acceptor;
-  // The final callback and user_data to invoke after the last handshaker.
-  grpc_handshaker_done_cb final_cb;
-  void* final_user_data;
-};
-
 struct grpc_handshake_manager {
+  gpr_mu mu;
+  gpr_refcount refs;
+  bool shutdown;
   // An array of handshakers added via grpc_handshake_manager_add().
   size_t count;
   grpc_handshaker** handshakers;
-  // State used while chaining handshakers.
-  struct grpc_handshaker_state* state;
+  // The index of the handshaker to invoke next and closure to invoke it.
+  size_t index;
+  grpc_closure call_next_handshaker;
+  // The acceptor to call the handshakers with.
+  grpc_tcp_server_acceptor* acceptor;
+  // Deadline timer across all handshakers.
+  grpc_timer deadline_timer;
+  // The final callback and user_data to invoke after the last handshaker.
+  grpc_closure on_handshake_done;
+  void* user_data;
+  // Handshaker args.
+  grpc_handshaker_args args;
 };
 
 grpc_handshake_manager* grpc_handshake_manager_create() {
   grpc_handshake_manager* mgr = gpr_malloc(sizeof(grpc_handshake_manager));
   memset(mgr, 0, sizeof(*mgr));
+  gpr_mu_init(&mgr->mu);
+  gpr_ref_init(&mgr->refs, 1);
   return mgr;
 }
 
@@ -106,6 +105,7 @@ static bool is_power_of_2(size_t n) { return (n & (n - 1)) == 0; }
 
 void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
                                 grpc_handshaker* handshaker) {
+  gpr_mu_lock(&mgr->mu);
   // To avoid allocating memory for each handshaker we add, we double
   // the number of elements every time we need more.
   size_t realloc_count = 0;
@@ -119,85 +119,117 @@ void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
         gpr_realloc(mgr->handshakers, realloc_count * sizeof(grpc_handshaker*));
   }
   mgr->handshakers[mgr->count++] = handshaker;
+  gpr_mu_unlock(&mgr->mu);
+}
+
+static void grpc_handshake_manager_unref(grpc_exec_ctx* exec_ctx,
+                                         grpc_handshake_manager* mgr) {
+  if (gpr_unref(&mgr->refs)) {
+    for (size_t i = 0; i < mgr->count; ++i) {
+      grpc_handshaker_destroy(exec_ctx, mgr->handshakers[i]);
+    }
+    gpr_free(mgr->handshakers);
+    gpr_mu_destroy(&mgr->mu);
+    gpr_free(mgr);
+  }
 }
 
 void grpc_handshake_manager_destroy(grpc_exec_ctx* exec_ctx,
                                     grpc_handshake_manager* mgr) {
-  for (size_t i = 0; i < mgr->count; ++i) {
-    grpc_handshaker_destroy(exec_ctx, mgr->handshakers[i]);
-  }
-  gpr_free(mgr->handshakers);
-  gpr_free(mgr);
+  grpc_handshake_manager_unref(exec_ctx, mgr);
 }
 
 void grpc_handshake_manager_shutdown(grpc_exec_ctx* exec_ctx,
                                      grpc_handshake_manager* mgr) {
-  for (size_t i = 0; i < mgr->count; ++i) {
-    grpc_handshaker_shutdown(exec_ctx, mgr->handshakers[i]);
+  gpr_mu_lock(&mgr->mu);
+  // Shutdown the handshaker that's currently in progress, if any.
+  if (!mgr->shutdown && mgr->index > 0) {
+    mgr->shutdown = true;
+    grpc_handshaker_shutdown(exec_ctx, mgr->handshakers[mgr->index - 1]);
   }
-  if (mgr->state != NULL) {
-    gpr_free(mgr->state);
-    mgr->state = NULL;
+  gpr_mu_unlock(&mgr->mu);
+}
+
+// Helper function to call either the next handshaker or the
+// on_handshake_done callback.
+// Returns true if we've scheduled the on_handshake_done callback.
+static bool call_next_handshaker_locked(grpc_exec_ctx* exec_ctx,
+                                        grpc_handshake_manager* mgr,
+                                        grpc_error* error) {
+  GPR_ASSERT(mgr->index <= mgr->count);
+  // If we got an error or we've been shut down or we're exiting early or
+  // we've finished the last handshaker, invoke the on_handshake_done
+  // callback.  Otherwise, call the next handshaker.
+  if (error != GRPC_ERROR_NONE || mgr->shutdown || mgr->args.exit_early ||
+      mgr->index == mgr->count) {
+    // Cancel deadline timer, since we're invoking the on_handshake_done
+    // callback now.
+    grpc_timer_cancel(exec_ctx, &mgr->deadline_timer);
+    grpc_exec_ctx_sched(exec_ctx, &mgr->on_handshake_done, error, NULL);
+    mgr->shutdown = true;
+  } else {
+    grpc_handshaker_do_handshake(exec_ctx, mgr->handshakers[mgr->index],
+                                 mgr->acceptor, &mgr->call_next_handshaker,
+                                 &mgr->args);
   }
+  ++mgr->index;
+  return mgr->shutdown;
 }
 
 // A function used as the handshaker-done callback when chaining
 // handshakers together.
-static void call_next_handshaker(grpc_exec_ctx* exec_ctx,
-                                 grpc_endpoint* endpoint,
-                                 grpc_channel_args* args,
-                                 grpc_slice_buffer* read_buffer,
-                                 void* user_data, grpc_error* error) {
-  grpc_handshake_manager* mgr = user_data;
-  GPR_ASSERT(mgr->state != NULL);
-  GPR_ASSERT(mgr->state->index < mgr->count);
-  // If we got an error, skip all remaining handshakers and invoke the
-  // caller-supplied callback immediately.
-  if (error != GRPC_ERROR_NONE) {
-    mgr->state->final_cb(exec_ctx, endpoint, args, read_buffer,
-                         mgr->state->final_user_data, error);
-    return;
+static void call_next_handshaker(grpc_exec_ctx* exec_ctx, void* arg,
+                                 grpc_error* error) {
+  grpc_handshake_manager* mgr = arg;
+  gpr_mu_lock(&mgr->mu);
+  bool done = call_next_handshaker_locked(exec_ctx, mgr, GRPC_ERROR_REF(error));
+  gpr_mu_unlock(&mgr->mu);
+  // If we're invoked the final callback, we won't be coming back
+  // to this function, so we can release our reference to the
+  // handshake manager.
+  if (done) {
+    grpc_handshake_manager_unref(exec_ctx, mgr);
   }
-  grpc_handshaker_done_cb cb = call_next_handshaker;
-  // If this is the last handshaker, use the caller-supplied callback
-  // and user_data instead of chaining back to this function again.
-  if (mgr->state->index == mgr->count - 1) {
-    cb = mgr->state->final_cb;
-    user_data = mgr->state->final_user_data;
-  }
-  // Invoke handshaker.
-  grpc_handshaker_do_handshake(
-      exec_ctx, mgr->handshakers[mgr->state->index], endpoint, args,
-      read_buffer, mgr->state->deadline, mgr->state->acceptor, cb, user_data);
-  ++mgr->state->index;
-  // If this is the last handshaker, clean up state.
-  if (mgr->state->index == mgr->count) {
-    gpr_free(mgr->state);
-    mgr->state = NULL;
+}
+
+// Callback invoked when deadline is exceeded.
+static void on_timeout(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
+  grpc_handshake_manager* mgr = arg;
+  if (error == GRPC_ERROR_NONE) {  // Timer fired, rather than being cancelled.
+    grpc_handshake_manager_shutdown(exec_ctx, mgr);
   }
+  grpc_handshake_manager_unref(exec_ctx, mgr);
 }
 
 void grpc_handshake_manager_do_handshake(
     grpc_exec_ctx* exec_ctx, grpc_handshake_manager* mgr,
-    grpc_endpoint* endpoint, const grpc_channel_args* args,
+    grpc_endpoint* endpoint, const grpc_channel_args* channel_args,
     gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor,
-    grpc_handshaker_done_cb cb, void* user_data) {
-  grpc_channel_args* args_copy = grpc_channel_args_copy(args);
-  grpc_slice_buffer* read_buffer = gpr_malloc(sizeof(*read_buffer));
-  grpc_slice_buffer_init(read_buffer);
-  if (mgr->count == 0) {
-    // No handshakers registered, so we just immediately call the done
-    // callback with the passed-in endpoint.
-    cb(exec_ctx, endpoint, args_copy, read_buffer, user_data, GRPC_ERROR_NONE);
-  } else {
-    GPR_ASSERT(mgr->state == NULL);
-    mgr->state = gpr_malloc(sizeof(struct grpc_handshaker_state));
-    memset(mgr->state, 0, sizeof(*mgr->state));
-    mgr->state->deadline = deadline;
-    mgr->state->acceptor = acceptor;
-    mgr->state->final_cb = cb;
-    mgr->state->final_user_data = user_data;
-    call_next_handshaker(exec_ctx, endpoint, args_copy, read_buffer, mgr,
-                         GRPC_ERROR_NONE);
+    grpc_iomgr_cb_func on_handshake_done, void* user_data) {
+  gpr_mu_lock(&mgr->mu);
+  GPR_ASSERT(mgr->index == 0);
+  GPR_ASSERT(!mgr->shutdown);
+  // Construct handshaker args.  These will be passed through all
+  // handshakers and eventually be freed by the on_handshake_done callback.
+  mgr->args.endpoint = endpoint;
+  mgr->args.args = grpc_channel_args_copy(channel_args);
+  mgr->args.user_data = user_data;
+  mgr->args.read_buffer = gpr_malloc(sizeof(*mgr->args.read_buffer));
+  grpc_slice_buffer_init(mgr->args.read_buffer);
+  // Initialize state needed for calling handshakers.
+  mgr->acceptor = acceptor;
+  grpc_closure_init(&mgr->call_next_handshaker, call_next_handshaker, mgr);
+  grpc_closure_init(&mgr->on_handshake_done, on_handshake_done, &mgr->args);
+  // Start deadline timer, which owns a ref.
+  gpr_ref(&mgr->refs);
+  grpc_timer_init(exec_ctx, &mgr->deadline_timer,
+                  gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC),
+                  on_timeout, mgr, gpr_now(GPR_CLOCK_MONOTONIC));
+  // Start first handshaker, which also owns a ref.
+  gpr_ref(&mgr->refs);
+  bool done = call_next_handshaker_locked(exec_ctx, mgr, GRPC_ERROR_NONE);
+  gpr_mu_unlock(&mgr->mu);
+  if (done) {
+    grpc_handshake_manager_unref(exec_ctx, mgr);
   }
 }

src/core/lib/channel/handshaker.h

@@ -54,15 +54,33 @@
 
 typedef struct grpc_handshaker grpc_handshaker;
 
-/// Callback type invoked when a handshaker is done.
-/// Takes ownership of \a args and \a read_buffer.
-typedef void (*grpc_handshaker_done_cb)(grpc_exec_ctx* exec_ctx,
-                                        grpc_endpoint* endpoint,
-                                        grpc_channel_args* args,
-                                        grpc_slice_buffer* read_buffer,
-                                        void* user_data, grpc_error* error);
-
-struct grpc_handshaker_vtable {
+/// Arguments passed through handshakers and to the on_handshake_done callback.
+///
+/// For handshakers, all members are input/output parameters; for
+/// example, a handshaker may read from or write to \a endpoint and
+/// then later replace it with a wrapped endpoint.  Similarly, a
+/// handshaker may modify \a args.
+///
+/// A handshaker takes ownership of the members while a handshake is in
+/// progress.  Upon failure or shutdown of an in-progress handshaker,
+/// the handshaker is responsible for destroying the members and setting
+/// them to NULL before invoking the on_handshake_done callback.
+///
+/// For the on_handshake_done callback, all members are input arguments,
+/// which the callback takes ownership of.
+typedef struct {
+  grpc_endpoint* endpoint;
+  grpc_channel_args* args;
+  grpc_slice_buffer* read_buffer;
+  // A handshaker may set this to true before invoking on_handshake_done
+  // to indicate that subsequent handshakers should be skipped.
+  bool exit_early;
+  // User data passed through the handshake manager.  Not used by
+  // individual handshakers.
+  void* user_data;
+} grpc_handshaker_args;
+
+typedef struct {
   /// Destroys the handshaker.
   void (*destroy)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker);
 
@@ -70,43 +88,35 @@ struct grpc_handshaker_vtable {
   /// aborted in the middle).
   void (*shutdown)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker);
 
-  /// Performs handshaking.  When finished, calls \a cb with \a user_data.
-  /// Takes ownership of \a args.
-  /// Takes ownership of \a read_buffer, which contains leftover bytes read
-  /// from the endpoint by the previous handshaker.
+  /// Performs handshaking, modifying \a args as needed (e.g., to
+  /// replace \a endpoint with a wrapped endpoint).
+  /// When finished, invokes \a on_handshake_done.
   /// \a acceptor will be NULL for client-side handshakers.
   void (*do_handshake)(grpc_exec_ctx* exec_ctx, grpc_handshaker* handshaker,
-                       grpc_endpoint* endpoint, grpc_channel_args* args,
-                       grpc_slice_buffer* read_buffer, gpr_timespec deadline,
                        grpc_tcp_server_acceptor* acceptor,
-                       grpc_handshaker_done_cb cb, void* user_data);
-};
+                       grpc_closure* on_handshake_done,
+                       grpc_handshaker_args* args);
+} grpc_handshaker_vtable;
 
 /// Base struct.  To subclass, make this the first member of the
 /// implementation struct.
 struct grpc_handshaker {
-  const struct grpc_handshaker_vtable* vtable;
+  const grpc_handshaker_vtable* vtable;
 };
 
 /// Called by concrete implementations to initialize the base struct.
-void grpc_handshaker_init(const struct grpc_handshaker_vtable* vtable,
+void grpc_handshaker_init(const grpc_handshaker_vtable* vtable,
                           grpc_handshaker* handshaker);
 
-/// Convenient wrappers for invoking methods via the vtable.
-/// These probably do not need to be called from anywhere but
-/// grpc_handshake_manager.
 void grpc_handshaker_destroy(grpc_exec_ctx* exec_ctx,
                              grpc_handshaker* handshaker);
 void grpc_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
                               grpc_handshaker* handshaker);
 void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx,
                                   grpc_handshaker* handshaker,
-                                  grpc_endpoint* endpoint,
-                                  grpc_channel_args* args,
-                                  grpc_slice_buffer* read_buffer,
-                                  gpr_timespec deadline,
                                   grpc_tcp_server_acceptor* acceptor,
-                                  grpc_handshaker_done_cb cb, void* user_data);
+                                  grpc_closure* on_handshake_done,
+                                  grpc_handshaker_args* args);
 
 ///
 /// grpc_handshake_manager
@@ -134,15 +144,21 @@ void grpc_handshake_manager_shutdown(grpc_exec_ctx* exec_ctx,
                                      grpc_handshake_manager* mgr);
 
 /// Invokes handshakers in the order they were added.
-/// Does NOT take ownership of \a args.  Instead, makes a copy before
+/// Takes ownership of \a endpoint, and then passes that ownership to
+/// the \a on_handshake_done callback.
+/// Does NOT take ownership of \a channel_args.  Instead, makes a copy before
 /// invoking the first handshaker.
 /// \a acceptor will be NULL for client-side handshakers.
-/// Invokes \a cb with \a user_data after either a handshaker fails or
-/// all handshakers have completed successfully.
+///
+/// When done, invokes \a on_handshake_done with a grpc_handshaker_args
+/// object as its argument.  If the callback is invoked with error !=
+/// GRPC_ERROR_NONE, then handshaking failed and the handshaker has done
+/// the necessary clean-up.  Otherwise, the callback takes ownership of
+/// the arguments.
 void grpc_handshake_manager_do_handshake(
     grpc_exec_ctx* exec_ctx, grpc_handshake_manager* mgr,
-    grpc_endpoint* endpoint, const grpc_channel_args* args,
+    grpc_endpoint* endpoint, const grpc_channel_args* channel_args,
     gpr_timespec deadline, grpc_tcp_server_acceptor* acceptor,
-    grpc_handshaker_done_cb cb, void* user_data);
+    grpc_iomgr_cb_func on_handshake_done, void* user_data);
 
 #endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_H */

src/core/lib/channel/http_client_filter.c

@@ -245,12 +245,15 @@ static void hc_mutate_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     message, and the payload is below the size threshold, and all the data
     for this request is immediately available. */
     grpc_mdelem *method = GRPC_MDELEM_METHOD_POST;
-    calld->send_message_blocked = false;
     if ((op->send_initial_metadata_flags &
          GRPC_INITIAL_METADATA_CACHEABLE_REQUEST) &&
         op->send_message != NULL &&
         op->send_message->length < channeld->max_payload_size_for_get) {
       method = GRPC_MDELEM_METHOD_GET;
+      /* The following write to calld->send_message_blocked isn't racy with
+      reads in hc_start_transport_op (which deals with SEND_MESSAGE ops) because
+      being here means ops->send_message is not NULL, which is primarily
+      guarding the read there. */
       calld->send_message_blocked = true;
     } else if (op->send_initial_metadata_flags &
                GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST) {
@@ -331,8 +334,7 @@ static void hc_start_transport_op(grpc_exec_ctx *exec_ctx,
   call_data *calld = elem->call_data;
   if (op->send_message != NULL && calld->send_message_blocked) {
     /* Don't forward the op. send_message contains slices that aren't ready
-    yet. The call will be forwarded by the op_complete of slice read call.
-    */
+    yet. The call will be forwarded by the op_complete of slice read call. */
   } else {
     grpc_call_next_op(exec_ctx, elem, op);
   }
@@ -347,6 +349,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   calld->on_done_recv_trailing_metadata = NULL;
   calld->on_complete = NULL;
   calld->payload_bytes = NULL;
+  calld->send_message_blocked = false;
   grpc_slice_buffer_init(&calld->slices);
   grpc_closure_init(&calld->hc_on_recv_initial_metadata,
                     hc_on_recv_initial_metadata, elem);

src/core/lib/http/httpcli_security_connector.c

@@ -38,7 +38,9 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/security/transport/handshake.h"
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/security/transport/security_handshaker.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
 
@@ -58,52 +60,43 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) {
   gpr_free(sc);
 }
 
-static void httpcli_ssl_do_handshake(grpc_exec_ctx *exec_ctx,
-                                     grpc_channel_security_connector *sc,
-                                     grpc_endpoint *nonsecure_endpoint,
-                                     grpc_slice_buffer *read_buffer,
-                                     gpr_timespec deadline,
-                                     grpc_security_handshake_done_cb cb,
-                                     void *user_data) {
+static void httpcli_ssl_add_handshakers(grpc_exec_ctx *exec_ctx,
+                                        grpc_channel_security_connector *sc,
+                                        grpc_handshake_manager *handshake_mgr) {
   grpc_httpcli_ssl_channel_security_connector *c =
       (grpc_httpcli_ssl_channel_security_connector *)sc;
-  tsi_result result = TSI_OK;
-  tsi_handshaker *handshaker;
-  if (c->handshaker_factory == NULL) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-    return;
-  }
-  result = tsi_ssl_handshaker_factory_create_handshaker(
-      c->handshaker_factory, c->secure_peer_name, &handshaker);
-  if (result != TSI_OK) {
-    gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.",
-            tsi_result_to_string(result));
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-  } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, read_buffer, deadline, cb,
-                               user_data);
+  tsi_handshaker *handshaker = NULL;
+  if (c->handshaker_factory != NULL) {
+    tsi_result result = tsi_ssl_handshaker_factory_create_handshaker(
+        c->handshaker_factory, c->secure_peer_name, &handshaker);
+    if (result != TSI_OK) {
+      gpr_log(GPR_ERROR, "Handshaker creation failed with error %s.",
+              tsi_result_to_string(result));
+    }
   }
+  grpc_handshake_manager_add(
+      handshake_mgr,
+      grpc_security_handshaker_create(exec_ctx, handshaker, &sc->base));
 }
 
 static void httpcli_ssl_check_peer(grpc_exec_ctx *exec_ctx,
                                    grpc_security_connector *sc, tsi_peer peer,
-                                   grpc_security_peer_check_cb cb,
-                                   void *user_data) {
+                                   grpc_auth_context **auth_context,
+                                   grpc_closure *on_peer_checked) {
   grpc_httpcli_ssl_channel_security_connector *c =
       (grpc_httpcli_ssl_channel_security_connector *)sc;
-  grpc_security_status status = GRPC_SECURITY_OK;
+  grpc_error *error = GRPC_ERROR_NONE;
 
   /* Check the peer name. */
   if (c->secure_peer_name != NULL &&
       !tsi_ssl_peer_matches_name(&peer, c->secure_peer_name)) {
-    gpr_log(GPR_ERROR, "Peer name %s is not in peer certificate",
-            c->secure_peer_name);
-    status = GRPC_SECURITY_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "Peer name %s is not in peer certificate",
+                 c->secure_peer_name);
+    error = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
   }
-  cb(exec_ctx, user_data, status, NULL);
+  grpc_exec_ctx_sched(exec_ctx, on_peer_checked, error, NULL);
   tsi_peer_destruct(&peer);
 }
 
@@ -140,7 +133,7 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create(
     *sc = NULL;
     return GRPC_SECURITY_ERROR;
   }
-  c->base.do_handshake = httpcli_ssl_do_handshake;
+  c->base.add_handshakers = httpcli_ssl_add_handshakers;
   *sc = &c->base;
   return GRPC_SECURITY_OK;
 }
@@ -150,19 +143,25 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create(
 typedef struct {
   void (*func)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint);
   void *arg;
+  grpc_handshake_manager *handshake_mgr;
 } on_done_closure;
 
-static void on_secure_transport_setup_done(grpc_exec_ctx *exec_ctx, void *rp,
-                                           grpc_security_status status,
-                                           grpc_endpoint *secure_endpoint,
-                                           grpc_auth_context *auth_context) {
-  on_done_closure *c = rp;
-  if (status != GRPC_SECURITY_OK) {
-    gpr_log(GPR_ERROR, "Secure transport setup failed with error %d.", status);
+static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error) {
+  grpc_handshaker_args *args = arg;
+  on_done_closure *c = args->user_data;
+  if (error != GRPC_ERROR_NONE) {
+    const char *msg = grpc_error_string(error);
+    gpr_log(GPR_ERROR, "Secure transport setup failed: %s", msg);
+    grpc_error_free_string(msg);
     c->func(exec_ctx, c->arg, NULL);
   } else {
-    c->func(exec_ctx, c->arg, secure_endpoint);
+    grpc_channel_args_destroy(args->args);
+    grpc_slice_buffer_destroy(args->read_buffer);
+    gpr_free(args->read_buffer);
+    c->func(exec_ctx, c->arg, args->endpoint);
   }
+  grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
   gpr_free(c);
 }
 
@@ -183,11 +182,15 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
   }
   c->func = on_done;
   c->arg = arg;
+  c->handshake_mgr = grpc_handshake_manager_create();
   GPR_ASSERT(httpcli_ssl_channel_security_connector_create(
                  pem_root_certs, pem_root_certs_size, host, &sc) ==
              GRPC_SECURITY_OK);
-  grpc_channel_security_connector_do_handshake(
-      exec_ctx, sc, tcp, NULL, deadline, on_secure_transport_setup_done, c);
+  grpc_channel_security_connector_add_handshakers(exec_ctx, sc,
+                                                  c->handshake_mgr);
+  grpc_handshake_manager_do_handshake(
+      exec_ctx, c->handshake_mgr, tcp, NULL /* channel_args */, deadline,
+      NULL /* acceptor */, on_handshake_done, c /* user_data */);
   GRPC_SECURITY_CONNECTOR_UNREF(&sc->base, "httpcli");
 }
 

src/core/lib/iomgr/combiner.c

@@ -90,6 +90,12 @@ static bool is_covered_by_poller(grpc_combiner *lock) {
          gpr_atm_acq_load(&lock->elements_covered_by_poller) > 0;
 }
 
+#define IS_COVERED_BY_POLLER_FMT "(final=%d elems=%" PRIdPTR ")->%d"
+#define IS_COVERED_BY_POLLER_ARGS(lock)                      \
+  (lock)->final_list_covered_by_poller,                      \
+      gpr_atm_acq_load(&(lock)->elements_covered_by_poller), \
+      is_covered_by_poller((lock))
+
 grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue) {
   grpc_combiner *lock = gpr_malloc(sizeof(*lock));
   lock->next_combiner_on_this_exec_ctx = NULL;
@@ -197,9 +203,10 @@ bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx) {
   GRPC_COMBINER_TRACE(
       gpr_log(GPR_DEBUG,
               "C:%p grpc_combiner_continue_exec_ctx workqueue=%p "
-              "is_covered_by_poller=%d exec_ctx_ready_to_finish=%d "
+              "is_covered_by_poller=" IS_COVERED_BY_POLLER_FMT
+              " exec_ctx_ready_to_finish=%d "
               "time_to_execute_final_list=%d",
-              lock, lock->optional_workqueue, is_covered_by_poller(lock),
+              lock, lock->optional_workqueue, IS_COVERED_BY_POLLER_ARGS(lock),
               grpc_exec_ctx_ready_to_finish(exec_ctx),
               lock->time_to_execute_final_list));
 

src/core/lib/iomgr/ev_epoll_linux.c

@@ -69,6 +69,9 @@ static int grpc_polling_trace = 0; /* Disabled by default */
     gpr_log(GPR_INFO, (fmt), __VA_ARGS__); \
   }
 
+/* Uncomment the following enable extra checks on poll_object operations */
+/* #define PO_DEBUG */
+
 static int grpc_wakeup_signal = -1;
 static bool is_grpc_wakeup_signal_initialized = false;
 
@@ -95,10 +98,42 @@ void grpc_use_signal(int signum) {
 
 struct polling_island;
 
+typedef enum {
+  POLL_OBJ_FD,
+  POLL_OBJ_POLLSET,
+  POLL_OBJ_POLLSET_SET
+} poll_obj_type;
+
+typedef struct poll_obj {
+#ifdef PO_DEBUG
+  poll_obj_type obj_type;
+#endif
+  gpr_mu mu;
+  struct polling_island *pi;
+} poll_obj;
+
+const char *poll_obj_string(poll_obj_type po_type) {
+  switch (po_type) {
+    case POLL_OBJ_FD:
+      return "fd";
+    case POLL_OBJ_POLLSET:
+      return "pollset";
+    case POLL_OBJ_POLLSET_SET:
+      return "pollset_set";
+  }
+
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
 /*******************************************************************************
  * Fd Declarations
  */
+
+#define FD_FROM_PO(po) ((grpc_fd *)(po))
+
 struct grpc_fd {
+  poll_obj po;
+
   int fd;
   /* refst format:
        bit 0    : 1=Active / 0=Orphaned
@@ -106,8 +141,6 @@ struct grpc_fd {
      Ref/Unref by two to avoid altering the orphaned bit */
   gpr_atm refst;
 
-  gpr_mu mu;
-
   /* Indicates that the fd is shutdown and that any pending read/write closures
      should fail */
   bool shutdown;
@@ -120,9 +153,6 @@ struct grpc_fd {
   grpc_closure *read_closure;
   grpc_closure *write_closure;
 
-  /* The polling island to which this fd belongs to (protected by mu) */
-  struct polling_island *polling_island;
-
   struct grpc_fd *freelist_next;
   grpc_closure *on_done_closure;
 
@@ -225,41 +255,21 @@ struct grpc_pollset_worker {
 };
 
 struct grpc_pollset {
-  gpr_mu mu;
+  poll_obj po;
+
   grpc_pollset_worker root_worker;
   bool kicked_without_pollers;
 
   bool shutting_down;          /* Is the pollset shutting down ? */
   bool finish_shutdown_called; /* Is the 'finish_shutdown_locked()' called ? */
   grpc_closure *shutdown_done; /* Called after after shutdown is complete */
-
-  /* The polling island to which this pollset belongs to */
-  struct polling_island *polling_island;
 };
 
 /*******************************************************************************
  * Pollset-set Declarations
  */
-/* TODO: sreek - Change the pollset_set implementation such that a pollset_set
- * directly points to a polling_island (and adding an fd/pollset/pollset_set to
- * the current pollset_set would result in polling island merges. This would
- * remove the need to maintain fd_count here. This will also significantly
- * simplify the grpc_fd structure since we would no longer need to explicitly
- * maintain the orphaned state */
 struct grpc_pollset_set {
-  gpr_mu mu;
-
-  size_t pollset_count;
-  size_t pollset_capacity;
-  grpc_pollset **pollsets;
-
-  size_t pollset_set_count;
-  size_t pollset_set_capacity;
-  struct grpc_pollset_set **pollset_sets;
-
-  size_t fd_count;
-  size_t fd_capacity;
-  grpc_fd **fds;
+  poll_obj po;
 };
 
 /*******************************************************************************
@@ -915,7 +925,7 @@ static void fd_global_shutdown(void) {
   while (fd_freelist != NULL) {
     grpc_fd *fd = fd_freelist;
     fd_freelist = fd_freelist->freelist_next;
-    gpr_mu_destroy(&fd->mu);
+    gpr_mu_destroy(&fd->po.mu);
     gpr_free(fd);
   }
   gpr_mu_destroy(&fd_freelist_mu);
@@ -933,13 +943,17 @@ static grpc_fd *fd_create(int fd, const char *name) {
 
   if (new_fd == NULL) {
     new_fd = gpr_malloc(sizeof(grpc_fd));
-    gpr_mu_init(&new_fd->mu);
+    gpr_mu_init(&new_fd->po.mu);
   }
 
-  /* Note: It is not really needed to get the new_fd->mu lock here. If this is a
-     newly created fd (or an fd we got from the freelist), no one else would be
-     holding a lock to it anyway. */
-  gpr_mu_lock(&new_fd->mu);
+  /* Note: It is not really needed to get the new_fd->po.mu lock here. If this
+   * is a newly created fd (or an fd we got from the freelist), no one else
+   * would be holding a lock to it anyway. */
+  gpr_mu_lock(&new_fd->po.mu);
+  new_fd->po.pi = NULL;
+#ifdef PO_DEBUG
+  new_fd->po.obj_type = POLL_OBJ_FD;
+#endif
 
   gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
   new_fd->fd = fd;
@@ -947,12 +961,11 @@ static grpc_fd *fd_create(int fd, const char *name) {
   new_fd->orphaned = false;
   new_fd->read_closure = CLOSURE_NOT_READY;
   new_fd->write_closure = CLOSURE_NOT_READY;
-  new_fd->polling_island = NULL;
   new_fd->freelist_next = NULL;
   new_fd->on_done_closure = NULL;
   new_fd->read_notifier_pollset = NULL;
 
-  gpr_mu_unlock(&new_fd->mu);
+  gpr_mu_unlock(&new_fd->po.mu);
 
   char *fd_name;
   gpr_asprintf(&fd_name, "%s fd=%d", name, fd);
@@ -964,17 +977,13 @@ static grpc_fd *fd_create(int fd, const char *name) {
   return new_fd;
 }
 
-static bool fd_is_orphaned(grpc_fd *fd) {
-  return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
-}
-
 static int fd_wrapped_fd(grpc_fd *fd) {
   int ret_fd = -1;
-  gpr_mu_lock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
   if (!fd->orphaned) {
     ret_fd = fd->fd;
   }
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
 
   return ret_fd;
 }
@@ -986,7 +995,7 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   grpc_error *error = GRPC_ERROR_NONE;
   polling_island *unref_pi = NULL;
 
-  gpr_mu_lock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
   fd->on_done_closure = on_done;
 
   /* If release_fd is not NULL, we should be relinquishing control of the file
@@ -1006,25 +1015,25 @@ static void fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 
   /* Remove the fd from the polling island:
      - Get a lock on the latest polling island (i.e the last island in the
-       linked list pointed by fd->polling_island). This is the island that
+       linked list pointed by fd->po.pi). This is the island that
        would actually contain the fd
      - Remove the fd from the latest polling island
      - Unlock the latest polling island
-     - Set fd->polling_island to NULL (but remove the ref on the polling island
+     - Set fd->po.pi to NULL (but remove the ref on the polling island
        before doing this.) */
-  if (fd->polling_island != NULL) {
-    polling_island *pi_latest = polling_island_lock(fd->polling_island);
+  if (fd->po.pi != NULL) {
+    polling_island *pi_latest = polling_island_lock(fd->po.pi);
     polling_island_remove_fd_locked(pi_latest, fd, is_fd_closed, &error);
     gpr_mu_unlock(&pi_latest->mu);
 
-    unref_pi = fd->polling_island;
-    fd->polling_island = NULL;
+    unref_pi = fd->po.pi;
+    fd->po.pi = NULL;
   }
 
   grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_REF(error),
                       NULL);
 
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
   UNREF_BY(fd, 2, reason); /* Drop the reference */
   if (unref_pi != NULL) {
     /* Unref stale polling island here, outside the fd lock above.
@@ -1089,23 +1098,23 @@ static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
                                                   grpc_fd *fd) {
   grpc_pollset *notifier = NULL;
 
-  gpr_mu_lock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
   notifier = fd->read_notifier_pollset;
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
 
   return notifier;
 }
 
 static bool fd_is_shutdown(grpc_fd *fd) {
-  gpr_mu_lock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
   const bool r = fd->shutdown;
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
   return r;
 }
 
 /* Might be called multiple times */
 static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  gpr_mu_lock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
   /* Do the actual shutdown only once */
   if (!fd->shutdown) {
     fd->shutdown = true;
@@ -1116,28 +1125,28 @@ static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
     set_ready_locked(exec_ctx, fd, &fd->read_closure);
     set_ready_locked(exec_ctx, fd, &fd->write_closure);
   }
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
 }
 
 static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                               grpc_closure *closure) {
-  gpr_mu_lock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
   notify_on_locked(exec_ctx, fd, &fd->read_closure, closure);
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
 }
 
 static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_closure *closure) {
-  gpr_mu_lock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
   notify_on_locked(exec_ctx, fd, &fd->write_closure, closure);
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
 }
 
 static grpc_workqueue *fd_get_workqueue(grpc_fd *fd) {
-  gpr_mu_lock(&fd->mu);
-  grpc_workqueue *workqueue = GRPC_WORKQUEUE_REF(
-      (grpc_workqueue *)fd->polling_island, "fd_get_workqueue");
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
+  grpc_workqueue *workqueue =
+      GRPC_WORKQUEUE_REF((grpc_workqueue *)fd->po.pi, "fd_get_workqueue");
+  gpr_mu_unlock(&fd->po.mu);
   return workqueue;
 }
 
@@ -1277,8 +1286,12 @@ static grpc_error *kick_poller(void) {
 }
 
 static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
-  gpr_mu_init(&pollset->mu);
-  *mu = &pollset->mu;
+  gpr_mu_init(&pollset->po.mu);
+  *mu = &pollset->po.mu;
+  pollset->po.pi = NULL;
+#ifdef PO_DEBUG
+  pollset->po.obj_type = POLL_OBJ_POLLSET;
+#endif
 
   pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
   pollset->kicked_without_pollers = false;
@@ -1286,8 +1299,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   pollset->shutting_down = false;
   pollset->finish_shutdown_called = false;
   pollset->shutdown_done = NULL;
-
-  pollset->polling_island = NULL;
 }
 
 /* Convert a timespec to milliseconds:
@@ -1317,26 +1328,26 @@ static int poll_deadline_to_millis_timeout(gpr_timespec deadline,
 
 static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                                grpc_pollset *notifier) {
-  /* Need the fd->mu since we might be racing with fd_notify_on_read */
-  gpr_mu_lock(&fd->mu);
+  /* Need the fd->po.mu since we might be racing with fd_notify_on_read */
+  gpr_mu_lock(&fd->po.mu);
   set_ready_locked(exec_ctx, fd, &fd->read_closure);
   fd->read_notifier_pollset = notifier;
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  /* Need the fd->mu since we might be racing with fd_notify_on_write */
-  gpr_mu_lock(&fd->mu);
+  /* Need the fd->po.mu since we might be racing with fd_notify_on_write */
+  gpr_mu_lock(&fd->po.mu);
   set_ready_locked(exec_ctx, fd, &fd->write_closure);
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_unlock(&fd->po.mu);
 }
 
 static void pollset_release_polling_island(grpc_exec_ctx *exec_ctx,
                                            grpc_pollset *ps, char *reason) {
-  if (ps->polling_island != NULL) {
-    PI_UNREF(exec_ctx, ps->polling_island, reason);
+  if (ps->po.pi != NULL) {
+    PI_UNREF(exec_ctx, ps->po.pi, reason);
   }
-  ps->polling_island = NULL;
+  ps->po.pi = NULL;
 }
 
 static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx,
@@ -1346,12 +1357,12 @@ static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx,
 
   pollset->finish_shutdown_called = true;
 
-  /* Release the ref and set pollset->polling_island to NULL */
+  /* Release the ref and set pollset->po.pi to NULL */
   pollset_release_polling_island(exec_ctx, pollset, "ps_shutdown");
   grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
 }
 
-/* pollset->mu lock must be held by the caller before calling this */
+/* pollset->po.mu lock must be held by the caller before calling this */
 static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                              grpc_closure *closure) {
   GPR_TIMER_BEGIN("pollset_shutdown", 0);
@@ -1376,7 +1387,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
  * here */
 static void pollset_destroy(grpc_pollset *pollset) {
   GPR_ASSERT(!pollset_has_workers(pollset));
-  gpr_mu_destroy(&pollset->mu);
+  gpr_mu_destroy(&pollset->po.mu);
 }
 
 static void pollset_reset(grpc_pollset *pollset) {
@@ -1386,7 +1397,7 @@ static void pollset_reset(grpc_pollset *pollset) {
   pollset->finish_shutdown_called = false;
   pollset->kicked_without_pollers = false;
   pollset->shutdown_done = NULL;
-  GPR_ASSERT(pollset->polling_island == NULL);
+  GPR_ASSERT(pollset->po.pi == NULL);
 }
 
 static bool maybe_do_workqueue_work(grpc_exec_ctx *exec_ctx,
@@ -1426,7 +1437,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
   GPR_TIMER_BEGIN("pollset_work_and_unlock", 0);
 
   /* We need to get the epoll_fd to wait on. The epoll_fd is in inside the
-     latest polling island pointed by pollset->polling_island.
+     latest polling island pointed by pollset->po.pi
 
      Since epoll_fd is immutable, we can read it without obtaining the polling
      island lock. There is however a possibility that the polling island (from
@@ -1435,36 +1446,36 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
      right-away from epoll_wait() and pick up the latest polling_island the next
      this function (i.e pollset_work_and_unlock()) is called */
 
-  if (pollset->polling_island == NULL) {
-    pollset->polling_island = polling_island_create(exec_ctx, NULL, error);
-    if (pollset->polling_island == NULL) {
+  if (pollset->po.pi == NULL) {
+    pollset->po.pi = polling_island_create(exec_ctx, NULL, error);
+    if (pollset->po.pi == NULL) {
       GPR_TIMER_END("pollset_work_and_unlock", 0);
       return; /* Fatal error. We cannot continue */
     }
 
-    PI_ADD_REF(pollset->polling_island, "ps");
+    PI_ADD_REF(pollset->po.pi, "ps");
     GRPC_POLLING_TRACE("pollset_work: pollset: %p created new pi: %p",
-                       (void *)pollset, (void *)pollset->polling_island);
+                       (void *)pollset, (void *)pollset->po.pi);
   }
 
-  pi = polling_island_maybe_get_latest(pollset->polling_island);
+  pi = polling_island_maybe_get_latest(pollset->po.pi);
   epoll_fd = pi->epoll_fd;
 
-  /* Update the pollset->polling_island since the island being pointed by
-     pollset->polling_island maybe older than the one pointed by pi) */
-  if (pollset->polling_island != pi) {
+  /* Update the pollset->po.pi since the island being pointed by
+     pollset->po.pi maybe older than the one pointed by pi) */
+  if (pollset->po.pi != pi) {
     /* Always do PI_ADD_REF before PI_UNREF because PI_UNREF may cause the
        polling island to be deleted */
     PI_ADD_REF(pi, "ps");
-    PI_UNREF(exec_ctx, pollset->polling_island, "ps");
-    pollset->polling_island = pi;
+    PI_UNREF(exec_ctx, pollset->po.pi, "ps");
+    pollset->po.pi = pi;
   }
 
   /* Add an extra ref so that the island does not get destroyed (which means
      the epoll_fd won't be closed) while we are are doing an epoll_wait() on the
      epoll_fd */
   PI_ADD_REF(pi, "ps_work");
-  gpr_mu_unlock(&pollset->mu);
+  gpr_mu_unlock(&pollset->po.mu);
 
   /* If we get some workqueue work to do, it might end up completing an item on
      the completion queue, so there's no need to poll... so we skip that and
@@ -1537,17 +1548,17 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(pi != NULL);
 
   /* Before leaving, release the extra ref we added to the polling island. It
-     is important to use "pi" here (i.e our old copy of pollset->polling_island
+     is important to use "pi" here (i.e our old copy of pollset->po.pi
      that we got before releasing the polling island lock). This is because
-     pollset->polling_island pointer might get udpated in other parts of the
+     pollset->po.pi pointer might get udpated in other parts of the
      code when there is an island merge while we are doing epoll_wait() above */
   PI_UNREF(exec_ctx, pi, "ps_work");
 
   GPR_TIMER_END("pollset_work_and_unlock", 0);
 }
 
-/* pollset->mu lock must be held by the caller before calling this.
-   The function pollset_work() may temporarily release the lock (pollset->mu)
+/* pollset->po.mu lock must be held by the caller before calling this.
+   The function pollset_work() may temporarily release the lock (pollset->po.mu)
    during the course of its execution but it will always re-acquire the lock and
    ensure that it is held by the time the function returns */
 static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -1617,7 +1628,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                             &g_orig_sigmask, &error);
     grpc_exec_ctx_flush(exec_ctx);
 
-    gpr_mu_lock(&pollset->mu);
+    gpr_mu_lock(&pollset->po.mu);
 
     /* Note: There is no need to reset worker.is_kicked to 0 since we are no
        longer going to use this worker */
@@ -1637,9 +1648,9 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     GPR_TIMER_MARK("pollset_work.finish_shutdown_locked", 0);
     finish_shutdown_locked(exec_ctx, pollset);
 
-    gpr_mu_unlock(&pollset->mu);
+    gpr_mu_unlock(&pollset->po.mu);
     grpc_exec_ctx_flush(exec_ctx);
-    gpr_mu_lock(&pollset->mu);
+    gpr_mu_lock(&pollset->po.mu);
   }
 
   *worker_hdl = NULL;
@@ -1653,130 +1664,160 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   return error;
 }
 
-static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
-                           grpc_fd *fd) {
-  GPR_TIMER_BEGIN("pollset_add_fd", 0);
-
-  grpc_error *error = GRPC_ERROR_NONE;
+static void add_poll_object(grpc_exec_ctx *exec_ctx, poll_obj *bag,
+                            poll_obj_type bag_type, poll_obj *item,
+                            poll_obj_type item_type) {
+  GPR_TIMER_BEGIN("add_poll_object", 0);
 
-  gpr_mu_lock(&pollset->mu);
-  gpr_mu_lock(&fd->mu);
+#ifdef PO_DEBUG
+  GPR_ASSERT(item->obj_type == item_type);
+  GPR_ASSERT(bag->obj_type == bag_type);
+#endif
 
+  grpc_error *error = GRPC_ERROR_NONE;
   polling_island *pi_new = NULL;
 
+  gpr_mu_lock(&bag->mu);
+  gpr_mu_lock(&item->mu);
+
 retry:
-  /* 1) If fd->polling_island and pollset->polling_island are both non-NULL and
-   *    equal, do nothing.
-   * 2) If fd->polling_island and pollset->polling_island are both NULL, create
-   *    a new polling island (with a refcount of 2) and make the polling_island
-   *    fields in both fd and pollset to point to the new island
-   * 3) If one of fd->polling_island or pollset->polling_island is NULL, update
-   *    the NULL polling_island field to point to the non-NULL polling_island
-   *    field (ensure that the refcount on the polling island is incremented by
-   *    1 to account for the newly added reference)
-   * 4) Finally, if fd->polling_island and pollset->polling_island are non-NULL
-   *    and different, merge both the polling islands and update the
-   *    polling_island fields in both fd and pollset to point to the merged
-   *    polling island.
+  /*
+   * 1) If item->pi and bag->pi are both non-NULL and equal, do nothing
+   * 2) If item->pi and bag->pi are both NULL, create a new polling island (with
+   *    a refcount of 2) and point item->pi and bag->pi to the new island
+   * 3) If exactly one of item->pi or bag->pi is NULL, update it to point to
+   *    the other's non-NULL pi
+   * 4) Finally if item->pi and bag-pi are non-NULL and not-equal, merge the
+   *    polling islands and update item->pi and bag->pi to point to the new
+   *    island
    */
 
-  if (fd->orphaned) {
-    gpr_mu_unlock(&fd->mu);
-    gpr_mu_unlock(&pollset->mu);
-    /* early out */
+  /* Early out if we are trying to add an 'fd' to a 'bag' but the fd is already
+   * orphaned */
+  if (item_type == POLL_OBJ_FD && (FD_FROM_PO(item))->orphaned) {
+    gpr_mu_unlock(&item->mu);
+    gpr_mu_unlock(&bag->mu);
     return;
   }
 
-  if (fd->polling_island == pollset->polling_island) {
-    pi_new = fd->polling_island;
+  if (item->pi == bag->pi) {
+    pi_new = item->pi;
     if (pi_new == NULL) {
-      /* Unlock before creating a new polling island: the polling island will
-         create a workqueue which creates a file descriptor, and holding an fd
-         lock here can eventually cause a loop to appear to TSAN (making it
-         unhappy). We don't think it's a real loop (there's an epoch point where
-         that loop possibility disappears), but the advantages of keeping TSAN
-         happy outweigh any performance advantage we might have by keeping the
-         lock held. */
-      gpr_mu_unlock(&fd->mu);
-      pi_new = polling_island_create(exec_ctx, fd, &error);
-      gpr_mu_lock(&fd->mu);
-      /* Need to reverify any assumptions made between the initial lock and
-         getting to this branch: if they've changed, we need to throw away our
-         work and figure things out again. */
-      if (fd->polling_island != NULL) {
-        GRPC_POLLING_TRACE(
-            "pollset_add_fd: Raced creating new polling island. pi_new: %p "
-            "(fd: %d, pollset: %p)",
-            (void *)pi_new, fd->fd, (void *)pollset);
-
-        /* No need to lock 'pi_new' here since this is a new polling island and
-         * no one has a reference to it yet */
-        polling_island_remove_all_fds_locked(pi_new, true, &error);
-
-        /* Ref and unref so that the polling island gets deleted during unref */
-        PI_ADD_REF(pi_new, "dance_of_destruction");
-        PI_UNREF(exec_ctx, pi_new, "dance_of_destruction");
-        goto retry;
+      /* GPR_ASSERT(item->pi == bag->pi == NULL) */
+
+      /* If we are adding an fd to a bag (i.e pollset or pollset_set), then
+       * we need to do some extra work to make TSAN happy */
+      if (item_type == POLL_OBJ_FD) {
+        /* Unlock before creating a new polling island: the polling island will
+           create a workqueue which creates a file descriptor, and holding an fd
+           lock here can eventually cause a loop to appear to TSAN (making it
+           unhappy). We don't think it's a real loop (there's an epoch point
+           where that loop possibility disappears), but the advantages of
+           keeping TSAN happy outweigh any performance advantage we might have
+           by keeping the lock held. */
+        gpr_mu_unlock(&item->mu);
+        pi_new = polling_island_create(exec_ctx, FD_FROM_PO(item), &error);
+        gpr_mu_lock(&item->mu);
+
+        /* Need to reverify any assumptions made between the initial lock and
+           getting to this branch: if they've changed, we need to throw away our
+           work and figure things out again. */
+        if (item->pi != NULL) {
+          GRPC_POLLING_TRACE(
+              "add_poll_object: Raced creating new polling island. pi_new: %p "
+              "(fd: %d, %s: %p)",
+              (void *)pi_new, FD_FROM_PO(item)->fd, poll_obj_string(bag_type),
+              (void *)bag);
+          /* No need to lock 'pi_new' here since this is a new polling island
+            * and no one has a reference to it yet */
+          polling_island_remove_all_fds_locked(pi_new, true, &error);
+
+          /* Ref and unref so that the polling island gets deleted during unref
+           */
+          PI_ADD_REF(pi_new, "dance_of_destruction");
+          PI_UNREF(exec_ctx, pi_new, "dance_of_destruction");
+          goto retry;
+        }
       } else {
-        GRPC_POLLING_TRACE(
-            "pollset_add_fd: Created new polling island. pi_new: %p (fd: %d, "
-            "pollset: %p)",
-            (void *)pi_new, fd->fd, (void *)pollset);
+        pi_new = polling_island_create(exec_ctx, NULL, &error);
       }
+
+      GRPC_POLLING_TRACE(
+          "add_poll_object: Created new polling island. pi_new: %p (%s: %p, "
+          "%s: %p)",
+          (void *)pi_new, poll_obj_string(item_type), (void *)item,
+          poll_obj_string(bag_type), (void *)bag);
+    } else {
+      GRPC_POLLING_TRACE(
+          "add_poll_object: Same polling island. pi: %p (%s, %s)",
+          (void *)pi_new, poll_obj_string(item_type),
+          poll_obj_string(bag_type));
+    }
+  } else if (item->pi == NULL) {
+    /* GPR_ASSERT(bag->pi != NULL) */
+    /* Make pi_new point to latest pi*/
+    pi_new = polling_island_lock(bag->pi);
+
+    if (item_type == POLL_OBJ_FD) {
+      grpc_fd *fd = FD_FROM_PO(item);
+      polling_island_add_fds_locked(pi_new, &fd, 1, true, &error);
     }
-  } else if (fd->polling_island == NULL) {
-    pi_new = polling_island_lock(pollset->polling_island);
-    polling_island_add_fds_locked(pi_new, &fd, 1, true, &error);
-    gpr_mu_unlock(&pi_new->mu);
 
+    gpr_mu_unlock(&pi_new->mu);
     GRPC_POLLING_TRACE(
-        "pollset_add_fd: fd->pi was NULL. pi_new: %p (fd: %d, pollset: %p, "
-        "pollset->pi: %p)",
-        (void *)pi_new, fd->fd, (void *)pollset,
-        (void *)pollset->polling_island);
-  } else if (pollset->polling_island == NULL) {
-    pi_new = polling_island_lock(fd->polling_island);
+        "add_poll_obj: item->pi was NULL. pi_new: %p (item(%s): %p, "
+        "bag(%s): %p)",
+        (void *)pi_new, poll_obj_string(item_type), (void *)item,
+        poll_obj_string(bag_type), (void *)bag);
+  } else if (bag->pi == NULL) {
+    /* GPR_ASSERT(item->pi != NULL) */
+    /* Make pi_new to point to latest pi */
+    pi_new = polling_island_lock(item->pi);
     gpr_mu_unlock(&pi_new->mu);
-
     GRPC_POLLING_TRACE(
-        "pollset_add_fd: pollset->pi was NULL. pi_new: %p (fd: %d, pollset: "
-        "%p, fd->pi: %p",
-        (void *)pi_new, fd->fd, (void *)pollset, (void *)fd->polling_island);
+        "add_poll_obj: bag->pi was NULL. pi_new: %p (item(%s): %p, "
+        "bag(%s): %p)",
+        (void *)pi_new, poll_obj_string(item_type), (void *)item,
+        poll_obj_string(bag_type), (void *)bag);
   } else {
-    pi_new = polling_island_merge(fd->polling_island, pollset->polling_island,
-                                  &error);
+    pi_new = polling_island_merge(item->pi, bag->pi, &error);
     GRPC_POLLING_TRACE(
-        "pollset_add_fd: polling islands merged. pi_new: %p (fd: %d, pollset: "
-        "%p, fd->pi: %p, pollset->pi: %p)",
-        (void *)pi_new, fd->fd, (void *)pollset, (void *)fd->polling_island,
-        (void *)pollset->polling_island);
+        "add_poll_obj: polling islands merged. pi_new: %p (item(%s): %p, "
+        "bag(%s): %p)",
+        (void *)pi_new, poll_obj_string(item_type), (void *)item,
+        poll_obj_string(bag_type), (void *)bag);
   }
 
-  /* At this point, pi_new is the polling island that both fd->polling_island
-     and pollset->polling_island must be pointing to */
+  /* At this point, pi_new is the polling island that both item->pi and bag->pi
+     MUST be pointing to */
 
-  if (fd->polling_island != pi_new) {
-    PI_ADD_REF(pi_new, "fd");
-    if (fd->polling_island != NULL) {
-      PI_UNREF(exec_ctx, fd->polling_island, "fd");
+  if (item->pi != pi_new) {
+    PI_ADD_REF(pi_new, poll_obj_string(item_type));
+    if (item->pi != NULL) {
+      PI_UNREF(exec_ctx, item->pi, poll_obj_string(item_type));
     }
-    fd->polling_island = pi_new;
+    item->pi = pi_new;
   }
 
-  if (pollset->polling_island != pi_new) {
-    PI_ADD_REF(pi_new, "ps");
-    if (pollset->polling_island != NULL) {
-      PI_UNREF(exec_ctx, pollset->polling_island, "ps");
+  if (bag->pi != pi_new) {
+    PI_ADD_REF(pi_new, poll_obj_string(bag_type));
+    if (bag->pi != NULL) {
+      PI_UNREF(exec_ctx, bag->pi, poll_obj_string(bag_type));
     }
-    pollset->polling_island = pi_new;
+    bag->pi = pi_new;
   }
 
-  gpr_mu_unlock(&fd->mu);
-  gpr_mu_unlock(&pollset->mu);
+  gpr_mu_unlock(&item->mu);
+  gpr_mu_unlock(&bag->mu);
 
-  GRPC_LOG_IF_ERROR("pollset_add_fd", error);
+  GRPC_LOG_IF_ERROR("add_poll_object", error);
+  GPR_TIMER_END("add_poll_object", 0);
+}
 
-  GPR_TIMER_END("pollset_add_fd", 0);
+static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
+                           grpc_fd *fd) {
+  add_poll_object(exec_ctx, &pollset->po, POLL_OBJ_POLLSET, &fd->po,
+                  POLL_OBJ_FD);
 }
 
 /*******************************************************************************
@@ -1784,142 +1825,60 @@ retry:
  */
 
 static grpc_pollset_set *pollset_set_create(void) {
-  grpc_pollset_set *pollset_set = gpr_malloc(sizeof(*pollset_set));
-  memset(pollset_set, 0, sizeof(*pollset_set));
-  gpr_mu_init(&pollset_set->mu);
-  return pollset_set;
+  grpc_pollset_set *pss = gpr_malloc(sizeof(*pss));
+  gpr_mu_init(&pss->po.mu);
+  pss->po.pi = NULL;
+#ifdef PO_DEBUG
+  pss->po.obj_type = POLL_OBJ_POLLSET_SET;
+#endif
+  return pss;
 }
 
-static void pollset_set_destroy(grpc_pollset_set *pollset_set) {
-  size_t i;
-  gpr_mu_destroy(&pollset_set->mu);
-  for (i = 0; i < pollset_set->fd_count; i++) {
-    GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
+static void pollset_set_destroy(grpc_pollset_set *pss) {
+  gpr_mu_destroy(&pss->po.mu);
+
+  if (pss->po.pi != NULL) {
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    PI_UNREF(&exec_ctx, pss->po.pi, "pss_destroy");
+    grpc_exec_ctx_finish(&exec_ctx);
   }
-  gpr_free(pollset_set->pollsets);
-  gpr_free(pollset_set->pollset_sets);
-  gpr_free(pollset_set->fds);
-  gpr_free(pollset_set);
+
+  gpr_free(pss);
 }
 
-static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  if (pollset_set->fd_count == pollset_set->fd_capacity) {
-    pollset_set->fd_capacity = GPR_MAX(8, 2 * pollset_set->fd_capacity);
-    pollset_set->fds = gpr_realloc(
-        pollset_set->fds, pollset_set->fd_capacity * sizeof(*pollset_set->fds));
-  }
-  GRPC_FD_REF(fd, "pollset_set");
-  pollset_set->fds[pollset_set->fd_count++] = fd;
-  for (i = 0; i < pollset_set->pollset_count; i++) {
-    pollset_add_fd(exec_ctx, pollset_set->pollsets[i], fd);
-  }
-  for (i = 0; i < pollset_set->pollset_set_count; i++) {
-    pollset_set_add_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
-  }
-  gpr_mu_unlock(&pollset_set->mu);
+static void pollset_set_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss,
+                               grpc_fd *fd) {
+  add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &fd->po,
+                  POLL_OBJ_FD);
 }
 
-static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx,
-                               grpc_pollset_set *pollset_set, grpc_fd *fd) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  for (i = 0; i < pollset_set->fd_count; i++) {
-    if (pollset_set->fds[i] == fd) {
-      pollset_set->fd_count--;
-      GPR_SWAP(grpc_fd *, pollset_set->fds[i],
-               pollset_set->fds[pollset_set->fd_count]);
-      GRPC_FD_UNREF(fd, "pollset_set");
-      break;
-    }
-  }
-  for (i = 0; i < pollset_set->pollset_set_count; i++) {
-    pollset_set_del_fd(exec_ctx, pollset_set->pollset_sets[i], fd);
-  }
-  gpr_mu_unlock(&pollset_set->mu);
+static void pollset_set_del_fd(grpc_exec_ctx *exec_ctx, grpc_pollset_set *pss,
+                               grpc_fd *fd) {
+  /* Nothing to do */
 }
 
 static void pollset_set_add_pollset(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset_set *pollset_set,
-                                    grpc_pollset *pollset) {
-  size_t i, j;
-  gpr_mu_lock(&pollset_set->mu);
-  if (pollset_set->pollset_count == pollset_set->pollset_capacity) {
-    pollset_set->pollset_capacity =
-        GPR_MAX(8, 2 * pollset_set->pollset_capacity);
-    pollset_set->pollsets =
-        gpr_realloc(pollset_set->pollsets, pollset_set->pollset_capacity *
-                                               sizeof(*pollset_set->pollsets));
-  }
-  pollset_set->pollsets[pollset_set->pollset_count++] = pollset;
-  for (i = 0, j = 0; i < pollset_set->fd_count; i++) {
-    if (fd_is_orphaned(pollset_set->fds[i])) {
-      GRPC_FD_UNREF(pollset_set->fds[i], "pollset_set");
-    } else {
-      pollset_add_fd(exec_ctx, pollset, pollset_set->fds[i]);
-      pollset_set->fds[j++] = pollset_set->fds[i];
-    }
-  }
-  pollset_set->fd_count = j;
-  gpr_mu_unlock(&pollset_set->mu);
+                                    grpc_pollset_set *pss, grpc_pollset *ps) {
+  add_poll_object(exec_ctx, &pss->po, POLL_OBJ_POLLSET_SET, &ps->po,
+                  POLL_OBJ_POLLSET);
 }
 
 static void pollset_set_del_pollset(grpc_exec_ctx *exec_ctx,
-                                    grpc_pollset_set *pollset_set,
-                                    grpc_pollset *pollset) {
-  size_t i;
-  gpr_mu_lock(&pollset_set->mu);
-  for (i = 0; i < pollset_set->pollset_count; i++) {
-    if (pollset_set->pollsets[i] == pollset) {
-      pollset_set->pollset_count--;
-      GPR_SWAP(grpc_pollset *, pollset_set->pollsets[i],
-               pollset_set->pollsets[pollset_set->pollset_count]);
-      break;
-    }
-  }
-  gpr_mu_unlock(&pollset_set->mu);
+                                    grpc_pollset_set *pss, grpc_pollset *ps) {
+  /* Nothing to do */
 }
 
 static void pollset_set_add_pollset_set(grpc_exec_ctx *exec_ctx,
                                         grpc_pollset_set *bag,
                                         grpc_pollset_set *item) {
-  size_t i, j;
-  gpr_mu_lock(&bag->mu);
-  if (bag->pollset_set_count == bag->pollset_set_capacity) {
-    bag->pollset_set_capacity = GPR_MAX(8, 2 * bag->pollset_set_capacity);
-    bag->pollset_sets =
-        gpr_realloc(bag->pollset_sets,
-                    bag->pollset_set_capacity * sizeof(*bag->pollset_sets));
-  }
-  bag->pollset_sets[bag->pollset_set_count++] = item;
-  for (i = 0, j = 0; i < bag->fd_count; i++) {
-    if (fd_is_orphaned(bag->fds[i])) {
-      GRPC_FD_UNREF(bag->fds[i], "pollset_set");
-    } else {
-      pollset_set_add_fd(exec_ctx, item, bag->fds[i]);
-      bag->fds[j++] = bag->fds[i];
-    }
-  }
-  bag->fd_count = j;
-  gpr_mu_unlock(&bag->mu);
+  add_poll_object(exec_ctx, &bag->po, POLL_OBJ_POLLSET_SET, &item->po,
+                  POLL_OBJ_POLLSET_SET);
 }
 
 static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
                                         grpc_pollset_set *bag,
                                         grpc_pollset_set *item) {
-  size_t i;
-  gpr_mu_lock(&bag->mu);
-  for (i = 0; i < bag->pollset_set_count; i++) {
-    if (bag->pollset_sets[i] == item) {
-      bag->pollset_set_count--;
-      GPR_SWAP(grpc_pollset_set *, bag->pollset_sets[i],
-               bag->pollset_sets[bag->pollset_set_count]);
-      break;
-    }
-  }
-  gpr_mu_unlock(&bag->mu);
+  /* Nothing to do */
 }
 
 /* Test helper functions
@@ -1927,9 +1886,9 @@ static void pollset_set_del_pollset_set(grpc_exec_ctx *exec_ctx,
 void *grpc_fd_get_polling_island(grpc_fd *fd) {
   polling_island *pi;
 
-  gpr_mu_lock(&fd->mu);
-  pi = fd->polling_island;
-  gpr_mu_unlock(&fd->mu);
+  gpr_mu_lock(&fd->po.mu);
+  pi = fd->po.pi;
+  gpr_mu_unlock(&fd->po.mu);
 
   return pi;
 }
@@ -1937,9 +1896,9 @@ void *grpc_fd_get_polling_island(grpc_fd *fd) {
 void *grpc_pollset_get_polling_island(grpc_pollset *ps) {
   polling_island *pi;
 
-  gpr_mu_lock(&ps->mu);
-  pi = ps->polling_island;
-  gpr_mu_unlock(&ps->mu);
+  gpr_mu_lock(&ps->po.mu);
+  pi = ps->po.pi;
+  gpr_mu_unlock(&ps->po.mu);
 
   return pi;
 }

src/core/lib/iomgr/iomgr.c

@@ -108,6 +108,7 @@ void grpc_iomgr_shutdown(void) {
                          NULL)) {
       gpr_mu_unlock(&g_mu);
       grpc_exec_ctx_flush(&exec_ctx);
+      grpc_iomgr_platform_flush();
       gpr_mu_lock(&g_mu);
       continue;
     }

src/core/lib/iomgr/resource_quota.c

@@ -104,6 +104,9 @@ struct grpc_resource_user {
   /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer
    */
   grpc_closure *reclaimers[2];
+  /* Reclaimers just posted: once we're in the combiner lock, we'll move them
+     to the array above */
+  grpc_closure *new_reclaimers[2];
   /* Trampoline closures to finish reclamation and re-enter the quota combiner
      lock */
   grpc_closure post_reclaimer_closure[2];
@@ -141,6 +144,12 @@ struct grpc_resource_quota {
   /* Closure around rq_reclamation_done */
   grpc_closure rq_reclamation_done_closure;
 
+  /* This is only really usable for debugging: it's always a stale pointer, but
+     a stale pointer that might just be fresh enough to guide us to where the
+     reclamation system is stuck */
+  grpc_closure *debug_only_last_initiated_reclaimer;
+  grpc_resource_user *debug_only_last_reclaimer_resource_user;
+
   /* Roots of all resource user lists */
   grpc_resource_user *roots[GRPC_RULIST_COUNT];
 
@@ -222,6 +231,7 @@ static void rulist_remove(grpc_resource_user *resource_user, grpc_rulist list) {
       resource_user->links[list].prev;
   resource_user->links[list].prev->links[list].next =
       resource_user->links[list].next;
+  resource_user->links[list].next = resource_user->links[list].prev = NULL;
 }
 
 /*******************************************************************************
@@ -337,6 +347,9 @@ static bool rq_reclaim(grpc_exec_ctx *exec_ctx,
   resource_quota->reclaiming = true;
   grpc_resource_quota_internal_ref(resource_quota);
   grpc_closure *c = resource_user->reclaimers[destructive];
+  GPR_ASSERT(c);
+  resource_quota->debug_only_last_reclaimer_resource_user = resource_user;
+  resource_quota->debug_only_last_initiated_reclaimer = c;
   resource_user->reclaimers[destructive] = NULL;
   grpc_closure_run(exec_ctx, c, GRPC_ERROR_NONE);
   return true;
@@ -418,9 +431,25 @@ static void ru_add_to_free_pool(grpc_exec_ctx *exec_ctx, void *ru,
   rulist_add_tail(resource_user, GRPC_RULIST_NON_EMPTY_FREE_POOL);
 }
 
+static bool ru_post_reclaimer(grpc_exec_ctx *exec_ctx,
+                              grpc_resource_user *resource_user,
+                              bool destructive) {
+  grpc_closure *closure = resource_user->new_reclaimers[destructive];
+  GPR_ASSERT(closure != NULL);
+  resource_user->new_reclaimers[destructive] = NULL;
+  GPR_ASSERT(resource_user->reclaimers[destructive] == NULL);
+  if (gpr_atm_acq_load(&resource_user->shutdown) > 0) {
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CANCELLED, NULL);
+    return false;
+  }
+  resource_user->reclaimers[destructive] = closure;
+  return true;
+}
+
 static void ru_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *ru,
                                      grpc_error *error) {
   grpc_resource_user *resource_user = ru;
+  if (!ru_post_reclaimer(exec_ctx, resource_user, false)) return;
   if (!rulist_empty(resource_user->resource_quota,
                     GRPC_RULIST_AWAITING_ALLOCATION) &&
       rulist_empty(resource_user->resource_quota,
@@ -435,6 +464,7 @@ static void ru_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *ru,
 static void ru_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *ru,
                                           grpc_error *error) {
   grpc_resource_user *resource_user = ru;
+  if (!ru_post_reclaimer(exec_ctx, resource_user, true)) return;
   if (!rulist_empty(resource_user->resource_quota,
                     GRPC_RULIST_AWAITING_ALLOCATION) &&
       rulist_empty(resource_user->resource_quota,
@@ -456,6 +486,8 @@ static void ru_shutdown(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) {
                       GRPC_ERROR_CANCELLED, NULL);
   resource_user->reclaimers[0] = NULL;
   resource_user->reclaimers[1] = NULL;
+  rulist_remove(resource_user, GRPC_RULIST_RECLAIMER_BENIGN);
+  rulist_remove(resource_user, GRPC_RULIST_RECLAIMER_DESTRUCTIVE);
 }
 
 static void ru_destroy(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) {
@@ -649,6 +681,8 @@ grpc_resource_user *grpc_resource_user_create(
   resource_user->added_to_free_pool = false;
   resource_user->reclaimers[0] = NULL;
   resource_user->reclaimers[1] = NULL;
+  resource_user->new_reclaimers[0] = NULL;
+  resource_user->new_reclaimers[1] = NULL;
   for (int i = 0; i < GRPC_RULIST_COUNT; i++) {
     resource_user->links[i].next = resource_user->links[i].prev = NULL;
   }
@@ -748,12 +782,8 @@ void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx,
                                        grpc_resource_user *resource_user,
                                        bool destructive,
                                        grpc_closure *closure) {
-  GPR_ASSERT(resource_user->reclaimers[destructive] == NULL);
-  if (gpr_atm_acq_load(&resource_user->shutdown) > 0) {
-    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CANCELLED, NULL);
-    return;
-  }
-  resource_user->reclaimers[destructive] = closure;
+  GPR_ASSERT(resource_user->new_reclaimers[destructive] == NULL);
+  resource_user->new_reclaimers[destructive] = closure;
   grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner,
                         &resource_user->post_reclaimer_closure[destructive],
                         GRPC_ERROR_NONE, false);

src/core/lib/iomgr/socket_windows.c

@@ -76,6 +76,14 @@ void grpc_winsocket_shutdown(grpc_winsocket *winsocket) {
   LPFN_DISCONNECTEX DisconnectEx;
   DWORD ioctl_num_bytes;
 
+  gpr_mu_lock(&winsocket->state_mu);
+  if (winsocket->shutdown_called) {
+    gpr_mu_unlock(&winsocket->state_mu);
+    return;
+  }
+  winsocket->shutdown_called = true;
+  gpr_mu_unlock(&winsocket->state_mu);
+
   status = WSAIoctl(winsocket->socket, SIO_GET_EXTENSION_FUNCTION_POINTER,
                     &guid, sizeof(guid), &DisconnectEx, sizeof(DisconnectEx),
                     &ioctl_num_bytes, NULL, NULL);

src/core/lib/iomgr/socket_windows.h

@@ -87,6 +87,7 @@ typedef struct grpc_winsocket {
   grpc_winsocket_callback_info read_info;
 
   gpr_mu state_mu;
+  bool shutdown_called;
 
   /* You can't add the same socket twice to the same IO Completion Port.
      This prevents that. */

src/core/lib/iomgr/tcp_client_windows.c

@@ -107,18 +107,22 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
 
   gpr_mu_lock(&ac->mu);
 
-  if (error == GRPC_ERROR_NONE && socket != NULL) {
-    DWORD transfered_bytes = 0;
-    DWORD flags;
-    BOOL wsa_success =
-        WSAGetOverlappedResult(socket->socket, &socket->write_info.overlapped,
-                               &transfered_bytes, FALSE, &flags);
-    GPR_ASSERT(transfered_bytes == 0);
-    if (!wsa_success) {
-      error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx");
+  if (error == GRPC_ERROR_NONE) {
+    if (socket != NULL) {
+      DWORD transfered_bytes = 0;
+      DWORD flags;
+      BOOL wsa_success =
+          WSAGetOverlappedResult(socket->socket, &socket->write_info.overlapped,
+                                 &transfered_bytes, FALSE, &flags);
+      GPR_ASSERT(transfered_bytes == 0);
+      if (!wsa_success) {
+        error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx");
+      } else {
+        *ep = grpc_tcp_create(socket, ac->resource_quota, ac->addr_name);
+        socket = NULL;
+      }
     } else {
-      *ep = grpc_tcp_create(socket, ac->resource_quota, ac->addr_name);
-      socket = NULL;
+      error = GRPC_ERROR_CREATE("socket is null");
     }
   }
 

src/core/lib/iomgr/tcp_posix.c

@@ -107,6 +107,12 @@ typedef struct {
   grpc_resource_user_slice_allocator slice_allocator;
 } grpc_tcp;
 
+static grpc_error *tcp_annotate_error(grpc_error *src_error, grpc_tcp *tcp) {
+  return grpc_error_set_str(
+      grpc_error_set_int(src_error, GRPC_ERROR_INT_FD, tcp->fd),
+      GRPC_ERROR_STR_TARGET_ADDRESS, tcp->peer_string);
+}
+
 static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
                             grpc_error *error);
 static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
@@ -230,13 +236,15 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
       grpc_fd_notify_on_read(exec_ctx, tcp->em_fd, &tcp->read_closure);
     } else {
       grpc_slice_buffer_reset_and_unref(tcp->incoming_buffer);
-      call_read_cb(exec_ctx, tcp, GRPC_OS_ERROR(errno, "recvmsg"));
+      call_read_cb(exec_ctx, tcp,
+                   tcp_annotate_error(GRPC_OS_ERROR(errno, "recvmsg"), tcp));
       TCP_UNREF(exec_ctx, tcp, "read");
     }
   } else if (read_bytes == 0) {
     /* 0 read size ==> end of stream */
     grpc_slice_buffer_reset_and_unref(tcp->incoming_buffer);
-    call_read_cb(exec_ctx, tcp, GRPC_ERROR_CREATE("Socket closed"));
+    call_read_cb(exec_ctx, tcp,
+                 tcp_annotate_error(GRPC_ERROR_CREATE("Socket closed"), tcp));
     TCP_UNREF(exec_ctx, tcp, "read");
   } else {
     GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length);
@@ -365,8 +373,13 @@ static bool tcp_flush(grpc_tcp *tcp, grpc_error **error) {
         tcp->outgoing_slice_idx = unwind_slice_idx;
         tcp->outgoing_byte_idx = unwind_byte_idx;
         return false;
+      } else if (errno == EPIPE) {
+        *error = grpc_error_set_int(GRPC_OS_ERROR(errno, "sendmsg"),
+                                    GRPC_ERROR_INT_GRPC_STATUS,
+                                    GRPC_STATUS_UNAVAILABLE);
+        return true;
       } else {
-        *error = GRPC_OS_ERROR(errno, "sendmsg");
+        *error = tcp_annotate_error(GRPC_OS_ERROR(errno, "sendmsg"), tcp);
         return true;
       }
     }
@@ -447,9 +460,10 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
   if (buf->length == 0) {
     GPR_TIMER_END("tcp_write", 0);
-    grpc_exec_ctx_sched(exec_ctx, cb, grpc_fd_is_shutdown(tcp->em_fd)
-                                          ? GRPC_ERROR_CREATE("EOF")
-                                          : GRPC_ERROR_NONE,
+    grpc_exec_ctx_sched(exec_ctx, cb,
+                        grpc_fd_is_shutdown(tcp->em_fd)
+                            ? tcp_annotate_error(GRPC_ERROR_CREATE("EOF"), tcp)
+                            : GRPC_ERROR_NONE,
                         NULL);
     return;
   }

src/core/lib/iomgr/tcp_server.h

@@ -52,7 +52,8 @@ typedef struct grpc_tcp_server_acceptor {
   unsigned fd_index;
 } grpc_tcp_server_acceptor;
 
-/* Called for newly connected TCP connections. */
+/* Called for newly connected TCP connections.
+   Takes ownership of acceptor. */
 typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_endpoint *ep,
                                    grpc_pollset *accepting_pollset,

src/core/lib/iomgr/tcp_server_posix.c

@@ -381,16 +381,12 @@ error:
 /* event manager callback when reads are ready */
 static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
   grpc_tcp_listener *sp = arg;
-  grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index,
-                                       sp->fd_index};
-  grpc_pollset *read_notifier_pollset = NULL;
-  grpc_fd *fdobj;
 
   if (err != GRPC_ERROR_NONE) {
     goto error;
   }
 
-  read_notifier_pollset =
+  grpc_pollset *read_notifier_pollset =
       sp->server->pollsets[(size_t)gpr_atm_no_barrier_fetch_add(
                                &sp->server->next_pollset_to_assign, 1) %
                            sp->server->pollset_count];
@@ -426,7 +422,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
       gpr_log(GPR_DEBUG, "SERVER_CONNECT: incoming connection: %s", addr_str);
     }
 
-    fdobj = grpc_fd_create(fd, name);
+    grpc_fd *fdobj = grpc_fd_create(fd, name);
 
     if (read_notifier_pollset == NULL) {
       gpr_log(GPR_ERROR, "Read notifier pollset is not set on the fd");
@@ -435,11 +431,17 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
 
     grpc_pollset_add_fd(exec_ctx, read_notifier_pollset, fdobj);
 
+    // Create acceptor.
+    grpc_tcp_server_acceptor *acceptor = gpr_malloc(sizeof(*acceptor));
+    acceptor->from_server = sp->server;
+    acceptor->port_index = sp->port_index;
+    acceptor->fd_index = sp->fd_index;
+
     sp->server->on_accept_cb(
         exec_ctx, sp->server->on_accept_cb_arg,
         grpc_tcp_create(fdobj, sp->server->resource_quota,
                         GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
-        read_notifier_pollset, &acceptor);
+        read_notifier_pollset, acceptor);
 
     gpr_free(name);
     gpr_free(addr_str);

src/core/lib/iomgr/tcp_server_uv.c

@@ -188,7 +188,6 @@ static void accepted_connection_close_cb(uv_handle_t *handle) {
 
 static void on_connect(uv_stream_t *server, int status) {
   grpc_tcp_listener *sp = (grpc_tcp_listener *)server->data;
-  grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index, 0};
   uv_tcp_t *client;
   grpc_endpoint *ep = NULL;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
@@ -201,6 +200,7 @@ static void on_connect(uv_stream_t *server, int status) {
             uv_strerror(status));
     return;
   }
+
   client = gpr_malloc(sizeof(uv_tcp_t));
   uv_tcp_init(uv_default_loop(), client);
   // UV documentation says this is guaranteed to succeed
@@ -220,8 +220,13 @@ static void on_connect(uv_stream_t *server, int status) {
       gpr_log(GPR_INFO, "uv_tcp_getpeername error: %s", uv_strerror(status));
     }
     ep = grpc_tcp_create(client, sp->server->resource_quota, peer_name_string);
+    // Create acceptor.
+    grpc_tcp_server_acceptor *acceptor = gpr_malloc(sizeof(*acceptor));
+    acceptor->from_server = sp->server;
+    acceptor->port_index = sp->port_index;
+    acceptor->fd_index = 0;
     sp->server->on_accept_cb(&exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
-                             &acceptor);
+                             acceptor);
     grpc_exec_ctx_finish(&exec_ctx);
   }
 }

src/core/lib/iomgr/tcp_server_windows.c

@@ -73,6 +73,7 @@ struct grpc_tcp_listener {
   /* The cached AcceptEx for that port. */
   LPFN_ACCEPTEX AcceptEx;
   int shutting_down;
+  int outstanding_calls;
   /* closure for socket notification of accept being ready */
   grpc_closure on_accept;
   /* linked list */
@@ -140,10 +141,9 @@ grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx,
   return GRPC_ERROR_NONE;
 }
 
-static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
-  if (s->shutdown_complete != NULL) {
-    grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
-  }
+static void destroy_server(grpc_exec_ctx *exec_ctx, void *arg,
+                           grpc_error *error) {
+  grpc_tcp_server *s = arg;
 
   /* Now that the accepts have been aborted, we can destroy the sockets.
      The IOCP won't get notified on these, so we can flag them as already
@@ -159,6 +159,16 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   gpr_free(s);
 }
 
+static void finish_shutdown_locked(grpc_exec_ctx *exec_ctx,
+                                   grpc_tcp_server *s) {
+  if (s->shutdown_complete != NULL) {
+    grpc_exec_ctx_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE, NULL);
+  }
+
+  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(destroy_server, s),
+                      GRPC_ERROR_NONE, NULL);
+}
+
 grpc_tcp_server *grpc_tcp_server_ref(grpc_tcp_server *s) {
   gpr_ref_non_zero(&s->refs);
   return s;
@@ -180,17 +190,14 @@ static void tcp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
   /* First, shutdown all fd's. This will queue abortion calls for all
      of the pending accepts due to the normal operation mechanism. */
   if (s->active_ports == 0) {
-    immediately_done = 1;
-  }
-  for (sp = s->head; sp; sp = sp->next) {
-    sp->shutting_down = 1;
-    grpc_winsocket_shutdown(sp->socket);
+    finish_shutdown_locked(exec_ctx, s);
+  } else {
+    for (sp = s->head; sp; sp = sp->next) {
+      sp->shutting_down = 1;
+      grpc_winsocket_shutdown(sp->socket);
+    }
   }
   gpr_mu_unlock(&s->mu);
-
-  if (immediately_done) {
-    finish_shutdown(exec_ctx, s);
-  }
 }
 
 void grpc_tcp_server_unref(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
@@ -251,31 +258,30 @@ failure:
   return error;
 }
 
-static void decrement_active_ports_and_notify(grpc_exec_ctx *exec_ctx,
-                                              grpc_tcp_listener *sp) {
+static void decrement_active_ports_and_notify_locked(grpc_exec_ctx *exec_ctx,
+                                                     grpc_tcp_listener *sp) {
   int notify = 0;
   sp->shutting_down = 0;
-  gpr_mu_lock(&sp->server->mu);
   GPR_ASSERT(sp->server->active_ports > 0);
   if (0 == --sp->server->active_ports) {
-    notify = 1;
-  }
-  gpr_mu_unlock(&sp->server->mu);
-  if (notify) {
-    finish_shutdown(exec_ctx, sp->server);
+    finish_shutdown_locked(exec_ctx, sp->server);
   }
 }
 
 /* In order to do an async accept, we need to create a socket first which
    will be the one assigned to the new incoming connection. */
-static grpc_error *start_accept(grpc_exec_ctx *exec_ctx,
-                                grpc_tcp_listener *port) {
+static grpc_error *start_accept_locked(grpc_exec_ctx *exec_ctx,
+                                       grpc_tcp_listener *port) {
   SOCKET sock = INVALID_SOCKET;
   BOOL success;
   DWORD addrlen = sizeof(struct sockaddr_in6) + 16;
   DWORD bytes_received = 0;
   grpc_error *error = GRPC_ERROR_NONE;
 
+  if (port->shutting_down) {
+    return GRPC_ERROR_NONE;
+  }
+
   sock = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, NULL, 0,
                    WSA_FLAG_OVERLAPPED);
   if (sock == INVALID_SOCKET) {
@@ -305,20 +311,11 @@ static grpc_error *start_accept(grpc_exec_ctx *exec_ctx,
      immediately process an accept that happened in the meantime. */
   port->new_socket = sock;
   grpc_socket_notify_on_read(exec_ctx, port->socket, &port->on_accept);
+  port->outstanding_calls++;
   return error;
 
 failure:
   GPR_ASSERT(error != GRPC_ERROR_NONE);
-  if (port->shutting_down) {
-    /* We are abandoning the listener port, take that into account to prevent
-       occasional hangs on shutdown. The hang happens when sp->shutting_down
-       change is not seen by on_accept and we proceed to trying new accept,
-       but we fail there because the listening port has been closed in the
-       meantime. */
-    decrement_active_ports_and_notify(exec_ctx, port);
-    GRPC_ERROR_UNREF(error);
-    return GRPC_ERROR_NONE;
-  }
   if (sock != INVALID_SOCKET) closesocket(sock);
   return error;
 }
@@ -326,7 +323,6 @@ failure:
 /* Event manager callback when reads are ready. */
 static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_tcp_listener *sp = arg;
-  grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index, 0};
   SOCKET sock = sp->new_socket;
   grpc_winsocket_callback_info *info = &sp->socket->read_info;
   grpc_endpoint *ep = NULL;
@@ -338,6 +334,8 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   BOOL wsa_success;
   int err;
 
+  gpr_mu_lock(&sp->server->mu);
+
   peer_name.len = sizeof(struct sockaddr_storage);
 
   /* The general mechanism for shutting down is to queue abortion calls. While
@@ -347,6 +345,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
     const char *msg = grpc_error_string(error);
     gpr_log(GPR_INFO, "Skipping on_accept due to error: %s", msg);
     grpc_error_free_string(msg);
+    gpr_mu_unlock(&sp->server->mu);
     return;
   }
 
@@ -356,17 +355,12 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   wsa_success = WSAGetOverlappedResult(sock, &info->overlapped,
                                        &transfered_bytes, FALSE, &flags);
   if (!wsa_success) {
-    if (sp->shutting_down) {
-      /* During the shutdown case, we ARE expecting an error. So that's well,
-         and we can wake up the shutdown thread. */
-      decrement_active_ports_and_notify(exec_ctx, sp);
-      return;
-    } else {
+    if (!sp->shutting_down) {
       char *utf8_message = gpr_format_message(WSAGetLastError());
       gpr_log(GPR_ERROR, "on_accept error: %s", utf8_message);
       gpr_free(utf8_message);
-      closesocket(sock);
     }
+    closesocket(sock);
   } else {
     if (!sp->shutting_down) {
       peer_name_string = NULL;
@@ -401,14 +395,24 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   /* The only time we should call our callback, is where we successfully
      managed to accept a connection, and created an endpoint. */
   if (ep) {
+    // Create acceptor.
+    grpc_tcp_server_acceptor *acceptor = gpr_malloc(sizeof(*acceptor));
+    acceptor->from_server = sp->server;
+    acceptor->port_index = sp->port_index;
+    acceptor->fd_index = 0;
     sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
-                             &acceptor);
+                             acceptor);
   }
   /* As we were notified from the IOCP of one and exactly one accept,
      the former socked we created has now either been destroy or assigned
      to the new connection. We need to create a new one for the next
      connection. */
-  GPR_ASSERT(GRPC_LOG_IF_ERROR("start_accept", start_accept(exec_ctx, sp)));
+  GPR_ASSERT(
+      GRPC_LOG_IF_ERROR("start_accept", start_accept_locked(exec_ctx, sp)));
+  if (0 == --sp->outstanding_calls) {
+    decrement_active_ports_and_notify_locked(exec_ctx, sp);
+  }
+  gpr_mu_unlock(&sp->server->mu);
 }
 
 static grpc_error *add_socket_to_server(grpc_tcp_server *s, SOCKET sock,
@@ -456,6 +460,7 @@ static grpc_error *add_socket_to_server(grpc_tcp_server *s, SOCKET sock,
   sp->server = s;
   sp->socket = grpc_winsocket_create(sock, "listener");
   sp->shutting_down = 0;
+  sp->outstanding_calls = 0;
   sp->AcceptEx = AcceptEx;
   sp->new_socket = INVALID_SOCKET;
   sp->port = port;
@@ -553,7 +558,8 @@ void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s,
   s->on_accept_cb = on_accept_cb;
   s->on_accept_cb_arg = on_accept_cb_arg;
   for (sp = s->head; sp; sp = sp->next) {
-    GPR_ASSERT(GRPC_LOG_IF_ERROR("start_accept", start_accept(exec_ctx, sp)));
+    GPR_ASSERT(
+        GRPC_LOG_IF_ERROR("start_accept", start_accept_locked(exec_ctx, sp)));
     s->active_ports++;
   }
   gpr_mu_unlock(&s->mu);

src/core/lib/iomgr/udp_server.c

@@ -388,7 +388,8 @@ int grpc_udp_server_add_port(grpc_udp_server *s,
     /* Try listening on IPv6 first. */
     addr = &wild6;
     // TODO(rjshade): Test and propagate the returned grpc_error*:
-    grpc_create_dualstack_socket(addr, SOCK_DGRAM, IPPROTO_UDP, &dsmode, &fd);
+    GRPC_ERROR_UNREF(grpc_create_dualstack_socket(addr, SOCK_DGRAM, IPPROTO_UDP,
+                                                  &dsmode, &fd));
     allocated_port1 = add_socket_to_server(s, fd, addr, read_cb, orphan_cb);
     if (fd >= 0 && dsmode == GRPC_DSMODE_DUALSTACK) {
       goto done;
@@ -402,7 +403,8 @@ int grpc_udp_server_add_port(grpc_udp_server *s,
   }
 
   // TODO(rjshade): Test and propagate the returned grpc_error*:
-  grpc_create_dualstack_socket(addr, SOCK_DGRAM, IPPROTO_UDP, &dsmode, &fd);
+  GRPC_ERROR_UNREF(grpc_create_dualstack_socket(addr, SOCK_DGRAM, IPPROTO_UDP,
+                                                &dsmode, &fd));
   if (fd < 0) {
     gpr_log(GPR_ERROR, "Unable to create socket: %s", strerror(errno));
   }

src/core/lib/security/transport/handshake.c

@@ -1,374 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/lib/security/transport/handshake.h"
-
-#include <stdbool.h>
-#include <string.h>
-
-#include <grpc/slice_buffer.h>
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include "src/core/lib/iomgr/timer.h"
-#include "src/core/lib/security/context/security_context.h"
-#include "src/core/lib/security/transport/secure_endpoint.h"
-#include "src/core/lib/security/transport/tsi_error.h"
-
-#define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
-
-typedef struct {
-  grpc_security_connector *connector;
-  tsi_handshaker *handshaker;
-  bool is_client_side;
-  unsigned char *handshake_buffer;
-  size_t handshake_buffer_size;
-  grpc_endpoint *wrapped_endpoint;
-  grpc_endpoint *secure_endpoint;
-  grpc_slice_buffer left_overs;
-  grpc_slice_buffer incoming;
-  grpc_slice_buffer outgoing;
-  grpc_security_handshake_done_cb cb;
-  void *user_data;
-  grpc_closure on_handshake_data_sent_to_peer;
-  grpc_closure on_handshake_data_received_from_peer;
-  grpc_auth_context *auth_context;
-  grpc_timer timer;
-  gpr_refcount refs;
-} grpc_security_handshake;
-
-static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
-                                                 void *setup,
-                                                 grpc_error *error);
-
-static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, void *setup,
-                                           grpc_error *error);
-
-static void security_connector_remove_handshake(grpc_security_handshake *h) {
-  GPR_ASSERT(!h->is_client_side);
-  grpc_security_connector_handshake_list *node;
-  grpc_security_connector_handshake_list *tmp;
-  grpc_server_security_connector *sc =
-      (grpc_server_security_connector *)h->connector;
-  gpr_mu_lock(&sc->mu);
-  node = sc->handshaking_handshakes;
-  if (node && node->handshake == h) {
-    sc->handshaking_handshakes = node->next;
-    gpr_free(node);
-    gpr_mu_unlock(&sc->mu);
-    return;
-  }
-  while (node) {
-    if (node->next->handshake == h) {
-      tmp = node->next;
-      node->next = node->next->next;
-      gpr_free(tmp);
-      gpr_mu_unlock(&sc->mu);
-      return;
-    }
-    node = node->next;
-  }
-  gpr_mu_unlock(&sc->mu);
-}
-
-static void unref_handshake(grpc_security_handshake *h) {
-  if (gpr_unref(&h->refs)) {
-    if (h->handshaker != NULL) tsi_handshaker_destroy(h->handshaker);
-    if (h->handshake_buffer != NULL) gpr_free(h->handshake_buffer);
-    grpc_slice_buffer_destroy(&h->left_overs);
-    grpc_slice_buffer_destroy(&h->outgoing);
-    grpc_slice_buffer_destroy(&h->incoming);
-    GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake");
-    GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake");
-    gpr_free(h);
-  }
-}
-
-static void security_handshake_done(grpc_exec_ctx *exec_ctx,
-                                    grpc_security_handshake *h,
-                                    grpc_error *error) {
-  grpc_timer_cancel(exec_ctx, &h->timer);
-  if (!h->is_client_side) {
-    security_connector_remove_handshake(h);
-  }
-  if (error == GRPC_ERROR_NONE) {
-    h->cb(exec_ctx, h->user_data, GRPC_SECURITY_OK, h->secure_endpoint,
-          h->auth_context);
-  } else {
-    const char *msg = grpc_error_string(error);
-    gpr_log(GPR_DEBUG, "Security handshake failed: %s", msg);
-    grpc_error_free_string(msg);
-
-    if (h->secure_endpoint != NULL) {
-      grpc_endpoint_shutdown(exec_ctx, h->secure_endpoint);
-      grpc_endpoint_destroy(exec_ctx, h->secure_endpoint);
-    } else {
-      grpc_endpoint_destroy(exec_ctx, h->wrapped_endpoint);
-    }
-    h->cb(exec_ctx, h->user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-  }
-  unref_handshake(h);
-  GRPC_ERROR_UNREF(error);
-}
-
-static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *user_data,
-                            grpc_security_status status,
-                            grpc_auth_context *auth_context) {
-  grpc_security_handshake *h = user_data;
-  tsi_frame_protector *protector;
-  tsi_result result;
-  if (status != GRPC_SECURITY_OK) {
-    security_handshake_done(
-        exec_ctx, h,
-        grpc_error_set_int(GRPC_ERROR_CREATE("Error checking peer."),
-                           GRPC_ERROR_INT_SECURITY_STATUS, status));
-    return;
-  }
-  h->auth_context = GRPC_AUTH_CONTEXT_REF(auth_context, "handshake");
-  result =
-      tsi_handshaker_create_frame_protector(h->handshaker, NULL, &protector);
-  if (result != TSI_OK) {
-    security_handshake_done(
-        exec_ctx, h,
-        grpc_set_tsi_error_result(
-            GRPC_ERROR_CREATE("Frame protector creation failed"), result));
-    return;
-  }
-  h->secure_endpoint =
-      grpc_secure_endpoint_create(protector, h->wrapped_endpoint,
-                                  h->left_overs.slices, h->left_overs.count);
-  h->left_overs.count = 0;
-  h->left_overs.length = 0;
-  security_handshake_done(exec_ctx, h, GRPC_ERROR_NONE);
-  return;
-}
-
-static void check_peer(grpc_exec_ctx *exec_ctx, grpc_security_handshake *h) {
-  tsi_peer peer;
-  tsi_result result = tsi_handshaker_extract_peer(h->handshaker, &peer);
-
-  if (result != TSI_OK) {
-    security_handshake_done(
-        exec_ctx, h, grpc_set_tsi_error_result(
-                         GRPC_ERROR_CREATE("Peer extraction failed"), result));
-    return;
-  }
-  grpc_security_connector_check_peer(exec_ctx, h->connector, peer,
-                                     on_peer_checked, h);
-}
-
-static void send_handshake_bytes_to_peer(grpc_exec_ctx *exec_ctx,
-                                         grpc_security_handshake *h) {
-  size_t offset = 0;
-  tsi_result result = TSI_OK;
-  grpc_slice to_send;
-
-  do {
-    size_t to_send_size = h->handshake_buffer_size - offset;
-    result = tsi_handshaker_get_bytes_to_send_to_peer(
-        h->handshaker, h->handshake_buffer + offset, &to_send_size);
-    offset += to_send_size;
-    if (result == TSI_INCOMPLETE_DATA) {
-      h->handshake_buffer_size *= 2;
-      h->handshake_buffer =
-          gpr_realloc(h->handshake_buffer, h->handshake_buffer_size);
-    }
-  } while (result == TSI_INCOMPLETE_DATA);
-
-  if (result != TSI_OK) {
-    security_handshake_done(exec_ctx, h,
-                            grpc_set_tsi_error_result(
-                                GRPC_ERROR_CREATE("Handshake failed"), result));
-    return;
-  }
-
-  to_send =
-      grpc_slice_from_copied_buffer((const char *)h->handshake_buffer, offset);
-  grpc_slice_buffer_reset_and_unref(&h->outgoing);
-  grpc_slice_buffer_add(&h->outgoing, to_send);
-  /* TODO(klempner,jboeuf): This should probably use the client setup
-     deadline */
-  grpc_endpoint_write(exec_ctx, h->wrapped_endpoint, &h->outgoing,
-                      &h->on_handshake_data_sent_to_peer);
-}
-
-static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
-                                                 void *handshake,
-                                                 grpc_error *error) {
-  grpc_security_handshake *h = handshake;
-  size_t consumed_slice_size = 0;
-  tsi_result result = TSI_OK;
-  size_t i;
-  size_t num_left_overs;
-  int has_left_overs_in_current_slice = 0;
-
-  if (error != GRPC_ERROR_NONE) {
-    security_handshake_done(
-        exec_ctx, h,
-        GRPC_ERROR_CREATE_REFERENCING("Handshake read failed", &error, 1));
-    return;
-  }
-
-  for (i = 0; i < h->incoming.count; i++) {
-    consumed_slice_size = GRPC_SLICE_LENGTH(h->incoming.slices[i]);
-    result = tsi_handshaker_process_bytes_from_peer(
-        h->handshaker, GRPC_SLICE_START_PTR(h->incoming.slices[i]),
-        &consumed_slice_size);
-    if (!tsi_handshaker_is_in_progress(h->handshaker)) break;
-  }
-
-  if (tsi_handshaker_is_in_progress(h->handshaker)) {
-    /* We may need more data. */
-    if (result == TSI_INCOMPLETE_DATA) {
-      grpc_endpoint_read(exec_ctx, h->wrapped_endpoint, &h->incoming,
-                         &h->on_handshake_data_received_from_peer);
-      return;
-    } else {
-      send_handshake_bytes_to_peer(exec_ctx, h);
-      return;
-    }
-  }
-
-  if (result != TSI_OK) {
-    security_handshake_done(exec_ctx, h,
-                            grpc_set_tsi_error_result(
-                                GRPC_ERROR_CREATE("Handshake failed"), result));
-    return;
-  }
-
-  /* Handshake is done and successful this point. */
-  has_left_overs_in_current_slice =
-      (consumed_slice_size < GRPC_SLICE_LENGTH(h->incoming.slices[i]));
-  num_left_overs =
-      (has_left_overs_in_current_slice ? 1 : 0) + h->incoming.count - i - 1;
-  if (num_left_overs == 0) {
-    check_peer(exec_ctx, h);
-    return;
-  }
-
-  /* Put the leftovers in our buffer (ownership transfered). */
-  if (has_left_overs_in_current_slice) {
-    grpc_slice_buffer_add(
-        &h->left_overs,
-        grpc_slice_split_tail(&h->incoming.slices[i], consumed_slice_size));
-    grpc_slice_unref(
-        h->incoming.slices[i]); /* split_tail above increments refcount. */
-  }
-  grpc_slice_buffer_addn(
-      &h->left_overs, &h->incoming.slices[i + 1],
-      num_left_overs - (size_t)has_left_overs_in_current_slice);
-  check_peer(exec_ctx, h);
-}
-
-/* If handshake is NULL, the handshake is done. */
-static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx,
-                                           void *handshake, grpc_error *error) {
-  grpc_security_handshake *h = handshake;
-
-  /* Make sure that write is OK. */
-  if (error != GRPC_ERROR_NONE) {
-    if (handshake != NULL)
-      security_handshake_done(
-          exec_ctx, h,
-          GRPC_ERROR_CREATE_REFERENCING("Handshake write failed", &error, 1));
-    return;
-  }
-
-  /* We may be done. */
-  if (tsi_handshaker_is_in_progress(h->handshaker)) {
-    /* TODO(klempner,jboeuf): This should probably use the client setup
-       deadline */
-    grpc_endpoint_read(exec_ctx, h->wrapped_endpoint, &h->incoming,
-                       &h->on_handshake_data_received_from_peer);
-  } else {
-    check_peer(exec_ctx, h);
-  }
-}
-
-static void on_timeout(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
-  grpc_security_handshake *h = arg;
-  if (error == GRPC_ERROR_NONE) {
-    grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint);
-  }
-  unref_handshake(h);
-}
-
-void grpc_do_security_handshake(
-    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
-    grpc_security_connector *connector, bool is_client_side,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb,
-    void *user_data) {
-  grpc_security_connector_handshake_list *handshake_node;
-  grpc_security_handshake *h = gpr_malloc(sizeof(grpc_security_handshake));
-  memset(h, 0, sizeof(grpc_security_handshake));
-  h->handshaker = handshaker;
-  h->connector = GRPC_SECURITY_CONNECTOR_REF(connector, "handshake");
-  h->is_client_side = is_client_side;
-  h->handshake_buffer_size = GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE;
-  h->handshake_buffer = gpr_malloc(h->handshake_buffer_size);
-  h->wrapped_endpoint = nonsecure_endpoint;
-  h->user_data = user_data;
-  h->cb = cb;
-  gpr_ref_init(&h->refs, 2); /* timer and handshake proper each get a ref */
-  grpc_closure_init(&h->on_handshake_data_sent_to_peer,
-                    on_handshake_data_sent_to_peer, h);
-  grpc_closure_init(&h->on_handshake_data_received_from_peer,
-                    on_handshake_data_received_from_peer, h);
-  grpc_slice_buffer_init(&h->left_overs);
-  grpc_slice_buffer_init(&h->outgoing);
-  grpc_slice_buffer_init(&h->incoming);
-  if (read_buffer != NULL) {
-    grpc_slice_buffer_move_into(read_buffer, &h->incoming);
-    gpr_free(read_buffer);
-  }
-  if (!is_client_side) {
-    grpc_server_security_connector *server_connector =
-        (grpc_server_security_connector *)connector;
-    handshake_node = gpr_malloc(sizeof(grpc_security_connector_handshake_list));
-    handshake_node->handshake = h;
-    gpr_mu_lock(&server_connector->mu);
-    handshake_node->next = server_connector->handshaking_handshakes;
-    server_connector->handshaking_handshakes = handshake_node;
-    gpr_mu_unlock(&server_connector->mu);
-  }
-  send_handshake_bytes_to_peer(exec_ctx, h);
-  grpc_timer_init(exec_ctx, &h->timer,
-                  gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC),
-                  on_timeout, h, gpr_now(GPR_CLOCK_MONOTONIC));
-}
-
-void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx,
-                                      void *handshake) {
-  grpc_security_handshake *h = handshake;
-  grpc_endpoint_shutdown(exec_ctx, h->wrapped_endpoint);
-}

src/core/lib/security/transport/security_connector.c

@@ -43,11 +43,12 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/alpn/alpn.h"
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/iomgr/load_file.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/credentials/credentials.h"
-#include "src/core/lib/security/transport/handshake.h"
 #include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/security_handshaker.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/fake_transport_security.h"
@@ -111,58 +112,34 @@ const tsi_peer_property *tsi_peer_get_property_by_name(const tsi_peer *peer,
   return NULL;
 }
 
-void grpc_server_security_connector_shutdown(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector) {
-  grpc_security_connector_handshake_list *tmp;
-  gpr_mu_lock(&connector->mu);
-  while (connector->handshaking_handshakes) {
-    tmp = connector->handshaking_handshakes;
-    grpc_security_handshake_shutdown(
-        exec_ctx, connector->handshaking_handshakes->handshake);
-    connector->handshaking_handshakes = tmp->next;
-    gpr_free(tmp);
+void grpc_channel_security_connector_add_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector,
+    grpc_handshake_manager *handshake_mgr) {
+  if (connector != NULL) {
+    connector->add_handshakers(exec_ctx, connector, handshake_mgr);
   }
-  gpr_mu_unlock(&connector->mu);
 }
 
-void grpc_channel_security_connector_do_handshake(
-    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb,
-    void *user_data) {
-  if (sc == NULL || nonsecure_endpoint == NULL) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-  } else {
-    sc->do_handshake(exec_ctx, sc, nonsecure_endpoint, read_buffer, deadline,
-                     cb, user_data);
-  }
-}
-
-void grpc_server_security_connector_do_handshake(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data) {
-  if (sc == NULL || nonsecure_endpoint == NULL) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL, NULL);
-  } else {
-    sc->do_handshake(exec_ctx, sc, acceptor, nonsecure_endpoint, read_buffer,
-                     deadline, cb, user_data);
+void grpc_server_security_connector_add_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector,
+    grpc_handshake_manager *handshake_mgr) {
+  if (connector != NULL) {
+    connector->add_handshakers(exec_ctx, connector, handshake_mgr);
   }
 }
 
 void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx,
                                         grpc_security_connector *sc,
                                         tsi_peer peer,
-                                        grpc_security_peer_check_cb cb,
-                                        void *user_data) {
+                                        grpc_auth_context **auth_context,
+                                        grpc_closure *on_peer_checked) {
   if (sc == NULL) {
-    cb(exec_ctx, user_data, GRPC_SECURITY_ERROR, NULL);
+    grpc_exec_ctx_sched(
+        exec_ctx, on_peer_checked,
+        GRPC_ERROR_CREATE("cannot check peer -- no security connector"), NULL);
     tsi_peer_destruct(&peer);
   } else {
-    sc->vtable->check_peer(exec_ctx, sc, peer, cb, user_data);
+    sc->vtable->check_peer(exec_ctx, sc, peer, auth_context, on_peer_checked);
   }
 }
 
@@ -262,45 +239,41 @@ static void fake_channel_destroy(grpc_security_connector *sc) {
   gpr_free(sc);
 }
 
-static void fake_server_destroy(grpc_security_connector *sc) {
-  grpc_server_security_connector *c = (grpc_server_security_connector *)sc;
-  gpr_mu_destroy(&c->mu);
-  gpr_free(sc);
-}
+static void fake_server_destroy(grpc_security_connector *sc) { gpr_free(sc); }
 
 static void fake_check_peer(grpc_exec_ctx *exec_ctx,
                             grpc_security_connector *sc, tsi_peer peer,
-                            grpc_security_peer_check_cb cb, void *user_data) {
+                            grpc_auth_context **auth_context,
+                            grpc_closure *on_peer_checked) {
   const char *prop_name;
-  grpc_security_status status = GRPC_SECURITY_OK;
-  grpc_auth_context *auth_context = NULL;
+  grpc_error *error = GRPC_ERROR_NONE;
+  *auth_context = NULL;
   if (peer.property_count != 1) {
-    gpr_log(GPR_ERROR, "Fake peers should only have 1 property.");
-    status = GRPC_SECURITY_ERROR;
+    error = GRPC_ERROR_CREATE("Fake peers should only have 1 property.");
     goto end;
   }
   prop_name = peer.properties[0].name;
   if (prop_name == NULL ||
       strcmp(prop_name, TSI_CERTIFICATE_TYPE_PEER_PROPERTY)) {
-    gpr_log(GPR_ERROR, "Unexpected property in fake peer: %s.",
-            prop_name == NULL ? "<EMPTY>" : prop_name);
-    status = GRPC_SECURITY_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "Unexpected property in fake peer: %s.",
+                 prop_name == NULL ? "<EMPTY>" : prop_name);
+    error = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
     goto end;
   }
   if (strncmp(peer.properties[0].value.data, TSI_FAKE_CERTIFICATE_TYPE,
               peer.properties[0].value.length)) {
-    gpr_log(GPR_ERROR, "Invalid value for cert type property.");
-    status = GRPC_SECURITY_ERROR;
+    error = GRPC_ERROR_CREATE("Invalid value for cert type property.");
     goto end;
   }
-  auth_context = grpc_auth_context_create(NULL);
+  *auth_context = grpc_auth_context_create(NULL);
   grpc_auth_context_add_cstring_property(
-      auth_context, GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME,
+      *auth_context, GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME,
       GRPC_FAKE_TRANSPORT_SECURITY_TYPE);
 
 end:
-  cb(exec_ctx, user_data, status, auth_context);
-  grpc_auth_context_unref(auth_context);
+  grpc_exec_ctx_sched(exec_ctx, on_peer_checked, error, NULL);
   tsi_peer_destruct(&peer);
 }
 
@@ -313,26 +286,24 @@ static void fake_channel_check_call_host(grpc_exec_ctx *exec_ctx,
   cb(exec_ctx, user_data, GRPC_SECURITY_OK);
 }
 
-static void fake_channel_do_handshake(grpc_exec_ctx *exec_ctx,
-                                      grpc_channel_security_connector *sc,
-                                      grpc_endpoint *nonsecure_endpoint,
-                                      grpc_slice_buffer *read_buffer,
-                                      gpr_timespec deadline,
-                                      grpc_security_handshake_done_cb cb,
-                                      void *user_data) {
-  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(1), &sc->base,
-                             true, nonsecure_endpoint, read_buffer, deadline,
-                             cb, user_data);
+static void fake_channel_add_handshakers(
+    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
+    grpc_handshake_manager *handshake_mgr) {
+  grpc_handshake_manager_add(
+      handshake_mgr,
+      grpc_security_handshaker_create(
+          exec_ctx, tsi_create_fake_handshaker(true /* is_client */),
+          &sc->base));
 }
 
-static void fake_server_do_handshake(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data) {
-  grpc_do_security_handshake(exec_ctx, tsi_create_fake_handshaker(0), &sc->base,
-                             false, nonsecure_endpoint, read_buffer, deadline,
-                             cb, user_data);
+static void fake_server_add_handshakers(grpc_exec_ctx *exec_ctx,
+                                        grpc_server_security_connector *sc,
+                                        grpc_handshake_manager *handshake_mgr) {
+  grpc_handshake_manager_add(
+      handshake_mgr,
+      grpc_security_handshaker_create(
+          exec_ctx, tsi_create_fake_handshaker(false /* is_client */),
+          &sc->base));
 }
 
 static grpc_security_connector_vtable fake_channel_vtable = {
@@ -350,7 +321,7 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create(
   c->base.vtable = &fake_channel_vtable;
   c->request_metadata_creds = grpc_call_credentials_ref(request_metadata_creds);
   c->check_call_host = fake_channel_check_call_host;
-  c->do_handshake = fake_channel_do_handshake;
+  c->add_handshakers = fake_channel_add_handshakers;
   return c;
 }
 
@@ -362,8 +333,7 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create(
   gpr_ref_init(&c->base.refcount, 1);
   c->base.vtable = &fake_server_vtable;
   c->base.url_scheme = GRPC_FAKE_SECURITY_URL_SCHEME;
-  c->do_handshake = fake_server_do_handshake;
-  gpr_mu_init(&c->mu);
+  c->add_handshakers = fake_server_add_handshakers;
   return c;
 }
 
@@ -396,11 +366,9 @@ static void ssl_channel_destroy(grpc_security_connector *sc) {
 static void ssl_server_destroy(grpc_security_connector *sc) {
   grpc_ssl_server_security_connector *c =
       (grpc_ssl_server_security_connector *)sc;
-
   if (c->handshaker_factory != NULL) {
     tsi_ssl_handshaker_factory_destroy(c->handshaker_factory);
   }
-  gpr_mu_destroy(&c->base.mu);
   gpr_free(sc);
 }
 
@@ -419,49 +387,35 @@ static grpc_security_status ssl_create_handshaker(
   return GRPC_SECURITY_OK;
 }
 
-static void ssl_channel_do_handshake(grpc_exec_ctx *exec_ctx,
-                                     grpc_channel_security_connector *sc,
-                                     grpc_endpoint *nonsecure_endpoint,
-                                     grpc_slice_buffer *read_buffer,
-                                     gpr_timespec deadline,
-                                     grpc_security_handshake_done_cb cb,
-                                     void *user_data) {
+static void ssl_channel_add_handshakers(grpc_exec_ctx *exec_ctx,
+                                        grpc_channel_security_connector *sc,
+                                        grpc_handshake_manager *handshake_mgr) {
   grpc_ssl_channel_security_connector *c =
       (grpc_ssl_channel_security_connector *)sc;
-  tsi_handshaker *handshaker;
-  grpc_security_status status = ssl_create_handshaker(
-      c->handshaker_factory, true,
-      c->overridden_target_name != NULL ? c->overridden_target_name
-                                        : c->target_name,
-      &handshaker);
-  if (status != GRPC_SECURITY_OK) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, status, NULL, NULL);
-  } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, true,
-                               nonsecure_endpoint, read_buffer, deadline, cb,
-                               user_data);
-  }
-}
-
-static void ssl_server_do_handshake(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data) {
+  // Instantiate TSI handshaker.
+  tsi_handshaker *tsi_hs = NULL;
+  ssl_create_handshaker(c->handshaker_factory, true /* is_client */,
+                        c->overridden_target_name != NULL
+                            ? c->overridden_target_name
+                            : c->target_name,
+                        &tsi_hs);
+  // Create handshakers.
+  grpc_handshake_manager_add(handshake_mgr, grpc_security_handshaker_create(
+                                                exec_ctx, tsi_hs, &sc->base));
+}
+
+static void ssl_server_add_handshakers(grpc_exec_ctx *exec_ctx,
+                                       grpc_server_security_connector *sc,
+                                       grpc_handshake_manager *handshake_mgr) {
   grpc_ssl_server_security_connector *c =
       (grpc_ssl_server_security_connector *)sc;
-  tsi_handshaker *handshaker;
-  grpc_security_status status =
-      ssl_create_handshaker(c->handshaker_factory, false, NULL, &handshaker);
-  if (status != GRPC_SECURITY_OK) {
-    gpr_free(read_buffer);
-    cb(exec_ctx, user_data, status, NULL, NULL);
-  } else {
-    grpc_do_security_handshake(exec_ctx, handshaker, &sc->base, false,
-                               nonsecure_endpoint, read_buffer, deadline, cb,
-                               user_data);
-  }
+  // Instantiate TSI handshaker.
+  tsi_handshaker *tsi_hs = NULL;
+  ssl_create_handshaker(c->handshaker_factory, false /* is_client */,
+                        NULL /* peer_name */, &tsi_hs);
+  // Create handshakers.
+  grpc_handshake_manager_add(handshake_mgr, grpc_security_handshaker_create(
+                                                exec_ctx, tsi_hs, &sc->base));
 }
 
 static int ssl_host_matches_name(const tsi_peer *peer, const char *peer_name) {
@@ -518,57 +472,53 @@ grpc_auth_context *tsi_ssl_peer_to_auth_context(const tsi_peer *peer) {
   return ctx;
 }
 
-static grpc_security_status ssl_check_peer(grpc_security_connector *sc,
-                                           const char *peer_name,
-                                           const tsi_peer *peer,
-                                           grpc_auth_context **auth_context) {
+static grpc_error *ssl_check_peer(grpc_security_connector *sc,
+                                  const char *peer_name, const tsi_peer *peer,
+                                  grpc_auth_context **auth_context) {
   /* Check the ALPN. */
   const tsi_peer_property *p =
       tsi_peer_get_property_by_name(peer, TSI_SSL_ALPN_SELECTED_PROTOCOL);
   if (p == NULL) {
-    gpr_log(GPR_ERROR, "Missing selected ALPN property.");
-    return GRPC_SECURITY_ERROR;
+    return GRPC_ERROR_CREATE(
+        "Cannot check peer: missing selected ALPN property.");
   }
   if (!grpc_chttp2_is_alpn_version_supported(p->value.data, p->value.length)) {
-    gpr_log(GPR_ERROR, "Invalid ALPN value.");
-    return GRPC_SECURITY_ERROR;
+    return GRPC_ERROR_CREATE("Cannot check peer: invalid ALPN value.");
   }
 
   /* Check the peer name if specified. */
   if (peer_name != NULL && !ssl_host_matches_name(peer, peer_name)) {
-    gpr_log(GPR_ERROR, "Peer name %s is not in peer certificate", peer_name);
-    return GRPC_SECURITY_ERROR;
+    char *msg;
+    gpr_asprintf(&msg, "Peer name %s is not in peer certificate", peer_name);
+    grpc_error *error = GRPC_ERROR_CREATE(msg);
+    gpr_free(msg);
+    return error;
   }
   *auth_context = tsi_ssl_peer_to_auth_context(peer);
-  return GRPC_SECURITY_OK;
+  return GRPC_ERROR_NONE;
 }
 
 static void ssl_channel_check_peer(grpc_exec_ctx *exec_ctx,
                                    grpc_security_connector *sc, tsi_peer peer,
-                                   grpc_security_peer_check_cb cb,
-                                   void *user_data) {
+                                   grpc_auth_context **auth_context,
+                                   grpc_closure *on_peer_checked) {
   grpc_ssl_channel_security_connector *c =
       (grpc_ssl_channel_security_connector *)sc;
-  grpc_security_status status;
-  grpc_auth_context *auth_context = NULL;
-  status = ssl_check_peer(sc, c->overridden_target_name != NULL
-                                  ? c->overridden_target_name
-                                  : c->target_name,
-                          &peer, &auth_context);
-  cb(exec_ctx, user_data, status, auth_context);
-  grpc_auth_context_unref(auth_context);
+  grpc_error *error = ssl_check_peer(sc, c->overridden_target_name != NULL
+                                             ? c->overridden_target_name
+                                             : c->target_name,
+                                     &peer, auth_context);
+  grpc_exec_ctx_sched(exec_ctx, on_peer_checked, error, NULL);
   tsi_peer_destruct(&peer);
 }
 
 static void ssl_server_check_peer(grpc_exec_ctx *exec_ctx,
                                   grpc_security_connector *sc, tsi_peer peer,
-                                  grpc_security_peer_check_cb cb,
-                                  void *user_data) {
-  grpc_auth_context *auth_context = NULL;
-  grpc_security_status status = ssl_check_peer(sc, NULL, &peer, &auth_context);
+                                  grpc_auth_context **auth_context,
+                                  grpc_closure *on_peer_checked) {
+  grpc_error *error = ssl_check_peer(sc, NULL, &peer, auth_context);
   tsi_peer_destruct(&peer);
-  cb(exec_ctx, user_data, status, auth_context);
-  grpc_auth_context_unref(auth_context);
+  grpc_exec_ctx_sched(exec_ctx, on_peer_checked, error, NULL);
 }
 
 static void add_shallow_auth_property_to_peer(tsi_peer *peer,
@@ -765,7 +715,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
   c->base.request_metadata_creds =
       grpc_call_credentials_ref(request_metadata_creds);
   c->base.check_call_host = ssl_channel_check_call_host;
-  c->base.do_handshake = ssl_channel_do_handshake;
+  c->base.add_handshakers = ssl_channel_add_handshakers;
   gpr_split_host_port(target_name, &c->target_name, &port);
   gpr_free(port);
   if (overridden_target_name != NULL) {
@@ -840,8 +790,7 @@ grpc_security_status grpc_ssl_server_security_connector_create(
     *sc = NULL;
     goto error;
   }
-  gpr_mu_init(&c->base.mu);
-  c->base.do_handshake = ssl_server_do_handshake;
+  c->base.add_handshakers = ssl_server_add_handshakers;
   *sc = &c->base;
   gpr_free((void *)alpn_protocol_strings);
   gpr_free(alpn_protocol_string_lengths);

src/core/lib/security/transport/security_connector.h

@@ -35,6 +35,8 @@
 #define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
 
 #include <grpc/grpc_security.h>
+
+#include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/tcp_server.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
@@ -57,21 +59,11 @@ typedef struct grpc_security_connector grpc_security_connector;
 
 #define GRPC_SECURITY_CONNECTOR_ARG "grpc.security_connector"
 
-typedef void (*grpc_security_peer_check_cb)(grpc_exec_ctx *exec_ctx,
-                                            void *user_data,
-                                            grpc_security_status status,
-                                            grpc_auth_context *auth_context);
-
-/* Ownership of the secure_endpoint is transfered. */
-typedef void (*grpc_security_handshake_done_cb)(
-    grpc_exec_ctx *exec_ctx, void *user_data, grpc_security_status status,
-    grpc_endpoint *secure_endpoint, grpc_auth_context *auth_context);
-
 typedef struct {
   void (*destroy)(grpc_security_connector *sc);
   void (*check_peer)(grpc_exec_ctx *exec_ctx, grpc_security_connector *sc,
-                     tsi_peer peer, grpc_security_peer_check_cb cb,
-                     void *user_data);
+                     tsi_peer peer, grpc_auth_context **auth_context,
+                     grpc_closure *on_peer_checked);
 } grpc_security_connector_vtable;
 
 typedef struct grpc_security_connector_handshake_list {
@@ -106,12 +98,12 @@ void grpc_security_connector_unref(grpc_security_connector *policy);
 #endif
 
 /* Check the peer. Callee takes ownership of the peer object.
-   The callback will include the resulting auth_context. */
+   When done, sets *auth_context and invokes on_peer_checked. */
 void grpc_security_connector_check_peer(grpc_exec_ctx *exec_ctx,
                                         grpc_security_connector *sc,
                                         tsi_peer peer,
-                                        grpc_security_peer_check_cb cb,
-                                        void *user_data);
+                                        grpc_auth_context **auth_context,
+                                        grpc_closure *on_peer_checked);
 
 /* Util to encapsulate the connector in a channel arg. */
 grpc_arg grpc_security_connector_to_arg(grpc_security_connector *sc);
@@ -141,11 +133,9 @@ struct grpc_channel_security_connector {
                           grpc_channel_security_connector *sc, const char *host,
                           grpc_auth_context *auth_context,
                           grpc_security_call_host_check_cb cb, void *user_data);
-  void (*do_handshake)(grpc_exec_ctx *exec_ctx,
-                       grpc_channel_security_connector *sc,
-                       grpc_endpoint *nonsecure_endpoint,
-                       grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-                       grpc_security_handshake_done_cb cb, void *user_data);
+  void (*add_handshakers)(grpc_exec_ctx *exec_ctx,
+                          grpc_channel_security_connector *sc,
+                          grpc_handshake_manager *handshake_mgr);
 };
 
 /* Checks that the host that will be set for a call is acceptable. */
@@ -154,11 +144,10 @@ void grpc_channel_security_connector_check_call_host(
     const char *host, grpc_auth_context *auth_context,
     grpc_security_call_host_check_cb cb, void *user_data);
 
-/* Handshake. */
-void grpc_channel_security_connector_do_handshake(
+/* Registers handshakers with \a handshake_mgr. */
+void grpc_channel_security_connector_add_handshakers(
     grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *connector,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data);
+    grpc_handshake_manager *handshake_mgr);
 
 /* --- server_security_connector object. ---
 
@@ -169,25 +158,14 @@ typedef struct grpc_server_security_connector grpc_server_security_connector;
 
 struct grpc_server_security_connector {
   grpc_security_connector base;
-  gpr_mu mu;
-  grpc_security_connector_handshake_list *handshaking_handshakes;
-  const grpc_channel_args *channel_args;
-  void (*do_handshake)(grpc_exec_ctx *exec_ctx,
-                       grpc_server_security_connector *sc,
-                       grpc_tcp_server_acceptor *acceptor,
-                       grpc_endpoint *nonsecure_endpoint,
-                       grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-                       grpc_security_handshake_done_cb cb, void *user_data);
+  void (*add_handshakers)(grpc_exec_ctx *exec_ctx,
+                          grpc_server_security_connector *sc,
+                          grpc_handshake_manager *handshake_mgr);
 };
 
-void grpc_server_security_connector_do_handshake(
+void grpc_server_security_connector_add_handshakers(
     grpc_exec_ctx *exec_ctx, grpc_server_security_connector *sc,
-    grpc_tcp_server_acceptor *acceptor, grpc_endpoint *nonsecure_endpoint,
-    grpc_slice_buffer *read_buffer, gpr_timespec deadline,
-    grpc_security_handshake_done_cb cb, void *user_data);
-
-void grpc_server_security_connector_shutdown(
-    grpc_exec_ctx *exec_ctx, grpc_server_security_connector *connector);
+    grpc_handshake_manager *handshake_mgr);
 
 /* --- Creation security connectors. --- */
 

src/core/lib/security/transport/security_handshaker.c

@@ -0,0 +1,450 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/transport/security_handshaker.h"
+
+#include <stdbool.h>
+#include <string.h>
+
+#include <grpc/slice_buffer.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/tsi_error.h"
+
+#define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
+
+typedef struct {
+  grpc_handshaker base;
+
+  // State set at creation time.
+  tsi_handshaker *handshaker;
+  grpc_security_connector *connector;
+
+  gpr_mu mu;
+  gpr_refcount refs;
+
+  bool shutdown;
+  // Endpoint and read buffer to destroy after a shutdown.
+  grpc_endpoint *endpoint_to_destroy;
+  grpc_slice_buffer *read_buffer_to_destroy;
+
+  // State saved while performing the handshake.
+  grpc_handshaker_args *args;
+  grpc_closure *on_handshake_done;
+
+  unsigned char *handshake_buffer;
+  size_t handshake_buffer_size;
+  grpc_slice_buffer left_overs;
+  grpc_slice_buffer outgoing;
+  grpc_closure on_handshake_data_sent_to_peer;
+  grpc_closure on_handshake_data_received_from_peer;
+  grpc_closure on_peer_checked;
+  grpc_auth_context *auth_context;
+} security_handshaker;
+
+static void security_handshaker_unref(grpc_exec_ctx *exec_ctx,
+                                      security_handshaker *h) {
+  if (gpr_unref(&h->refs)) {
+    gpr_mu_destroy(&h->mu);
+    tsi_handshaker_destroy(h->handshaker);
+    if (h->endpoint_to_destroy != NULL) {
+      grpc_endpoint_destroy(exec_ctx, h->endpoint_to_destroy);
+    }
+    if (h->read_buffer_to_destroy != NULL) {
+      grpc_slice_buffer_destroy(h->read_buffer_to_destroy);
+      gpr_free(h->read_buffer_to_destroy);
+    }
+    gpr_free(h->handshake_buffer);
+    grpc_slice_buffer_destroy(&h->left_overs);
+    grpc_slice_buffer_destroy(&h->outgoing);
+    GRPC_AUTH_CONTEXT_UNREF(h->auth_context, "handshake");
+    GRPC_SECURITY_CONNECTOR_UNREF(h->connector, "handshake");
+    gpr_free(h);
+  }
+}
+
+// Set args fields to NULL, saving the endpoint and read buffer for
+// later destruction.
+static void cleanup_args_for_failure_locked(security_handshaker *h) {
+  h->endpoint_to_destroy = h->args->endpoint;
+  h->args->endpoint = NULL;
+  h->read_buffer_to_destroy = h->args->read_buffer;
+  h->args->read_buffer = NULL;
+  grpc_channel_args_destroy(h->args->args);
+  h->args->args = NULL;
+}
+
+// If the handshake failed or we're shutting down, clean up and invoke the
+// callback with the error.
+static void security_handshake_failed_locked(grpc_exec_ctx *exec_ctx,
+                                             security_handshaker *h,
+                                             grpc_error *error) {
+  if (error == GRPC_ERROR_NONE) {
+    // If we were shut down after the handshake succeeded but before an
+    // endpoint callback was invoked, we need to generate our own error.
+    error = GRPC_ERROR_CREATE("Handshaker shutdown");
+  }
+  const char *msg = grpc_error_string(error);
+  gpr_log(GPR_DEBUG, "Security handshake failed: %s", msg);
+  grpc_error_free_string(msg);
+  if (!h->shutdown) {
+    // TODO(ctiller): It is currently necessary to shutdown endpoints
+    // before destroying them, even if we know that there are no
+    // pending read/write callbacks.  This should be fixed, at which
+    // point this can be removed.
+    grpc_endpoint_shutdown(exec_ctx, h->args->endpoint);
+    // Not shutting down, so the write failed.  Clean up before
+    // invoking the callback.
+    cleanup_args_for_failure_locked(h);
+    // Set shutdown to true so that subsequent calls to
+    // security_handshaker_shutdown() do nothing.
+    h->shutdown = true;
+  }
+  // Invoke callback.
+  grpc_exec_ctx_sched(exec_ctx, h->on_handshake_done, error, NULL);
+}
+
+static void on_peer_checked(grpc_exec_ctx *exec_ctx, void *arg,
+                            grpc_error *error) {
+  security_handshaker *h = arg;
+  gpr_mu_lock(&h->mu);
+  if (error != GRPC_ERROR_NONE || h->shutdown) {
+    security_handshake_failed_locked(exec_ctx, h, GRPC_ERROR_REF(error));
+    goto done;
+  }
+  // Get frame protector.
+  tsi_frame_protector *protector;
+  tsi_result result =
+      tsi_handshaker_create_frame_protector(h->handshaker, NULL, &protector);
+  if (result != TSI_OK) {
+    error = grpc_set_tsi_error_result(
+        GRPC_ERROR_CREATE("Frame protector creation failed"), result);
+    security_handshake_failed_locked(exec_ctx, h, error);
+    goto done;
+  }
+  // Success.
+  // Create secure endpoint.
+  h->args->endpoint = grpc_secure_endpoint_create(
+      protector, h->args->endpoint, h->left_overs.slices, h->left_overs.count);
+  h->left_overs.count = 0;
+  h->left_overs.length = 0;
+  // Clear out the read buffer before it gets passed to the transport,
+  // since any excess bytes were already copied to h->left_overs.
+  grpc_slice_buffer_reset_and_unref(h->args->read_buffer);
+  // Add auth context to channel args.
+  grpc_arg auth_context_arg = grpc_auth_context_to_arg(h->auth_context);
+  grpc_channel_args *tmp_args = h->args->args;
+  h->args->args =
+      grpc_channel_args_copy_and_add(tmp_args, &auth_context_arg, 1);
+  grpc_channel_args_destroy(tmp_args);
+  // Invoke callback.
+  grpc_exec_ctx_sched(exec_ctx, h->on_handshake_done, GRPC_ERROR_NONE, NULL);
+  // Set shutdown to true so that subsequent calls to
+  // security_handshaker_shutdown() do nothing.
+  h->shutdown = true;
+done:
+  gpr_mu_unlock(&h->mu);
+  security_handshaker_unref(exec_ctx, h);
+}
+
+static grpc_error *check_peer_locked(grpc_exec_ctx *exec_ctx,
+                                     security_handshaker *h) {
+  tsi_peer peer;
+  tsi_result result = tsi_handshaker_extract_peer(h->handshaker, &peer);
+  if (result != TSI_OK) {
+    return grpc_set_tsi_error_result(
+        GRPC_ERROR_CREATE("Peer extraction failed"), result);
+  }
+  grpc_security_connector_check_peer(exec_ctx, h->connector, peer,
+                                     &h->auth_context, &h->on_peer_checked);
+  return GRPC_ERROR_NONE;
+}
+
+static grpc_error *send_handshake_bytes_to_peer_locked(grpc_exec_ctx *exec_ctx,
+                                                       security_handshaker *h) {
+  // Get data to send.
+  tsi_result result = TSI_OK;
+  size_t offset = 0;
+  do {
+    size_t to_send_size = h->handshake_buffer_size - offset;
+    result = tsi_handshaker_get_bytes_to_send_to_peer(
+        h->handshaker, h->handshake_buffer + offset, &to_send_size);
+    offset += to_send_size;
+    if (result == TSI_INCOMPLETE_DATA) {
+      h->handshake_buffer_size *= 2;
+      h->handshake_buffer =
+          gpr_realloc(h->handshake_buffer, h->handshake_buffer_size);
+    }
+  } while (result == TSI_INCOMPLETE_DATA);
+  if (result != TSI_OK) {
+    return grpc_set_tsi_error_result(GRPC_ERROR_CREATE("Handshake failed"),
+                                     result);
+  }
+  // Send data.
+  grpc_slice to_send =
+      grpc_slice_from_copied_buffer((const char *)h->handshake_buffer, offset);
+  grpc_slice_buffer_reset_and_unref(&h->outgoing);
+  grpc_slice_buffer_add(&h->outgoing, to_send);
+  grpc_endpoint_write(exec_ctx, h->args->endpoint, &h->outgoing,
+                      &h->on_handshake_data_sent_to_peer);
+  return GRPC_ERROR_NONE;
+}
+
+static void on_handshake_data_received_from_peer(grpc_exec_ctx *exec_ctx,
+                                                 void *arg, grpc_error *error) {
+  security_handshaker *h = arg;
+  gpr_mu_lock(&h->mu);
+  if (error != GRPC_ERROR_NONE || h->shutdown) {
+    security_handshake_failed_locked(
+        exec_ctx, h,
+        GRPC_ERROR_CREATE_REFERENCING("Handshake read failed", &error, 1));
+    gpr_mu_unlock(&h->mu);
+    security_handshaker_unref(exec_ctx, h);
+    return;
+  }
+  // Process received data.
+  tsi_result result = TSI_OK;
+  size_t consumed_slice_size = 0;
+  size_t i;
+  for (i = 0; i < h->args->read_buffer->count; i++) {
+    consumed_slice_size = GRPC_SLICE_LENGTH(h->args->read_buffer->slices[i]);
+    result = tsi_handshaker_process_bytes_from_peer(
+        h->handshaker, GRPC_SLICE_START_PTR(h->args->read_buffer->slices[i]),
+        &consumed_slice_size);
+    if (!tsi_handshaker_is_in_progress(h->handshaker)) break;
+  }
+  if (tsi_handshaker_is_in_progress(h->handshaker)) {
+    /* We may need more data. */
+    if (result == TSI_INCOMPLETE_DATA) {
+      grpc_endpoint_read(exec_ctx, h->args->endpoint, h->args->read_buffer,
+                         &h->on_handshake_data_received_from_peer);
+      goto done;
+    } else {
+      error = send_handshake_bytes_to_peer_locked(exec_ctx, h);
+      if (error != GRPC_ERROR_NONE) {
+        security_handshake_failed_locked(exec_ctx, h, error);
+        gpr_mu_unlock(&h->mu);
+        security_handshaker_unref(exec_ctx, h);
+        return;
+      }
+      goto done;
+    }
+  }
+  if (result != TSI_OK) {
+    security_handshake_failed_locked(
+        exec_ctx, h, grpc_set_tsi_error_result(
+                         GRPC_ERROR_CREATE("Handshake failed"), result));
+    gpr_mu_unlock(&h->mu);
+    security_handshaker_unref(exec_ctx, h);
+    return;
+  }
+  /* Handshake is done and successful this point. */
+  bool has_left_overs_in_current_slice =
+      (consumed_slice_size <
+       GRPC_SLICE_LENGTH(h->args->read_buffer->slices[i]));
+  size_t num_left_overs = (has_left_overs_in_current_slice ? 1 : 0) +
+                          h->args->read_buffer->count - i - 1;
+  if (num_left_overs > 0) {
+    /* Put the leftovers in our buffer (ownership transfered). */
+    if (has_left_overs_in_current_slice) {
+      grpc_slice_buffer_add(
+          &h->left_overs,
+          grpc_slice_split_tail(&h->args->read_buffer->slices[i],
+                                consumed_slice_size));
+      /* split_tail above increments refcount. */
+      grpc_slice_unref(h->args->read_buffer->slices[i]);
+    }
+    grpc_slice_buffer_addn(
+        &h->left_overs, &h->args->read_buffer->slices[i + 1],
+        num_left_overs - (size_t)has_left_overs_in_current_slice);
+  }
+  // Check peer.
+  error = check_peer_locked(exec_ctx, h);
+  if (error != GRPC_ERROR_NONE) {
+    security_handshake_failed_locked(exec_ctx, h, error);
+    gpr_mu_unlock(&h->mu);
+    security_handshaker_unref(exec_ctx, h);
+    return;
+  }
+done:
+  gpr_mu_unlock(&h->mu);
+}
+
+static void on_handshake_data_sent_to_peer(grpc_exec_ctx *exec_ctx, void *arg,
+                                           grpc_error *error) {
+  security_handshaker *h = arg;
+  gpr_mu_lock(&h->mu);
+  if (error != GRPC_ERROR_NONE || h->shutdown) {
+    security_handshake_failed_locked(
+        exec_ctx, h,
+        GRPC_ERROR_CREATE_REFERENCING("Handshake write failed", &error, 1));
+    gpr_mu_unlock(&h->mu);
+    security_handshaker_unref(exec_ctx, h);
+    return;
+  }
+  /* We may be done. */
+  if (tsi_handshaker_is_in_progress(h->handshaker)) {
+    grpc_endpoint_read(exec_ctx, h->args->endpoint, h->args->read_buffer,
+                       &h->on_handshake_data_received_from_peer);
+  } else {
+    error = check_peer_locked(exec_ctx, h);
+    if (error != GRPC_ERROR_NONE) {
+      security_handshake_failed_locked(exec_ctx, h, error);
+      gpr_mu_unlock(&h->mu);
+      security_handshaker_unref(exec_ctx, h);
+      return;
+    }
+  }
+  gpr_mu_unlock(&h->mu);
+}
+
+//
+// public handshaker API
+//
+
+static void security_handshaker_destroy(grpc_exec_ctx *exec_ctx,
+                                        grpc_handshaker *handshaker) {
+  security_handshaker *h = (security_handshaker *)handshaker;
+  security_handshaker_unref(exec_ctx, h);
+}
+
+static void security_handshaker_shutdown(grpc_exec_ctx *exec_ctx,
+                                         grpc_handshaker *handshaker) {
+  security_handshaker *h = (security_handshaker *)handshaker;
+  gpr_mu_lock(&h->mu);
+  if (!h->shutdown) {
+    h->shutdown = true;
+    grpc_endpoint_shutdown(exec_ctx, h->args->endpoint);
+    cleanup_args_for_failure_locked(h);
+  }
+  gpr_mu_unlock(&h->mu);
+}
+
+static void security_handshaker_do_handshake(grpc_exec_ctx *exec_ctx,
+                                             grpc_handshaker *handshaker,
+                                             grpc_tcp_server_acceptor *acceptor,
+                                             grpc_closure *on_handshake_done,
+                                             grpc_handshaker_args *args) {
+  security_handshaker *h = (security_handshaker *)handshaker;
+  gpr_mu_lock(&h->mu);
+  h->args = args;
+  h->on_handshake_done = on_handshake_done;
+  gpr_ref(&h->refs);
+  grpc_error *error = send_handshake_bytes_to_peer_locked(exec_ctx, h);
+  if (error != GRPC_ERROR_NONE) {
+    security_handshake_failed_locked(exec_ctx, h, error);
+    gpr_mu_unlock(&h->mu);
+    security_handshaker_unref(exec_ctx, h);
+    return;
+  }
+  gpr_mu_unlock(&h->mu);
+}
+
+static const grpc_handshaker_vtable security_handshaker_vtable = {
+    security_handshaker_destroy, security_handshaker_shutdown,
+    security_handshaker_do_handshake};
+
+static grpc_handshaker *security_handshaker_create(
+    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
+    grpc_security_connector *connector) {
+  security_handshaker *h = gpr_malloc(sizeof(security_handshaker));
+  memset(h, 0, sizeof(security_handshaker));
+  grpc_handshaker_init(&security_handshaker_vtable, &h->base);
+  h->handshaker = handshaker;
+  h->connector = GRPC_SECURITY_CONNECTOR_REF(connector, "handshake");
+  gpr_mu_init(&h->mu);
+  gpr_ref_init(&h->refs, 1);
+  h->handshake_buffer_size = GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE;
+  h->handshake_buffer = gpr_malloc(h->handshake_buffer_size);
+  grpc_closure_init(&h->on_handshake_data_sent_to_peer,
+                    on_handshake_data_sent_to_peer, h);
+  grpc_closure_init(&h->on_handshake_data_received_from_peer,
+                    on_handshake_data_received_from_peer, h);
+  grpc_closure_init(&h->on_peer_checked, on_peer_checked, h);
+  grpc_slice_buffer_init(&h->left_overs);
+  grpc_slice_buffer_init(&h->outgoing);
+  return &h->base;
+}
+
+//
+// fail_handshaker
+//
+
+static void fail_handshaker_destroy(grpc_exec_ctx *exec_ctx,
+                                    grpc_handshaker *handshaker) {
+  gpr_free(handshaker);
+}
+
+static void fail_handshaker_shutdown(grpc_exec_ctx *exec_ctx,
+                                     grpc_handshaker *handshaker) {}
+
+static void fail_handshaker_do_handshake(grpc_exec_ctx *exec_ctx,
+                                         grpc_handshaker *handshaker,
+                                         grpc_tcp_server_acceptor *acceptor,
+                                         grpc_closure *on_handshake_done,
+                                         grpc_handshaker_args *args) {
+  grpc_exec_ctx_sched(exec_ctx, on_handshake_done,
+                      GRPC_ERROR_CREATE("Failed to create security handshaker"),
+                      NULL);
+}
+
+static const grpc_handshaker_vtable fail_handshaker_vtable = {
+    fail_handshaker_destroy, fail_handshaker_shutdown,
+    fail_handshaker_do_handshake};
+
+static grpc_handshaker *fail_handshaker_create() {
+  grpc_handshaker *h = gpr_malloc(sizeof(*h));
+  grpc_handshaker_init(&fail_handshaker_vtable, h);
+  return h;
+}
+
+//
+// exported functions
+//
+
+grpc_handshaker *grpc_security_handshaker_create(
+    grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
+    grpc_security_connector *connector) {
+  // If no TSI handshaker was created, return a handshaker that always fails.
+  // Otherwise, return a real security handshaker.
+  if (handshaker == NULL) {
+    return fail_handshaker_create();
+  } else {
+    return security_handshaker_create(exec_ctx, handshaker, connector);
+  }
+}

src/core/lib/security/transport/handshake.h → src/core/lib/security/transport/security_handshaker.h

@@ -31,20 +31,16 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
-#define GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_HANDSHAKER_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_HANDSHAKER_H
 
-#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/channel/handshaker.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/security/transport/security_connector.h"
 
-/* Calls the callback upon completion. Takes owership of handshaker and
- * read_buffer. */
-void grpc_do_security_handshake(
+/// Creates a security handshaker using \a handshaker.
+grpc_handshaker *grpc_security_handshaker_create(
     grpc_exec_ctx *exec_ctx, tsi_handshaker *handshaker,
-    grpc_security_connector *connector, bool is_client_side,
-    grpc_endpoint *nonsecure_endpoint, grpc_slice_buffer *read_buffer,
-    gpr_timespec deadline, grpc_security_handshake_done_cb cb, void *user_data);
+    grpc_security_connector *connector);
 
-void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
-
-#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_HANDSHAKER_H */

src/core/lib/support/backoff.c

@@ -35,8 +35,10 @@
 
 #include <grpc/support/useful.h>
 
-void gpr_backoff_init(gpr_backoff *backoff, double multiplier, double jitter,
+void gpr_backoff_init(gpr_backoff *backoff, int64_t initial_connect_timeout,
+                      double multiplier, double jitter,
                       int64_t min_timeout_millis, int64_t max_timeout_millis) {
+  backoff->initial_connect_timeout = initial_connect_timeout;
   backoff->multiplier = multiplier;
   backoff->jitter = jitter;
   backoff->min_timeout_millis = min_timeout_millis;
@@ -45,9 +47,10 @@ void gpr_backoff_init(gpr_backoff *backoff, double multiplier, double jitter,
 }
 
 gpr_timespec gpr_backoff_begin(gpr_backoff *backoff, gpr_timespec now) {
-  backoff->current_timeout_millis = backoff->min_timeout_millis;
-  return gpr_time_add(
-      now, gpr_time_from_millis(backoff->current_timeout_millis, GPR_TIMESPAN));
+  backoff->current_timeout_millis = backoff->initial_connect_timeout;
+  const int64_t first_timeout =
+      GPR_MAX(backoff->current_timeout_millis, backoff->min_timeout_millis);
+  return gpr_time_add(now, gpr_time_from_millis(first_timeout, GPR_TIMESPAN));
 }
 
 /* Generate a random number between 0 and 1. */
@@ -57,20 +60,28 @@ static double generate_uniform_random_number(uint32_t *rng_state) {
 }
 
 gpr_timespec gpr_backoff_step(gpr_backoff *backoff, gpr_timespec now) {
-  double new_timeout_millis =
+  const double new_timeout_millis =
       backoff->multiplier * (double)backoff->current_timeout_millis;
-  double jitter_range = backoff->jitter * new_timeout_millis;
-  double jitter =
+  backoff->current_timeout_millis =
+      GPR_MIN((int64_t)new_timeout_millis, backoff->max_timeout_millis);
+
+  const double jitter_range_width = backoff->jitter * new_timeout_millis;
+  const double jitter =
       (2 * generate_uniform_random_number(&backoff->rng_state) - 1) *
-      jitter_range;
+      jitter_range_width;
+
   backoff->current_timeout_millis =
-      GPR_CLAMP((int64_t)(new_timeout_millis + jitter),
-                backoff->min_timeout_millis, backoff->max_timeout_millis);
-  return gpr_time_add(
+      (int64_t)((double)(backoff->current_timeout_millis) + jitter);
+
+  const gpr_timespec current_deadline = gpr_time_add(
       now, gpr_time_from_millis(backoff->current_timeout_millis, GPR_TIMESPAN));
+
+  const gpr_timespec min_deadline = gpr_time_add(
+      now, gpr_time_from_millis(backoff->min_timeout_millis, GPR_TIMESPAN));
+
+  return gpr_time_max(current_deadline, min_deadline);
 }
 
 void gpr_backoff_reset(gpr_backoff *backoff) {
-  // forces step() to return a timeout of min_timeout_millis
-  backoff->current_timeout_millis = 0;
+  backoff->current_timeout_millis = backoff->initial_connect_timeout;
 }

src/core/lib/support/backoff.h

@@ -37,7 +37,9 @@
 #include <grpc/support/time.h>
 
 typedef struct {
-  /// const: multiplier between retry attempts
+  /// const:  how long to wait after the first failure before retrying
+  int64_t initial_connect_timeout;
+  /// const: factor with which to multiply backoff after a failed retry
   double multiplier;
   /// const: amount to randomize backoffs
   double jitter;
@@ -54,7 +56,8 @@ typedef struct {
 } gpr_backoff;
 
 /// Initialize backoff machinery - does not need to be destroyed
-void gpr_backoff_init(gpr_backoff *backoff, double multiplier, double jitter,
+void gpr_backoff_init(gpr_backoff *backoff, int64_t initial_connect_timeout,
+                      double multiplier, double jitter,
                       int64_t min_timeout_millis, int64_t max_timeout_millis);
 
 /// Begin retry loop: returns a timespec for the NEXT retry

src/core/lib/support/subprocess_posix.c

@@ -40,6 +40,7 @@
 #include <assert.h>
 #include <errno.h>
 #include <signal.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -52,7 +53,7 @@
 
 struct gpr_subprocess {
   int pid;
-  int joined;
+  bool joined;
 };
 
 const char *gpr_subprocess_binary_extension() { return ""; }
@@ -97,9 +98,11 @@ retry:
     if (errno == EINTR) {
       goto retry;
     }
-    gpr_log(GPR_ERROR, "waitpid failed: %s", strerror(errno));
+    gpr_log(GPR_ERROR, "waitpid failed for pid %d: %s", p->pid,
+            strerror(errno));
     return -1;
   }
+  p->joined = true;
   return status;
 }
 

src/core/lib/surface/call.c

@@ -1551,6 +1551,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
           error = GRPC_CALL_ERROR_TOO_MANY_OPERATIONS;
           goto done_with_error;
         }
+        /* IF this is a server, then GRPC_OP_RECV_INITIAL_METADATA *must* come
+           from server.c. In that case, it's coming from accept_stream, and in
+           that case we're not necessarily covered by a poller. */
+        stream_op->covered_by_poller = call->is_client;
         call->received_initial_metadata = 1;
         call->buffered_metadata[0] = op->data.recv_initial_metadata;
         grpc_closure_init(&call->receiving_initial_metadata_ready,

src/core/lib/surface/completion_queue.c

@@ -354,11 +354,13 @@ static void dump_pending_tags(grpc_completion_queue *cc) {
   gpr_strvec v;
   gpr_strvec_init(&v);
   gpr_strvec_add(&v, gpr_strdup("PENDING TAGS:"));
+  gpr_mu_lock(cc->mu);
   for (size_t i = 0; i < cc->outstanding_tag_count; i++) {
     char *s;
     gpr_asprintf(&s, " %p", cc->outstanding_tags[i]);
     gpr_strvec_add(&v, s);
   }
+  gpr_mu_unlock(cc->mu);
   char *out = gpr_strvec_flatten(&v, NULL);
   gpr_strvec_destroy(&v);
   gpr_log(GPR_DEBUG, "%s", out);

src/core/lib/transport/connectivity_state.c

@@ -100,7 +100,12 @@ grpc_connectivity_state grpc_connectivity_state_check(
   return tracker->current_state;
 }
 
-int grpc_connectivity_state_notify_on_state_change(
+bool grpc_connectivity_state_has_watchers(
+    grpc_connectivity_state_tracker *connectivity_state) {
+  return connectivity_state->watchers != NULL;
+}
+
+bool grpc_connectivity_state_notify_on_state_change(
     grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
     grpc_connectivity_state *current, grpc_closure *notify) {
   if (grpc_connectivity_state_trace) {
@@ -119,7 +124,7 @@ int grpc_connectivity_state_notify_on_state_change(
       grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
       tracker->watchers = w->next;
       gpr_free(w);
-      return 0;
+      return false;
     }
     while (w != NULL) {
       grpc_connectivity_state_watcher *rm_candidate = w->next;
@@ -127,11 +132,11 @@ int grpc_connectivity_state_notify_on_state_change(
         grpc_exec_ctx_sched(exec_ctx, notify, GRPC_ERROR_CANCELLED, NULL);
         w->next = w->next->next;
         gpr_free(rm_candidate);
-        return 0;
+        return false;
       }
       w = w->next;
     }
-    return 0;
+    return false;
   } else {
     if (tracker->current_state != *current) {
       *current = tracker->current_state;

src/core/lib/transport/connectivity_state.h

@@ -75,13 +75,16 @@ void grpc_connectivity_state_set(grpc_exec_ctx *exec_ctx,
                                  grpc_error *associated_error,
                                  const char *reason);
 
+bool grpc_connectivity_state_has_watchers(
+    grpc_connectivity_state_tracker *tracker);
+
 grpc_connectivity_state grpc_connectivity_state_check(
     grpc_connectivity_state_tracker *tracker, grpc_error **current_error);
 
 /** Return 1 if the channel should start connecting, 0 otherwise.
     If current==NULL cancel notify if it is already queued (success==0 in that
     case) */
-int grpc_connectivity_state_notify_on_state_change(
+bool grpc_connectivity_state_notify_on_state_change(
     grpc_exec_ctx *exec_ctx, grpc_connectivity_state_tracker *tracker,
     grpc_connectivity_state *current, grpc_closure *notify);
 

src/cpp/common/completion_queue_cc.cc

@@ -43,11 +43,21 @@ namespace grpc {
 
 static internal::GrpcLibraryInitializer g_gli_initializer;
 
-CompletionQueue::CompletionQueue(grpc_completion_queue* take) : cq_(take) {}
+CompletionQueue::CompletionQueue(grpc_completion_queue* take) : cq_(take) {
+  InitialAvalanching();
+}
 
 void CompletionQueue::Shutdown() {
   g_gli_initializer.summon();
-  grpc_completion_queue_shutdown(cq_);
+  CompleteAvalanching();
+}
+
+void CompletionQueue::CompleteAvalanching() {
+  // Check if this was the last avalanching operation
+  if (gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_,
+                                   static_cast<gpr_atm>(-1)) == 1) {
+    grpc_completion_queue_shutdown(cq_);
+  }
 }
 
 CompletionQueue::NextStatus CompletionQueue::AsyncNextInternal(

src/cpp/server/server_cc.cc

@@ -510,12 +510,6 @@ void Server::ShutdownInternal(gpr_timespec deadline) {
     ShutdownTag shutdown_tag;  // Dummy shutdown tag
     grpc_server_shutdown_and_notify(server_, shutdown_cq.cq(), &shutdown_tag);
 
-    // Shutdown all ThreadManagers. This will try to gracefully stop all the
-    // threads in the ThreadManagers (once they process any inflight requests)
-    for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
-      (*it)->Shutdown();  // ThreadManager's Shutdown()
-    }
-
     shutdown_cq.Shutdown();
 
     void* tag;
@@ -531,6 +525,12 @@ void Server::ShutdownInternal(gpr_timespec deadline) {
     // Else in case of SHUTDOWN or GOT_EVENT, it means that the server has
     // successfully shutdown
 
+    // Shutdown all ThreadManagers. This will try to gracefully stop all the
+    // threads in the ThreadManagers (once they process any inflight requests)
+    for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
+      (*it)->Shutdown();  // ThreadManager's Shutdown()
+    }
+
     // Wait for threads in all ThreadManagers to terminate
     for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
       (*it)->Wait();
@@ -575,9 +575,14 @@ ServerInterface::BaseAsyncRequest::BaseAsyncRequest(
       tag_(tag),
       delete_on_finalize_(delete_on_finalize),
       call_(nullptr) {
+  call_cq_->RegisterAvalanching();  // This op will trigger more ops
   memset(&initial_metadata_array_, 0, sizeof(initial_metadata_array_));
 }
 
+ServerInterface::BaseAsyncRequest::~BaseAsyncRequest() {
+  call_cq_->CompleteAvalanching();
+}
+
 bool ServerInterface::BaseAsyncRequest::FinalizeResult(void** tag,
                                                        bool* status) {
   if (*status) {

src/csharp/Grpc.Auth/Grpc.Auth.csproj

@@ -29,15 +29,6 @@
     <WarningLevel>4</WarningLevel>
     <ConsolePause>false</ConsolePause>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Net" />
@@ -87,7 +78,6 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
-    <None Include="Grpc.Auth.nuspec" />
     <None Include="Grpc.Auth.project.json" />
     <None Include="packages.config" />
   </ItemGroup>

src/csharp/Grpc.Auth/Grpc.Auth.nuspec

@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<package>
-  <metadata>
-    <id>Grpc.Auth</id>
-    <title>gRPC C# Auth</title>
-    <summary>Auth library for C# implementation of gRPC - an RPC library and framework</summary>
-    <description>Auth library for C# implementation of gRPC - an RPC library and framework. See project site for more info.</description>
-    <version>$version$</version>
-    <authors>Google Inc.</authors>
-    <owners>grpc-packages</owners>
-    <licenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</licenseUrl>
-    <projectUrl>https://github.com/grpc/grpc</projectUrl>
-    <requireLicenseAcceptance>false</requireLicenseAcceptance>
-    <releaseNotes>Release $version$ of gRPC C#</releaseNotes>
-    <copyright>Copyright 2015, Google Inc.</copyright>
-    <tags>gRPC RPC Protocol HTTP/2 Auth OAuth2</tags>
-	<dependencies>
-	  <dependency id="Google.Apis.Auth" version="1.15.0" />
-	  <dependency id="Grpc.Core" version="$version$" />
-    </dependencies>
-  </metadata>
-  <files>
-    <file src="bin/ReleaseSigned/Grpc.Auth.dll" target="lib/net45" />
-	<file src="bin/ReleaseSigned/Grpc.Auth.pdb" target="lib/net45" />
-	<file src="bin/ReleaseSigned/Grpc.Auth.xml" target="lib/net45" />
-	<file src="**\*.cs" target="src" />
-  </files>
-</package>

src/csharp/Grpc.Core.Tests/CallOptionsTest.cs

@@ -67,6 +67,9 @@ namespace Grpc.Core.Tests
             var credentials = new FakeCallCredentials();
             Assert.AreSame(credentials, options.WithCredentials(credentials).Credentials);
 
+            var flags = CallFlags.WaitForReady | CallFlags.CacheableRequest;
+            Assert.AreEqual(flags, options.WithFlags(flags).Flags);
+
             // Check that the original instance is unchanged.
             Assert.IsNull(options.Headers);
             Assert.IsNull(options.Deadline);
@@ -74,6 +77,7 @@ namespace Grpc.Core.Tests
             Assert.IsNull(options.WriteOptions);
             Assert.IsNull(options.PropagationToken);
             Assert.IsNull(options.Credentials);
+            Assert.AreEqual(default(CallFlags), options.Flags);
         }
 
         [Test]
@@ -94,5 +98,16 @@ namespace Grpc.Core.Tests
             Assert.AreEqual(token, new CallOptions(propagationToken: propagationToken2).Normalize().CancellationToken);
             Assert.Throws(typeof(ArgumentException), () => new CallOptions(cancellationToken: token, propagationToken: propagationToken2).Normalize());
         }
+
+        [Test]
+        public void WaitForReady()
+        {
+            var callOptions = new CallOptions();
+            Assert.IsFalse(callOptions.IsWaitForReady);
+
+            Assert.AreEqual(CallFlags.WaitForReady, callOptions.WithWaitForReady().Flags);
+            Assert.IsTrue(callOptions.WithWaitForReady().IsWaitForReady);
+            Assert.IsFalse(callOptions.WithWaitForReady(true).WithWaitForReady(false).IsWaitForReady);
+        }
     }
 }

src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj

@@ -25,15 +25,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="nunit.framework">

src/csharp/Grpc.Core.Tests/Internal/FakeNativeCall.cs

@@ -115,27 +115,27 @@ namespace Grpc.Core.Internal.Tests
             return "PEER";
         }
 
-        public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags)
+        public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             UnaryResponseClientHandler = callback;
         }
 
-        public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags)
+        public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             throw new NotImplementedException();
         }
 
-        public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray)
+        public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             UnaryResponseClientHandler = callback;
         }
 
-        public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags)
+        public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             ReceivedStatusOnClientHandler = callback;
         }
 
-        public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray)
+        public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             ReceivedStatusOnClientHandler = callback;
         }

src/csharp/Grpc.Core.Tests/SanityTest.cs

@@ -115,7 +115,8 @@ namespace Grpc.Core.Tests
             var otherAssemblies = new[] {
                 "Grpc.Examples.Tests",
                 "Grpc.HealthCheck.Tests",
-                "Grpc.IntegrationTesting"
+                "Grpc.IntegrationTesting",
+                "Grpc.Reflection.Tests",
             };
             foreach (var assemblyName in otherAssemblies)
             {

src/csharp/Grpc.Core/CallOptions.cs

@@ -50,6 +50,7 @@ namespace Grpc.Core
         WriteOptions writeOptions;
         ContextPropagationToken propagationToken;
         CallCredentials credentials;
+        CallFlags flags;
 
         /// <summary>
         /// Creates a new instance of <c>CallOptions</c> struct.
@@ -69,6 +70,7 @@ namespace Grpc.Core
             this.writeOptions = writeOptions;
             this.propagationToken = propagationToken;
             this.credentials = credentials;
+            this.flags = default(CallFlags);
         }
 
         /// <summary>
@@ -125,6 +127,24 @@ namespace Grpc.Core
             get { return this.credentials; }
         }
 
+        /// <summary>
+        /// If <c>true</c> and and channel is in <c>ChannelState.TransientFailure</c>, the call will attempt waiting for the channel to recover
+        /// instead of failing immediately (which is the default "FailFast" semantics).
+        /// Note: experimental API that can change or be removed without any prior notice.
+        /// </summary>
+        public bool IsWaitForReady
+        {
+            get { return (this.flags & CallFlags.WaitForReady) == CallFlags.WaitForReady; }
+        }
+
+        /// <summary>
+        /// Flags to use for this call.
+        /// </summary>
+        internal CallFlags Flags
+        {
+            get { return this.flags; }
+        }
+
         /// <summary>
         /// Returns new instance of <see cref="CallOptions"/> with
         /// <c>Headers</c> set to the value provided. Values of all other fields are preserved.
@@ -197,6 +217,32 @@ namespace Grpc.Core
             return newOptions;
         }
 
+        /// <summary>
+        /// Returns new instance of <see cref="CallOptions"/> with "WaitForReady" semantics enabled/disabled.
+        /// <see cref="IsWaitForReady"/>.
+        /// Note: experimental API that can change or be removed without any prior notice.
+        /// </summary>
+        public CallOptions WithWaitForReady(bool waitForReady = true)
+        {
+            if (waitForReady)
+            {
+                return WithFlags(this.flags | CallFlags.WaitForReady);
+            }
+            return WithFlags(this.flags & ~CallFlags.WaitForReady);
+        }
+
+        /// <summary>
+        /// Returns new instance of <see cref="CallOptions"/> with
+        /// <c>Flags</c> set to the value provided. Values of all other fields are preserved.
+        /// </summary>
+        /// <param name="flags">The call flags.</param>
+        internal CallOptions WithFlags(CallFlags flags)
+        {
+            var newOptions = this;
+            newOptions.flags = flags;
+            return newOptions;
+        }
+
         /// <summary>
         /// Returns a new instance of <see cref="CallOptions"/> with 
         /// all previously unset values set to their defaults and deadline and cancellation

src/csharp/Grpc.Core/Grpc.Core.csproj

@@ -27,16 +27,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <DefineConstants>SIGNED</DefineConstants>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Interactive.Async">
@@ -140,9 +130,9 @@
     <Compile Include="Logging\LogLevelFilterLogger.cs" />
     <Compile Include="Internal\RequestCallContextSafeHandle.cs" />
     <Compile Include="Utils\TaskUtils.cs" />
+    <Compile Include="Internal\CallFlags.cs" />
   </ItemGroup>
   <ItemGroup>
-    <None Include="Grpc.Core.nuspec" />
     <None Include="Grpc.Core.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
@@ -154,4 +144,4 @@
       <Link>roots.pem</Link>
     </EmbeddedResource>
   </ItemGroup>
-</Project>
+</Project>

src/csharp/Grpc.Core/Grpc.Core.nuspec

@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<package>
-  <metadata>
-    <id>Grpc.Core</id>
-    <title>gRPC C# Core</title>
-    <summary>Core C# implementation of gRPC - an RPC library and framework</summary>
-    <description>Core C# implementation of gRPC - an RPC library and framework. See project site for more info.</description>
-    <version>$version$</version>
-    <authors>Google Inc.</authors>
-    <owners>grpc-packages</owners>
-    <licenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</licenseUrl>
-    <projectUrl>https://github.com/grpc/grpc</projectUrl>
-    <requireLicenseAcceptance>false</requireLicenseAcceptance>
-    <releaseNotes>Release $version$ of gRPC C#</releaseNotes>
-    <copyright>Copyright 2015, Google Inc.</copyright>
-    <tags>gRPC RPC Protocol HTTP/2</tags>
-    <dependencies>
-      <dependency id="System.Interactive.Async" version="3.1.1" />
-    </dependencies>
-  </metadata>
-  <files>
-    <file src="bin/ReleaseSigned/Grpc.Core.dll" target="lib/net45" />
-    <file src="bin/ReleaseSigned/Grpc.Core.pdb" target="lib/net45" />
-    <file src="bin/ReleaseSigned/Grpc.Core.xml" target="lib/net45" />
-    <file src="**\*.cs" target="src" />
-    <file src="Grpc.Core.targets" target="\build\net45\Grpc.Core.targets" />
-    <!-- without backslashes in the the source path, nuget won't copy the files -->
-    <file src="..\nativelibs\windows_x86\grpc_csharp_ext.dll" target="/runtimes/win/native/grpc_csharp_ext.x86.dll" />
-    <file src="..\nativelibs\windows_x64\grpc_csharp_ext.dll" target="/runtimes/win/native/grpc_csharp_ext.x64.dll" />
-    <file src="..\nativelibs\linux_x86\libgrpc_csharp_ext.so" target="/runtimes/linux/native/libgrpc_csharp_ext.x86.so" />
-    <file src="..\nativelibs\linux_x64\libgrpc_csharp_ext.so" target="/runtimes/linux/native/libgrpc_csharp_ext.x64.so" />
-    <file src="..\nativelibs\macosx_x86\libgrpc_csharp_ext.dylib" target="/runtimes/osx/native/libgrpc_csharp_ext.x86.dylib" />
-    <file src="..\nativelibs\macosx_x64\libgrpc_csharp_ext.dylib" target="/runtimes/osx/native/libgrpc_csharp_ext.x64.dylib" />
-  </files>
-</package>

src/csharp/Grpc.Core/Internal/AsyncCall.cs

@@ -106,7 +106,7 @@ namespace Grpc.Core.Internal
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 using (var ctx = BatchContextSafeHandle.Create())
                 {
-                    call.StartUnary(ctx, payload, metadataArray, GetWriteFlagsForCall());
+                    call.StartUnary(ctx, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags);
 
                     var ev = cq.Pluck(ctx.Handle);
 
@@ -150,7 +150,7 @@ namespace Grpc.Core.Internal
                 unaryResponseTcs = new TaskCompletionSource<TResponse>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartUnary(HandleUnaryResponse, payload, metadataArray, GetWriteFlagsForCall());
+                    call.StartUnary(HandleUnaryResponse, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags);
                 }
                 return unaryResponseTcs.Task;
             }
@@ -174,7 +174,7 @@ namespace Grpc.Core.Internal
                 unaryResponseTcs = new TaskCompletionSource<TResponse>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartClientStreaming(HandleUnaryResponse, metadataArray);
+                    call.StartClientStreaming(HandleUnaryResponse, metadataArray, details.Options.Flags);
                 }
 
                 return unaryResponseTcs.Task;
@@ -200,7 +200,7 @@ namespace Grpc.Core.Internal
                 streamingResponseCallFinishedTcs = new TaskCompletionSource<object>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartServerStreaming(HandleFinished, payload, metadataArray, GetWriteFlagsForCall());
+                    call.StartServerStreaming(HandleFinished, payload, GetWriteFlagsForCall(), metadataArray, details.Options.Flags);
                 }
                 call.StartReceiveInitialMetadata(HandleReceivedResponseHeaders);
             }
@@ -222,7 +222,7 @@ namespace Grpc.Core.Internal
                 streamingResponseCallFinishedTcs = new TaskCompletionSource<object>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
-                    call.StartDuplexStreaming(HandleFinished, metadataArray);
+                    call.StartDuplexStreaming(HandleFinished, metadataArray, details.Options.Flags);
                 }
                 call.StartReceiveInitialMetadata(HandleReceivedResponseHeaders);
             }

src/csharp/Grpc.Core/Internal/CallFlags.cs

@@ -0,0 +1,60 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+
+namespace Grpc.Core.Internal
+{
+    /// <summary>
+    /// Flags to enable special call behaviors (client-side only).
+    /// </summary>
+    [Flags]
+    internal enum CallFlags
+    {
+        /// <summary>
+        /// The call is idempotent (retrying the call doesn't change the outcome of the operation).
+        /// </summary>
+        IdempotentRequest = 0x10,
+
+        /// <summary>
+        /// If channel is in <c>ChannelState.TransientFailure</c>, attempt waiting for the channel to recover
+        /// instead of failing the call immediately.
+        /// </summary>
+        WaitForReady = 0x20,
+
+        /// <summary>
+        /// The call is cacheable. gRPC is free to use GET verb */
+        /// </summary>
+        CacheableRequest = 0x40
+    }
+}

src/csharp/Grpc.Core/Internal/CallSafeHandle.cs

@@ -63,50 +63,50 @@ namespace Grpc.Core.Internal
             Native.grpcsharp_call_set_credentials(this, credentials).CheckOk();
         }
 
-        public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags)
+        public void StartUnary(UnaryResponseClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
                 completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
-                Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags)
+                Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, metadataArray, callFlags)
                     .CheckOk();
             }
         }
 
-        public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags)
+        public void StartUnary(BatchContextSafeHandle ctx, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
-            Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags)
+            Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, metadataArray, callFlags)
                 .CheckOk();
         }
 
-        public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray)
+        public void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
                 completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
-                Native.grpcsharp_call_start_client_streaming(this, ctx, metadataArray).CheckOk();
+                Native.grpcsharp_call_start_client_streaming(this, ctx, metadataArray, callFlags).CheckOk();
             }
         }
 
-        public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags)
+        public void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
                 completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
-                Native.grpcsharp_call_start_server_streaming(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags).CheckOk();
+                Native.grpcsharp_call_start_server_streaming(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, metadataArray, callFlags).CheckOk();
             }
         }
 
-        public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray)
+        public void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags)
         {
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
                 completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
-                Native.grpcsharp_call_start_duplex_streaming(this, ctx, metadataArray).CheckOk();
+                Native.grpcsharp_call_start_duplex_streaming(this, ctx, metadataArray, callFlags).CheckOk();
             }
         }
 

src/csharp/Grpc.Core/Internal/INativeCall.cs

@@ -31,6 +31,7 @@
 #endregion
 
 using System;
+using Grpc.Core;
 
 namespace Grpc.Core.Internal
 {
@@ -54,19 +55,19 @@ namespace Grpc.Core.Internal
     {
         void Cancel();
 
-        void CancelWithStatus(Grpc.Core.Status status);
+        void CancelWithStatus(Status status);
 
         string GetPeer();
 
-        void StartUnary(UnaryResponseClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, Grpc.Core.WriteFlags writeFlags);
+        void StartUnary(UnaryResponseClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartUnary(BatchContextSafeHandle ctx, byte[] payload, MetadataArraySafeHandle metadataArray, Grpc.Core.WriteFlags writeFlags);
+        void StartUnary(BatchContextSafeHandle ctx, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray);
+        void StartClientStreaming(UnaryResponseClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, MetadataArraySafeHandle metadataArray, Grpc.Core.WriteFlags writeFlags);
+        void StartServerStreaming(ReceivedStatusOnClientHandler callback, byte[] payload, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
-        void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray);
+        void StartDuplexStreaming(ReceivedStatusOnClientHandler callback, MetadataArraySafeHandle metadataArray, CallFlags callFlags);
 
         void StartReceiveMessage(ReceivedMessageHandler callback);
 
@@ -74,11 +75,11 @@ namespace Grpc.Core.Internal
 
         void StartSendInitialMetadata(SendCompletionHandler callback, MetadataArraySafeHandle metadataArray);
 
-        void StartSendMessage(SendCompletionHandler callback, byte[] payload, Grpc.Core.WriteFlags writeFlags, bool sendEmptyInitialMetadata);
+        void StartSendMessage(SendCompletionHandler callback, byte[] payload, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
 
         void StartSendCloseFromClient(SendCompletionHandler callback);
 
-        void StartSendStatusFromServer(SendCompletionHandler callback, Grpc.Core.Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, byte[] optionalPayload, Grpc.Core.WriteFlags writeFlags);
+        void StartSendStatusFromServer(SendCompletionHandler callback, Status status, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata, byte[] optionalPayload, WriteFlags writeFlags);
 
         void StartServerSide(ReceivedCloseOnServerHandler callback);
     }

src/csharp/Grpc.Core/Internal/NativeMethods.cs

@@ -325,14 +325,14 @@ namespace Grpc.Core.Internal
             public delegate CallError grpcsharp_call_cancel_delegate(CallSafeHandle call);
             public delegate CallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description);
             public delegate CallError grpcsharp_call_start_unary_delegate(CallSafeHandle call,
-                BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
+                BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, MetadataArraySafeHandle metadataArray, CallFlags metadataFlags);
             public delegate CallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call,
-                BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
+                BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray, CallFlags metadataFlags);
             public delegate CallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call,
-                BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen,
-                MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
+                BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags,
+                MetadataArraySafeHandle metadataArray, CallFlags metadataFlags);
             public delegate CallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call,
-                BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
+                BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray, CallFlags metadataFlags);
             public delegate CallError grpcsharp_call_send_message_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
             public delegate CallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call,

src/csharp/Grpc.Core/NativeDeps.targets

@@ -4,13 +4,11 @@
   <PropertyGroup Condition=" '$(NativeDependenciesConfiguration)' == '' ">
     <NativeDependenciesConfiguration Condition=" '$(Configuration)' == 'Debug' ">Debug</NativeDependenciesConfiguration>
     <NativeDependenciesConfiguration Condition=" '$(Configuration)' == 'Release' ">Release</NativeDependenciesConfiguration>
-    <NativeDependenciesConfiguration Condition=" '$(Configuration)' == 'ReleaseSigned' ">Release</NativeDependenciesConfiguration>
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(NativeDependenciesConfigurationUnix)' == '' ">
     <NativeDependenciesConfigurationUnix Condition=" '$(Configuration)' == 'Debug' ">dbg</NativeDependenciesConfigurationUnix>
     <NativeDependenciesConfigurationUnix Condition=" '$(Configuration)' == 'Release' ">opt</NativeDependenciesConfigurationUnix>
-    <NativeDependenciesConfigurationUnix Condition=" '$(Configuration)' == 'ReleaseSigned' ">opt</NativeDependenciesConfigurationUnix>
   </PropertyGroup>
 
   <!-- Autodetect platform -->

src/csharp/Grpc.Dotnet.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 14
-VisualStudioVersion = 14.0.25123.0
+VisualStudioVersion = 14.0.25420.1
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Core", "Grpc.Core\Grpc.Core.xproj", "{DC9908B6-F291-4FC8-A46D-2EA2551790EC}"
 EndProject
@@ -31,6 +31,10 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.IntegrationTesting.Ser
 EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.IntegrationTesting.StressClient", "Grpc.IntegrationTesting.StressClient\Grpc.IntegrationTesting.StressClient.xproj", "{0EBC910B-8867-4D3E-8686-91F34183D839}"
 EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Reflection", "Grpc.Reflection\Grpc.Reflection.xproj", "{2B372155-80BA-4CF9-82D6-4B938E8EC3A0}"
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Grpc.Reflection.Tests", "Grpc.Reflection.Tests\Grpc.Reflection.Tests.xproj", "{FE90181D-A4B3-4A5C-8490-F07561E18E3B}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -93,6 +97,14 @@ Global
 		{0EBC910B-8867-4D3E-8686-91F34183D839}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0EBC910B-8867-4D3E-8686-91F34183D839}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0EBC910B-8867-4D3E-8686-91F34183D839}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2B372155-80BA-4CF9-82D6-4B938E8EC3A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2B372155-80BA-4CF9-82D6-4B938E8EC3A0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2B372155-80BA-4CF9-82D6-4B938E8EC3A0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2B372155-80BA-4CF9-82D6-4B938E8EC3A0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FE90181D-A4B3-4A5C-8490-F07561E18E3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FE90181D-A4B3-4A5C-8490-F07561E18E3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FE90181D-A4B3-4A5C-8490-F07561E18E3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FE90181D-A4B3-4A5C-8490-F07561E18E3B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj

@@ -27,15 +27,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
   </ItemGroup>

src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj

@@ -27,15 +27,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
   </ItemGroup>

src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj

@@ -25,15 +25,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="nunit.framework">

src/csharp/Grpc.Examples/Grpc.Examples.csproj

@@ -27,15 +27,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="nunit.framework">
       <HintPath>..\packages\NUnit.3.2.0\lib\net45\nunit.framework.dll</HintPath>

src/csharp/Grpc.Examples/Math.cs

@@ -1,5 +1,5 @@
 // Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: math.proto
+// source: math/math.proto
 #pragma warning disable 1591, 0612, 3021
 #region Designer generated code
 
@@ -9,11 +9,11 @@ using pbr = global::Google.Protobuf.Reflection;
 using scg = global::System.Collections.Generic;
 namespace Math {
 
-  /// <summary>Holder for reflection information generated from math.proto</summary>
+  /// <summary>Holder for reflection information generated from math/math.proto</summary>
   public static partial class MathReflection {
 
     #region Descriptor
-    /// <summary>File descriptor for math.proto</summary>
+    /// <summary>File descriptor for math/math.proto</summary>
     public static pbr::FileDescriptor Descriptor {
       get { return descriptor; }
     }
@@ -22,15 +22,15 @@ namespace Math {
     static MathReflection() {
       byte[] descriptorData = global::System.Convert.FromBase64String(
           string.Concat(
-            "CgptYXRoLnByb3RvEgRtYXRoIiwKB0RpdkFyZ3MSEAoIZGl2aWRlbmQYASAB",
-            "KAMSDwoHZGl2aXNvchgCIAEoAyIvCghEaXZSZXBseRIQCghxdW90aWVudBgB",
-            "IAEoAxIRCglyZW1haW5kZXIYAiABKAMiGAoHRmliQXJncxINCgVsaW1pdBgB",
-            "IAEoAyISCgNOdW0SCwoDbnVtGAEgASgDIhkKCEZpYlJlcGx5Eg0KBWNvdW50",
-            "GAEgASgDMqQBCgRNYXRoEiYKA0RpdhINLm1hdGguRGl2QXJncxoOLm1hdGgu",
-            "RGl2UmVwbHkiABIuCgdEaXZNYW55Eg0ubWF0aC5EaXZBcmdzGg4ubWF0aC5E",
-            "aXZSZXBseSIAKAEwARIjCgNGaWISDS5tYXRoLkZpYkFyZ3MaCS5tYXRoLk51",
-            "bSIAMAESHwoDU3VtEgkubWF0aC5OdW0aCS5tYXRoLk51bSIAKAFiBnByb3Rv",
-            "Mw=="));
+            "Cg9tYXRoL21hdGgucHJvdG8SBG1hdGgiLAoHRGl2QXJncxIQCghkaXZpZGVu",
+            "ZBgBIAEoAxIPCgdkaXZpc29yGAIgASgDIi8KCERpdlJlcGx5EhAKCHF1b3Rp",
+            "ZW50GAEgASgDEhEKCXJlbWFpbmRlchgCIAEoAyIYCgdGaWJBcmdzEg0KBWxp",
+            "bWl0GAEgASgDIhIKA051bRILCgNudW0YASABKAMiGQoIRmliUmVwbHkSDQoF",
+            "Y291bnQYASABKAMypAEKBE1hdGgSJgoDRGl2Eg0ubWF0aC5EaXZBcmdzGg4u",
+            "bWF0aC5EaXZSZXBseSIAEi4KB0Rpdk1hbnkSDS5tYXRoLkRpdkFyZ3MaDi5t",
+            "YXRoLkRpdlJlcGx5IgAoATABEiMKA0ZpYhINLm1hdGguRmliQXJncxoJLm1h",
+            "dGguTnVtIgAwARIfCgNTdW0SCS5tYXRoLk51bRoJLm1hdGguTnVtIgAoAWIG",
+            "cHJvdG8z"));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
           new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {

src/csharp/Grpc.Examples/MathGrpc.cs

@@ -1,5 +1,5 @@
 // Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: math.proto
+// source: math/math.proto
 // Original file comments:
 // Copyright 2015, Google Inc.
 // All rights reserved.
@@ -85,39 +85,53 @@ namespace Math {
     public abstract partial class MathBase
     {
       /// <summary>
-      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
-      ///  and remainder.
+      /// Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      /// and remainder.
       /// </summary>
+      /// <param name="request">The request received from the client.</param>
+      /// <param name="context">The context of the server-side call handler being invoked.</param>
+      /// <returns>The response to send back to the client (wrapped by a task).</returns>
       public virtual global::System.Threading.Tasks.Task<global::Math.DivReply> Div(global::Math.DivArgs request, ServerCallContext context)
       {
         throw new RpcException(new Status(StatusCode.Unimplemented, ""));
       }
 
       /// <summary>
-      ///  DivMany accepts an arbitrary number of division args from the client stream
-      ///  and sends back the results in the reply stream.  The stream continues until
-      ///  the client closes its end; the server does the same after sending all the
-      ///  replies.  The stream ends immediately if either end aborts.
+      /// DivMany accepts an arbitrary number of division args from the client stream
+      /// and sends back the results in the reply stream.  The stream continues until
+      /// the client closes its end; the server does the same after sending all the
+      /// replies.  The stream ends immediately if either end aborts.
       /// </summary>
+      /// <param name="requestStream">Used for reading requests from the client.</param>
+      /// <param name="responseStream">Used for sending responses back to the client.</param>
+      /// <param name="context">The context of the server-side call handler being invoked.</param>
+      /// <returns>A task indicating completion of the handler.</returns>
       public virtual global::System.Threading.Tasks.Task DivMany(IAsyncStreamReader<global::Math.DivArgs> requestStream, IServerStreamWriter<global::Math.DivReply> responseStream, ServerCallContext context)
       {
         throw new RpcException(new Status(StatusCode.Unimplemented, ""));
       }
 
       /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
-      ///  generates up to limit numbers; otherwise it continues until the call is
-      ///  canceled.  Unlike Fib above, Fib has no final FibReply.
+      /// Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
+      /// generates up to limit numbers; otherwise it continues until the call is
+      /// canceled.  Unlike Fib above, Fib has no final FibReply.
       /// </summary>
+      /// <param name="request">The request received from the client.</param>
+      /// <param name="responseStream">Used for sending responses back to the client.</param>
+      /// <param name="context">The context of the server-side call handler being invoked.</param>
+      /// <returns>A task indicating completion of the handler.</returns>
       public virtual global::System.Threading.Tasks.Task Fib(global::Math.FibArgs request, IServerStreamWriter<global::Math.Num> responseStream, ServerCallContext context)
       {
         throw new RpcException(new Status(StatusCode.Unimplemented, ""));
       }
 
       /// <summary>
-      ///  Sum sums a stream of numbers, returning the final result once the stream
-      ///  is closed.
+      /// Sum sums a stream of numbers, returning the final result once the stream
+      /// is closed.
       /// </summary>
+      /// <param name="requestStream">Used for reading requests from the client.</param>
+      /// <param name="context">The context of the server-side call handler being invoked.</param>
+      /// <returns>The response to send back to the client (wrapped by a task).</returns>
       public virtual global::System.Threading.Tasks.Task<global::Math.Num> Sum(IAsyncStreamReader<global::Math.Num> requestStream, ServerCallContext context)
       {
         throw new RpcException(new Status(StatusCode.Unimplemented, ""));
@@ -149,87 +163,123 @@ namespace Math {
       }
 
       /// <summary>
-      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
-      ///  and remainder.
+      /// Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      /// and remainder.
       /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="headers">The initial metadata to send with the call. This parameter is optional.</param>
+      /// <param name="deadline">An optional deadline for the call. The call will be cancelled if deadline is hit.</param>
+      /// <param name="cancellationToken">An optional token for canceling the call.</param>
+      /// <returns>The response received from the server.</returns>
       public virtual global::Math.DivReply Div(global::Math.DivArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
       {
         return Div(request, new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
-      ///  and remainder.
+      /// Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      /// and remainder.
       /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="options">The options for the call.</param>
+      /// <returns>The response received from the server.</returns>
       public virtual global::Math.DivReply Div(global::Math.DivArgs request, CallOptions options)
       {
         return CallInvoker.BlockingUnaryCall(__Method_Div, null, options, request);
       }
       /// <summary>
-      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
-      ///  and remainder.
+      /// Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      /// and remainder.
       /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="headers">The initial metadata to send with the call. This parameter is optional.</param>
+      /// <param name="deadline">An optional deadline for the call. The call will be cancelled if deadline is hit.</param>
+      /// <param name="cancellationToken">An optional token for canceling the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncUnaryCall<global::Math.DivReply> DivAsync(global::Math.DivArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
       {
         return DivAsync(request, new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
-      ///  and remainder.
+      /// Div divides DivArgs.dividend by DivArgs.divisor and returns the quotient
+      /// and remainder.
       /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="options">The options for the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncUnaryCall<global::Math.DivReply> DivAsync(global::Math.DivArgs request, CallOptions options)
       {
         return CallInvoker.AsyncUnaryCall(__Method_Div, null, options, request);
       }
       /// <summary>
-      ///  DivMany accepts an arbitrary number of division args from the client stream
-      ///  and sends back the results in the reply stream.  The stream continues until
-      ///  the client closes its end; the server does the same after sending all the
-      ///  replies.  The stream ends immediately if either end aborts.
+      /// DivMany accepts an arbitrary number of division args from the client stream
+      /// and sends back the results in the reply stream.  The stream continues until
+      /// the client closes its end; the server does the same after sending all the
+      /// replies.  The stream ends immediately if either end aborts.
       /// </summary>
+      /// <param name="headers">The initial metadata to send with the call. This parameter is optional.</param>
+      /// <param name="deadline">An optional deadline for the call. The call will be cancelled if deadline is hit.</param>
+      /// <param name="cancellationToken">An optional token for canceling the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncDuplexStreamingCall<global::Math.DivArgs, global::Math.DivReply> DivMany(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
       {
         return DivMany(new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  DivMany accepts an arbitrary number of division args from the client stream
-      ///  and sends back the results in the reply stream.  The stream continues until
-      ///  the client closes its end; the server does the same after sending all the
-      ///  replies.  The stream ends immediately if either end aborts.
+      /// DivMany accepts an arbitrary number of division args from the client stream
+      /// and sends back the results in the reply stream.  The stream continues until
+      /// the client closes its end; the server does the same after sending all the
+      /// replies.  The stream ends immediately if either end aborts.
       /// </summary>
+      /// <param name="options">The options for the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncDuplexStreamingCall<global::Math.DivArgs, global::Math.DivReply> DivMany(CallOptions options)
       {
         return CallInvoker.AsyncDuplexStreamingCall(__Method_DivMany, null, options);
       }
       /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
-      ///  generates up to limit numbers; otherwise it continues until the call is
-      ///  canceled.  Unlike Fib above, Fib has no final FibReply.
+      /// Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
+      /// generates up to limit numbers; otherwise it continues until the call is
+      /// canceled.  Unlike Fib above, Fib has no final FibReply.
       /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="headers">The initial metadata to send with the call. This parameter is optional.</param>
+      /// <param name="deadline">An optional deadline for the call. The call will be cancelled if deadline is hit.</param>
+      /// <param name="cancellationToken">An optional token for canceling the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncServerStreamingCall<global::Math.Num> Fib(global::Math.FibArgs request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
       {
         return Fib(request, new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
-      ///  generates up to limit numbers; otherwise it continues until the call is
-      ///  canceled.  Unlike Fib above, Fib has no final FibReply.
+      /// Fib generates numbers in the Fibonacci sequence.  If FibArgs.limit > 0, Fib
+      /// generates up to limit numbers; otherwise it continues until the call is
+      /// canceled.  Unlike Fib above, Fib has no final FibReply.
       /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="options">The options for the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncServerStreamingCall<global::Math.Num> Fib(global::Math.FibArgs request, CallOptions options)
       {
         return CallInvoker.AsyncServerStreamingCall(__Method_Fib, null, options, request);
       }
       /// <summary>
-      ///  Sum sums a stream of numbers, returning the final result once the stream
-      ///  is closed.
+      /// Sum sums a stream of numbers, returning the final result once the stream
+      /// is closed.
       /// </summary>
+      /// <param name="headers">The initial metadata to send with the call. This parameter is optional.</param>
+      /// <param name="deadline">An optional deadline for the call. The call will be cancelled if deadline is hit.</param>
+      /// <param name="cancellationToken">An optional token for canceling the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncClientStreamingCall<global::Math.Num, global::Math.Num> Sum(Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
       {
         return Sum(new CallOptions(headers, deadline, cancellationToken));
       }
       /// <summary>
-      ///  Sum sums a stream of numbers, returning the final result once the stream
-      ///  is closed.
+      /// Sum sums a stream of numbers, returning the final result once the stream
+      /// is closed.
       /// </summary>
+      /// <param name="options">The options for the call.</param>
+      /// <returns>The call object.</returns>
       public virtual AsyncClientStreamingCall<global::Math.Num, global::Math.Num> Sum(CallOptions options)
       {
         return CallInvoker.AsyncClientStreamingCall(__Method_Sum, null, options);
@@ -242,6 +292,7 @@ namespace Math {
     }
 
     /// <summary>Creates service definition that can be registered with a server</summary>
+    /// <param name="serviceImpl">An object implementing the server-side handling logic.</param>
     public static ServerServiceDefinition BindService(MathBase serviceImpl)
     {
       return ServerServiceDefinition.CreateBuilder()

src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj

@@ -27,15 +27,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />

src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -28,15 +28,6 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Core" />
@@ -62,13 +53,12 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <None Include="Grpc.HealthCheck.nuspec" />
     <None Include="Grpc.HealthCheck.project.json" />
     <None Include="packages.config" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Grpc.Core\Grpc.Core.csproj">
-      <Project>{ccc4440e-49f7-4790-b0af-feabb0837ae7}</Project>
+      <Project>{CCC4440E-49F7-4790-B0AF-FEABB0837AE7}</Project>
       <Name>Grpc.Core</Name>
     </ProjectReference>
   </ItemGroup>

src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec

@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<package>
-  <metadata>
-    <id>Grpc.HealthCheck</id>
-    <title>gRPC C# Healthchecking</title>
-    <summary>Implementation of gRPC health service</summary>
-    <description>Example implementation of grpc.health.v1 service that can be used for health-checking.</description>
-    <version>$version$</version>
-    <authors>Google Inc.</authors>
-    <owners>grpc-packages</owners>
-    <licenseUrl>https://github.com/grpc/grpc/blob/master/LICENSE</licenseUrl>
-    <projectUrl>https://github.com/grpc/grpc</projectUrl>
-    <requireLicenseAcceptance>false</requireLicenseAcceptance>
-    <copyright>Copyright 2015, Google Inc.</copyright>
-    <tags>gRPC health check</tags>
-	<dependencies>
-	  <dependency id="Google.Protobuf" version="$ProtobufVersion$" />
-	  <dependency id="Grpc.Core" version="$version$" />
-	  <dependency id="System.Interactive.Async" version="3.1.1" />
-    </dependencies>
-  </metadata>
-  <files>
-    <file src="bin/ReleaseSigned/Grpc.HealthCheck.dll" target="lib/net45" />
-	<file src="bin/ReleaseSigned/Grpc.HealthCheck.pdb" target="lib/net45" />
-	<file src="bin/ReleaseSigned/Grpc.HealthCheck.xml" target="lib/net45" />
-	<file src="**\*.cs" target="src" />
-  </files>
-</package>

src/csharp/Grpc.HealthCheck/Health.cs

@@ -1,5 +1,5 @@
 // Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: health.proto
+// source: grpc/health/v1/health.proto
 #pragma warning disable 1591, 0612, 3021
 #region Designer generated code
 
@@ -9,11 +9,11 @@ using pbr = global::Google.Protobuf.Reflection;
 using scg = global::System.Collections.Generic;
 namespace Grpc.Health.V1 {
 
-  /// <summary>Holder for reflection information generated from health.proto</summary>
+  /// <summary>Holder for reflection information generated from grpc/health/v1/health.proto</summary>
   public static partial class HealthReflection {
 
     #region Descriptor
-    /// <summary>File descriptor for health.proto</summary>
+    /// <summary>File descriptor for grpc/health/v1/health.proto</summary>
     public static pbr::FileDescriptor Descriptor {
       get { return descriptor; }
     }
@@ -22,14 +22,14 @@ namespace Grpc.Health.V1 {
     static HealthReflection() {
       byte[] descriptorData = global::System.Convert.FromBase64String(
           string.Concat(
-            "CgxoZWFsdGgucHJvdG8SDmdycGMuaGVhbHRoLnYxIiUKEkhlYWx0aENoZWNr",
-            "UmVxdWVzdBIPCgdzZXJ2aWNlGAEgASgJIpQBChNIZWFsdGhDaGVja1Jlc3Bv",
-            "bnNlEkEKBnN0YXR1cxgBIAEoDjIxLmdycGMuaGVhbHRoLnYxLkhlYWx0aENo",
-            "ZWNrUmVzcG9uc2UuU2VydmluZ1N0YXR1cyI6Cg1TZXJ2aW5nU3RhdHVzEgsK",
-            "B1VOS05PV04QABILCgdTRVJWSU5HEAESDwoLTk9UX1NFUlZJTkcQAjJaCgZI",
-            "ZWFsdGgSUAoFQ2hlY2sSIi5ncnBjLmhlYWx0aC52MS5IZWFsdGhDaGVja1Jl",
-            "cXVlc3QaIy5ncnBjLmhlYWx0aC52MS5IZWFsdGhDaGVja1Jlc3BvbnNlQhGq",
-            "Ag5HcnBjLkhlYWx0aC5WMWIGcHJvdG8z"));
+            "ChtncnBjL2hlYWx0aC92MS9oZWFsdGgucHJvdG8SDmdycGMuaGVhbHRoLnYx",
+            "IiUKEkhlYWx0aENoZWNrUmVxdWVzdBIPCgdzZXJ2aWNlGAEgASgJIpQBChNI",
+            "ZWFsdGhDaGVja1Jlc3BvbnNlEkEKBnN0YXR1cxgBIAEoDjIxLmdycGMuaGVh",
+            "bHRoLnYxLkhlYWx0aENoZWNrUmVzcG9uc2UuU2VydmluZ1N0YXR1cyI6Cg1T",
+            "ZXJ2aW5nU3RhdHVzEgsKB1VOS05PV04QABILCgdTRVJWSU5HEAESDwoLTk9U",
+            "X1NFUlZJTkcQAjJaCgZIZWFsdGgSUAoFQ2hlY2sSIi5ncnBjLmhlYWx0aC52",
+            "MS5IZWFsdGhDaGVja1JlcXVlc3QaIy5ncnBjLmhlYWx0aC52MS5IZWFsdGhD",
+            "aGVja1Jlc3BvbnNlQhGqAg5HcnBjLkhlYWx0aC5WMWIGcHJvdG8z"));
       descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
           new pbr::FileDescriptor[] { },
           new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {

src/csharp/Grpc.HealthCheck/HealthGrpc.cs

@@ -1,5 +1,5 @@
 // Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: health.proto
+// source: grpc/health/v1/health.proto
 // Original file comments:
 // Copyright 2015, Google Inc.
 // All rights reserved.
@@ -115,6 +115,7 @@ namespace Grpc.Health.V1 {
     }
 
     /// <summary>Creates service definition that can be registered with a server</summary>
+    /// <param name="serviceImpl">An object implementing the server-side handling logic.</param>
     public static ServerServiceDefinition BindService(HealthBase serviceImpl)
     {
       return ServerServiceDefinition.CreateBuilder()

src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj

@@ -29,15 +29,6 @@
     <WarningLevel>4</WarningLevel>
     <PlatformTarget>AnyCPU</PlatformTarget>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Net" />

src/csharp/Grpc.IntegrationTesting.QpsWorker/Grpc.IntegrationTesting.QpsWorker.csproj

@@ -27,15 +27,6 @@
     <WarningLevel>4</WarningLevel>
     <PlatformTarget>AnyCPU</PlatformTarget>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
   </ItemGroup>

src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj

@@ -29,15 +29,6 @@
     <WarningLevel>4</WarningLevel>
     <PlatformTarget>AnyCPU</PlatformTarget>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
     <Reference Include="System.Net" />

src/csharp/Grpc.IntegrationTesting.StressClient/Grpc.IntegrationTesting.StressClient.csproj

@@ -27,15 +27,6 @@
     <WarningLevel>4</WarningLevel>
     <PlatformTarget>AnyCPU</PlatformTarget>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'ReleaseSigned|AnyCPU' ">
-    <DebugType>pdbonly</DebugType>
-    <Optimize>true</Optimize>
-    <OutputPath>bin\ReleaseSigned</OutputPath>
-    <ErrorReport>prompt</ErrorReport>
-    <WarningLevel>4</WarningLevel>
-    <SignAssembly>True</SignAssembly>
-    <AssemblyOriginatorKeyFile>..\keys\Grpc.snk</AssemblyOriginatorKeyFile>
-  </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
   </ItemGroup>