Merge branch 'master' into master

BUILD

@@ -701,7 +701,6 @@ grpc_cc_library(
         "src/core/lib/channel/channelz_registry.cc",
         "src/core/lib/channel/connected_channel.cc",
         "src/core/lib/channel/handshaker.cc",
-        "src/core/lib/channel/handshaker_factory.cc",
         "src/core/lib/channel/handshaker_registry.cc",
         "src/core/lib/channel/status_util.cc",
         "src/core/lib/compression/compression.cc",
@@ -1071,10 +1070,10 @@ grpc_cc_library(
         "src/core/ext/filters/client_channel/parse_address.cc",
         "src/core/ext/filters/client_channel/proxy_mapper.cc",
         "src/core/ext/filters/client_channel/proxy_mapper_registry.cc",
-        "src/core/ext/filters/client_channel/request_routing.cc",
         "src/core/ext/filters/client_channel/resolver.cc",
         "src/core/ext/filters/client_channel/resolver_registry.cc",
         "src/core/ext/filters/client_channel/resolver_result_parsing.cc",
+        "src/core/ext/filters/client_channel/resolving_lb_policy.cc",
         "src/core/ext/filters/client_channel/retry_throttle.cc",
         "src/core/ext/filters/client_channel/server_address.cc",
         "src/core/ext/filters/client_channel/subchannel.cc",
@@ -1097,11 +1096,11 @@ grpc_cc_library(
         "src/core/ext/filters/client_channel/parse_address.h",
         "src/core/ext/filters/client_channel/proxy_mapper.h",
         "src/core/ext/filters/client_channel/proxy_mapper_registry.h",
-        "src/core/ext/filters/client_channel/request_routing.h",
         "src/core/ext/filters/client_channel/resolver.h",
         "src/core/ext/filters/client_channel/resolver_factory.h",
         "src/core/ext/filters/client_channel/resolver_registry.h",
         "src/core/ext/filters/client_channel/resolver_result_parsing.h",
+        "src/core/ext/filters/client_channel/resolving_lb_policy.h",
         "src/core/ext/filters/client_channel/retry_throttle.h",
         "src/core/ext/filters/client_channel/server_address.h",
         "src/core/ext/filters/client_channel/subchannel.h",

CMakeLists.txt

@@ -538,6 +538,9 @@ add_dependencies(buildtests_cxx auth_property_iterator_test)
 add_dependencies(buildtests_cxx backoff_test)
 add_dependencies(buildtests_cxx bdp_estimator_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+add_dependencies(buildtests_cxx bm_alarm)
+endif()
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx bm_arena)
 endif()
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
@@ -968,7 +971,6 @@ add_library(grpc
   src/core/lib/channel/channelz_registry.cc
   src/core/lib/channel/connected_channel.cc
   src/core/lib/channel/handshaker.cc
-  src/core/lib/channel/handshaker_factory.cc
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
@@ -1230,10 +1232,10 @@ add_library(grpc
   src/core/ext/filters/client_channel/parse_address.cc
   src/core/ext/filters/client_channel/proxy_mapper.cc
   src/core/ext/filters/client_channel/proxy_mapper_registry.cc
-  src/core/ext/filters/client_channel/request_routing.cc
   src/core/ext/filters/client_channel/resolver.cc
   src/core/ext/filters/client_channel/resolver_registry.cc
   src/core/ext/filters/client_channel/resolver_result_parsing.cc
+  src/core/ext/filters/client_channel/resolving_lb_policy.cc
   src/core/ext/filters/client_channel/retry_throttle.cc
   src/core/ext/filters/client_channel/server_address.cc
   src/core/ext/filters/client_channel/subchannel.cc
@@ -1394,7 +1396,6 @@ add_library(grpc_cronet
   src/core/lib/channel/channelz_registry.cc
   src/core/lib/channel/connected_channel.cc
   src/core/lib/channel/handshaker.cc
-  src/core/lib/channel/handshaker_factory.cc
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
@@ -1586,10 +1587,10 @@ add_library(grpc_cronet
   src/core/ext/filters/client_channel/parse_address.cc
   src/core/ext/filters/client_channel/proxy_mapper.cc
   src/core/ext/filters/client_channel/proxy_mapper_registry.cc
-  src/core/ext/filters/client_channel/request_routing.cc
   src/core/ext/filters/client_channel/resolver.cc
   src/core/ext/filters/client_channel/resolver_registry.cc
   src/core/ext/filters/client_channel/resolver_result_parsing.cc
+  src/core/ext/filters/client_channel/resolving_lb_policy.cc
   src/core/ext/filters/client_channel/retry_throttle.cc
   src/core/ext/filters/client_channel/server_address.cc
   src/core/ext/filters/client_channel/subchannel.cc
@@ -1805,7 +1806,6 @@ add_library(grpc_test_util
   src/core/lib/channel/channelz_registry.cc
   src/core/lib/channel/connected_channel.cc
   src/core/lib/channel/handshaker.cc
-  src/core/lib/channel/handshaker_factory.cc
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
@@ -1965,10 +1965,10 @@ add_library(grpc_test_util
   src/core/ext/filters/client_channel/parse_address.cc
   src/core/ext/filters/client_channel/proxy_mapper.cc
   src/core/ext/filters/client_channel/proxy_mapper_registry.cc
-  src/core/ext/filters/client_channel/request_routing.cc
   src/core/ext/filters/client_channel/resolver.cc
   src/core/ext/filters/client_channel/resolver_registry.cc
   src/core/ext/filters/client_channel/resolver_result_parsing.cc
+  src/core/ext/filters/client_channel/resolving_lb_policy.cc
   src/core/ext/filters/client_channel/retry_throttle.cc
   src/core/ext/filters/client_channel/server_address.cc
   src/core/ext/filters/client_channel/subchannel.cc
@@ -2131,7 +2131,6 @@ add_library(grpc_test_util_unsecure
   src/core/lib/channel/channelz_registry.cc
   src/core/lib/channel/connected_channel.cc
   src/core/lib/channel/handshaker.cc
-  src/core/lib/channel/handshaker_factory.cc
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
@@ -2291,10 +2290,10 @@ add_library(grpc_test_util_unsecure
   src/core/ext/filters/client_channel/parse_address.cc
   src/core/ext/filters/client_channel/proxy_mapper.cc
   src/core/ext/filters/client_channel/proxy_mapper_registry.cc
-  src/core/ext/filters/client_channel/request_routing.cc
   src/core/ext/filters/client_channel/resolver.cc
   src/core/ext/filters/client_channel/resolver_registry.cc
   src/core/ext/filters/client_channel/resolver_result_parsing.cc
+  src/core/ext/filters/client_channel/resolving_lb_policy.cc
   src/core/ext/filters/client_channel/retry_throttle.cc
   src/core/ext/filters/client_channel/server_address.cc
   src/core/ext/filters/client_channel/subchannel.cc
@@ -2433,7 +2432,6 @@ add_library(grpc_unsecure
   src/core/lib/channel/channelz_registry.cc
   src/core/lib/channel/connected_channel.cc
   src/core/lib/channel/handshaker.cc
-  src/core/lib/channel/handshaker_factory.cc
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
@@ -2628,10 +2626,10 @@ add_library(grpc_unsecure
   src/core/ext/filters/client_channel/parse_address.cc
   src/core/ext/filters/client_channel/proxy_mapper.cc
   src/core/ext/filters/client_channel/proxy_mapper_registry.cc
-  src/core/ext/filters/client_channel/request_routing.cc
   src/core/ext/filters/client_channel/resolver.cc
   src/core/ext/filters/client_channel/resolver_registry.cc
   src/core/ext/filters/client_channel/resolver_result_parsing.cc
+  src/core/ext/filters/client_channel/resolving_lb_policy.cc
   src/core/ext/filters/client_channel/retry_throttle.cc
   src/core/ext/filters/client_channel/server_address.cc
   src/core/ext/filters/client_channel/subchannel.cc
@@ -3321,7 +3319,6 @@ add_library(grpc++_cronet
   src/core/lib/channel/channelz_registry.cc
   src/core/lib/channel/connected_channel.cc
   src/core/lib/channel/handshaker.cc
-  src/core/lib/channel/handshaker_factory.cc
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
@@ -3486,10 +3483,10 @@ add_library(grpc++_cronet
   src/core/ext/filters/client_channel/parse_address.cc
   src/core/ext/filters/client_channel/proxy_mapper.cc
   src/core/ext/filters/client_channel/proxy_mapper_registry.cc
-  src/core/ext/filters/client_channel/request_routing.cc
   src/core/ext/filters/client_channel/resolver.cc
   src/core/ext/filters/client_channel/resolver_registry.cc
   src/core/ext/filters/client_channel/resolver_result_parsing.cc
+  src/core/ext/filters/client_channel/resolving_lb_policy.cc
   src/core/ext/filters/client_channel/retry_throttle.cc
   src/core/ext/filters/client_channel/server_address.cc
   src/core/ext/filters/client_channel/subchannel.cc
@@ -11177,6 +11174,51 @@ endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 
+add_executable(bm_alarm
+  test/cpp/microbenchmarks/bm_alarm.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(bm_alarm
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(bm_alarm
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_benchmark
+  ${_gRPC_BENCHMARK_LIBRARIES}
+  grpc++_test_util_unsecure
+  grpc_test_util_unsecure
+  grpc++_unsecure
+  grpc_unsecure
+  gpr
+  grpc++_test_config
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
+endif()
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
+
 add_executable(bm_arena
   test/cpp/microbenchmarks/bm_arena.cc
   third_party/googletest/googletest/src/gtest-all.cc

Makefile

@@ -1137,6 +1137,7 @@ async_end2end_test: $(BINDIR)/$(CONFIG)/async_end2end_test
 auth_property_iterator_test: $(BINDIR)/$(CONFIG)/auth_property_iterator_test
 backoff_test: $(BINDIR)/$(CONFIG)/backoff_test
 bdp_estimator_test: $(BINDIR)/$(CONFIG)/bdp_estimator_test
+bm_alarm: $(BINDIR)/$(CONFIG)/bm_alarm
 bm_arena: $(BINDIR)/$(CONFIG)/bm_arena
 bm_byte_buffer: $(BINDIR)/$(CONFIG)/bm_byte_buffer
 bm_call_create: $(BINDIR)/$(CONFIG)/bm_call_create
@@ -1654,6 +1655,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/auth_property_iterator_test \
   $(BINDIR)/$(CONFIG)/backoff_test \
   $(BINDIR)/$(CONFIG)/bdp_estimator_test \
+  $(BINDIR)/$(CONFIG)/bm_alarm \
   $(BINDIR)/$(CONFIG)/bm_arena \
   $(BINDIR)/$(CONFIG)/bm_byte_buffer \
   $(BINDIR)/$(CONFIG)/bm_call_create \
@@ -1842,6 +1844,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/auth_property_iterator_test \
   $(BINDIR)/$(CONFIG)/backoff_test \
   $(BINDIR)/$(CONFIG)/bdp_estimator_test \
+  $(BINDIR)/$(CONFIG)/bm_alarm \
   $(BINDIR)/$(CONFIG)/bm_arena \
   $(BINDIR)/$(CONFIG)/bm_byte_buffer \
   $(BINDIR)/$(CONFIG)/bm_call_create \
@@ -2285,6 +2288,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/backoff_test || ( echo test backoff_test failed ; exit 1 )
 	$(E) "[RUN]     Testing bdp_estimator_test"
 	$(Q) $(BINDIR)/$(CONFIG)/bdp_estimator_test || ( echo test bdp_estimator_test failed ; exit 1 )
+	$(E) "[RUN]     Testing bm_alarm"
+	$(Q) $(BINDIR)/$(CONFIG)/bm_alarm || ( echo test bm_alarm failed ; exit 1 )
 	$(E) "[RUN]     Testing bm_arena"
 	$(Q) $(BINDIR)/$(CONFIG)/bm_arena || ( echo test bm_arena failed ; exit 1 )
 	$(E) "[RUN]     Testing bm_byte_buffer"
@@ -3492,7 +3497,6 @@ LIBGRPC_SRC = \
     src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
-    src/core/lib/channel/handshaker_factory.cc \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
@@ -3754,10 +3758,10 @@ LIBGRPC_SRC = \
     src/core/ext/filters/client_channel/parse_address.cc \
     src/core/ext/filters/client_channel/proxy_mapper.cc \
     src/core/ext/filters/client_channel/proxy_mapper_registry.cc \
-    src/core/ext/filters/client_channel/request_routing.cc \
     src/core/ext/filters/client_channel/resolver.cc \
     src/core/ext/filters/client_channel/resolver_registry.cc \
     src/core/ext/filters/client_channel/resolver_result_parsing.cc \
+    src/core/ext/filters/client_channel/resolving_lb_policy.cc \
     src/core/ext/filters/client_channel/retry_throttle.cc \
     src/core/ext/filters/client_channel/server_address.cc \
     src/core/ext/filters/client_channel/subchannel.cc \
@@ -3912,7 +3916,6 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
-    src/core/lib/channel/handshaker_factory.cc \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
@@ -4104,10 +4107,10 @@ LIBGRPC_CRONET_SRC = \
     src/core/ext/filters/client_channel/parse_address.cc \
     src/core/ext/filters/client_channel/proxy_mapper.cc \
     src/core/ext/filters/client_channel/proxy_mapper_registry.cc \
-    src/core/ext/filters/client_channel/request_routing.cc \
     src/core/ext/filters/client_channel/resolver.cc \
     src/core/ext/filters/client_channel/resolver_registry.cc \
     src/core/ext/filters/client_channel/resolver_result_parsing.cc \
+    src/core/ext/filters/client_channel/resolving_lb_policy.cc \
     src/core/ext/filters/client_channel/retry_throttle.cc \
     src/core/ext/filters/client_channel/server_address.cc \
     src/core/ext/filters/client_channel/subchannel.cc \
@@ -4316,7 +4319,6 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
-    src/core/lib/channel/handshaker_factory.cc \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
@@ -4476,10 +4478,10 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/ext/filters/client_channel/parse_address.cc \
     src/core/ext/filters/client_channel/proxy_mapper.cc \
     src/core/ext/filters/client_channel/proxy_mapper_registry.cc \
-    src/core/ext/filters/client_channel/request_routing.cc \
     src/core/ext/filters/client_channel/resolver.cc \
     src/core/ext/filters/client_channel/resolver_registry.cc \
     src/core/ext/filters/client_channel/resolver_result_parsing.cc \
+    src/core/ext/filters/client_channel/resolving_lb_policy.cc \
     src/core/ext/filters/client_channel/retry_throttle.cc \
     src/core/ext/filters/client_channel/server_address.cc \
     src/core/ext/filters/client_channel/subchannel.cc \
@@ -4629,7 +4631,6 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
-    src/core/lib/channel/handshaker_factory.cc \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
@@ -4789,10 +4790,10 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     src/core/ext/filters/client_channel/parse_address.cc \
     src/core/ext/filters/client_channel/proxy_mapper.cc \
     src/core/ext/filters/client_channel/proxy_mapper_registry.cc \
-    src/core/ext/filters/client_channel/request_routing.cc \
     src/core/ext/filters/client_channel/resolver.cc \
     src/core/ext/filters/client_channel/resolver_registry.cc \
     src/core/ext/filters/client_channel/resolver_result_parsing.cc \
+    src/core/ext/filters/client_channel/resolving_lb_policy.cc \
     src/core/ext/filters/client_channel/retry_throttle.cc \
     src/core/ext/filters/client_channel/server_address.cc \
     src/core/ext/filters/client_channel/subchannel.cc \
@@ -4905,7 +4906,6 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
-    src/core/lib/channel/handshaker_factory.cc \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
@@ -5100,10 +5100,10 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/ext/filters/client_channel/parse_address.cc \
     src/core/ext/filters/client_channel/proxy_mapper.cc \
     src/core/ext/filters/client_channel/proxy_mapper_registry.cc \
-    src/core/ext/filters/client_channel/request_routing.cc \
     src/core/ext/filters/client_channel/resolver.cc \
     src/core/ext/filters/client_channel/resolver_registry.cc \
     src/core/ext/filters/client_channel/resolver_result_parsing.cc \
+    src/core/ext/filters/client_channel/resolving_lb_policy.cc \
     src/core/ext/filters/client_channel/retry_throttle.cc \
     src/core/ext/filters/client_channel/server_address.cc \
     src/core/ext/filters/client_channel/subchannel.cc \
@@ -5770,7 +5770,6 @@ LIBGRPC++_CRONET_SRC = \
     src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
-    src/core/lib/channel/handshaker_factory.cc \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
@@ -5935,10 +5934,10 @@ LIBGRPC++_CRONET_SRC = \
     src/core/ext/filters/client_channel/parse_address.cc \
     src/core/ext/filters/client_channel/proxy_mapper.cc \
     src/core/ext/filters/client_channel/proxy_mapper_registry.cc \
-    src/core/ext/filters/client_channel/request_routing.cc \
     src/core/ext/filters/client_channel/resolver.cc \
     src/core/ext/filters/client_channel/resolver_registry.cc \
     src/core/ext/filters/client_channel/resolver_result_parsing.cc \
+    src/core/ext/filters/client_channel/resolving_lb_policy.cc \
     src/core/ext/filters/client_channel/retry_throttle.cc \
     src/core/ext/filters/client_channel/server_address.cc \
     src/core/ext/filters/client_channel/subchannel.cc \
@@ -16186,6 +16185,50 @@ endif
 endif
 
 
+BM_ALARM_SRC = \
+    test/cpp/microbenchmarks/bm_alarm.cc \
+
+BM_ALARM_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_ALARM_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/bm_alarm: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+.
+
+$(BINDIR)/$(CONFIG)/bm_alarm: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/bm_alarm: $(PROTOBUF_DEP) $(BM_ALARM_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(BM_ALARM_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_alarm
+
+endif
+
+endif
+
+$(BM_ALARM_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX
+$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_alarm.o:  $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+
+deps_bm_alarm: $(BM_ALARM_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(BM_ALARM_OBJS:.o=.dep)
+endif
+endif
+
+
 BM_ARENA_SRC = \
     test/cpp/microbenchmarks/bm_arena.cc \
 

bazel/grpc_deps.bzl

@@ -185,6 +185,14 @@ def grpc_deps():
             sha256 = "ee854b5de299138c1f4a2edb5573d22b21d975acfc7aa938f36d30b49ef97498",
         )
 
+    if "bazel_skylib" not in native.existing_rules():
+        http_archive(
+            name = "bazel_skylib",
+            sha256 = "ba5d15ca230efca96320085d8e4d58da826d1f81b444ef8afccd8b23e0799b52",
+            strip_prefix = "bazel-skylib-f83cb8dd6f5658bc574ccd873e25197055265d1c",
+            url = "https://github.com/bazelbuild/bazel-skylib/archive/f83cb8dd6f5658bc574ccd873e25197055265d1c.tar.gz",
+        )
+
     if "io_opencensus_cpp" not in native.existing_rules():
         http_archive(
             name = "io_opencensus_cpp",

build.yaml

@@ -242,7 +242,6 @@ filegroups:
   - src/core/lib/channel/channelz_registry.cc
   - src/core/lib/channel/connected_channel.cc
   - src/core/lib/channel/handshaker.cc
-  - src/core/lib/channel/handshaker_factory.cc
   - src/core/lib/channel/handshaker_registry.cc
   - src/core/lib/channel/status_util.cc
   - src/core/lib/compression/compression.cc
@@ -588,11 +587,11 @@ filegroups:
   - src/core/ext/filters/client_channel/parse_address.h
   - src/core/ext/filters/client_channel/proxy_mapper.h
   - src/core/ext/filters/client_channel/proxy_mapper_registry.h
-  - src/core/ext/filters/client_channel/request_routing.h
   - src/core/ext/filters/client_channel/resolver.h
   - src/core/ext/filters/client_channel/resolver_factory.h
   - src/core/ext/filters/client_channel/resolver_registry.h
   - src/core/ext/filters/client_channel/resolver_result_parsing.h
+  - src/core/ext/filters/client_channel/resolving_lb_policy.h
   - src/core/ext/filters/client_channel/retry_throttle.h
   - src/core/ext/filters/client_channel/server_address.h
   - src/core/ext/filters/client_channel/subchannel.h
@@ -615,10 +614,10 @@ filegroups:
   - src/core/ext/filters/client_channel/parse_address.cc
   - src/core/ext/filters/client_channel/proxy_mapper.cc
   - src/core/ext/filters/client_channel/proxy_mapper_registry.cc
-  - src/core/ext/filters/client_channel/request_routing.cc
   - src/core/ext/filters/client_channel/resolver.cc
   - src/core/ext/filters/client_channel/resolver_registry.cc
   - src/core/ext/filters/client_channel/resolver_result_parsing.cc
+  - src/core/ext/filters/client_channel/resolving_lb_policy.cc
   - src/core/ext/filters/client_channel/retry_throttle.cc
   - src/core/ext/filters/client_channel/server_address.cc
   - src/core/ext/filters/client_channel/subchannel.cc
@@ -3936,6 +3935,26 @@ targets:
   - grpc
   - gpr
   uses_polling: false
+- name: bm_alarm
+  build: test
+  language: c++
+  src:
+  - test/cpp/microbenchmarks/bm_alarm.cc
+  deps:
+  - grpc_benchmark
+  - benchmark
+  - grpc++_test_util_unsecure
+  - grpc_test_util_unsecure
+  - grpc++_unsecure
+  - grpc_unsecure
+  - gpr
+  - grpc++_test_config
+  benchmark: true
+  defaults: benchmark
+  platforms:
+  - mac
+  - linux
+  - posix
 - name: bm_arena
   build: test
   language: c++

config.m4

@@ -94,7 +94,6 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/channel/channelz_registry.cc \
     src/core/lib/channel/connected_channel.cc \
     src/core/lib/channel/handshaker.cc \
-    src/core/lib/channel/handshaker_factory.cc \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
@@ -356,10 +355,10 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/filters/client_channel/parse_address.cc \
     src/core/ext/filters/client_channel/proxy_mapper.cc \
     src/core/ext/filters/client_channel/proxy_mapper_registry.cc \
-    src/core/ext/filters/client_channel/request_routing.cc \
     src/core/ext/filters/client_channel/resolver.cc \
     src/core/ext/filters/client_channel/resolver_registry.cc \
     src/core/ext/filters/client_channel/resolver_result_parsing.cc \
+    src/core/ext/filters/client_channel/resolving_lb_policy.cc \
     src/core/ext/filters/client_channel/retry_throttle.cc \
     src/core/ext/filters/client_channel/server_address.cc \
     src/core/ext/filters/client_channel/subchannel.cc \

config.w32

@@ -69,7 +69,6 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\channel\\channelz_registry.cc " +
     "src\\core\\lib\\channel\\connected_channel.cc " +
     "src\\core\\lib\\channel\\handshaker.cc " +
-    "src\\core\\lib\\channel\\handshaker_factory.cc " +
     "src\\core\\lib\\channel\\handshaker_registry.cc " +
     "src\\core\\lib\\channel\\status_util.cc " +
     "src\\core\\lib\\compression\\compression.cc " +
@@ -331,10 +330,10 @@ if (PHP_GRPC != "no") {
     "src\\core\\ext\\filters\\client_channel\\parse_address.cc " +
     "src\\core\\ext\\filters\\client_channel\\proxy_mapper.cc " +
     "src\\core\\ext\\filters\\client_channel\\proxy_mapper_registry.cc " +
-    "src\\core\\ext\\filters\\client_channel\\request_routing.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver_registry.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver_result_parsing.cc " +
+    "src\\core\\ext\\filters\\client_channel\\resolving_lb_policy.cc " +
     "src\\core\\ext\\filters\\client_channel\\retry_throttle.cc " +
     "src\\core\\ext\\filters\\client_channel\\server_address.cc " +
     "src\\core\\ext\\filters\\client_channel\\subchannel.cc " +

doc/compression.md

@@ -30,7 +30,7 @@ configured:
    therefore the compression that SHALL be used in the absence of per-RPC
    compression configuration.
 +  At response time, via:
-   +  For unary RPCs, the {Client,Server}Context instance. 
+   +  For unary RPCs, the {Client,Server}Context instance.
    +  For streaming RPCs, the {Client,Server}Writer instance. In this case,
       configuration is reduced to disabling compression altogether.
 
@@ -41,14 +41,14 @@ of the request, including not performing any compression, regardless of channel
 and RPC settings (for example, if compression would result in small or negative
 gains).
 
-When a message from a client compressed with an unsupported algorithm is
-processed by a server, it WILL result in an `UNIMPLEMENTED` error status on the
-server. The server will then include in its response a `grpc-accept-encoding`
-header specifying the algorithms it does accept. If an `UNIMPLEMENTED` error
-status is returned from the server despite having used one of the algorithms
-from the `grpc-accept-encoding` header, the cause MUST NOT be related to
-compression. Data sent from a server compressed with an algorithm not supported
-by the client WILL result in an `INTERNAL` error status on the client side.
+If a client message is compressed by an algorithm that is not supported
+by a server, the message WILL result in an `UNIMPLEMENTED` error status on the
+server. The server will then include a `grpc-accept-encoding` response
+header which specifies the algorithms that the server accepts. If the client
+message is compressed using one of the algorithms from the `grpc-accept-encoding` header
+and an `UNIMPLEMENTED` error status is returned from the server, the cause of the error
+MUST NOT be related to compression. If a server sent data which is compressed by an algorithm
+that is not supported by the client, an `INTERNAL` error status will occur on the client side.
 
 Note that a peer MAY choose to not disclose all the encodings it supports.
 However, if it receives a message compressed in an undisclosed but supported
@@ -57,7 +57,7 @@ header.
 
 For every message a server is requested to compress using an algorithm it knows
 the client doesn't support (as indicated by the last `grpc-accept-encoding`
-header received from the client), it SHALL send the message uncompressed. 
+header received from the client), it SHALL send the message uncompressed.
 
 ### Specific Disabling of Compression
 

doc/keepalive.md

@@ -5,12 +5,14 @@ The keepalive ping is a way to check if a channel is currently working by sendin
 This guide documents the knobs within gRPC core to control the current behavior of the keepalive ping.
 
 The keepalive ping is controlled by two important channel arguments -
+
 * **GRPC_ARG_KEEPALIVE_TIME_MS**
   * This channel argument controls the period (in milliseconds) after which a keepalive ping is sent on the transport.
 * **GRPC_ARG_KEEPALIVE_TIMEOUT_MS**
-  * This channel argument controls the amount of time (in milliseconds), the sender of the keepalive ping waits for an acknowledgement. If it does not receive an acknowledgement within this time, it will close the connection.
+  * This channel argument controls the amount of time (in milliseconds) the sender of the keepalive ping waits for an acknowledgement. If it does not receive an acknowledgment within this time, it will close the connection.
 
 The above two channel arguments should be sufficient for most users, but the following arguments can also be useful in certain use cases.
+
 * **GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS**
   * This channel argument if set to 1 (0 : false; 1 : true), allows keepalive pings to be sent even if there are no calls in flight. 
 * **GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA**
@@ -39,12 +41,12 @@ GRPC_ARG_HTTP2_MAX_PING_STRIKES|N/A|2
 * When is the keepalive timer started?
   * The keepalive timer is started when a transport is done connecting (after handshake).
 * What happens when the keepalive timer fires?
-  * When the keepalive timer fires, gRPC Core would try to send a keepalive ping on the transport. This ping can be blocked if -
+  * When the keepalive timer fires, gRPC Core will try to send a keepalive ping on the transport. This ping can be blocked if -
     * there is no active call on that transport and GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS is false.
     * the number of pings already sent on the transport without any data has already exceeded GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA.
-    * the time expired since the previous ping is less than GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS.
-  * If a keepalive ping is not blocked and is sent on the transport, then the keepalive watchdog timer is started which would close the transport if the ping is not acknowledged before it fires.
+    * the time elapsed since the previous ping is less than GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS.
+  * If a keepalive ping is not blocked and is sent on the transport, then the keepalive watchdog timer is started which will close the transport if the ping is not acknowledged before it fires.
 * Why am I receiving a GOAWAY with error code ENHANCE_YOUR_CALM?
   * A server sends a GOAWAY with ENHANCE_YOUR_CALM if the client sends too many misbehaving pings. For example -
-    * if a server has GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS set to false, and the client sends pings without there being any call in flight.
+    * if a server has GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS set to false and the client sends pings without there being any call in flight.
     * if the client's GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS setting is lower than the server's GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS.

examples/csharp/HelloworldUnity/.gitignore

@@ -0,0 +1,52 @@
+[Ll]ibrary/
+[Tt]emp/
+[Oo]bj/
+[Bb]uild/
+[Bb]uilds/
+[Ll]ogs/
+
+# Never ignore Asset meta data
+![Aa]ssets/**/*.meta
+
+# Uncomment this line if you wish to ignore the asset store tools plugin
+# [Aa]ssets/AssetStoreTools*
+
+# Visual Studio cache directory
+.vs/
+
+# Gradle cache directory
+.gradle/
+
+# Autogenerated VS/MD/Consulo solution and project files
+ExportedObj/
+.consulo/
+*.csproj
+*.unityproj
+*.sln
+*.suo
+*.tmp
+*.user
+*.userprefs
+*.pidb
+*.booproj
+*.svd
+*.pdb
+*.mdb
+*.opendb
+*.VC.db
+
+# Unity3D generated meta files
+*.pidb.meta
+*.pdb.meta
+*.mdb.meta
+
+# Unity3D generated file on crash reports
+sysinfo.txt
+
+# Builds
+*.apk
+*.unitypackage
+
+# Crashlytics generated file
+crashlytics-build.properties
+

examples/csharp/HelloworldUnity/Assets/Plugins.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 9e39cea189b0245c4a39113ff6459d24
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/Assets/Scenes.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 131a6b21c8605f84396be9f6751fb6e3
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/Assets/Scenes/SampleScene.unity

@@ -0,0 +1,586 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!29 &1
+OcclusionCullingSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 2
+  m_OcclusionBakeSettings:
+    smallestOccluder: 5
+    smallestHole: 0.25
+    backfaceThreshold: 100
+  m_SceneGUID: 00000000000000000000000000000000
+  m_OcclusionCullingData: {fileID: 0}
+--- !u!104 &2
+RenderSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 9
+  m_Fog: 0
+  m_FogColor: {r: 0.5, g: 0.5, b: 0.5, a: 1}
+  m_FogMode: 3
+  m_FogDensity: 0.01
+  m_LinearFogStart: 0
+  m_LinearFogEnd: 300
+  m_AmbientSkyColor: {r: 0.212, g: 0.227, b: 0.259, a: 1}
+  m_AmbientEquatorColor: {r: 0.114, g: 0.125, b: 0.133, a: 1}
+  m_AmbientGroundColor: {r: 0.047, g: 0.043, b: 0.035, a: 1}
+  m_AmbientIntensity: 1
+  m_AmbientMode: 3
+  m_SubtractiveShadowColor: {r: 0.42, g: 0.478, b: 0.627, a: 1}
+  m_SkyboxMaterial: {fileID: 0}
+  m_HaloStrength: 0.5
+  m_FlareStrength: 1
+  m_FlareFadeSpeed: 3
+  m_HaloTexture: {fileID: 0}
+  m_SpotCookie: {fileID: 10001, guid: 0000000000000000e000000000000000, type: 0}
+  m_DefaultReflectionMode: 0
+  m_DefaultReflectionResolution: 128
+  m_ReflectionBounces: 1
+  m_ReflectionIntensity: 1
+  m_CustomReflection: {fileID: 0}
+  m_Sun: {fileID: 0}
+  m_IndirectSpecularColor: {r: 0, g: 0, b: 0, a: 1}
+  m_UseRadianceAmbientProbe: 0
+--- !u!157 &3
+LightmapSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 11
+  m_GIWorkflowMode: 1
+  m_GISettings:
+    serializedVersion: 2
+    m_BounceScale: 1
+    m_IndirectOutputScale: 1
+    m_AlbedoBoost: 1
+    m_TemporalCoherenceThreshold: 1
+    m_EnvironmentLightingMode: 0
+    m_EnableBakedLightmaps: 0
+    m_EnableRealtimeLightmaps: 0
+  m_LightmapEditorSettings:
+    serializedVersion: 10
+    m_Resolution: 2
+    m_BakeResolution: 40
+    m_AtlasSize: 1024
+    m_AO: 0
+    m_AOMaxDistance: 1
+    m_CompAOExponent: 1
+    m_CompAOExponentDirect: 0
+    m_Padding: 2
+    m_LightmapParameters: {fileID: 0}
+    m_LightmapsBakeMode: 1
+    m_TextureCompression: 1
+    m_FinalGather: 0
+    m_FinalGatherFiltering: 1
+    m_FinalGatherRayCount: 256
+    m_ReflectionCompression: 2
+    m_MixedBakeMode: 2
+    m_BakeBackend: 0
+    m_PVRSampling: 1
+    m_PVRDirectSampleCount: 32
+    m_PVRSampleCount: 500
+    m_PVRBounces: 2
+    m_PVRFilterTypeDirect: 0
+    m_PVRFilterTypeIndirect: 0
+    m_PVRFilterTypeAO: 0
+    m_PVRFilteringMode: 1
+    m_PVRCulling: 1
+    m_PVRFilteringGaussRadiusDirect: 1
+    m_PVRFilteringGaussRadiusIndirect: 5
+    m_PVRFilteringGaussRadiusAO: 2
+    m_PVRFilteringAtrousPositionSigmaDirect: 0.5
+    m_PVRFilteringAtrousPositionSigmaIndirect: 2
+    m_PVRFilteringAtrousPositionSigmaAO: 1
+    m_ShowResolutionOverlay: 1
+  m_LightingDataAsset: {fileID: 0}
+  m_UseShadowmask: 1
+--- !u!196 &4
+NavMeshSettings:
+  serializedVersion: 2
+  m_ObjectHideFlags: 0
+  m_BuildSettings:
+    serializedVersion: 2
+    agentTypeID: 0
+    agentRadius: 0.5
+    agentHeight: 2
+    agentSlope: 45
+    agentClimb: 0.4
+    ledgeDropHeight: 0
+    maxJumpAcrossDistance: 0
+    minRegionArea: 2
+    manualCellSize: 0
+    cellSize: 0.16666667
+    manualTileSize: 0
+    tileSize: 256
+    accuratePlacement: 0
+    debug:
+      m_Flags: 0
+  m_NavMeshData: {fileID: 0}
+--- !u!1 &519420028
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 519420032}
+  - component: {fileID: 519420031}
+  - component: {fileID: 519420029}
+  m_Layer: 0
+  m_Name: Main Camera
+  m_TagString: MainCamera
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!81 &519420029
+AudioListener:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 519420028}
+  m_Enabled: 1
+--- !u!20 &519420031
+Camera:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 519420028}
+  m_Enabled: 1
+  serializedVersion: 2
+  m_ClearFlags: 2
+  m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0}
+  m_projectionMatrixMode: 1
+  m_SensorSize: {x: 36, y: 24}
+  m_LensShift: {x: 0, y: 0}
+  m_FocalLength: 50
+  m_NormalizedViewPortRect:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+  near clip plane: 0.3
+  far clip plane: 1000
+  field of view: 60
+  orthographic: 1
+  orthographic size: 5
+  m_Depth: -1
+  m_CullingMask:
+    serializedVersion: 2
+    m_Bits: 4294967295
+  m_RenderingPath: -1
+  m_TargetTexture: {fileID: 0}
+  m_TargetDisplay: 0
+  m_TargetEye: 0
+  m_HDR: 1
+  m_AllowMSAA: 0
+  m_AllowDynamicResolution: 0
+  m_ForceIntoRT: 0
+  m_OcclusionCulling: 0
+  m_StereoConvergence: 10
+  m_StereoSeparation: 0.022
+--- !u!4 &519420032
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 519420028}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: -10}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!1 &785253852
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 785253855}
+  - component: {fileID: 785253854}
+  - component: {fileID: 785253853}
+  m_Layer: 0
+  m_Name: EventSystem
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!114 &785253853
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 785253852}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 1077351063, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_HorizontalAxis: Horizontal
+  m_VerticalAxis: Vertical
+  m_SubmitButton: Submit
+  m_CancelButton: Cancel
+  m_InputActionsPerSecond: 10
+  m_RepeatDelay: 0.5
+  m_ForceModuleActive: 0
+--- !u!114 &785253854
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 785253852}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: -619905303, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_FirstSelected: {fileID: 0}
+  m_sendNavigationEvents: 1
+  m_DragThreshold: 10
+--- !u!4 &785253855
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 785253852}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 2
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!1 &1639505844
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 1639505846}
+  - component: {fileID: 1639505845}
+  m_Layer: 0
+  m_Name: UIManager
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!114 &1639505845
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1639505844}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: d62381e23356a4203b3e54cc6c2e3a4f, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+--- !u!4 &1639505846
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1639505844}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 3
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!1 &1729899994
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 1729899995}
+  - component: {fileID: 1729899997}
+  - component: {fileID: 1729899996}
+  m_Layer: 5
+  m_Name: Text
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &1729899995
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1729899994}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 2040475500}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0, y: 0}
+  m_AnchorMax: {x: 1, y: 1}
+  m_AnchoredPosition: {x: 0, y: 0}
+  m_SizeDelta: {x: 0, y: 0}
+  m_Pivot: {x: 0.5, y: 0.5}
+--- !u!114 &1729899996
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1729899994}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 708705254, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1}
+  m_RaycastTarget: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+    m_TypeName: UnityEngine.UI.MaskableGraphic+CullStateChangedEvent, UnityEngine.UI,
+      Version=1.0.0.0, Culture=neutral, PublicKeyToken=null
+  m_FontData:
+    m_Font: {fileID: 10102, guid: 0000000000000000e000000000000000, type: 0}
+    m_FontSize: 14
+    m_FontStyle: 0
+    m_BestFit: 0
+    m_MinSize: 10
+    m_MaxSize: 40
+    m_Alignment: 4
+    m_AlignByGeometry: 0
+    m_RichText: 1
+    m_HorizontalOverflow: 0
+    m_VerticalOverflow: 0
+    m_LineSpacing: 1
+  m_Text: Hello gRPC!!!
+--- !u!222 &1729899997
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 1729899994}
+  m_CullTransparentMesh: 0
+--- !u!1 &2040475499
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 2040475500}
+  - component: {fileID: 2040475503}
+  - component: {fileID: 2040475502}
+  - component: {fileID: 2040475501}
+  m_Layer: 5
+  m_Name: Button
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!224 &2040475500
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2040475499}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children:
+  - {fileID: 1729899995}
+  m_Father: {fileID: 2066701619}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0.5, y: 0.5}
+  m_AnchorMax: {x: 0.5, y: 0.5}
+  m_AnchoredPosition: {x: 0, y: 0}
+  m_SizeDelta: {x: 500, y: 150}
+  m_Pivot: {x: 0.5, y: 0.5}
+--- !u!114 &2040475501
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2040475499}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 1392445389, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Navigation:
+    m_Mode: 3
+    m_SelectOnUp: {fileID: 0}
+    m_SelectOnDown: {fileID: 0}
+    m_SelectOnLeft: {fileID: 0}
+    m_SelectOnRight: {fileID: 0}
+  m_Transition: 1
+  m_Colors:
+    m_NormalColor: {r: 1, g: 1, b: 1, a: 1}
+    m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
+    m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1}
+    m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608}
+    m_ColorMultiplier: 1
+    m_FadeDuration: 0.1
+  m_SpriteState:
+    m_HighlightedSprite: {fileID: 0}
+    m_PressedSprite: {fileID: 0}
+    m_DisabledSprite: {fileID: 0}
+  m_AnimationTriggers:
+    m_NormalTrigger: Normal
+    m_HighlightedTrigger: Highlighted
+    m_PressedTrigger: Pressed
+    m_DisabledTrigger: Disabled
+  m_Interactable: 1
+  m_TargetGraphic: {fileID: 2040475502}
+  m_OnClick:
+    m_PersistentCalls:
+      m_Calls:
+      - m_Target: {fileID: 1639505845}
+        m_MethodName: RunHelloWorld
+        m_Mode: 2
+        m_Arguments:
+          m_ObjectArgument: {fileID: 1729899996}
+          m_ObjectArgumentAssemblyTypeName: UnityEngine.UI.Text, UnityEngine.UI
+          m_IntArgument: 0
+          m_FloatArgument: 0
+          m_StringArgument: 
+          m_BoolArgument: 0
+        m_CallState: 2
+    m_TypeName: UnityEngine.UI.Button+ButtonClickedEvent, UnityEngine.UI, Version=1.0.0.0,
+      Culture=neutral, PublicKeyToken=null
+--- !u!114 &2040475502
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2040475499}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: -765806418, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_Material: {fileID: 0}
+  m_Color: {r: 0.34157702, g: 0.6037736, b: 0.093983635, a: 1}
+  m_RaycastTarget: 1
+  m_OnCullStateChanged:
+    m_PersistentCalls:
+      m_Calls: []
+    m_TypeName: UnityEngine.UI.MaskableGraphic+CullStateChangedEvent, UnityEngine.UI,
+      Version=1.0.0.0, Culture=neutral, PublicKeyToken=null
+  m_Sprite: {fileID: 10905, guid: 0000000000000000f000000000000000, type: 0}
+  m_Type: 1
+  m_PreserveAspect: 0
+  m_FillCenter: 1
+  m_FillMethod: 4
+  m_FillAmount: 1
+  m_FillClockwise: 1
+  m_FillOrigin: 0
+--- !u!222 &2040475503
+CanvasRenderer:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2040475499}
+  m_CullTransparentMesh: 0
+--- !u!1 &2066701615
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 2066701619}
+  - component: {fileID: 2066701618}
+  - component: {fileID: 2066701617}
+  - component: {fileID: 2066701616}
+  m_Layer: 5
+  m_Name: Canvas
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!114 &2066701616
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2066701615}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 1301386320, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_IgnoreReversedGraphics: 1
+  m_BlockingObjects: 0
+  m_BlockingMask:
+    serializedVersion: 2
+    m_Bits: 4294967295
+--- !u!114 &2066701617
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2066701615}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 1980459831, guid: f70555f144d8491a825f0804e09c671c, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  m_UiScaleMode: 0
+  m_ReferencePixelsPerUnit: 100
+  m_ScaleFactor: 1
+  m_ReferenceResolution: {x: 800, y: 600}
+  m_ScreenMatchMode: 0
+  m_MatchWidthOrHeight: 0
+  m_PhysicalUnit: 3
+  m_FallbackScreenDPI: 96
+  m_DefaultSpriteDPI: 96
+  m_DynamicPixelsPerUnit: 1
+--- !u!223 &2066701618
+Canvas:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2066701615}
+  m_Enabled: 1
+  serializedVersion: 3
+  m_RenderMode: 0
+  m_Camera: {fileID: 0}
+  m_PlaneDistance: 100
+  m_PixelPerfect: 0
+  m_ReceivesEvents: 1
+  m_OverrideSorting: 0
+  m_OverridePixelPerfect: 0
+  m_SortingBucketNormalizedSize: 0
+  m_AdditionalShaderChannelsFlag: 0
+  m_SortingLayerID: 0
+  m_SortingOrder: 0
+  m_TargetDisplay: 0
+--- !u!224 &2066701619
+RectTransform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 0}
+  m_GameObject: {fileID: 2066701615}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 0, y: 0, z: 0}
+  m_Children:
+  - {fileID: 2040475500}
+  m_Father: {fileID: 0}
+  m_RootOrder: 1
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+  m_AnchorMin: {x: 0, y: 0}
+  m_AnchorMax: {x: 0, y: 0}
+  m_AnchoredPosition: {x: 0, y: 0}
+  m_SizeDelta: {x: 0, y: 0}
+  m_Pivot: {x: 0, y: 0}

examples/csharp/HelloworldUnity/Assets/Scenes/SampleScene.unity.meta

@@ -0,0 +1,7 @@
+fileFormatVersion: 2
+guid: 2cda990e2423bbf4892e6590ba056729
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/Assets/Scripts.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 55598493aa3774a6dad4b7a4974826ff
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/Assets/Scripts/HelloWorldScript.cs

@@ -0,0 +1,38 @@
+#region Copyright notice and license
+
+// Copyright 2019 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#endregion
+
+using UnityEngine;
+using UnityEngine.UI;
+
+public class HelloWorldScript : MonoBehaviour {
+  int counter = 1;
+
+  // Use this for initialization
+  void Start () {}
+
+  // Update is called once per frame
+  void Update() {}
+
+  // Ran when button is clicked
+  public void RunHelloWorld(Text text)
+  {
+    var reply = HelloWorldTest.Greet("Unity " + counter);
+    text.text = "Greeting: " + reply.Message;
+    counter++;
+  }
+}

examples/csharp/HelloworldUnity/Assets/Scripts/HelloWorldScript.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: d62381e23356a4203b3e54cc6c2e3a4f
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/Assets/Scripts/HelloWorldTest.cs

@@ -0,0 +1,81 @@
+#region Copyright notice and license
+
+// Copyright 2019 The gRPC Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#endregion
+
+using UnityEngine;
+using System.Threading.Tasks;
+using System;
+using Grpc.Core;
+using Helloworld;
+
+class HelloWorldTest
+{
+  // Can be run from commandline.
+  // Example command:
+  // "/Applications/Unity/Unity.app/Contents/MacOS/Unity -quit -batchmode -nographics -executeMethod HelloWorldTest.RunHelloWorld -logfile"
+  public static void RunHelloWorld()
+  {
+    Application.SetStackTraceLogType(LogType.Log, StackTraceLogType.None);
+
+    Debug.Log("==============================================================");
+    Debug.Log("Starting tests");
+    Debug.Log("==============================================================");
+
+    Debug.Log("Application.platform: " + Application.platform);
+    Debug.Log("Environment.OSVersion: " + Environment.OSVersion);
+
+    var reply = Greet("Unity");
+    Debug.Log("Greeting: " + reply.Message);
+
+    Debug.Log("==============================================================");
+    Debug.Log("Tests finished successfully.");
+    Debug.Log("==============================================================");
+  }
+
+  public static HelloReply Greet(string greeting)
+  {
+    const int Port = 50051;
+
+    Server server = new Server
+    {
+      Services = { Greeter.BindService(new GreeterImpl()) },
+      Ports = { new ServerPort("localhost", Port, ServerCredentials.Insecure) }
+    };
+    server.Start();
+
+    Channel channel = new Channel("127.0.0.1:50051", ChannelCredentials.Insecure);
+
+    var client = new Greeter.GreeterClient(channel);
+
+    var reply = client.SayHello(new HelloRequest { Name = greeting });
+
+    channel.ShutdownAsync().Wait();
+
+    server.ShutdownAsync().Wait();
+
+    return reply;
+  }
+
+  class GreeterImpl : Greeter.GreeterBase
+  {
+    // Server side handler of the SayHello RPC
+    public override Task<HelloReply> SayHello(HelloRequest request, ServerCallContext context)
+    {
+      return Task.FromResult(new HelloReply { Message = "Hello " + request.Name });
+    }
+  }
+}

examples/csharp/HelloworldUnity/Assets/Scripts/HelloWorldTest.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 8c088e5dee11c45fc95e41b9281d55e2
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/Assets/Scripts/Helloworld.cs

@@ -0,0 +1,286 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: helloworld.proto
+#pragma warning disable 1591, 0612, 3021
+#region Designer generated code
+
+using pb = global::Google.Protobuf;
+using pbc = global::Google.Protobuf.Collections;
+using pbr = global::Google.Protobuf.Reflection;
+using scg = global::System.Collections.Generic;
+namespace Helloworld {
+
+  /// <summary>Holder for reflection information generated from helloworld.proto</summary>
+  public static partial class HelloworldReflection {
+
+    #region Descriptor
+    /// <summary>File descriptor for helloworld.proto</summary>
+    public static pbr::FileDescriptor Descriptor {
+      get { return descriptor; }
+    }
+    private static pbr::FileDescriptor descriptor;
+
+    static HelloworldReflection() {
+      byte[] descriptorData = global::System.Convert.FromBase64String(
+          string.Concat(
+            "ChBoZWxsb3dvcmxkLnByb3RvEgpoZWxsb3dvcmxkIhwKDEhlbGxvUmVxdWVz",
+            "dBIMCgRuYW1lGAEgASgJIh0KCkhlbGxvUmVwbHkSDwoHbWVzc2FnZRgBIAEo",
+            "CTJJCgdHcmVldGVyEj4KCFNheUhlbGxvEhguaGVsbG93b3JsZC5IZWxsb1Jl",
+            "cXVlc3QaFi5oZWxsb3dvcmxkLkhlbGxvUmVwbHkiAEI2Chtpby5ncnBjLmV4",
+            "YW1wbGVzLmhlbGxvd29ybGRCD0hlbGxvV29ybGRQcm90b1ABogIDSExXYgZw",
+            "cm90bzM="));
+      descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
+          new pbr::FileDescriptor[] { },
+          new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] {
+            new pbr::GeneratedClrTypeInfo(typeof(global::Helloworld.HelloRequest), global::Helloworld.HelloRequest.Parser, new[]{ "Name" }, null, null, null),
+            new pbr::GeneratedClrTypeInfo(typeof(global::Helloworld.HelloReply), global::Helloworld.HelloReply.Parser, new[]{ "Message" }, null, null, null)
+          }));
+    }
+    #endregion
+
+  }
+  #region Messages
+  /// <summary>
+  /// The request message containing the user's name.
+  /// </summary>
+  public sealed partial class HelloRequest : pb::IMessage<HelloRequest> {
+    private static readonly pb::MessageParser<HelloRequest> _parser = new pb::MessageParser<HelloRequest>(() => new HelloRequest());
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public static pb::MessageParser<HelloRequest> Parser { get { return _parser; } }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public static pbr::MessageDescriptor Descriptor {
+      get { return global::Helloworld.HelloworldReflection.Descriptor.MessageTypes[0]; }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    pbr::MessageDescriptor pb::IMessage.Descriptor {
+      get { return Descriptor; }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public HelloRequest() {
+      OnConstruction();
+    }
+
+    partial void OnConstruction();
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public HelloRequest(HelloRequest other) : this() {
+      name_ = other.name_;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public HelloRequest Clone() {
+      return new HelloRequest(this);
+    }
+
+    /// <summary>Field number for the "name" field.</summary>
+    public const int NameFieldNumber = 1;
+    private string name_ = "";
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public string Name {
+      get { return name_; }
+      set {
+        name_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override bool Equals(object other) {
+      return Equals(other as HelloRequest);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public bool Equals(HelloRequest other) {
+      if (ReferenceEquals(other, null)) {
+        return false;
+      }
+      if (ReferenceEquals(other, this)) {
+        return true;
+      }
+      if (Name != other.Name) return false;
+      return true;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override int GetHashCode() {
+      int hash = 1;
+      if (Name.Length != 0) hash ^= Name.GetHashCode();
+      return hash;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override string ToString() {
+      return pb::JsonFormatter.ToDiagnosticString(this);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void WriteTo(pb::CodedOutputStream output) {
+      if (Name.Length != 0) {
+        output.WriteRawTag(10);
+        output.WriteString(Name);
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public int CalculateSize() {
+      int size = 0;
+      if (Name.Length != 0) {
+        size += 1 + pb::CodedOutputStream.ComputeStringSize(Name);
+      }
+      return size;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void MergeFrom(HelloRequest other) {
+      if (other == null) {
+        return;
+      }
+      if (other.Name.Length != 0) {
+        Name = other.Name;
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void MergeFrom(pb::CodedInputStream input) {
+      uint tag;
+      while ((tag = input.ReadTag()) != 0) {
+        switch(tag) {
+          default:
+            input.SkipLastField();
+            break;
+          case 10: {
+            Name = input.ReadString();
+            break;
+          }
+        }
+      }
+    }
+
+  }
+
+  /// <summary>
+  /// The response message containing the greetings
+  /// </summary>
+  public sealed partial class HelloReply : pb::IMessage<HelloReply> {
+    private static readonly pb::MessageParser<HelloReply> _parser = new pb::MessageParser<HelloReply>(() => new HelloReply());
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public static pb::MessageParser<HelloReply> Parser { get { return _parser; } }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public static pbr::MessageDescriptor Descriptor {
+      get { return global::Helloworld.HelloworldReflection.Descriptor.MessageTypes[1]; }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    pbr::MessageDescriptor pb::IMessage.Descriptor {
+      get { return Descriptor; }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public HelloReply() {
+      OnConstruction();
+    }
+
+    partial void OnConstruction();
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public HelloReply(HelloReply other) : this() {
+      message_ = other.message_;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public HelloReply Clone() {
+      return new HelloReply(this);
+    }
+
+    /// <summary>Field number for the "message" field.</summary>
+    public const int MessageFieldNumber = 1;
+    private string message_ = "";
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public string Message {
+      get { return message_; }
+      set {
+        message_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override bool Equals(object other) {
+      return Equals(other as HelloReply);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public bool Equals(HelloReply other) {
+      if (ReferenceEquals(other, null)) {
+        return false;
+      }
+      if (ReferenceEquals(other, this)) {
+        return true;
+      }
+      if (Message != other.Message) return false;
+      return true;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override int GetHashCode() {
+      int hash = 1;
+      if (Message.Length != 0) hash ^= Message.GetHashCode();
+      return hash;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public override string ToString() {
+      return pb::JsonFormatter.ToDiagnosticString(this);
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void WriteTo(pb::CodedOutputStream output) {
+      if (Message.Length != 0) {
+        output.WriteRawTag(10);
+        output.WriteString(Message);
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public int CalculateSize() {
+      int size = 0;
+      if (Message.Length != 0) {
+        size += 1 + pb::CodedOutputStream.ComputeStringSize(Message);
+      }
+      return size;
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void MergeFrom(HelloReply other) {
+      if (other == null) {
+        return;
+      }
+      if (other.Message.Length != 0) {
+        Message = other.Message;
+      }
+    }
+
+    [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
+    public void MergeFrom(pb::CodedInputStream input) {
+      uint tag;
+      while ((tag = input.ReadTag()) != 0) {
+        switch(tag) {
+          default:
+            input.SkipLastField();
+            break;
+          case 10: {
+            Message = input.ReadString();
+            break;
+          }
+        }
+      }
+    }
+
+  }
+
+  #endregion
+
+}
+
+#endregion Designer generated code

examples/csharp/HelloworldUnity/Assets/Scripts/Helloworld.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 8bfcdd9a5979d4cc7b76d17be585e778
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/Assets/Scripts/HelloworldGrpc.cs

@@ -0,0 +1,150 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: helloworld.proto
+// Original file comments:
+// Copyright 2015 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+#pragma warning disable 1591
+#region Designer generated code
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using grpc = global::Grpc.Core;
+
+namespace Helloworld {
+  /// <summary>
+  /// The greeting service definition.
+  /// </summary>
+  public static partial class Greeter
+  {
+    static readonly string __ServiceName = "helloworld.Greeter";
+
+    static readonly grpc::Marshaller<global::Helloworld.HelloRequest> __Marshaller_HelloRequest = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Helloworld.HelloRequest.Parser.ParseFrom);
+    static readonly grpc::Marshaller<global::Helloworld.HelloReply> __Marshaller_HelloReply = grpc::Marshallers.Create((arg) => global::Google.Protobuf.MessageExtensions.ToByteArray(arg), global::Helloworld.HelloReply.Parser.ParseFrom);
+
+    static readonly grpc::Method<global::Helloworld.HelloRequest, global::Helloworld.HelloReply> __Method_SayHello = new grpc::Method<global::Helloworld.HelloRequest, global::Helloworld.HelloReply>(
+        grpc::MethodType.Unary,
+        __ServiceName,
+        "SayHello",
+        __Marshaller_HelloRequest,
+        __Marshaller_HelloReply);
+
+    /// <summary>Service descriptor</summary>
+    public static global::Google.Protobuf.Reflection.ServiceDescriptor Descriptor
+    {
+      get { return global::Helloworld.HelloworldReflection.Descriptor.Services[0]; }
+    }
+
+    /// <summary>Base class for server-side implementations of Greeter</summary>
+    public abstract partial class GreeterBase
+    {
+      /// <summary>
+      /// Sends a greeting
+      /// </summary>
+      /// <param name="request">The request received from the client.</param>
+      /// <param name="context">The context of the server-side call handler being invoked.</param>
+      /// <returns>The response to send back to the client (wrapped by a task).</returns>
+      public virtual global::System.Threading.Tasks.Task<global::Helloworld.HelloReply> SayHello(global::Helloworld.HelloRequest request, grpc::ServerCallContext context)
+      {
+        throw new grpc::RpcException(new grpc::Status(grpc::StatusCode.Unimplemented, ""));
+      }
+
+    }
+
+    /// <summary>Client for Greeter</summary>
+    public partial class GreeterClient : grpc::ClientBase<GreeterClient>
+    {
+      /// <summary>Creates a new client for Greeter</summary>
+      /// <param name="channel">The channel to use to make remote calls.</param>
+      public GreeterClient(grpc::Channel channel) : base(channel)
+      {
+      }
+      /// <summary>Creates a new client for Greeter that uses a custom <c>CallInvoker</c>.</summary>
+      /// <param name="callInvoker">The callInvoker to use to make remote calls.</param>
+      public GreeterClient(grpc::CallInvoker callInvoker) : base(callInvoker)
+      {
+      }
+      /// <summary>Protected parameterless constructor to allow creation of test doubles.</summary>
+      protected GreeterClient() : base()
+      {
+      }
+      /// <summary>Protected constructor to allow creation of configured clients.</summary>
+      /// <param name="configuration">The client configuration.</param>
+      protected GreeterClient(ClientBaseConfiguration configuration) : base(configuration)
+      {
+      }
+
+      /// <summary>
+      /// Sends a greeting
+      /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="headers">The initial metadata to send with the call. This parameter is optional.</param>
+      /// <param name="deadline">An optional deadline for the call. The call will be cancelled if deadline is hit.</param>
+      /// <param name="cancellationToken">An optional token for canceling the call.</param>
+      /// <returns>The response received from the server.</returns>
+      public virtual global::Helloworld.HelloReply SayHello(global::Helloworld.HelloRequest request, grpc::Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
+      {
+        return SayHello(request, new grpc::CallOptions(headers, deadline, cancellationToken));
+      }
+      /// <summary>
+      /// Sends a greeting
+      /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="options">The options for the call.</param>
+      /// <returns>The response received from the server.</returns>
+      public virtual global::Helloworld.HelloReply SayHello(global::Helloworld.HelloRequest request, grpc::CallOptions options)
+      {
+        return CallInvoker.BlockingUnaryCall(__Method_SayHello, null, options, request);
+      }
+      /// <summary>
+      /// Sends a greeting
+      /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="headers">The initial metadata to send with the call. This parameter is optional.</param>
+      /// <param name="deadline">An optional deadline for the call. The call will be cancelled if deadline is hit.</param>
+      /// <param name="cancellationToken">An optional token for canceling the call.</param>
+      /// <returns>The call object.</returns>
+      public virtual grpc::AsyncUnaryCall<global::Helloworld.HelloReply> SayHelloAsync(global::Helloworld.HelloRequest request, grpc::Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default(CancellationToken))
+      {
+        return SayHelloAsync(request, new grpc::CallOptions(headers, deadline, cancellationToken));
+      }
+      /// <summary>
+      /// Sends a greeting
+      /// </summary>
+      /// <param name="request">The request to send to the server.</param>
+      /// <param name="options">The options for the call.</param>
+      /// <returns>The call object.</returns>
+      public virtual grpc::AsyncUnaryCall<global::Helloworld.HelloReply> SayHelloAsync(global::Helloworld.HelloRequest request, grpc::CallOptions options)
+      {
+        return CallInvoker.AsyncUnaryCall(__Method_SayHello, null, options, request);
+      }
+      /// <summary>Creates a new instance of client from given <c>ClientBaseConfiguration</c>.</summary>
+      protected override GreeterClient NewInstance(ClientBaseConfiguration configuration)
+      {
+        return new GreeterClient(configuration);
+      }
+    }
+
+    /// <summary>Creates service definition that can be registered with a server</summary>
+    /// <param name="serviceImpl">An object implementing the server-side handling logic.</param>
+    public static grpc::ServerServiceDefinition BindService(GreeterBase serviceImpl)
+    {
+      return grpc::ServerServiceDefinition.CreateBuilder()
+          .AddMethod(__Method_SayHello, serviceImpl.SayHello).Build();
+    }
+
+  }
+}
+#endregion

examples/csharp/HelloworldUnity/Assets/Scripts/HelloworldGrpc.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: cf9b820c371a143ce96df8edaebb3fe2
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

examples/csharp/HelloworldUnity/ProjectSettings/AudioManager.asset

@@ -0,0 +1,17 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!11 &1
+AudioManager:
+  m_ObjectHideFlags: 0
+  m_Volume: 1
+  Rolloff Scale: 1
+  Doppler Factor: 1
+  Default Speaker Mode: 2
+  m_SampleRate: 0
+  m_DSPBufferSize: 1024
+  m_VirtualVoiceCount: 512
+  m_RealVoiceCount: 32
+  m_SpatializerPlugin: 
+  m_AmbisonicDecoderPlugin: 
+  m_DisableAudio: 0
+  m_VirtualizeEffects: 1

examples/csharp/HelloworldUnity/ProjectSettings/ClusterInputManager.asset

@@ -0,0 +1,6 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!236 &1
+ClusterInputManager:
+  m_ObjectHideFlags: 0
+  m_Inputs: []

examples/csharp/HelloworldUnity/ProjectSettings/DynamicsManager.asset

@@ -0,0 +1,29 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!55 &1
+PhysicsManager:
+  m_ObjectHideFlags: 0
+  serializedVersion: 7
+  m_Gravity: {x: 0, y: -9.81, z: 0}
+  m_DefaultMaterial: {fileID: 0}
+  m_BounceThreshold: 2
+  m_SleepThreshold: 0.005
+  m_DefaultContactOffset: 0.01
+  m_DefaultSolverIterations: 6
+  m_DefaultSolverVelocityIterations: 1
+  m_QueriesHitBackfaces: 0
+  m_QueriesHitTriggers: 1
+  m_EnableAdaptiveForce: 0
+  m_ClothInterCollisionDistance: 0
+  m_ClothInterCollisionStiffness: 0
+  m_ContactsGeneration: 1
+  m_LayerCollisionMatrix: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+  m_AutoSimulation: 1
+  m_AutoSyncTransforms: 1
+  m_ClothInterCollisionSettingsToggle: 0
+  m_ContactPairsMode: 0
+  m_BroadphaseType: 0
+  m_WorldBounds:
+    m_Center: {x: 0, y: 0, z: 0}
+    m_Extent: {x: 250, y: 250, z: 250}
+  m_WorldSubdivisions: 8

examples/csharp/HelloworldUnity/ProjectSettings/EditorBuildSettings.asset

@@ -0,0 +1,11 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!1045 &1
+EditorBuildSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 2
+  m_Scenes:
+  - enabled: 1
+    path: Assets/Scenes/SampleScene.unity
+    guid: 2cda990e2423bbf4892e6590ba056729
+  m_configObjects: {}

examples/csharp/HelloworldUnity/ProjectSettings/EditorSettings.asset

@@ -0,0 +1,21 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!159 &1
+EditorSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 7
+  m_ExternalVersionControlSupport: Visible Meta Files
+  m_SerializationMode: 2
+  m_LineEndingsForNewScripts: 2
+  m_DefaultBehaviorMode: 1
+  m_SpritePackerMode: 4
+  m_SpritePackerPaddingPower: 1
+  m_EtcTextureCompressorBehavior: 1
+  m_EtcTextureFastCompressor: 1
+  m_EtcTextureNormalCompressor: 2
+  m_EtcTextureBestCompressor: 4
+  m_ProjectGenerationIncludedExtensions: txt;xml;fnt;cd
+  m_ProjectGenerationRootNamespace: 
+  m_UserGeneratedProjectSuffix: 
+  m_CollabEditorSettings:
+    inProgressEnabled: 1

examples/csharp/HelloworldUnity/ProjectSettings/GraphicsSettings.asset

@@ -0,0 +1,60 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!30 &1
+GraphicsSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 12
+  m_Deferred:
+    m_Mode: 1
+    m_Shader: {fileID: 69, guid: 0000000000000000f000000000000000, type: 0}
+  m_DeferredReflections:
+    m_Mode: 1
+    m_Shader: {fileID: 74, guid: 0000000000000000f000000000000000, type: 0}
+  m_ScreenSpaceShadows:
+    m_Mode: 1
+    m_Shader: {fileID: 64, guid: 0000000000000000f000000000000000, type: 0}
+  m_LegacyDeferred:
+    m_Mode: 1
+    m_Shader: {fileID: 63, guid: 0000000000000000f000000000000000, type: 0}
+  m_DepthNormals:
+    m_Mode: 1
+    m_Shader: {fileID: 62, guid: 0000000000000000f000000000000000, type: 0}
+  m_MotionVectors:
+    m_Mode: 1
+    m_Shader: {fileID: 75, guid: 0000000000000000f000000000000000, type: 0}
+  m_LightHalo:
+    m_Mode: 1
+    m_Shader: {fileID: 105, guid: 0000000000000000f000000000000000, type: 0}
+  m_LensFlare:
+    m_Mode: 1
+    m_Shader: {fileID: 102, guid: 0000000000000000f000000000000000, type: 0}
+  m_AlwaysIncludedShaders:
+  - {fileID: 10753, guid: 0000000000000000f000000000000000, type: 0}
+  - {fileID: 10770, guid: 0000000000000000f000000000000000, type: 0}
+  - {fileID: 17000, guid: 0000000000000000f000000000000000, type: 0}
+  - {fileID: 16000, guid: 0000000000000000f000000000000000, type: 0}
+  - {fileID: 16002, guid: 0000000000000000f000000000000000, type: 0}
+  m_PreloadedShaders: []
+  m_SpritesDefaultMaterial: {fileID: 10754, guid: 0000000000000000f000000000000000,
+    type: 0}
+  m_CustomRenderPipeline: {fileID: 0}
+  m_TransparencySortMode: 0
+  m_TransparencySortAxis: {x: 0, y: 0, z: 1}
+  m_DefaultRenderingPath: 1
+  m_DefaultMobileRenderingPath: 1
+  m_TierSettings: []
+  m_LightmapStripping: 0
+  m_FogStripping: 0
+  m_InstancingStripping: 0
+  m_LightmapKeepPlain: 1
+  m_LightmapKeepDirCombined: 1
+  m_LightmapKeepDynamicPlain: 1
+  m_LightmapKeepDynamicDirCombined: 1
+  m_LightmapKeepShadowMask: 1
+  m_LightmapKeepSubtractive: 1
+  m_FogKeepLinear: 1
+  m_FogKeepExp: 1
+  m_FogKeepExp2: 1
+  m_AlbedoSwatchInfos: []
+  m_LightsUseLinearIntensity: 0
+  m_LightsUseColorTemperature: 0

examples/csharp/HelloworldUnity/ProjectSettings/InputManager.asset

@@ -0,0 +1,295 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!13 &1
+InputManager:
+  m_ObjectHideFlags: 0
+  serializedVersion: 2
+  m_Axes:
+  - serializedVersion: 3
+    m_Name: Horizontal
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: left
+    positiveButton: right
+    altNegativeButton: a
+    altPositiveButton: d
+    gravity: 3
+    dead: 0.001
+    sensitivity: 3
+    snap: 1
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Vertical
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: down
+    positiveButton: up
+    altNegativeButton: s
+    altPositiveButton: w
+    gravity: 3
+    dead: 0.001
+    sensitivity: 3
+    snap: 1
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Fire1
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: left ctrl
+    altNegativeButton: 
+    altPositiveButton: mouse 0
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Fire2
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: left alt
+    altNegativeButton: 
+    altPositiveButton: mouse 1
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Fire3
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: left shift
+    altNegativeButton: 
+    altPositiveButton: mouse 2
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Jump
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: space
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Mouse X
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: 
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 0
+    dead: 0
+    sensitivity: 0.1
+    snap: 0
+    invert: 0
+    type: 1
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Mouse Y
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: 
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 0
+    dead: 0
+    sensitivity: 0.1
+    snap: 0
+    invert: 0
+    type: 1
+    axis: 1
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Mouse ScrollWheel
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: 
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 0
+    dead: 0
+    sensitivity: 0.1
+    snap: 0
+    invert: 0
+    type: 1
+    axis: 2
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Horizontal
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: 
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 0
+    dead: 0.19
+    sensitivity: 1
+    snap: 0
+    invert: 0
+    type: 2
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Vertical
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: 
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 0
+    dead: 0.19
+    sensitivity: 1
+    snap: 0
+    invert: 1
+    type: 2
+    axis: 1
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Fire1
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: joystick button 0
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Fire2
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: joystick button 1
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Fire3
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: joystick button 2
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Jump
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: joystick button 3
+    altNegativeButton: 
+    altPositiveButton: 
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Submit
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: return
+    altNegativeButton: 
+    altPositiveButton: joystick button 0
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Submit
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: enter
+    altNegativeButton: 
+    altPositiveButton: space
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0
+  - serializedVersion: 3
+    m_Name: Cancel
+    descriptiveName: 
+    descriptiveNegativeName: 
+    negativeButton: 
+    positiveButton: escape
+    altNegativeButton: 
+    altPositiveButton: joystick button 1
+    gravity: 1000
+    dead: 0.001
+    sensitivity: 1000
+    snap: 0
+    invert: 0
+    type: 0
+    axis: 0
+    joyNum: 0

examples/csharp/HelloworldUnity/ProjectSettings/NavMeshAreas.asset

@@ -0,0 +1,91 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!126 &1
+NavMeshProjectSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 2
+  areas:
+  - name: Walkable
+    cost: 1
+  - name: Not Walkable
+    cost: 1
+  - name: Jump
+    cost: 2
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  - name: 
+    cost: 1
+  m_LastAgentTypeID: -887442657
+  m_Settings:
+  - serializedVersion: 2
+    agentTypeID: 0
+    agentRadius: 0.5
+    agentHeight: 2
+    agentSlope: 45
+    agentClimb: 0.75
+    ledgeDropHeight: 0
+    maxJumpAcrossDistance: 0
+    minRegionArea: 2
+    manualCellSize: 0
+    cellSize: 0.16666667
+    manualTileSize: 0
+    tileSize: 256
+    accuratePlacement: 0
+    debug:
+      m_Flags: 0
+  m_SettingNames:
+  - Humanoid

examples/csharp/HelloworldUnity/ProjectSettings/NetworkManager.asset

@@ -0,0 +1,8 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!149 &1
+NetworkManager:
+  m_ObjectHideFlags: 0
+  m_DebugLevel: 0
+  m_Sendrate: 15
+  m_AssetToPrefab: {}

examples/csharp/HelloworldUnity/ProjectSettings/Physics2DSettings.asset

@@ -0,0 +1,55 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!19 &1
+Physics2DSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 3
+  m_Gravity: {x: 0, y: -9.81}
+  m_DefaultMaterial: {fileID: 0}
+  m_VelocityIterations: 8
+  m_PositionIterations: 3
+  m_VelocityThreshold: 1
+  m_MaxLinearCorrection: 0.2
+  m_MaxAngularCorrection: 8
+  m_MaxTranslationSpeed: 100
+  m_MaxRotationSpeed: 360
+  m_BaumgarteScale: 0.2
+  m_BaumgarteTimeOfImpactScale: 0.75
+  m_TimeToSleep: 0.5
+  m_LinearSleepTolerance: 0.01
+  m_AngularSleepTolerance: 2
+  m_DefaultContactOffset: 0.01
+  m_JobOptions:
+    serializedVersion: 2
+    useMultithreading: 0
+    useConsistencySorting: 0
+    m_InterpolationPosesPerJob: 100
+    m_NewContactsPerJob: 30
+    m_CollideContactsPerJob: 100
+    m_ClearFlagsPerJob: 200
+    m_ClearBodyForcesPerJob: 200
+    m_SyncDiscreteFixturesPerJob: 50
+    m_SyncContinuousFixturesPerJob: 50
+    m_FindNearestContactsPerJob: 100
+    m_UpdateTriggerContactsPerJob: 100
+    m_IslandSolverCostThreshold: 100
+    m_IslandSolverBodyCostScale: 1
+    m_IslandSolverContactCostScale: 10
+    m_IslandSolverJointCostScale: 10
+    m_IslandSolverBodiesPerJob: 50
+    m_IslandSolverContactsPerJob: 50
+  m_AutoSimulation: 1
+  m_QueriesHitTriggers: 1
+  m_QueriesStartInColliders: 1
+  m_CallbacksOnDisable: 1
+  m_AutoSyncTransforms: 1
+  m_AlwaysShowColliders: 0
+  m_ShowColliderSleep: 1
+  m_ShowColliderContacts: 0
+  m_ShowColliderAABB: 0
+  m_ContactArrowScale: 0.2
+  m_ColliderAwakeColor: {r: 0.5686275, g: 0.95686275, b: 0.54509807, a: 0.7529412}
+  m_ColliderAsleepColor: {r: 0.5686275, g: 0.95686275, b: 0.54509807, a: 0.36078432}
+  m_ColliderContactColor: {r: 1, g: 0, b: 1, a: 0.6862745}
+  m_ColliderAABBColor: {r: 1, g: 1, b: 0, a: 0.2509804}
+  m_LayerCollisionMatrix: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff

examples/csharp/HelloworldUnity/ProjectSettings/PresetManager.asset

@@ -0,0 +1,13 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!1386491679 &1
+PresetManager:
+  m_ObjectHideFlags: 0
+  m_DefaultList:
+  - type:
+      m_NativeTypeID: 20
+      m_ManagedTypePPtr: {fileID: 0}
+      m_ManagedTypeFallback: 
+    defaultPresets:
+    - m_Preset: {fileID: 2655988077585873504, guid: bfcfc320427f8224bbb7a96f3d3aebad,
+        type: 2}

examples/csharp/HelloworldUnity/ProjectSettings/ProjectSettings.asset

@@ -0,0 +1,656 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!129 &1
+PlayerSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 15
+  productGUID: 2ed9f077cb8c7421b9d7c7fa18f3c25d
+  AndroidProfiler: 0
+  AndroidFilterTouchesWhenObscured: 0
+  AndroidEnableSustainedPerformanceMode: 0
+  defaultScreenOrientation: 4
+  targetDevice: 2
+  useOnDemandResources: 0
+  accelerometerFrequency: 60
+  companyName: com.grpc.examples
+  productName: HelloworldUnity
+  defaultCursor: {fileID: 0}
+  cursorHotspot: {x: 0, y: 0}
+  m_SplashScreenBackgroundColor: {r: 0.13725491, g: 0.12156863, b: 0.1254902, a: 1}
+  m_ShowUnitySplashScreen: 1
+  m_ShowUnitySplashLogo: 1
+  m_SplashScreenOverlayOpacity: 1
+  m_SplashScreenAnimation: 1
+  m_SplashScreenLogoStyle: 1
+  m_SplashScreenDrawMode: 0
+  m_SplashScreenBackgroundAnimationZoom: 1
+  m_SplashScreenLogoAnimationZoom: 1
+  m_SplashScreenBackgroundLandscapeAspect: 1
+  m_SplashScreenBackgroundPortraitAspect: 1
+  m_SplashScreenBackgroundLandscapeUvs:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+  m_SplashScreenBackgroundPortraitUvs:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+  m_SplashScreenLogos: []
+  m_VirtualRealitySplashScreen: {fileID: 0}
+  m_HolographicTrackingLossScreen: {fileID: 0}
+  defaultScreenWidth: 1024
+  defaultScreenHeight: 768
+  defaultScreenWidthWeb: 960
+  defaultScreenHeightWeb: 600
+  m_StereoRenderingPath: 0
+  m_ActiveColorSpace: 0
+  m_MTRendering: 1
+  m_StackTraceTypes: 010000000100000001000000010000000100000001000000
+  iosShowActivityIndicatorOnLoading: -1
+  androidShowActivityIndicatorOnLoading: -1
+  iosAppInBackgroundBehavior: 0
+  displayResolutionDialog: 1
+  iosAllowHTTPDownload: 1
+  allowedAutorotateToPortrait: 1
+  allowedAutorotateToPortraitUpsideDown: 1
+  allowedAutorotateToLandscapeRight: 1
+  allowedAutorotateToLandscapeLeft: 1
+  useOSAutorotation: 1
+  use32BitDisplayBuffer: 1
+  preserveFramebufferAlpha: 0
+  disableDepthAndStencilBuffers: 0
+  androidStartInFullscreen: 1
+  androidRenderOutsideSafeArea: 0
+  androidBlitType: 0
+  defaultIsNativeResolution: 1
+  macRetinaSupport: 1
+  runInBackground: 1
+  captureSingleScreen: 0
+  muteOtherAudioSources: 0
+  Prepare IOS For Recording: 0
+  Force IOS Speakers When Recording: 0
+  deferSystemGesturesMode: 0
+  hideHomeButton: 0
+  submitAnalytics: 1
+  usePlayerLog: 1
+  bakeCollisionMeshes: 0
+  forceSingleInstance: 0
+  resizableWindow: 0
+  useMacAppStoreValidation: 0
+  macAppStoreCategory: public.app-category.games
+  gpuSkinning: 0
+  graphicsJobs: 0
+  xboxPIXTextureCapture: 0
+  xboxEnableAvatar: 0
+  xboxEnableKinect: 0
+  xboxEnableKinectAutoTracking: 0
+  xboxEnableFitness: 0
+  visibleInBackground: 1
+  allowFullscreenSwitch: 1
+  graphicsJobMode: 0
+  fullscreenMode: 1
+  xboxSpeechDB: 0
+  xboxEnableHeadOrientation: 0
+  xboxEnableGuest: 0
+  xboxEnablePIXSampling: 0
+  metalFramebufferOnly: 0
+  xboxOneResolution: 0
+  xboxOneSResolution: 0
+  xboxOneXResolution: 3
+  xboxOneMonoLoggingLevel: 0
+  xboxOneLoggingLevel: 1
+  xboxOneDisableEsram: 0
+  xboxOnePresentImmediateThreshold: 0
+  switchQueueCommandMemory: 0
+  vulkanEnableSetSRGBWrite: 0
+  m_SupportedAspectRatios:
+    4:3: 1
+    5:4: 1
+    16:10: 1
+    16:9: 1
+    Others: 1
+  bundleVersion: 0.1
+  preloadedAssets: []
+  metroInputSource: 0
+  wsaTransparentSwapchain: 0
+  m_HolographicPauseOnTrackingLoss: 1
+  xboxOneDisableKinectGpuReservation: 0
+  xboxOneEnable7thCore: 0
+  isWsaHolographicRemotingEnabled: 0
+  vrSettings:
+    cardboard:
+      depthFormat: 0
+      enableTransitionView: 0
+    daydream:
+      depthFormat: 0
+      useSustainedPerformanceMode: 0
+      enableVideoLayer: 0
+      useProtectedVideoMemory: 0
+      minimumSupportedHeadTracking: 0
+      maximumSupportedHeadTracking: 1
+    hololens:
+      depthFormat: 1
+      depthBufferSharingEnabled: 0
+    oculus:
+      sharedDepthBuffer: 0
+      dashSupport: 0
+    enable360StereoCapture: 0
+  protectGraphicsMemory: 0
+  enableFrameTimingStats: 0
+  useHDRDisplay: 0
+  m_ColorGamuts: 00000000
+  targetPixelDensity: 30
+  resolutionScalingMode: 0
+  androidSupportedAspectRatio: 1
+  androidMaxAspectRatio: 2.1
+  applicationIdentifier:
+    Android: com.grpc.examples
+    Standalone: com.Company.ProductName
+    iOS: com.jattermusch.grpc.example
+  buildNumber: {}
+  AndroidBundleVersionCode: 1
+  AndroidMinSdkVersion: 16
+  AndroidTargetSdkVersion: 0
+  AndroidPreferredInstallLocation: 1
+  aotOptions: 
+  stripEngineCode: 1
+  iPhoneStrippingLevel: 0
+  iPhoneScriptCallOptimization: 0
+  ForceInternetPermission: 0
+  ForceSDCardPermission: 0
+  CreateWallpaper: 0
+  APKExpansionFiles: 0
+  keepLoadedShadersAlive: 0
+  StripUnusedMeshComponents: 1
+  VertexChannelCompressionMask: 4054
+  iPhoneSdkVersion: 989
+  iOSTargetOSVersionString: 9.0
+  tvOSSdkVersion: 0
+  tvOSRequireExtendedGameController: 0
+  tvOSTargetOSVersionString: 9.0
+  uIPrerenderedIcon: 0
+  uIRequiresPersistentWiFi: 0
+  uIRequiresFullScreen: 1
+  uIStatusBarHidden: 1
+  uIExitOnSuspend: 0
+  uIStatusBarStyle: 0
+  iPhoneSplashScreen: {fileID: 0}
+  iPhoneHighResSplashScreen: {fileID: 0}
+  iPhoneTallHighResSplashScreen: {fileID: 0}
+  iPhone47inSplashScreen: {fileID: 0}
+  iPhone55inPortraitSplashScreen: {fileID: 0}
+  iPhone55inLandscapeSplashScreen: {fileID: 0}
+  iPhone58inPortraitSplashScreen: {fileID: 0}
+  iPhone58inLandscapeSplashScreen: {fileID: 0}
+  iPadPortraitSplashScreen: {fileID: 0}
+  iPadHighResPortraitSplashScreen: {fileID: 0}
+  iPadLandscapeSplashScreen: {fileID: 0}
+  iPadHighResLandscapeSplashScreen: {fileID: 0}
+  appleTVSplashScreen: {fileID: 0}
+  appleTVSplashScreen2x: {fileID: 0}
+  tvOSSmallIconLayers: []
+  tvOSSmallIconLayers2x: []
+  tvOSLargeIconLayers: []
+  tvOSLargeIconLayers2x: []
+  tvOSTopShelfImageLayers: []
+  tvOSTopShelfImageLayers2x: []
+  tvOSTopShelfImageWideLayers: []
+  tvOSTopShelfImageWideLayers2x: []
+  iOSLaunchScreenType: 0
+  iOSLaunchScreenPortrait: {fileID: 0}
+  iOSLaunchScreenLandscape: {fileID: 0}
+  iOSLaunchScreenBackgroundColor:
+    serializedVersion: 2
+    rgba: 0
+  iOSLaunchScreenFillPct: 100
+  iOSLaunchScreenSize: 100
+  iOSLaunchScreenCustomXibPath: 
+  iOSLaunchScreeniPadType: 0
+  iOSLaunchScreeniPadImage: {fileID: 0}
+  iOSLaunchScreeniPadBackgroundColor:
+    serializedVersion: 2
+    rgba: 0
+  iOSLaunchScreeniPadFillPct: 100
+  iOSLaunchScreeniPadSize: 100
+  iOSLaunchScreeniPadCustomXibPath: 
+  iOSUseLaunchScreenStoryboard: 0
+  iOSLaunchScreenCustomStoryboardPath: 
+  iOSDeviceRequirements: []
+  iOSURLSchemes: []
+  iOSBackgroundModes: 0
+  iOSMetalForceHardShadows: 0
+  metalEditorSupport: 1
+  metalAPIValidation: 1
+  iOSRenderExtraFrameOnPause: 0
+  appleDeveloperTeamID: 
+  iOSManualSigningProvisioningProfileID: 
+  tvOSManualSigningProvisioningProfileID: 
+  iOSManualSigningProvisioningProfileType: 0
+  tvOSManualSigningProvisioningProfileType: 0
+  appleEnableAutomaticSigning: 0
+  iOSRequireARKit: 0
+  appleEnableProMotion: 0
+  clonedFromGUID: 5f34be1353de5cf4398729fda238591b
+  templatePackageId: com.unity.template.2d@1.0.1
+  templateDefaultScene: Assets/Scenes/SampleScene.unity
+  AndroidTargetArchitectures: 5
+  AndroidSplashScreenScale: 0
+  androidSplashScreen: {fileID: 0}
+  AndroidKeystoreName: 
+  AndroidKeyaliasName: 
+  AndroidBuildApkPerCpuArchitecture: 0
+  AndroidTVCompatibility: 1
+  AndroidIsGame: 1
+  AndroidEnableTango: 0
+  androidEnableBanner: 1
+  androidUseLowAccuracyLocation: 0
+  m_AndroidBanners:
+  - width: 320
+    height: 180
+    banner: {fileID: 0}
+  androidGamepadSupportLevel: 0
+  resolutionDialogBanner: {fileID: 0}
+  m_BuildTargetIcons: []
+  m_BuildTargetPlatformIcons:
+  - m_BuildTarget: Android
+    m_Icons:
+    - m_Textures: []
+      m_Width: 432
+      m_Height: 432
+      m_Kind: 2
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 324
+      m_Height: 324
+      m_Kind: 2
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 216
+      m_Height: 216
+      m_Kind: 2
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 162
+      m_Height: 162
+      m_Kind: 2
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 108
+      m_Height: 108
+      m_Kind: 2
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 81
+      m_Height: 81
+      m_Kind: 2
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 192
+      m_Height: 192
+      m_Kind: 1
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 144
+      m_Height: 144
+      m_Kind: 1
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 96
+      m_Height: 96
+      m_Kind: 1
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 72
+      m_Height: 72
+      m_Kind: 1
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 48
+      m_Height: 48
+      m_Kind: 1
+      m_SubKind: 
+    - m_Textures: []
+      m_Width: 36
+      m_Height: 36
+      m_Kind: 1
+      m_SubKind: 
+  m_BuildTargetBatching: []
+  m_BuildTargetGraphicsAPIs: []
+  m_BuildTargetVRSettings: []
+  m_BuildTargetEnableVuforiaSettings: []
+  openGLRequireES31: 0
+  openGLRequireES31AEP: 0
+  m_TemplateCustomTags: {}
+  mobileMTRendering:
+    Android: 1
+    iPhone: 1
+    tvOS: 1
+  m_BuildTargetGroupLightmapEncodingQuality: []
+  m_BuildTargetGroupLightmapSettings: []
+  playModeTestRunnerEnabled: 0
+  runPlayModeTestAsEditModeTest: 0
+  actionOnDotNetUnhandledException: 1
+  enableInternalProfiler: 0
+  logObjCUncaughtExceptions: 1
+  enableCrashReportAPI: 0
+  cameraUsageDescription: 
+  locationUsageDescription: 
+  microphoneUsageDescription: 
+  switchNetLibKey: 
+  switchSocketMemoryPoolSize: 6144
+  switchSocketAllocatorPoolSize: 128
+  switchSocketConcurrencyLimit: 14
+  switchScreenResolutionBehavior: 2
+  switchUseCPUProfiler: 0
+  switchApplicationID: 0x01004b9000490000
+  switchNSODependencies: 
+  switchTitleNames_0: 
+  switchTitleNames_1: 
+  switchTitleNames_2: 
+  switchTitleNames_3: 
+  switchTitleNames_4: 
+  switchTitleNames_5: 
+  switchTitleNames_6: 
+  switchTitleNames_7: 
+  switchTitleNames_8: 
+  switchTitleNames_9: 
+  switchTitleNames_10: 
+  switchTitleNames_11: 
+  switchTitleNames_12: 
+  switchTitleNames_13: 
+  switchTitleNames_14: 
+  switchPublisherNames_0: 
+  switchPublisherNames_1: 
+  switchPublisherNames_2: 
+  switchPublisherNames_3: 
+  switchPublisherNames_4: 
+  switchPublisherNames_5: 
+  switchPublisherNames_6: 
+  switchPublisherNames_7: 
+  switchPublisherNames_8: 
+  switchPublisherNames_9: 
+  switchPublisherNames_10: 
+  switchPublisherNames_11: 
+  switchPublisherNames_12: 
+  switchPublisherNames_13: 
+  switchPublisherNames_14: 
+  switchIcons_0: {fileID: 0}
+  switchIcons_1: {fileID: 0}
+  switchIcons_2: {fileID: 0}
+  switchIcons_3: {fileID: 0}
+  switchIcons_4: {fileID: 0}
+  switchIcons_5: {fileID: 0}
+  switchIcons_6: {fileID: 0}
+  switchIcons_7: {fileID: 0}
+  switchIcons_8: {fileID: 0}
+  switchIcons_9: {fileID: 0}
+  switchIcons_10: {fileID: 0}
+  switchIcons_11: {fileID: 0}
+  switchIcons_12: {fileID: 0}
+  switchIcons_13: {fileID: 0}
+  switchIcons_14: {fileID: 0}
+  switchSmallIcons_0: {fileID: 0}
+  switchSmallIcons_1: {fileID: 0}
+  switchSmallIcons_2: {fileID: 0}
+  switchSmallIcons_3: {fileID: 0}
+  switchSmallIcons_4: {fileID: 0}
+  switchSmallIcons_5: {fileID: 0}
+  switchSmallIcons_6: {fileID: 0}
+  switchSmallIcons_7: {fileID: 0}
+  switchSmallIcons_8: {fileID: 0}
+  switchSmallIcons_9: {fileID: 0}
+  switchSmallIcons_10: {fileID: 0}
+  switchSmallIcons_11: {fileID: 0}
+  switchSmallIcons_12: {fileID: 0}
+  switchSmallIcons_13: {fileID: 0}
+  switchSmallIcons_14: {fileID: 0}
+  switchManualHTML: 
+  switchAccessibleURLs: 
+  switchLegalInformation: 
+  switchMainThreadStackSize: 1048576
+  switchPresenceGroupId: 
+  switchLogoHandling: 0
+  switchReleaseVersion: 0
+  switchDisplayVersion: 1.0.0
+  switchStartupUserAccount: 0
+  switchTouchScreenUsage: 0
+  switchSupportedLanguagesMask: 0
+  switchLogoType: 0
+  switchApplicationErrorCodeCategory: 
+  switchUserAccountSaveDataSize: 0
+  switchUserAccountSaveDataJournalSize: 0
+  switchApplicationAttribute: 0
+  switchCardSpecSize: -1
+  switchCardSpecClock: -1
+  switchRatingsMask: 0
+  switchRatingsInt_0: 0
+  switchRatingsInt_1: 0
+  switchRatingsInt_2: 0
+  switchRatingsInt_3: 0
+  switchRatingsInt_4: 0
+  switchRatingsInt_5: 0
+  switchRatingsInt_6: 0
+  switchRatingsInt_7: 0
+  switchRatingsInt_8: 0
+  switchRatingsInt_9: 0
+  switchRatingsInt_10: 0
+  switchRatingsInt_11: 0
+  switchLocalCommunicationIds_0: 
+  switchLocalCommunicationIds_1: 
+  switchLocalCommunicationIds_2: 
+  switchLocalCommunicationIds_3: 
+  switchLocalCommunicationIds_4: 
+  switchLocalCommunicationIds_5: 
+  switchLocalCommunicationIds_6: 
+  switchLocalCommunicationIds_7: 
+  switchParentalControl: 0
+  switchAllowsScreenshot: 1
+  switchAllowsVideoCapturing: 1
+  switchAllowsRuntimeAddOnContentInstall: 0
+  switchDataLossConfirmation: 0
+  switchUserAccountLockEnabled: 0
+  switchSupportedNpadStyles: 3
+  switchNativeFsCacheSize: 32
+  switchIsHoldTypeHorizontal: 0
+  switchSupportedNpadCount: 8
+  switchSocketConfigEnabled: 0
+  switchTcpInitialSendBufferSize: 32
+  switchTcpInitialReceiveBufferSize: 64
+  switchTcpAutoSendBufferSizeMax: 256
+  switchTcpAutoReceiveBufferSizeMax: 256
+  switchUdpSendBufferSize: 9
+  switchUdpReceiveBufferSize: 42
+  switchSocketBufferEfficiency: 4
+  switchSocketInitializeEnabled: 1
+  switchNetworkInterfaceManagerInitializeEnabled: 1
+  switchPlayerConnectionEnabled: 1
+  ps4NPAgeRating: 12
+  ps4NPTitleSecret: 
+  ps4NPTrophyPackPath: 
+  ps4ParentalLevel: 11
+  ps4ContentID: ED1633-NPXX51362_00-0000000000000000
+  ps4Category: 0
+  ps4MasterVersion: 01.00
+  ps4AppVersion: 01.00
+  ps4AppType: 0
+  ps4ParamSfxPath: 
+  ps4VideoOutPixelFormat: 0
+  ps4VideoOutInitialWidth: 1920
+  ps4VideoOutBaseModeInitialWidth: 1920
+  ps4VideoOutReprojectionRate: 60
+  ps4PronunciationXMLPath: 
+  ps4PronunciationSIGPath: 
+  ps4BackgroundImagePath: 
+  ps4StartupImagePath: 
+  ps4StartupImagesFolder: 
+  ps4IconImagesFolder: 
+  ps4SaveDataImagePath: 
+  ps4SdkOverride: 
+  ps4BGMPath: 
+  ps4ShareFilePath: 
+  ps4ShareOverlayImagePath: 
+  ps4PrivacyGuardImagePath: 
+  ps4NPtitleDatPath: 
+  ps4RemotePlayKeyAssignment: -1
+  ps4RemotePlayKeyMappingDir: 
+  ps4PlayTogetherPlayerCount: 0
+  ps4EnterButtonAssignment: 1
+  ps4ApplicationParam1: 0
+  ps4ApplicationParam2: 0
+  ps4ApplicationParam3: 0
+  ps4ApplicationParam4: 0
+  ps4DownloadDataSize: 0
+  ps4GarlicHeapSize: 2048
+  ps4ProGarlicHeapSize: 2560
+  ps4Passcode: frAQBc8Wsa1xVPfvJcrgRYwTiizs2trQ
+  ps4pnSessions: 1
+  ps4pnPresence: 1
+  ps4pnFriends: 1
+  ps4pnGameCustomData: 1
+  playerPrefsSupport: 0
+  enableApplicationExit: 0
+  resetTempFolder: 1
+  restrictedAudioUsageRights: 0
+  ps4UseResolutionFallback: 0
+  ps4ReprojectionSupport: 0
+  ps4UseAudio3dBackend: 0
+  ps4SocialScreenEnabled: 0
+  ps4ScriptOptimizationLevel: 0
+  ps4Audio3dVirtualSpeakerCount: 14
+  ps4attribCpuUsage: 0
+  ps4PatchPkgPath: 
+  ps4PatchLatestPkgPath: 
+  ps4PatchChangeinfoPath: 
+  ps4PatchDayOne: 0
+  ps4attribUserManagement: 0
+  ps4attribMoveSupport: 0
+  ps4attrib3DSupport: 0
+  ps4attribShareSupport: 0
+  ps4attribExclusiveVR: 0
+  ps4disableAutoHideSplash: 0
+  ps4videoRecordingFeaturesUsed: 0
+  ps4contentSearchFeaturesUsed: 0
+  ps4attribEyeToEyeDistanceSettingVR: 0
+  ps4IncludedModules: []
+  monoEnv: 
+  splashScreenBackgroundSourceLandscape: {fileID: 0}
+  splashScreenBackgroundSourcePortrait: {fileID: 0}
+  spritePackerPolicy: 
+  webGLMemorySize: 256
+  webGLExceptionSupport: 1
+  webGLNameFilesAsHashes: 0
+  webGLDataCaching: 1
+  webGLDebugSymbols: 0
+  webGLEmscriptenArgs: 
+  webGLModulesDirectory: 
+  webGLTemplate: APPLICATION:Default
+  webGLAnalyzeBuildSize: 0
+  webGLUseEmbeddedResources: 0
+  webGLCompressionFormat: 1
+  webGLLinkerTarget: 1
+  webGLThreadsSupport: 0
+  scriptingDefineSymbols: {}
+  platformArchitecture: {}
+  scriptingBackend:
+    Android: 1
+  il2cppCompilerConfiguration: {}
+  managedStrippingLevel: {}
+  incrementalIl2cppBuild: {}
+  allowUnsafeCode: 0
+  additionalIl2CppArgs: 
+  scriptingRuntimeVersion: 1
+  apiCompatibilityLevelPerPlatform:
+    Android: 3
+  m_RenderingPath: 1
+  m_MobileRenderingPath: 1
+  metroPackageName: Template_2D
+  metroPackageVersion: 
+  metroCertificatePath: 
+  metroCertificatePassword: 
+  metroCertificateSubject: 
+  metroCertificateIssuer: 
+  metroCertificateNotAfter: 0000000000000000
+  metroApplicationDescription: Template_2D
+  wsaImages: {}
+  metroTileShortName: 
+  metroTileShowName: 0
+  metroMediumTileShowName: 0
+  metroLargeTileShowName: 0
+  metroWideTileShowName: 0
+  metroSupportStreamingInstall: 0
+  metroLastRequiredScene: 0
+  metroDefaultTileSize: 1
+  metroTileForegroundText: 2
+  metroTileBackgroundColor: {r: 0.13333334, g: 0.17254902, b: 0.21568628, a: 0}
+  metroSplashScreenBackgroundColor: {r: 0.12941177, g: 0.17254902, b: 0.21568628,
+    a: 1}
+  metroSplashScreenUseBackgroundColor: 0
+  platformCapabilities: {}
+  metroTargetDeviceFamilies: {}
+  metroFTAName: 
+  metroFTAFileTypes: []
+  metroProtocolName: 
+  metroCompilationOverrides: 1
+  XboxOneProductId: 
+  XboxOneUpdateKey: 
+  XboxOneSandboxId: 
+  XboxOneContentId: 
+  XboxOneTitleId: 
+  XboxOneSCId: 
+  XboxOneGameOsOverridePath: 
+  XboxOnePackagingOverridePath: 
+  XboxOneAppManifestOverridePath: 
+  XboxOneVersion: 1.0.0.0
+  XboxOnePackageEncryption: 0
+  XboxOnePackageUpdateGranularity: 2
+  XboxOneDescription: 
+  XboxOneLanguage:
+  - enus
+  XboxOneCapability: []
+  XboxOneGameRating: {}
+  XboxOneIsContentPackage: 0
+  XboxOneEnableGPUVariability: 0
+  XboxOneSockets: {}
+  XboxOneSplashScreen: {fileID: 0}
+  XboxOneAllowedProductIds: []
+  XboxOnePersistentLocalStorageSize: 0
+  XboxOneXTitleMemory: 8
+  xboxOneScriptCompiler: 0
+  XboxOneOverrideIdentityName: 
+  vrEditorSettings:
+    daydream:
+      daydreamIconForeground: {fileID: 0}
+      daydreamIconBackground: {fileID: 0}
+  cloudServicesEnabled:
+    UNet: 1
+  luminIcon:
+    m_Name: 
+    m_ModelFolderPath: 
+    m_PortalFolderPath: 
+  luminCert:
+    m_CertPath: 
+    m_PrivateKeyPath: 
+  luminIsChannelApp: 0
+  luminVersion:
+    m_VersionCode: 1
+    m_VersionName: 
+  facebookSdkVersion: 7.9.4
+  facebookAppId: 
+  facebookCookies: 1
+  facebookLogging: 1
+  facebookStatus: 1
+  facebookXfbml: 0
+  facebookFrictionlessRequests: 1
+  apiCompatibilityLevel: 3
+  cloudProjectId: 
+  framebufferDepthMemorylessMode: 0
+  projectName: 
+  organizationId: 
+  cloudEnabled: 0
+  enableNativePlatformBackendsForNewInputSystem: 0
+  disableOldInputManagerSupport: 0
+  legacyClampBlendShapeWeights: 1

examples/csharp/HelloworldUnity/ProjectSettings/ProjectVersion.txt

@@ -0,0 +1 @@
+m_EditorVersion: 2018.3.5f1

examples/csharp/HelloworldUnity/ProjectSettings/QualitySettings.asset

@@ -0,0 +1,191 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!47 &1
+QualitySettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 5
+  m_CurrentQuality: 3
+  m_QualitySettings:
+  - serializedVersion: 2
+    name: Very Low
+    pixelLightCount: 0
+    shadows: 0
+    shadowResolution: 0
+    shadowProjection: 1
+    shadowCascades: 1
+    shadowDistance: 15
+    shadowNearPlaneOffset: 3
+    shadowCascade2Split: 0.33333334
+    shadowCascade4Split: {x: 0.06666667, y: 0.2, z: 0.46666667}
+    shadowmaskMode: 0
+    blendWeights: 1
+    textureQuality: 1
+    anisotropicTextures: 0
+    antiAliasing: 0
+    softParticles: 0
+    softVegetation: 0
+    realtimeReflectionProbes: 0
+    billboardsFaceCameraPosition: 0
+    vSyncCount: 0
+    lodBias: 0.3
+    maximumLODLevel: 0
+    particleRaycastBudget: 4
+    asyncUploadTimeSlice: 2
+    asyncUploadBufferSize: 4
+    resolutionScalingFixedDPIFactor: 1
+    excludedTargetPlatforms: []
+  - serializedVersion: 2
+    name: Low
+    pixelLightCount: 0
+    shadows: 0
+    shadowResolution: 0
+    shadowProjection: 1
+    shadowCascades: 1
+    shadowDistance: 20
+    shadowNearPlaneOffset: 3
+    shadowCascade2Split: 0.33333334
+    shadowCascade4Split: {x: 0.06666667, y: 0.2, z: 0.46666667}
+    shadowmaskMode: 0
+    blendWeights: 2
+    textureQuality: 0
+    anisotropicTextures: 0
+    antiAliasing: 0
+    softParticles: 0
+    softVegetation: 0
+    realtimeReflectionProbes: 0
+    billboardsFaceCameraPosition: 0
+    vSyncCount: 0
+    lodBias: 0.4
+    maximumLODLevel: 0
+    particleRaycastBudget: 16
+    asyncUploadTimeSlice: 2
+    asyncUploadBufferSize: 4
+    resolutionScalingFixedDPIFactor: 1
+    excludedTargetPlatforms: []
+  - serializedVersion: 2
+    name: Medium
+    pixelLightCount: 1
+    shadows: 0
+    shadowResolution: 0
+    shadowProjection: 1
+    shadowCascades: 1
+    shadowDistance: 20
+    shadowNearPlaneOffset: 3
+    shadowCascade2Split: 0.33333334
+    shadowCascade4Split: {x: 0.06666667, y: 0.2, z: 0.46666667}
+    shadowmaskMode: 0
+    blendWeights: 2
+    textureQuality: 0
+    anisotropicTextures: 0
+    antiAliasing: 0
+    softParticles: 0
+    softVegetation: 0
+    realtimeReflectionProbes: 0
+    billboardsFaceCameraPosition: 0
+    vSyncCount: 1
+    lodBias: 0.7
+    maximumLODLevel: 0
+    particleRaycastBudget: 64
+    asyncUploadTimeSlice: 2
+    asyncUploadBufferSize: 4
+    resolutionScalingFixedDPIFactor: 1
+    excludedTargetPlatforms: []
+  - serializedVersion: 2
+    name: High
+    pixelLightCount: 2
+    shadows: 0
+    shadowResolution: 1
+    shadowProjection: 1
+    shadowCascades: 2
+    shadowDistance: 40
+    shadowNearPlaneOffset: 3
+    shadowCascade2Split: 0.33333334
+    shadowCascade4Split: {x: 0.06666667, y: 0.2, z: 0.46666667}
+    shadowmaskMode: 1
+    blendWeights: 2
+    textureQuality: 0
+    anisotropicTextures: 0
+    antiAliasing: 0
+    softParticles: 0
+    softVegetation: 1
+    realtimeReflectionProbes: 0
+    billboardsFaceCameraPosition: 0
+    vSyncCount: 1
+    lodBias: 1
+    maximumLODLevel: 0
+    particleRaycastBudget: 256
+    asyncUploadTimeSlice: 2
+    asyncUploadBufferSize: 4
+    resolutionScalingFixedDPIFactor: 1
+    excludedTargetPlatforms: []
+  - serializedVersion: 2
+    name: Very High
+    pixelLightCount: 3
+    shadows: 0
+    shadowResolution: 2
+    shadowProjection: 1
+    shadowCascades: 2
+    shadowDistance: 70
+    shadowNearPlaneOffset: 3
+    shadowCascade2Split: 0.33333334
+    shadowCascade4Split: {x: 0.06666667, y: 0.2, z: 0.46666667}
+    shadowmaskMode: 1
+    blendWeights: 4
+    textureQuality: 0
+    anisotropicTextures: 0
+    antiAliasing: 0
+    softParticles: 0
+    softVegetation: 1
+    realtimeReflectionProbes: 0
+    billboardsFaceCameraPosition: 0
+    vSyncCount: 1
+    lodBias: 1.5
+    maximumLODLevel: 0
+    particleRaycastBudget: 1024
+    asyncUploadTimeSlice: 2
+    asyncUploadBufferSize: 4
+    resolutionScalingFixedDPIFactor: 1
+    excludedTargetPlatforms: []
+  - serializedVersion: 2
+    name: Ultra
+    pixelLightCount: 4
+    shadows: 0
+    shadowResolution: 0
+    shadowProjection: 1
+    shadowCascades: 4
+    shadowDistance: 150
+    shadowNearPlaneOffset: 3
+    shadowCascade2Split: 0.33333334
+    shadowCascade4Split: {x: 0.06666667, y: 0.2, z: 0.46666667}
+    shadowmaskMode: 1
+    blendWeights: 4
+    textureQuality: 0
+    anisotropicTextures: 0
+    antiAliasing: 0
+    softParticles: 0
+    softVegetation: 1
+    realtimeReflectionProbes: 0
+    billboardsFaceCameraPosition: 0
+    vSyncCount: 1
+    lodBias: 2
+    maximumLODLevel: 0
+    particleRaycastBudget: 4096
+    asyncUploadTimeSlice: 2
+    asyncUploadBufferSize: 4
+    resolutionScalingFixedDPIFactor: 1
+    excludedTargetPlatforms: []
+  m_PerPlatformDefaultQuality:
+    Android: 2
+    Nintendo 3DS: 5
+    Nintendo Switch: 5
+    PS4: 5
+    PSM: 5
+    PSP2: 2
+    Standalone: 5
+    Tizen: 2
+    WebGL: 3
+    WiiU: 5
+    Windows Store Apps: 5
+    XboxOne: 5
+    iPhone: 2
+    tvOS: 2

examples/csharp/HelloworldUnity/ProjectSettings/TagManager.asset

@@ -0,0 +1,43 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!78 &1
+TagManager:
+  serializedVersion: 2
+  tags: []
+  layers:
+  - Default
+  - TransparentFX
+  - Ignore Raycast
+  - 
+  - Water
+  - UI
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  - 
+  m_SortingLayers:
+  - name: Default
+    uniqueID: 0
+    locked: 0

examples/csharp/HelloworldUnity/ProjectSettings/TimeManager.asset

@@ -0,0 +1,9 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!5 &1
+TimeManager:
+  m_ObjectHideFlags: 0
+  Fixed Timestep: 0.02
+  Maximum Allowed Timestep: 0.1
+  m_TimeScale: 1
+  Maximum Particle Timestep: 0.03

examples/csharp/HelloworldUnity/ProjectSettings/UnityConnectSettings.asset

@@ -0,0 +1,34 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!310 &1
+UnityConnectSettings:
+  m_ObjectHideFlags: 0
+  serializedVersion: 1
+  m_Enabled: 1
+  m_TestMode: 0
+  m_EventOldUrl: https://api.uca.cloud.unity3d.com/v1/events
+  m_EventUrl: https://cdp.cloud.unity3d.com/v1/events
+  m_ConfigUrl: https://config.uca.cloud.unity3d.com
+  m_TestInitMode: 0
+  CrashReportingSettings:
+    m_EventUrl: https://perf-events.cloud.unity3d.com
+    m_Enabled: 0
+    m_LogBufferSize: 10
+    m_CaptureEditorExceptions: 1
+  UnityPurchasingSettings:
+    m_Enabled: 0
+    m_TestMode: 0
+  UnityAnalyticsSettings:
+    m_Enabled: 1
+    m_TestMode: 0
+    m_InitializeOnStartup: 1
+  UnityAdsSettings:
+    m_Enabled: 0
+    m_InitializeOnStartup: 1
+    m_TestMode: 0
+    m_IosGameId: 
+    m_AndroidGameId: 
+    m_GameIds: {}
+    m_GameId: 
+  PerformanceReportingSettings:
+    m_Enabled: 0

examples/csharp/HelloworldUnity/ProjectSettings/VFXManager.asset

@@ -0,0 +1,11 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!937362698 &1
+VFXManager:
+  m_ObjectHideFlags: 0
+  m_IndirectShader: {fileID: 0}
+  m_CopyBufferShader: {fileID: 0}
+  m_SortShader: {fileID: 0}
+  m_RenderPipeSettingsPath: 
+  m_FixedTimeStep: 0.016666668
+  m_MaxDeltaTime: 0.05

examples/csharp/HelloworldUnity/README.md

@@ -0,0 +1,19 @@
+gRPC C# on Unity
+========================
+
+EXPERIMENTAL ONLY
+-------------
+Support of the Unity platform is currently experimental.
+
+PREREQUISITES
+-------------
+
+- Unity 2018.3.5f1
+
+BUILD
+-------
+
+- Follow instructions in https://github.com/grpc/grpc/tree/master/src/csharp/experimental#unity to obtain the grpc_csharp_unity.zip
+  that contains gRPC C# for Unity. Unzip it under `Assets/Plugins` directory.
+- Open the `HelloworldUnity.sln` in Unity Editor.
+- Build using Unity Editor.

examples/csharp/HelloworldUnity/UIElementsSchema/UIElements.xsd

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xs:schema xmlns:engine="UnityEngine.Experimental.UIElements" xmlns:editor="UnityEditor.Experimental.UIElements" elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <xs:import schemaLocation="UnityEngine.Experimental.UIElements.xsd" namespace="UnityEngine.Experimental.UIElements" />
+  <xs:import schemaLocation="UnityEditor.PackageManager.UI.xsd" namespace="UnityEditor.PackageManager.UI" />
+  <xs:import schemaLocation="UnityEditor.Experimental.UIElements.xsd" namespace="UnityEditor.Experimental.UIElements" />
+</xs:schema>

examples/csharp/HelloworldUnity/UIElementsSchema/UnityEditor.Experimental.UIElements.xsd

@@ -0,0 +1,228 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xs:schema xmlns:engine="UnityEngine.Experimental.UIElements" xmlns:editor="UnityEditor.Experimental.UIElements" elementFormDefault="qualified" targetNamespace="UnityEditor.Experimental.UIElements" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <xs:import schemaLocation="UnityEngine.Experimental.UIElements.xsd" namespace="UnityEngine.Experimental.UIElements" />
+  <xs:complexType name="FloatFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:attribute default="-1" name="maxLength" type="xs:int" use="optional" />
+        <xs:attribute default="false" name="password" type="xs:boolean" use="optional" />
+        <xs:attribute default="*" name="maskCharacter" type="xs:string" use="optional" />
+        <xs:attribute default="0" name="value" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="FloatField" substitutionGroup="engine:VisualElement" type="editor:FloatFieldType" />
+  <xs:complexType name="DoubleFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:attribute default="-1" name="maxLength" type="xs:int" use="optional" />
+        <xs:attribute default="false" name="password" type="xs:boolean" use="optional" />
+        <xs:attribute default="*" name="maskCharacter" type="xs:string" use="optional" />
+        <xs:attribute default="0" name="value" type="xs:double" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="DoubleField" substitutionGroup="engine:VisualElement" type="editor:DoubleFieldType" />
+  <xs:complexType name="IntegerFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:attribute default="-1" name="maxLength" type="xs:int" use="optional" />
+        <xs:attribute default="false" name="password" type="xs:boolean" use="optional" />
+        <xs:attribute default="*" name="maskCharacter" type="xs:string" use="optional" />
+        <xs:attribute default="0" name="value" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="IntegerField" substitutionGroup="engine:VisualElement" type="editor:IntegerFieldType" />
+  <xs:complexType name="LongFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:attribute default="-1" name="maxLength" type="xs:int" use="optional" />
+        <xs:attribute default="false" name="password" type="xs:boolean" use="optional" />
+        <xs:attribute default="*" name="maskCharacter" type="xs:string" use="optional" />
+        <xs:attribute default="0" name="value" type="xs:long" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="LongField" substitutionGroup="engine:VisualElement" type="editor:LongFieldType" />
+  <xs:complexType name="CurveFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="CurveField" substitutionGroup="engine:VisualElement" type="editor:CurveFieldType" />
+  <xs:complexType name="ObjectFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="true" name="allowSceneObjects" type="xs:boolean" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="ObjectField" substitutionGroup="engine:VisualElement" type="editor:ObjectFieldType" />
+  <xs:complexType name="ColorFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="RGBA(0.000, 0.000, 0.000, 1.000)" name="value" type="xs:string" use="optional" />
+        <xs:attribute default="true" name="showEyeDropper" type="xs:boolean" use="optional" />
+        <xs:attribute default="true" name="showAlpha" type="xs:boolean" use="optional" />
+        <xs:attribute default="false" name="hdr" type="xs:boolean" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="ColorField" substitutionGroup="engine:VisualElement" type="editor:ColorFieldType" />
+  <xs:complexType name="EnumFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:attribute name="type" type="xs:string" use="required" />
+        <xs:attribute default="" name="value" type="xs:string" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="EnumField" substitutionGroup="engine:VisualElement" type="editor:EnumFieldType" />
+  <xs:complexType name="GradientFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="GradientField" substitutionGroup="engine:VisualElement" type="editor:GradientFieldType" />
+  <xs:complexType name="RectFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="x" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="y" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="w" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="h" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="RectField" substitutionGroup="engine:VisualElement" type="editor:RectFieldType" />
+  <xs:complexType name="Vector2FieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="x" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="y" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Vector2Field" substitutionGroup="engine:VisualElement" type="editor:Vector2FieldType" />
+  <xs:complexType name="Vector3FieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="x" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="y" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="z" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Vector3Field" substitutionGroup="engine:VisualElement" type="editor:Vector3FieldType" />
+  <xs:complexType name="Vector4FieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="x" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="y" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="z" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="w" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Vector4Field" substitutionGroup="engine:VisualElement" type="editor:Vector4FieldType" />
+  <xs:complexType name="BoundsFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="cx" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="cy" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="cz" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="ex" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="ey" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="ez" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="BoundsField" substitutionGroup="engine:VisualElement" type="editor:BoundsFieldType" />
+  <xs:simpleType name="PropertyControl_typeOf_Type">
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="Long" />
+      <xs:enumeration value="Double" />
+      <xs:enumeration value="Int" />
+      <xs:enumeration value="Float" />
+      <xs:enumeration value="String" />
+    </xs:restriction>
+  </xs:simpleType>
+  <xs:complexType name="PropertyControlType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute name="typeOf" type="editor:PropertyControl_typeOf_Type" use="required" />
+        <xs:attribute default="" name="value" type="xs:string" use="optional" />
+        <xs:attribute default="" name="label" type="xs:string" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PropertyControl" substitutionGroup="engine:VisualElement" type="editor:PropertyControlType" />
+</xs:schema>

examples/csharp/HelloworldUnity/UIElementsSchema/UnityEditor.PackageManager.UI.xsd

@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xs:schema xmlns:engine="UnityEngine.Experimental.UIElements" xmlns:editor="UnityEditor.Experimental.UIElements" elementFormDefault="qualified" targetNamespace="UnityEditor.PackageManager.UI" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <xs:import schemaLocation="UnityEngine.Experimental.UIElements.xsd" namespace="UnityEngine.Experimental.UIElements" />
+  <xs:complexType name="AlertType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Alert" substitutionGroup="engine:VisualElement" xmlns:q1="UnityEditor.PackageManager.UI" type="q1:AlertType" />
+  <xs:complexType name="LoadingSpinnerType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="LoadingSpinner" substitutionGroup="engine:VisualElement" xmlns:q2="UnityEditor.PackageManager.UI" type="q2:LoadingSpinnerType" />
+  <xs:complexType name="PackageDetailsType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PackageDetails" substitutionGroup="engine:VisualElement" xmlns:q3="UnityEditor.PackageManager.UI" type="q3:PackageDetailsType" />
+  <xs:complexType name="PackageGroupType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PackageGroup" substitutionGroup="engine:VisualElement" xmlns:q4="UnityEditor.PackageManager.UI" type="q4:PackageGroupType" />
+  <xs:complexType name="PackageItemType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PackageItem" substitutionGroup="engine:VisualElement" xmlns:q5="UnityEditor.PackageManager.UI" type="q5:PackageItemType" />
+  <xs:complexType name="PackageListType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PackageList" substitutionGroup="engine:VisualElement" xmlns:q6="UnityEditor.PackageManager.UI" type="q6:PackageListType" />
+  <xs:complexType name="PackageSearchFilterTabsType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PackageSearchFilterTabs" substitutionGroup="engine:VisualElement" xmlns:q7="UnityEditor.PackageManager.UI" type="q7:PackageSearchFilterTabsType" />
+  <xs:complexType name="PackagesLoadingType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PackagesLoading" substitutionGroup="engine:VisualElement" xmlns:q8="UnityEditor.PackageManager.UI" type="q8:PackagesLoadingType" />
+</xs:schema>

examples/csharp/HelloworldUnity/UIElementsSchema/UnityEngine.Experimental.UIElements.xsd

@@ -0,0 +1,269 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xs:schema xmlns:engine="UnityEngine.Experimental.UIElements" xmlns:editor="UnityEditor.Experimental.UIElements" elementFormDefault="qualified" targetNamespace="UnityEngine.Experimental.UIElements" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <xs:complexType name="UXMLType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="xs:anyType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="UXML" type="engine:UXMLType" />
+  <xs:simpleType name="VisualElement_pickingMode_Type">
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="Position" />
+      <xs:enumeration value="Ignore" />
+    </xs:restriction>
+  </xs:simpleType>
+  <xs:complexType name="VisualElementType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="xs:anyType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="VisualElement" type="engine:VisualElementType" />
+  <xs:complexType name="IMGUIContainerType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="IMGUIContainer" substitutionGroup="engine:VisualElement" type="engine:IMGUIContainerType" />
+  <xs:complexType name="ImageType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Image" substitutionGroup="engine:VisualElement" type="engine:ImageType" />
+  <xs:complexType name="LabelType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Label" substitutionGroup="engine:VisualElement" type="engine:LabelType" />
+  <xs:complexType name="RepeatButtonType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:attribute default="0" name="delay" type="xs:long" use="optional" />
+        <xs:attribute default="0" name="interval" type="xs:long" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="RepeatButton" substitutionGroup="engine:VisualElement" type="engine:RepeatButtonType" />
+  <xs:complexType name="ScrollerButtonType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="delay" type="xs:long" use="optional" />
+        <xs:attribute default="0" name="interval" type="xs:long" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="ScrollerButton" substitutionGroup="engine:VisualElement" type="engine:ScrollerButtonType" />
+  <xs:complexType name="ScrollViewType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="false" name="showHorizontalScroller" type="xs:boolean" use="optional" />
+        <xs:attribute default="false" name="showVerticalScroller" type="xs:boolean" use="optional" />
+        <xs:attribute default="0" name="horizontalLowValue" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="horizontalHighValue" type="xs:float" use="optional" />
+        <xs:attribute default="10" name="horizontalPageSize" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="horizontalValue" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="verticalLowValue" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="verticalHighValue" type="xs:float" use="optional" />
+        <xs:attribute default="10" name="verticalPageSize" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="verticalValue" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="ScrollView" substitutionGroup="engine:VisualElement" type="engine:ScrollViewType" />
+  <xs:simpleType name="Scroller_direction_Type">
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="Horizontal" />
+      <xs:enumeration value="Vertical" />
+    </xs:restriction>
+  </xs:simpleType>
+  <xs:complexType name="ScrollerType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="lowValue" type="xs:float" use="optional" />
+        <xs:attribute default="0" name="highValue" type="xs:float" use="optional" />
+        <xs:attribute default="Vertical" name="direction" type="engine:Scroller_direction_Type" use="optional" />
+        <xs:attribute default="0" name="value" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Scroller" substitutionGroup="engine:VisualElement" type="engine:ScrollerType" />
+  <xs:simpleType name="Slider_direction_Type">
+    <xs:restriction base="xs:string">
+      <xs:enumeration value="Horizontal" />
+      <xs:enumeration value="Vertical" />
+    </xs:restriction>
+  </xs:simpleType>
+  <xs:complexType name="SliderType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="0" name="lowValue" type="xs:float" use="optional" />
+        <xs:attribute default="10" name="highValue" type="xs:float" use="optional" />
+        <xs:attribute default="10" name="pageSize" type="xs:float" use="optional" />
+        <xs:attribute default="Vertical" name="direction" type="engine:Slider_direction_Type" use="optional" />
+        <xs:attribute default="0" name="value" type="xs:float" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Slider" substitutionGroup="engine:VisualElement" type="engine:SliderType" />
+  <xs:complexType name="TextFieldType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:attribute default="-1" name="maxLength" type="xs:int" use="optional" />
+        <xs:attribute default="false" name="password" type="xs:boolean" use="optional" />
+        <xs:attribute default="*" name="maskCharacter" type="xs:string" use="optional" />
+        <xs:attribute default="false" name="multiline" type="xs:boolean" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="TextField" substitutionGroup="engine:VisualElement" type="engine:TextFieldType" />
+  <xs:complexType name="ToggleType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="0" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="label" type="xs:string" use="optional" />
+        <xs:attribute default="false" name="value" type="xs:boolean" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Toggle" substitutionGroup="engine:VisualElement" type="engine:ToggleType" />
+  <xs:complexType name="VisualContainerType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="VisualContainer" substitutionGroup="engine:VisualElement" type="engine:VisualContainerType" />
+  <xs:complexType name="TemplateContainerType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute name="template" type="xs:string" use="required" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="TemplateContainer" substitutionGroup="engine:VisualElement" type="engine:TemplateContainerType" />
+  <xs:complexType name="BoxType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Box" substitutionGroup="engine:VisualElement" type="engine:BoxType" />
+  <xs:complexType name="PopupWindowType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:sequence minOccurs="0" maxOccurs="unbounded">
+          <xs:element ref="engine:VisualElement" />
+        </xs:sequence>
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="PopupWindow" substitutionGroup="engine:VisualElement" type="engine:PopupWindowType" />
+  <xs:complexType name="ListViewType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="30" name="itemHeight" type="xs:int" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="ListView" substitutionGroup="engine:VisualElement" type="engine:ListViewType" />
+  <xs:complexType name="ButtonType">
+    <xs:complexContent mixed="false">
+      <xs:restriction base="engine:VisualElementType">
+        <xs:attribute default="" name="name" type="xs:string" use="optional" />
+        <xs:attribute default="Position" name="pickingMode" type="engine:VisualElement_pickingMode_Type" use="optional" />
+        <xs:attribute default="-1" name="focusIndex" type="xs:int" use="optional" />
+        <xs:attribute default="" name="text" type="xs:string" use="optional" />
+        <xs:anyAttribute processContents="lax" />
+      </xs:restriction>
+    </xs:complexContent>
+  </xs:complexType>
+  <xs:element name="Button" substitutionGroup="engine:VisualElement" type="engine:ButtonType" />
+</xs:schema>

examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj

@@ -50,12 +50,12 @@
     <Reference Include="System.Interactive.Async">
       <HintPath>..\packages\System.Interactive.Async.3.1.1\lib\netstandard1.3\System.Interactive.Async.dll</HintPath>
     </Reference>
-    <Reference Include="Grpc.Core">
-      <HintPath>..\packages\Grpc.Core.1.15.0-dev\lib\netstandard1.5\Grpc.Core.dll</HintPath>
-    </Reference>
     <Reference Include="Google.Protobuf">
       <HintPath>..\packages\Google.Protobuf.3.6.0\lib\netstandard1.0\Google.Protobuf.dll</HintPath>
     </Reference>
+    <Reference Include="Grpc.Core">
+      <HintPath>..\packages\Grpc.Core.1.18.0\lib\netstandard1.5\Grpc.Core.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="MainActivity.cs" />
@@ -79,5 +79,5 @@
   </ItemGroup>
   <Import Project="..\HelloworldXamarin\HelloworldXamarin.projitems" Label="Shared" Condition="Exists('..\HelloworldXamarin\HelloworldXamarin.projitems')" />
   <Import Project="$(MSBuildExtensionsPath)\Xamarin\Android\Xamarin.Android.CSharp.targets" />
-  <Import Project="..\packages\Grpc.Core.1.15.0-dev\build\MonoAndroid\Grpc.Core.targets" Condition="Exists('..\packages\Grpc.Core.1.15.0-dev\build\MonoAndroid\Grpc.Core.targets')" />
+  <Import Project="..\packages\Grpc.Core.1.18.0\build\MonoAndroid10\Grpc.Core.targets" Condition="Exists('..\packages\Grpc.Core.1.18.0\build\MonoAndroid10\Grpc.Core.targets')" />
 </Project>

examples/csharp/HelloworldXamarin/Droid/packages.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="Google.Protobuf" version="3.6.0" targetFramework="monoandroid81" />
-  <package id="Grpc.Core" version="1.15.0-dev" targetFramework="monoandroid81" />
+  <package id="Grpc.Core" version="1.18.0" targetFramework="monoandroid81" />
   <package id="Microsoft.NETCore.Platforms" version="1.1.0" targetFramework="monoandroid81" />
   <package id="Microsoft.Win32.Primitives" version="4.3.0" targetFramework="monoandroid81" />
   <package id="NETStandard.Library" version="1.6.1" targetFramework="monoandroid81" />

examples/csharp/HelloworldXamarin/README.md

@@ -4,11 +4,6 @@ gRPC C# on Xamarin
 EXPERIMENTAL ONLY
 -------------
 Support of the Xamarin platform is currently experimental.
-The example depends on experimental Grpc.Core nuget package that hasn't
-been officially released and is only available via the [daily builds](https://packages.grpc.io/)
-source.
-
-HINT: To download the package, please manually download the latest `.nupkg` packages from "Daily Builds" in [packages.grpc.io](https://packages.grpc.io/) into a local directory. Then add a nuget source that points to that directory (That can be [done in Visual Studio](https://docs.microsoft.com/en-us/nuget/tools/package-manager-ui#package-sources) or Visual Studio for Mac via "Configure nuget sources"). After that, nuget will also explore that directory when looking for packages.
 
 BACKGROUND
 -------------

examples/csharp/HelloworldXamarin/iOS/HelloworldXamarin.iOS.csproj

@@ -89,12 +89,12 @@
     <Reference Include="System.Interactive.Async">
       <HintPath>..\packages\System.Interactive.Async.3.1.1\lib\netstandard1.3\System.Interactive.Async.dll</HintPath>
     </Reference>
-    <Reference Include="Grpc.Core">
-      <HintPath>..\packages\Grpc.Core.1.15.0-dev\lib\netstandard1.5\Grpc.Core.dll</HintPath>
-    </Reference>
     <Reference Include="Google.Protobuf">
       <HintPath>..\packages\Google.Protobuf.3.6.0\lib\netstandard1.0\Google.Protobuf.dll</HintPath>
     </Reference>
+    <Reference Include="Grpc.Core">
+      <HintPath>..\packages\Grpc.Core.1.18.0\lib\netstandard1.5\Grpc.Core.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <ImageAsset Include="Assets.xcassets\AppIcon.appiconset\Contents.json" />
@@ -122,5 +122,5 @@
   </ItemGroup>
   <Import Project="..\HelloworldXamarin\HelloworldXamarin.projitems" Label="Shared" Condition="Exists('..\HelloworldXamarin\HelloworldXamarin.projitems')" />
   <Import Project="$(MSBuildExtensionsPath)\Xamarin\iOS\Xamarin.iOS.CSharp.targets" />
-  <Import Project="..\packages\Grpc.Core.1.15.0-dev\build\Xamarin.iOS\Grpc.Core.targets" Condition="Exists('..\packages\Grpc.Core.1.15.0-dev\build\Xamarin.iOS\Grpc.Core.targets')" />
+  <Import Project="..\packages\Grpc.Core.1.18.0\build\Xamarin.iOS10\Grpc.Core.targets" Condition="Exists('..\packages\Grpc.Core.1.18.0\build\Xamarin.iOS10\Grpc.Core.targets')" />
 </Project>

examples/csharp/HelloworldXamarin/iOS/packages.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="Google.Protobuf" version="3.6.0" targetFramework="xamarinios10" />
-  <package id="Grpc.Core" version="1.15.0-dev" targetFramework="xamarinios10" />
+  <package id="Grpc.Core" version="1.18.0" targetFramework="xamarinios10" />
   <package id="Microsoft.NETCore.Platforms" version="1.1.0" targetFramework="xamarinios10" />
   <package id="Microsoft.Win32.Primitives" version="4.3.0" targetFramework="xamarinios10" />
   <package id="NETStandard.Library" version="1.6.1" targetFramework="xamarinios10" />

gRPC-C++.podspec

@@ -360,11 +360,11 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/parse_address.h',
                       'src/core/ext/filters/client_channel/proxy_mapper.h',
                       'src/core/ext/filters/client_channel/proxy_mapper_registry.h',
-                      'src/core/ext/filters/client_channel/request_routing.h',
                       'src/core/ext/filters/client_channel/resolver.h',
                       'src/core/ext/filters/client_channel/resolver_factory.h',
                       'src/core/ext/filters/client_channel/resolver_registry.h',
                       'src/core/ext/filters/client_channel/resolver_result_parsing.h',
+                      'src/core/ext/filters/client_channel/resolving_lb_policy.h',
                       'src/core/ext/filters/client_channel/retry_throttle.h',
                       'src/core/ext/filters/client_channel/server_address.h',
                       'src/core/ext/filters/client_channel/subchannel.h',

gRPC-Core.podspec

@@ -354,11 +354,11 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/parse_address.h',
                       'src/core/ext/filters/client_channel/proxy_mapper.h',
                       'src/core/ext/filters/client_channel/proxy_mapper_registry.h',
-                      'src/core/ext/filters/client_channel/request_routing.h',
                       'src/core/ext/filters/client_channel/resolver.h',
                       'src/core/ext/filters/client_channel/resolver_factory.h',
                       'src/core/ext/filters/client_channel/resolver_registry.h',
                       'src/core/ext/filters/client_channel/resolver_result_parsing.h',
+                      'src/core/ext/filters/client_channel/resolving_lb_policy.h',
                       'src/core/ext/filters/client_channel/retry_throttle.h',
                       'src/core/ext/filters/client_channel/server_address.h',
                       'src/core/ext/filters/client_channel/subchannel.h',
@@ -543,7 +543,6 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/channelz_registry.cc',
                       'src/core/lib/channel/connected_channel.cc',
                       'src/core/lib/channel/handshaker.cc',
-                      'src/core/lib/channel/handshaker_factory.cc',
                       'src/core/lib/channel/handshaker_registry.cc',
                       'src/core/lib/channel/status_util.cc',
                       'src/core/lib/compression/compression.cc',
@@ -802,10 +801,10 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/parse_address.cc',
                       'src/core/ext/filters/client_channel/proxy_mapper.cc',
                       'src/core/ext/filters/client_channel/proxy_mapper_registry.cc',
-                      'src/core/ext/filters/client_channel/request_routing.cc',
                       'src/core/ext/filters/client_channel/resolver.cc',
                       'src/core/ext/filters/client_channel/resolver_registry.cc',
                       'src/core/ext/filters/client_channel/resolver_result_parsing.cc',
+                      'src/core/ext/filters/client_channel/resolving_lb_policy.cc',
                       'src/core/ext/filters/client_channel/retry_throttle.cc',
                       'src/core/ext/filters/client_channel/server_address.cc',
                       'src/core/ext/filters/client_channel/subchannel.cc',
@@ -985,11 +984,11 @@ Pod::Spec.new do |s|
                               'src/core/ext/filters/client_channel/parse_address.h',
                               'src/core/ext/filters/client_channel/proxy_mapper.h',
                               'src/core/ext/filters/client_channel/proxy_mapper_registry.h',
-                              'src/core/ext/filters/client_channel/request_routing.h',
                               'src/core/ext/filters/client_channel/resolver.h',
                               'src/core/ext/filters/client_channel/resolver_factory.h',
                               'src/core/ext/filters/client_channel/resolver_registry.h',
                               'src/core/ext/filters/client_channel/resolver_result_parsing.h',
+                              'src/core/ext/filters/client_channel/resolving_lb_policy.h',
                               'src/core/ext/filters/client_channel/retry_throttle.h',
                               'src/core/ext/filters/client_channel/server_address.h',
                               'src/core/ext/filters/client_channel/subchannel.h',

grpc.gemspec

@@ -288,11 +288,11 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/client_channel/parse_address.h )
   s.files += %w( src/core/ext/filters/client_channel/proxy_mapper.h )
   s.files += %w( src/core/ext/filters/client_channel/proxy_mapper_registry.h )
-  s.files += %w( src/core/ext/filters/client_channel/request_routing.h )
   s.files += %w( src/core/ext/filters/client_channel/resolver.h )
   s.files += %w( src/core/ext/filters/client_channel/resolver_factory.h )
   s.files += %w( src/core/ext/filters/client_channel/resolver_registry.h )
   s.files += %w( src/core/ext/filters/client_channel/resolver_result_parsing.h )
+  s.files += %w( src/core/ext/filters/client_channel/resolving_lb_policy.h )
   s.files += %w( src/core/ext/filters/client_channel/retry_throttle.h )
   s.files += %w( src/core/ext/filters/client_channel/server_address.h )
   s.files += %w( src/core/ext/filters/client_channel/subchannel.h )
@@ -477,7 +477,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/channel/channelz_registry.cc )
   s.files += %w( src/core/lib/channel/connected_channel.cc )
   s.files += %w( src/core/lib/channel/handshaker.cc )
-  s.files += %w( src/core/lib/channel/handshaker_factory.cc )
   s.files += %w( src/core/lib/channel/handshaker_registry.cc )
   s.files += %w( src/core/lib/channel/status_util.cc )
   s.files += %w( src/core/lib/compression/compression.cc )
@@ -739,10 +738,10 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/client_channel/parse_address.cc )
   s.files += %w( src/core/ext/filters/client_channel/proxy_mapper.cc )
   s.files += %w( src/core/ext/filters/client_channel/proxy_mapper_registry.cc )
-  s.files += %w( src/core/ext/filters/client_channel/request_routing.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver_registry.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver_result_parsing.cc )
+  s.files += %w( src/core/ext/filters/client_channel/resolving_lb_policy.cc )
   s.files += %w( src/core/ext/filters/client_channel/retry_throttle.cc )
   s.files += %w( src/core/ext/filters/client_channel/server_address.cc )
   s.files += %w( src/core/ext/filters/client_channel/subchannel.cc )

grpc.gyp

@@ -276,7 +276,6 @@
         'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
-        'src/core/lib/channel/handshaker_factory.cc',
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
@@ -538,10 +537,10 @@
         'src/core/ext/filters/client_channel/parse_address.cc',
         'src/core/ext/filters/client_channel/proxy_mapper.cc',
         'src/core/ext/filters/client_channel/proxy_mapper_registry.cc',
-        'src/core/ext/filters/client_channel/request_routing.cc',
         'src/core/ext/filters/client_channel/resolver.cc',
         'src/core/ext/filters/client_channel/resolver_registry.cc',
         'src/core/ext/filters/client_channel/resolver_result_parsing.cc',
+        'src/core/ext/filters/client_channel/resolving_lb_policy.cc',
         'src/core/ext/filters/client_channel/retry_throttle.cc',
         'src/core/ext/filters/client_channel/server_address.cc',
         'src/core/ext/filters/client_channel/subchannel.cc',
@@ -643,7 +642,6 @@
         'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
-        'src/core/lib/channel/handshaker_factory.cc',
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
@@ -803,10 +801,10 @@
         'src/core/ext/filters/client_channel/parse_address.cc',
         'src/core/ext/filters/client_channel/proxy_mapper.cc',
         'src/core/ext/filters/client_channel/proxy_mapper_registry.cc',
-        'src/core/ext/filters/client_channel/request_routing.cc',
         'src/core/ext/filters/client_channel/resolver.cc',
         'src/core/ext/filters/client_channel/resolver_registry.cc',
         'src/core/ext/filters/client_channel/resolver_result_parsing.cc',
+        'src/core/ext/filters/client_channel/resolving_lb_policy.cc',
         'src/core/ext/filters/client_channel/retry_throttle.cc',
         'src/core/ext/filters/client_channel/server_address.cc',
         'src/core/ext/filters/client_channel/subchannel.cc',
@@ -889,7 +887,6 @@
         'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
-        'src/core/lib/channel/handshaker_factory.cc',
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
@@ -1049,10 +1046,10 @@
         'src/core/ext/filters/client_channel/parse_address.cc',
         'src/core/ext/filters/client_channel/proxy_mapper.cc',
         'src/core/ext/filters/client_channel/proxy_mapper_registry.cc',
-        'src/core/ext/filters/client_channel/request_routing.cc',
         'src/core/ext/filters/client_channel/resolver.cc',
         'src/core/ext/filters/client_channel/resolver_registry.cc',
         'src/core/ext/filters/client_channel/resolver_result_parsing.cc',
+        'src/core/ext/filters/client_channel/resolving_lb_policy.cc',
         'src/core/ext/filters/client_channel/retry_throttle.cc',
         'src/core/ext/filters/client_channel/server_address.cc',
         'src/core/ext/filters/client_channel/subchannel.cc',
@@ -1111,7 +1108,6 @@
         'src/core/lib/channel/channelz_registry.cc',
         'src/core/lib/channel/connected_channel.cc',
         'src/core/lib/channel/handshaker.cc',
-        'src/core/lib/channel/handshaker_factory.cc',
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
@@ -1306,10 +1302,10 @@
         'src/core/ext/filters/client_channel/parse_address.cc',
         'src/core/ext/filters/client_channel/proxy_mapper.cc',
         'src/core/ext/filters/client_channel/proxy_mapper_registry.cc',
-        'src/core/ext/filters/client_channel/request_routing.cc',
         'src/core/ext/filters/client_channel/resolver.cc',
         'src/core/ext/filters/client_channel/resolver_registry.cc',
         'src/core/ext/filters/client_channel/resolver_result_parsing.cc',
+        'src/core/ext/filters/client_channel/resolving_lb_policy.cc',
         'src/core/ext/filters/client_channel/retry_throttle.cc',
         'src/core/ext/filters/client_channel/server_address.cc',
         'src/core/ext/filters/client_channel/subchannel.cc',

include/grpcpp/impl/codegen/interceptor.h

@@ -45,6 +45,10 @@ namespace experimental {
 /// PRE_RECV means an interception between the time that a certain
 /// operation has been requested and it is available. POST_RECV means that a
 /// result is available but has not yet been passed back to the application.
+/// A batch of interception points will only contain either PRE or POST hooks
+/// but not both types. For example, a batch with PRE_SEND hook points will not
+/// contain POST_RECV or POST_SEND ops. Likewise, a batch with POST_* ops can
+/// not contain PRE_* ops.
 enum class InterceptionHookPoints {
   /// The first three in this list are for clients and servers
   PRE_SEND_INITIAL_METADATA,
@@ -52,8 +56,8 @@ enum class InterceptionHookPoints {
   POST_SEND_MESSAGE,
   PRE_SEND_STATUS,  // server only
   PRE_SEND_CLOSE,   // client only: WritesDone for stream; after write in unary
-  /// The following three are for hijacked clients only and can only be
-  /// registered by the global interceptor
+  /// The following three are for hijacked clients only. A batch with PRE_RECV_*
+  /// hook points will never contain hook points of other types.
   PRE_RECV_INITIAL_METADATA,
   PRE_RECV_MESSAGE,
   PRE_RECV_STATUS,

package.xml

@@ -293,11 +293,11 @@
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/parse_address.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/proxy_mapper.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/proxy_mapper_registry.h" role="src" />
-    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/request_routing.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver_factory.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver_registry.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver_result_parsing.h" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolving_lb_policy.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/retry_throttle.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/server_address.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/subchannel.h" role="src" />
@@ -482,7 +482,6 @@
     <file baseinstalldir="/" name="src/core/lib/channel/channelz_registry.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/connected_channel.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/handshaker.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/channel/handshaker_factory.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/handshaker_registry.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/status_util.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/compression.cc" role="src" />
@@ -744,10 +743,10 @@
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/parse_address.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/proxy_mapper.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/proxy_mapper_registry.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/request_routing.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver_registry.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver_result_parsing.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolving_lb_policy.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/retry_throttle.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/server_address.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/subchannel.cc" role="src" />

src/core/ext/filters/client_channel/channel_connectivity.cc

@@ -35,6 +35,7 @@ grpc_connectivity_state grpc_channel_check_connectivity_state(
   /* forward through to the underlying client channel */
   grpc_channel_element* client_channel_elem =
       grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   grpc_connectivity_state state;
   GRPC_API_TRACE(
@@ -202,6 +203,7 @@ void grpc_channel_watch_connectivity_state(
     gpr_timespec deadline, grpc_completion_queue* cq, void* tag) {
   grpc_channel_element* client_channel_elem =
       grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   state_watcher* w = static_cast<state_watcher*>(gpr_malloc(sizeof(*w)));
 

src/core/ext/filters/client_channel/client_channel.cc

@@ -32,12 +32,14 @@
 #include <grpc/support/sync.h>
 
 #include "src/core/ext/filters/client_channel/backup_poller.h"
+#include "src/core/ext/filters/client_channel/global_subchannel_pool.h"
 #include "src/core/ext/filters/client_channel/http_connect_handshaker.h"
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
+#include "src/core/ext/filters/client_channel/local_subchannel_pool.h"
 #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
-#include "src/core/ext/filters/client_channel/request_routing.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/ext/filters/client_channel/resolver_result_parsing.h"
+#include "src/core/ext/filters/client_channel/resolving_lb_policy.h"
 #include "src/core/ext/filters/client_channel/retry_throttle.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/ext/filters/deadline/deadline_filter.h"
@@ -68,6 +70,8 @@ using grpc_core::internal::ClientChannelMethodParamsTable;
 using grpc_core::internal::ProcessedResolverResult;
 using grpc_core::internal::ServerRetryThrottleData;
 
+using grpc_core::LoadBalancingPolicy;
+
 /* Client channel implementation */
 
 // By default, we buffer 256 KiB per RPC for retries.
@@ -86,44 +90,171 @@ grpc_core::TraceFlag grpc_client_channel_trace(false, "client_channel");
 
 struct external_connectivity_watcher;
 
-typedef struct client_channel_channel_data {
-  grpc_core::ManualConstructor<grpc_core::RequestRouter> request_router;
+struct QueuedPick {
+  LoadBalancingPolicy::PickState pick;
+  grpc_call_element* elem;
+  QueuedPick* next = nullptr;
+};
 
+typedef struct client_channel_channel_data {
   bool deadline_checking_enabled;
   bool enable_retries;
   size_t per_rpc_retry_buffer_size;
 
   /** combiner protecting all variables below in this data structure */
   grpc_combiner* combiner;
-  /** retry throttle data */
-  grpc_core::RefCountedPtr<ServerRetryThrottleData> retry_throttle_data;
-  /** maps method names to method_parameters structs */
-  grpc_core::RefCountedPtr<ClientChannelMethodParamsTable> method_params_table;
   /** owning stack */
   grpc_channel_stack* owning_stack;
   /** interested parties (owned) */
   grpc_pollset_set* interested_parties;
-
-  /* external_connectivity_watcher_list head is guarded by its own mutex, since
-   * counts need to be grabbed immediately without polling on a cq */
-  gpr_mu external_connectivity_watcher_list_mu;
-  struct external_connectivity_watcher* external_connectivity_watcher_list_head;
+  // Client channel factory.  Holds a ref.
+  grpc_client_channel_factory* client_channel_factory;
+  // Subchannel pool.
+  grpc_core::RefCountedPtr<grpc_core::SubchannelPoolInterface> subchannel_pool;
+
+  grpc_core::channelz::ClientChannelNode* channelz_node;
+
+  // Resolving LB policy.
+  grpc_core::OrphanablePtr<LoadBalancingPolicy> resolving_lb_policy;
+  // Subchannel picker from LB policy.
+  grpc_core::UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker;
+  // Linked list of queued picks.
+  QueuedPick* queued_picks;
+
+  bool have_service_config;
+  /** retry throttle data from service config */
+  grpc_core::RefCountedPtr<ServerRetryThrottleData> retry_throttle_data;
+  /** per-method service config data */
+  grpc_core::RefCountedPtr<ClientChannelMethodParamsTable> method_params_table;
 
   /* the following properties are guarded by a mutex since APIs require them
      to be instantaneously available */
   gpr_mu info_mu;
   grpc_core::UniquePtr<char> info_lb_policy_name;
-  /** service config in JSON form */
   grpc_core::UniquePtr<char> info_service_config_json;
+
+  grpc_connectivity_state_tracker state_tracker;
+  grpc_error* disconnect_error;
+
+  /* external_connectivity_watcher_list head is guarded by its own mutex, since
+   * counts need to be grabbed immediately without polling on a cq */
+  gpr_mu external_connectivity_watcher_list_mu;
+  struct external_connectivity_watcher* external_connectivity_watcher_list_head;
 } channel_data;
 
-// Synchronous callback from chand->request_router to process a resolver
+// Forward declarations.
+static void start_pick_locked(void* arg, grpc_error* ignored);
+static void maybe_apply_service_config_to_call_locked(grpc_call_element* elem);
+
+static const char* get_channel_connectivity_state_change_string(
+    grpc_connectivity_state state) {
+  switch (state) {
+    case GRPC_CHANNEL_IDLE:
+      return "Channel state change to IDLE";
+    case GRPC_CHANNEL_CONNECTING:
+      return "Channel state change to CONNECTING";
+    case GRPC_CHANNEL_READY:
+      return "Channel state change to READY";
+    case GRPC_CHANNEL_TRANSIENT_FAILURE:
+      return "Channel state change to TRANSIENT_FAILURE";
+    case GRPC_CHANNEL_SHUTDOWN:
+      return "Channel state change to SHUTDOWN";
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
+static void set_connectivity_state_and_picker_locked(
+    channel_data* chand, grpc_connectivity_state state, grpc_error* state_error,
+    const char* reason,
+    grpc_core::UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker) {
+  // Update connectivity state.
+  grpc_connectivity_state_set(&chand->state_tracker, state, state_error,
+                              reason);
+  if (chand->channelz_node != nullptr) {
+    chand->channelz_node->AddTraceEvent(
+        grpc_core::channelz::ChannelTrace::Severity::Info,
+        grpc_slice_from_static_string(
+            get_channel_connectivity_state_change_string(state)));
+  }
+  // Update picker.
+  chand->picker = std::move(picker);
+  // Re-process queued picks.
+  for (QueuedPick* pick = chand->queued_picks; pick != nullptr;
+       pick = pick->next) {
+    start_pick_locked(pick->elem, GRPC_ERROR_NONE);
+  }
+}
+
+namespace grpc_core {
+namespace {
+
+class ClientChannelControlHelper
+    : public LoadBalancingPolicy::ChannelControlHelper {
+ public:
+  explicit ClientChannelControlHelper(channel_data* chand) : chand_(chand) {
+    GRPC_CHANNEL_STACK_REF(chand_->owning_stack, "ClientChannelControlHelper");
+  }
+
+  ~ClientChannelControlHelper() override {
+    GRPC_CHANNEL_STACK_UNREF(chand_->owning_stack,
+                             "ClientChannelControlHelper");
+  }
+
+  Subchannel* CreateSubchannel(const grpc_channel_args& args) override {
+    grpc_arg arg = SubchannelPoolInterface::CreateChannelArg(
+        chand_->subchannel_pool.get());
+    grpc_channel_args* new_args =
+        grpc_channel_args_copy_and_add(&args, &arg, 1);
+    Subchannel* subchannel = grpc_client_channel_factory_create_subchannel(
+        chand_->client_channel_factory, new_args);
+    grpc_channel_args_destroy(new_args);
+    return subchannel;
+  }
+
+  grpc_channel* CreateChannel(const char* target, grpc_client_channel_type type,
+                              const grpc_channel_args& args) override {
+    return grpc_client_channel_factory_create_channel(
+        chand_->client_channel_factory, target, type, &args);
+  }
+
+  void UpdateState(
+      grpc_connectivity_state state, grpc_error* state_error,
+      UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker) override {
+    if (grpc_client_channel_trace.enabled()) {
+      const char* extra = chand_->disconnect_error == GRPC_ERROR_NONE
+                              ? ""
+                              : " (ignoring -- channel shutting down)";
+      gpr_log(GPR_INFO, "chand=%p: update: state=%s error=%s picker=%p%s",
+              chand_, grpc_connectivity_state_name(state),
+              grpc_error_string(state_error), picker.get(), extra);
+    }
+    // Do update only if not shutting down.
+    if (chand_->disconnect_error == GRPC_ERROR_NONE) {
+      set_connectivity_state_and_picker_locked(chand_, state, state_error,
+                                               "helper", std::move(picker));
+    } else {
+      GRPC_ERROR_UNREF(state_error);
+    }
+  }
+
+  // No-op -- we should never get this from ResolvingLoadBalancingPolicy.
+  void RequestReresolution() override {}
+
+ private:
+  channel_data* chand_;
+};
+
+}  // namespace
+}  // namespace grpc_core
+
+// Synchronous callback from chand->resolving_lb_policy to process a resolver
 // result update.
 static bool process_resolver_result_locked(void* arg,
                                            const grpc_channel_args& args,
                                            const char** lb_policy_name,
                                            grpc_json** lb_policy_config) {
   channel_data* chand = static_cast<channel_data*>(arg);
+  chand->have_service_config = true;
   ProcessedResolverResult resolver_result(args, chand->enable_retries);
   grpc_core::UniquePtr<char> service_config_json =
       resolver_result.service_config_json();
@@ -148,9 +279,38 @@ static bool process_resolver_result_locked(void* arg,
   // Return results.
   *lb_policy_name = chand->info_lb_policy_name.get();
   *lb_policy_config = resolver_result.lb_policy_config();
+  // Apply service config to queued picks.
+  for (QueuedPick* pick = chand->queued_picks; pick != nullptr;
+       pick = pick->next) {
+    maybe_apply_service_config_to_call_locked(pick->elem);
+  }
   return service_config_changed;
 }
 
+static grpc_error* do_ping_locked(channel_data* chand, grpc_transport_op* op) {
+  grpc_error* error = GRPC_ERROR_NONE;
+  grpc_connectivity_state state =
+      grpc_connectivity_state_get(&chand->state_tracker, &error);
+  if (state != GRPC_CHANNEL_READY) {
+    grpc_error* new_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+        "channel not connected", &error, 1);
+    GRPC_ERROR_UNREF(error);
+    return new_error;
+  }
+  LoadBalancingPolicy::PickState pick;
+  chand->picker->Pick(&pick, &error);
+  if (pick.connected_subchannel != nullptr) {
+    pick.connected_subchannel->Ping(op->send_ping.on_initiate,
+                                    op->send_ping.on_ack);
+  } else {
+    if (error == GRPC_ERROR_NONE) {
+      error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "LB policy dropped call on ping");
+    }
+  }
+  return error;
+}
+
 static void start_transport_op_locked(void* arg, grpc_error* error_ignored) {
   grpc_transport_op* op = static_cast<grpc_transport_op*>(arg);
   grpc_channel_element* elem =
@@ -158,47 +318,40 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
 
   if (op->on_connectivity_state_change != nullptr) {
-    chand->request_router->NotifyOnConnectivityStateChange(
-        op->connectivity_state, op->on_connectivity_state_change);
+    grpc_connectivity_state_notify_on_state_change(
+        &chand->state_tracker, op->connectivity_state,
+        op->on_connectivity_state_change);
     op->on_connectivity_state_change = nullptr;
     op->connectivity_state = nullptr;
   }
 
   if (op->send_ping.on_initiate != nullptr || op->send_ping.on_ack != nullptr) {
-    if (chand->request_router->lb_policy() == nullptr) {
-      grpc_error* error =
-          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Ping with no load balancing");
+    grpc_error* error = do_ping_locked(chand, op);
+    if (error != GRPC_ERROR_NONE) {
       GRPC_CLOSURE_SCHED(op->send_ping.on_initiate, GRPC_ERROR_REF(error));
       GRPC_CLOSURE_SCHED(op->send_ping.on_ack, error);
-    } else {
-      grpc_error* error = GRPC_ERROR_NONE;
-      grpc_core::LoadBalancingPolicy::PickState pick_state;
-      // Pick must return synchronously, because pick_state.on_complete is null.
-      GPR_ASSERT(
-          chand->request_router->lb_policy()->PickLocked(&pick_state, &error));
-      if (pick_state.connected_subchannel != nullptr) {
-        pick_state.connected_subchannel->Ping(op->send_ping.on_initiate,
-                                              op->send_ping.on_ack);
-      } else {
-        if (error == GRPC_ERROR_NONE) {
-          error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-              "LB policy dropped call on ping");
-        }
-        GRPC_CLOSURE_SCHED(op->send_ping.on_initiate, GRPC_ERROR_REF(error));
-        GRPC_CLOSURE_SCHED(op->send_ping.on_ack, error);
-      }
-      op->bind_pollset = nullptr;
     }
+    op->bind_pollset = nullptr;
     op->send_ping.on_initiate = nullptr;
     op->send_ping.on_ack = nullptr;
   }
 
-  if (op->disconnect_with_error != GRPC_ERROR_NONE) {
-    chand->request_router->ShutdownLocked(op->disconnect_with_error);
+  if (op->reset_connect_backoff) {
+    chand->resolving_lb_policy->ResetBackoffLocked();
   }
 
-  if (op->reset_connect_backoff) {
-    chand->request_router->ResetConnectionBackoffLocked();
+  if (op->disconnect_with_error != GRPC_ERROR_NONE) {
+    chand->disconnect_error = op->disconnect_with_error;
+    grpc_pollset_set_del_pollset_set(
+        chand->resolving_lb_policy->interested_parties(),
+        chand->interested_parties);
+    chand->resolving_lb_policy.reset();
+    set_connectivity_state_and_picker_locked(
+        chand, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(op->disconnect_with_error),
+        "shutdown from API",
+        grpc_core::UniquePtr<LoadBalancingPolicy::SubchannelPicker>(
+            grpc_core::New<LoadBalancingPolicy::TransientFailurePicker>(
+                GRPC_ERROR_REF(op->disconnect_with_error))));
   }
 
   GRPC_CHANNEL_STACK_UNREF(chand->owning_stack, "start_transport_op");
@@ -244,6 +397,9 @@ static grpc_error* cc_init_channel_elem(grpc_channel_element* elem,
   GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
   // Initialize data members.
   chand->combiner = grpc_combiner_create();
+  grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
+                               "client_channel");
+  chand->disconnect_error = GRPC_ERROR_NONE;
   gpr_mu_init(&chand->info_mu);
   gpr_mu_init(&chand->external_connectivity_watcher_list_mu);
 
@@ -275,8 +431,9 @@ static grpc_error* cc_init_channel_elem(grpc_channel_element* elem,
     return GRPC_ERROR_CREATE_FROM_STATIC_STRING(
         "client channel factory arg must be a pointer");
   }
-  grpc_client_channel_factory* client_channel_factory =
+  chand->client_channel_factory =
       static_cast<grpc_client_channel_factory*>(arg->value.pointer.p);
+  grpc_client_channel_factory_ref(chand->client_channel_factory);
   // Get server name to resolve, using proxy mapper if needed.
   arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVER_URI);
   if (arg == nullptr) {
@@ -291,26 +448,71 @@ static grpc_error* cc_init_channel_elem(grpc_channel_element* elem,
   grpc_channel_args* new_args = nullptr;
   grpc_proxy_mappers_map_name(arg->value.string, args->channel_args,
                               &proxy_name, &new_args);
-  // Instantiate request router.
-  grpc_client_channel_factory_ref(client_channel_factory);
+  grpc_core::UniquePtr<char> target_uri(
+      proxy_name != nullptr ? proxy_name : gpr_strdup(arg->value.string));
+  // Instantiate subchannel pool.
+  arg = grpc_channel_args_find(args->channel_args,
+                               GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL);
+  if (grpc_channel_arg_get_bool(arg, false)) {
+    chand->subchannel_pool =
+        grpc_core::MakeRefCounted<grpc_core::LocalSubchannelPool>();
+  } else {
+    chand->subchannel_pool = grpc_core::GlobalSubchannelPool::instance();
+  }
+  // Instantiate resolving LB policy.
+  LoadBalancingPolicy::Args lb_args;
+  lb_args.combiner = chand->combiner;
+  lb_args.channel_control_helper =
+      grpc_core::UniquePtr<LoadBalancingPolicy::ChannelControlHelper>(
+          grpc_core::New<grpc_core::ClientChannelControlHelper>(chand));
+  lb_args.args = new_args != nullptr ? new_args : args->channel_args;
   grpc_error* error = GRPC_ERROR_NONE;
-  chand->request_router.Init(
-      chand->owning_stack, chand->combiner, client_channel_factory,
-      chand->interested_parties, &grpc_client_channel_trace,
-      process_resolver_result_locked, chand,
-      proxy_name != nullptr ? proxy_name : arg->value.string /* target_uri */,
-      new_args != nullptr ? new_args : args->channel_args, &error);
-  gpr_free(proxy_name);
+  chand->resolving_lb_policy.reset(
+      grpc_core::New<grpc_core::ResolvingLoadBalancingPolicy>(
+          std::move(lb_args), &grpc_client_channel_trace, std::move(target_uri),
+          process_resolver_result_locked, chand, &error));
   grpc_channel_args_destroy(new_args);
+  if (error != GRPC_ERROR_NONE) {
+    // Orphan the resolving LB policy and flush the exec_ctx to ensure
+    // that it finishes shutting down.  This ensures that if we are
+    // failing, we destroy the ClientChannelControlHelper (and thus
+    // unref the channel stack) before we return.
+    // TODO(roth): This is not a complete solution, because it only
+    // catches the case where channel stack initialization fails in this
+    // particular filter.  If there is a failure in a different filter, we
+    // will leave a dangling ref here, which can cause a crash.  Fortunately,
+    // in practice, there are no other filters that can cause failures in
+    // channel stack initialization, so this works for now.
+    chand->resolving_lb_policy.reset();
+    grpc_core::ExecCtx::Get()->Flush();
+  } else {
+    grpc_pollset_set_add_pollset_set(
+        chand->resolving_lb_policy->interested_parties(),
+        chand->interested_parties);
+    if (grpc_client_channel_trace.enabled()) {
+      gpr_log(GPR_INFO, "chand=%p: created resolving_lb_policy=%p", chand,
+              chand->resolving_lb_policy.get());
+    }
+  }
   return error;
 }
 
 /* Destructor for channel_data */
 static void cc_destroy_channel_elem(grpc_channel_element* elem) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  chand->request_router.Destroy();
+  if (chand->resolving_lb_policy != nullptr) {
+    grpc_pollset_set_del_pollset_set(
+        chand->resolving_lb_policy->interested_parties(),
+        chand->interested_parties);
+    chand->resolving_lb_policy.reset();
+  }
   // TODO(roth): Once we convert the filter API to C++, there will no
   // longer be any need to explicitly reset these smart pointer data members.
+  chand->picker.reset();
+  chand->subchannel_pool.reset();
+  if (chand->client_channel_factory != nullptr) {
+    grpc_client_channel_factory_unref(chand->client_channel_factory);
+  }
   chand->info_lb_policy_name.reset();
   chand->info_service_config_json.reset();
   chand->retry_throttle_data.reset();
@@ -318,6 +520,8 @@ static void cc_destroy_channel_elem(grpc_channel_element* elem) {
   grpc_client_channel_stop_backup_polling(chand->interested_parties);
   grpc_pollset_set_destroy(chand->interested_parties);
   GRPC_COMBINER_UNREF(chand->combiner, "client_channel");
+  GRPC_ERROR_UNREF(chand->disconnect_error);
+  grpc_connectivity_state_destroy(&chand->state_tracker);
   gpr_mu_destroy(&chand->info_mu);
   gpr_mu_destroy(&chand->external_connectivity_watcher_list_mu);
 }
@@ -371,6 +575,12 @@ static void cc_destroy_channel_elem(grpc_channel_element* elem) {
 //   (census filter is on top of this one)
 // - add census stats for retries
 
+namespace grpc_core {
+namespace {
+class QueuedPickCanceller;
+}  // namespace
+}  // namespace grpc_core
+
 namespace {
 
 struct call_data;
@@ -509,8 +719,11 @@ struct call_data {
     for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches); ++i) {
       GPR_ASSERT(pending_batches[i].batch == nullptr);
     }
-    if (have_request) {
-      request.Destroy();
+    for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
+      if (pick.pick.subchannel_call_context[i].destroy != nullptr) {
+        pick.pick.subchannel_call_context[i].destroy(
+            pick.pick.subchannel_call_context[i].value);
+      }
     }
   }
 
@@ -537,8 +750,10 @@ struct call_data {
   // Set when we get a cancel_stream op.
   grpc_error* cancel_error = GRPC_ERROR_NONE;
 
-  grpc_core::ManualConstructor<grpc_core::RequestRouter::Request> request;
-  bool have_request = false;
+  QueuedPick pick;
+  bool pick_queued = false;
+  bool service_config_applied = false;
+  grpc_core::QueuedPickCanceller* pick_canceller = nullptr;
   grpc_closure pick_closure;
 
   grpc_polling_entity* pollent = nullptr;
@@ -600,7 +815,7 @@ static void retry_commit(grpc_call_element* elem,
 static void start_internal_recv_trailing_metadata(grpc_call_element* elem);
 static void on_complete(void* arg, grpc_error* error);
 static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored);
-static void start_pick_locked(void* arg, grpc_error* ignored);
+static void remove_call_from_queued_picks_locked(grpc_call_element* elem);
 
 //
 // send op data caching
@@ -728,7 +943,7 @@ static void free_cached_send_op_data_for_completed_batch(
 //
 
 void maybe_inject_recv_trailing_metadata_ready_for_lb(
-    const grpc_core::LoadBalancingPolicy::PickState& pick,
+    const LoadBalancingPolicy::PickState& pick,
     grpc_transport_stream_op_batch* batch) {
   if (pick.recv_trailing_metadata_ready != nullptr) {
     *pick.original_recv_trailing_metadata_ready =
@@ -846,10 +1061,25 @@ static void fail_pending_batch_in_call_combiner(void* arg, grpc_error* error) {
 }
 
 // This is called via the call combiner, so access to calld is synchronized.
-// If yield_call_combiner is true, assumes responsibility for yielding
-// the call combiner.
-static void pending_batches_fail(grpc_call_element* elem, grpc_error* error,
-                                 bool yield_call_combiner) {
+// If yield_call_combiner_predicate returns true, assumes responsibility for
+// yielding the call combiner.
+typedef bool (*YieldCallCombinerPredicate)(
+    const grpc_core::CallCombinerClosureList& closures);
+static bool yield_call_combiner(
+    const grpc_core::CallCombinerClosureList& closures) {
+  return true;
+}
+static bool no_yield_call_combiner(
+    const grpc_core::CallCombinerClosureList& closures) {
+  return false;
+}
+static bool yield_call_combiner_if_pending_batches_found(
+    const grpc_core::CallCombinerClosureList& closures) {
+  return closures.size() > 0;
+}
+static void pending_batches_fail(
+    grpc_call_element* elem, grpc_error* error,
+    YieldCallCombinerPredicate yield_call_combiner_predicate) {
   GPR_ASSERT(error != GRPC_ERROR_NONE);
   call_data* calld = static_cast<call_data*>(elem->call_data);
   if (grpc_client_channel_trace.enabled()) {
@@ -866,9 +1096,9 @@ static void pending_batches_fail(grpc_call_element* elem, grpc_error* error,
     pending_batch* pending = &calld->pending_batches[i];
     grpc_transport_stream_op_batch* batch = pending->batch;
     if (batch != nullptr) {
-      if (batch->recv_trailing_metadata && calld->have_request) {
-        maybe_inject_recv_trailing_metadata_ready_for_lb(
-            *calld->request->pick(), batch);
+      if (batch->recv_trailing_metadata) {
+        maybe_inject_recv_trailing_metadata_ready_for_lb(calld->pick.pick,
+                                                         batch);
       }
       batch->handler_private.extra_arg = calld;
       GRPC_CLOSURE_INIT(&batch->handler_private.closure,
@@ -879,7 +1109,7 @@ static void pending_batches_fail(grpc_call_element* elem, grpc_error* error,
       pending_batch_clear(calld, pending);
     }
   }
-  if (yield_call_combiner) {
+  if (yield_call_combiner_predicate(closures)) {
     closures.RunClosures(calld->call_combiner);
   } else {
     closures.RunClosuresWithoutYielding(calld->call_combiner);
@@ -923,8 +1153,8 @@ static void pending_batches_resume(grpc_call_element* elem) {
     grpc_transport_stream_op_batch* batch = pending->batch;
     if (batch != nullptr) {
       if (batch->recv_trailing_metadata) {
-        maybe_inject_recv_trailing_metadata_ready_for_lb(
-            *calld->request->pick(), batch);
+        maybe_inject_recv_trailing_metadata_ready_for_lb(calld->pick.pick,
+                                                         batch);
       }
       batch->handler_private.extra_arg = calld->subchannel_call.get();
       GRPC_CLOSURE_INIT(&batch->handler_private.closure,
@@ -1015,11 +1245,9 @@ static void do_retry(grpc_call_element* elem,
   const ClientChannelMethodParams::RetryPolicy* retry_policy =
       calld->method_params->retry_policy();
   GPR_ASSERT(retry_policy != nullptr);
+  // Reset subchannel call and connected subchannel.
   calld->subchannel_call.reset();
-  if (calld->have_request) {
-    calld->have_request = false;
-    calld->request.Destroy();
-  }
+  calld->pick.pick.connected_subchannel.reset();
   // Compute backoff delay.
   grpc_millis next_attempt_time;
   if (server_pushback_ms >= 0) {
@@ -1938,7 +2166,7 @@ static void add_retriable_recv_trailing_metadata_op(
   batch_data->batch.payload->recv_trailing_metadata
       .recv_trailing_metadata_ready =
       &retry_state->recv_trailing_metadata_ready;
-  maybe_inject_recv_trailing_metadata_ready_for_lb(*calld->request->pick(),
+  maybe_inject_recv_trailing_metadata_ready_for_lb(calld->pick.pick,
                                                    &batch_data->batch);
 }
 
@@ -2207,41 +2435,38 @@ static void start_retriable_subchannel_batches(void* arg, grpc_error* ignored) {
 // LB pick
 //
 
-static void create_subchannel_call(grpc_call_element* elem, grpc_error* error) {
+static void create_subchannel_call(grpc_call_element* elem) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
   const size_t parent_data_size =
       calld->enable_retries ? sizeof(subchannel_call_retry_state) : 0;
   const grpc_core::ConnectedSubchannel::CallArgs call_args = {
-      calld->pollent,                                   // pollent
-      calld->path,                                      // path
-      calld->call_start_time,                           // start_time
-      calld->deadline,                                  // deadline
-      calld->arena,                                     // arena
-      calld->request->pick()->subchannel_call_context,  // context
-      calld->call_combiner,                             // call_combiner
-      parent_data_size                                  // parent_data_size
+      calld->pollent,                            // pollent
+      calld->path,                               // path
+      calld->call_start_time,                    // start_time
+      calld->deadline,                           // deadline
+      calld->arena,                              // arena
+      calld->pick.pick.subchannel_call_context,  // context
+      calld->call_combiner,                      // call_combiner
+      parent_data_size                           // parent_data_size
   };
-  grpc_error* new_error = GRPC_ERROR_NONE;
+  grpc_error* error = GRPC_ERROR_NONE;
   calld->subchannel_call =
-      calld->request->pick()->connected_subchannel->CreateCall(call_args,
-                                                               &new_error);
+      calld->pick.pick.connected_subchannel->CreateCall(call_args, &error);
   if (grpc_client_channel_trace.enabled()) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
             chand, calld, calld->subchannel_call.get(),
-            grpc_error_string(new_error));
+            grpc_error_string(error));
   }
-  if (GPR_UNLIKELY(new_error != GRPC_ERROR_NONE)) {
-    new_error = grpc_error_add_child(new_error, error);
-    pending_batches_fail(elem, new_error, true /* yield_call_combiner */);
+  if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) {
+    pending_batches_fail(elem, error, yield_call_combiner);
   } else {
     if (parent_data_size > 0) {
-      new (calld->subchannel_call->GetParentData()) subchannel_call_retry_state(
-          calld->request->pick()->subchannel_call_context);
+      new (calld->subchannel_call->GetParentData())
+          subchannel_call_retry_state(calld->pick.pick.subchannel_call_context);
     }
     pending_batches_resume(elem);
   }
-  GRPC_ERROR_UNREF(error);
 }
 
 // Invoked when a pick is completed, on both success or failure.
@@ -2249,54 +2474,106 @@ static void pick_done(void* arg, grpc_error* error) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  if (GPR_UNLIKELY(calld->request->pick()->connected_subchannel == nullptr)) {
-    // Failed to create subchannel.
-    // If there was no error, this is an LB policy drop, in which case
-    // we return an error; otherwise, we may retry.
-    grpc_status_code status = GRPC_STATUS_OK;
-    grpc_error_get_status(error, calld->deadline, &status, nullptr, nullptr,
-                          nullptr);
-    if (error == GRPC_ERROR_NONE || !calld->enable_retries ||
-        !maybe_retry(elem, nullptr /* batch_data */, status,
-                     nullptr /* server_pushback_md */)) {
-      grpc_error* new_error =
-          error == GRPC_ERROR_NONE
-              ? GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                    "Call dropped by load balancing policy")
-              : GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                    "Failed to create subchannel", &error, 1);
+  if (error != GRPC_ERROR_NONE) {
+    if (grpc_client_channel_trace.enabled()) {
+      gpr_log(GPR_INFO,
+              "chand=%p calld=%p: failed to pick subchannel: error=%s", chand,
+              calld, grpc_error_string(error));
+    }
+    pending_batches_fail(elem, GRPC_ERROR_REF(error), yield_call_combiner);
+    return;
+  }
+  create_subchannel_call(elem);
+}
+
+namespace grpc_core {
+namespace {
+
+// A class to handle the call combiner cancellation callback for a
+// queued pick.
+class QueuedPickCanceller {
+ public:
+  explicit QueuedPickCanceller(grpc_call_element* elem) : elem_(elem) {
+    auto* calld = static_cast<call_data*>(elem->call_data);
+    auto* chand = static_cast<channel_data*>(elem->channel_data);
+    GRPC_CALL_STACK_REF(calld->owning_call, "QueuedPickCanceller");
+    GRPC_CLOSURE_INIT(&closure_, &CancelLocked, this,
+                      grpc_combiner_scheduler(chand->combiner));
+    grpc_call_combiner_set_notify_on_cancel(calld->call_combiner, &closure_);
+  }
+
+ private:
+  static void CancelLocked(void* arg, grpc_error* error) {
+    auto* self = static_cast<QueuedPickCanceller*>(arg);
+    auto* chand = static_cast<channel_data*>(self->elem_->channel_data);
+    auto* calld = static_cast<call_data*>(self->elem_->call_data);
+    if (grpc_client_channel_trace.enabled()) {
+      gpr_log(GPR_INFO,
+              "chand=%p calld=%p: cancelling queued pick: "
+              "error=%s self=%p calld->pick_canceller=%p",
+              chand, calld, grpc_error_string(error), self,
+              calld->pick_canceller);
+    }
+    if (calld->pick_canceller == self && error != GRPC_ERROR_NONE) {
+      // Remove pick from list of queued picks.
+      remove_call_from_queued_picks_locked(self->elem_);
+      // Fail pending batches on the call.
+      pending_batches_fail(self->elem_, GRPC_ERROR_REF(error),
+                           yield_call_combiner_if_pending_batches_found);
+    }
+    GRPC_CALL_STACK_UNREF(calld->owning_call, "QueuedPickCanceller");
+    Delete(self);
+  }
+
+  grpc_call_element* elem_;
+  grpc_closure closure_;
+};
+
+}  // namespace
+}  // namespace grpc_core
+
+// Removes the call from the channel's list of queued picks.
+static void remove_call_from_queued_picks_locked(grpc_call_element* elem) {
+  auto* chand = static_cast<channel_data*>(elem->channel_data);
+  auto* calld = static_cast<call_data*>(elem->call_data);
+  for (QueuedPick** pick = &chand->queued_picks; *pick != nullptr;
+       pick = &(*pick)->next) {
+    if (*pick == &calld->pick) {
       if (grpc_client_channel_trace.enabled()) {
-        gpr_log(GPR_INFO,
-                "chand=%p calld=%p: failed to create subchannel: error=%s",
-                chand, calld, grpc_error_string(new_error));
+        gpr_log(GPR_INFO, "chand=%p calld=%p: removing from queued picks list",
+                chand, calld);
       }
-      pending_batches_fail(elem, new_error, true /* yield_call_combiner */);
+      calld->pick_queued = false;
+      *pick = calld->pick.next;
+      // Remove call's pollent from channel's interested_parties.
+      grpc_polling_entity_del_from_pollset_set(calld->pollent,
+                                               chand->interested_parties);
+      // Lame the call combiner canceller.
+      calld->pick_canceller = nullptr;
+      break;
     }
-  } else {
-    /* Create call on subchannel. */
-    create_subchannel_call(elem, GRPC_ERROR_REF(error));
   }
 }
 
-// If the channel is in TRANSIENT_FAILURE and the call is not
-// wait_for_ready=true, fails the call and returns true.
-static bool fail_call_if_in_transient_failure(grpc_call_element* elem) {
-  channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  call_data* calld = static_cast<call_data*>(elem->call_data);
-  grpc_transport_stream_op_batch* batch = calld->pending_batches[0].batch;
-  if (chand->request_router->GetConnectivityState() ==
-          GRPC_CHANNEL_TRANSIENT_FAILURE &&
-      (batch->payload->send_initial_metadata.send_initial_metadata_flags &
-       GRPC_INITIAL_METADATA_WAIT_FOR_READY) == 0) {
-    pending_batches_fail(
-        elem,
-        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                               "channel is in state TRANSIENT_FAILURE"),
-                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
-        true /* yield_call_combiner */);
-    return true;
+// Adds the call to the channel's list of queued picks.
+static void add_call_to_queued_picks_locked(grpc_call_element* elem) {
+  auto* chand = static_cast<channel_data*>(elem->channel_data);
+  auto* calld = static_cast<call_data*>(elem->call_data);
+  if (grpc_client_channel_trace.enabled()) {
+    gpr_log(GPR_INFO, "chand=%p calld=%p: adding to queued picks list", chand,
+            calld);
   }
-  return false;
+  calld->pick_queued = true;
+  // Add call to queued picks list.
+  calld->pick.elem = elem;
+  calld->pick.next = chand->queued_picks;
+  chand->queued_picks = &calld->pick;
+  // Add call's pollent to channel's interested_parties, so that I/O
+  // can be done under the call's CQ.
+  grpc_polling_entity_add_to_pollset_set(calld->pollent,
+                                         chand->interested_parties);
+  // Register call combiner cancellation callback.
+  calld->pick_canceller = grpc_core::New<grpc_core::QueuedPickCanceller>(elem);
 }
 
 // Applies service config to the call.  Must be invoked once we know
@@ -2356,36 +2633,37 @@ static void apply_service_config_to_call_locked(grpc_call_element* elem) {
 }
 
 // Invoked once resolver results are available.
-static bool maybe_apply_service_config_to_call_locked(void* arg) {
-  grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
+static void maybe_apply_service_config_to_call_locked(grpc_call_element* elem) {
+  channel_data* chand = static_cast<channel_data*>(elem->channel_data);
   call_data* calld = static_cast<call_data*>(elem->call_data);
-  // Only get service config data on the first attempt.
-  if (GPR_LIKELY(calld->num_attempts_completed == 0)) {
+  // Apply service config data to the call only once, and only if the
+  // channel has the data available.
+  if (GPR_LIKELY(chand->have_service_config &&
+                 !calld->service_config_applied)) {
+    calld->service_config_applied = true;
     apply_service_config_to_call_locked(elem);
-    // Check this after applying service config, since it may have
-    // affected the call's wait_for_ready value.
-    if (fail_call_if_in_transient_failure(elem)) return false;
   }
-  return true;
 }
 
-static void start_pick_locked(void* arg, grpc_error* ignored) {
+static const char* pick_result_name(
+    LoadBalancingPolicy::SubchannelPicker::PickResult result) {
+  switch (result) {
+    case LoadBalancingPolicy::SubchannelPicker::PICK_COMPLETE:
+      return "COMPLETE";
+    case LoadBalancingPolicy::SubchannelPicker::PICK_QUEUE:
+      return "QUEUE";
+    case LoadBalancingPolicy::SubchannelPicker::PICK_TRANSIENT_FAILURE:
+      return "TRANSIENT_FAILURE";
+  }
+  GPR_UNREACHABLE_CODE(return "UNKNOWN");
+}
+
+static void start_pick_locked(void* arg, grpc_error* error) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   call_data* calld = static_cast<call_data*>(elem->call_data);
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  GPR_ASSERT(!calld->have_request);
+  GPR_ASSERT(calld->pick.pick.connected_subchannel == nullptr);
   GPR_ASSERT(calld->subchannel_call == nullptr);
-  // Normally, we want to do this check until after we've processed the
-  // service config, so that we can honor the wait_for_ready setting in
-  // the service config.  However, if the channel is in TRANSIENT_FAILURE
-  // and we don't have an LB policy at this point, that means that the
-  // resolver has returned a failure, so we're not going to get a service
-  // config right away.  In that case, we fail the call now based on the
-  // wait_for_ready value passed in from the application.
-  if (chand->request_router->lb_policy() == nullptr &&
-      fail_call_if_in_transient_failure(elem)) {
-    return;
-  }
   // If this is a retry, use the send_initial_metadata payload that
   // we've cached; otherwise, use the pending batch.  The
   // send_initial_metadata batch will be the first pending batch in the
@@ -2396,25 +2674,78 @@ static void start_pick_locked(void* arg, grpc_error* ignored) {
   // allocate the subchannel batch earlier so that we can give the
   // subchannel's copy of the metadata batch (which is copied for each
   // attempt) to the LB policy instead the one from the parent channel.
-  grpc_metadata_batch* initial_metadata =
+  calld->pick.pick.initial_metadata =
       calld->seen_send_initial_metadata
           ? &calld->send_initial_metadata
           : calld->pending_batches[0]
                 .batch->payload->send_initial_metadata.send_initial_metadata;
-  uint32_t* initial_metadata_flags =
+  uint32_t* send_initial_metadata_flags =
       calld->seen_send_initial_metadata
           ? &calld->send_initial_metadata_flags
           : &calld->pending_batches[0]
                  .batch->payload->send_initial_metadata
                  .send_initial_metadata_flags;
+  // Apply service config to call if needed.
+  maybe_apply_service_config_to_call_locked(elem);
+  // When done, we schedule this closure to leave the channel combiner.
   GRPC_CLOSURE_INIT(&calld->pick_closure, pick_done, elem,
                     grpc_schedule_on_exec_ctx);
-  calld->request.Init(calld->owning_call, calld->call_combiner, calld->pollent,
-                      initial_metadata, initial_metadata_flags,
-                      maybe_apply_service_config_to_call_locked, elem,
-                      &calld->pick_closure);
-  calld->have_request = true;
-  chand->request_router->RouteCallLocked(calld->request.get());
+  // Attempt pick.
+  error = GRPC_ERROR_NONE;
+  auto pick_result = chand->picker->Pick(&calld->pick.pick, &error);
+  if (grpc_client_channel_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "chand=%p calld=%p: LB pick returned %s (connected_subchannel=%p, "
+            "error=%s)",
+            chand, calld, pick_result_name(pick_result),
+            calld->pick.pick.connected_subchannel.get(),
+            grpc_error_string(error));
+  }
+  switch (pick_result) {
+    case LoadBalancingPolicy::SubchannelPicker::PICK_TRANSIENT_FAILURE:
+      // If we're shutting down, fail all RPCs.
+      if (chand->disconnect_error != GRPC_ERROR_NONE) {
+        GRPC_ERROR_UNREF(error);
+        GRPC_CLOSURE_SCHED(&calld->pick_closure,
+                           GRPC_ERROR_REF(chand->disconnect_error));
+        break;
+      }
+      // If wait_for_ready is false, then the error indicates the RPC
+      // attempt's final status.
+      if ((*send_initial_metadata_flags &
+           GRPC_INITIAL_METADATA_WAIT_FOR_READY) == 0) {
+        // Retry if appropriate; otherwise, fail.
+        grpc_status_code status = GRPC_STATUS_OK;
+        grpc_error_get_status(error, calld->deadline, &status, nullptr, nullptr,
+                              nullptr);
+        if (!calld->enable_retries ||
+            !maybe_retry(elem, nullptr /* batch_data */, status,
+                         nullptr /* server_pushback_md */)) {
+          grpc_error* new_error =
+              GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                  "Failed to create subchannel", &error, 1);
+          GRPC_ERROR_UNREF(error);
+          GRPC_CLOSURE_SCHED(&calld->pick_closure, new_error);
+        }
+        if (calld->pick_queued) remove_call_from_queued_picks_locked(elem);
+        break;
+      }
+      // If wait_for_ready is true, then queue to retry when we get a new
+      // picker.
+      GRPC_ERROR_UNREF(error);
+      // Fallthrough
+    case LoadBalancingPolicy::SubchannelPicker::PICK_QUEUE:
+      if (!calld->pick_queued) add_call_to_queued_picks_locked(elem);
+      break;
+    default:  // PICK_COMPLETE
+      // Handle drops.
+      if (GPR_UNLIKELY(calld->pick.pick.connected_subchannel == nullptr)) {
+        error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "Call dropped by load balancing policy");
+      }
+      GRPC_CLOSURE_SCHED(&calld->pick_closure, error);
+      if (calld->pick_queued) remove_call_from_queued_picks_locked(elem);
+  }
 }
 
 //
@@ -2458,8 +2789,10 @@ static void cc_start_transport_stream_op_batch(
     // been started), fail all pending batches.  Otherwise, send the
     // cancellation down to the subchannel call.
     if (calld->subchannel_call == nullptr) {
+      // TODO(roth): If there is a pending retry callback, do we need to
+      // cancel it here?
       pending_batches_fail(elem, GRPC_ERROR_REF(calld->cancel_error),
-                           false /* yield_call_combiner */);
+                           no_yield_call_combiner);
       // Note: This will release the call combiner.
       grpc_transport_stream_op_batch_finish_with_failure(
           batch, GRPC_ERROR_REF(calld->cancel_error), calld->call_combiner);
@@ -2556,7 +2889,8 @@ const grpc_channel_filter grpc_client_channel_filter = {
 void grpc_client_channel_set_channelz_node(
     grpc_channel_element* elem, grpc_core::channelz::ClientChannelNode* node) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  chand->request_router->set_channelz_node(node);
+  chand->channelz_node = node;
+  chand->resolving_lb_policy->set_channelz_node(node->Ref());
 }
 
 void grpc_client_channel_populate_child_refs(
@@ -2564,22 +2898,23 @@ void grpc_client_channel_populate_child_refs(
     grpc_core::channelz::ChildRefsList* child_subchannels,
     grpc_core::channelz::ChildRefsList* child_channels) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  if (chand->request_router->lb_policy() != nullptr) {
-    chand->request_router->lb_policy()->FillChildRefsForChannelz(
-        child_subchannels, child_channels);
+  if (chand->resolving_lb_policy != nullptr) {
+    chand->resolving_lb_policy->FillChildRefsForChannelz(child_subchannels,
+                                                         child_channels);
   }
 }
 
 static void try_to_connect_locked(void* arg, grpc_error* error_ignored) {
   channel_data* chand = static_cast<channel_data*>(arg);
-  chand->request_router->ExitIdleLocked();
+  chand->resolving_lb_policy->ExitIdleLocked();
   GRPC_CHANNEL_STACK_UNREF(chand->owning_stack, "try_to_connect");
 }
 
 grpc_connectivity_state grpc_client_channel_check_connectivity_state(
     grpc_channel_element* elem, int try_to_connect) {
   channel_data* chand = static_cast<channel_data*>(elem->channel_data);
-  grpc_connectivity_state out = chand->request_router->GetConnectivityState();
+  grpc_connectivity_state out =
+      grpc_connectivity_state_check(&chand->state_tracker);
   if (out == GRPC_CHANNEL_IDLE && try_to_connect) {
     GRPC_CHANNEL_STACK_REF(chand->owning_stack, "try_to_connect");
     GRPC_CLOSURE_SCHED(
@@ -2688,15 +3023,15 @@ static void watch_connectivity_state_locked(void* arg,
     GRPC_CLOSURE_RUN(w->watcher_timer_init, GRPC_ERROR_NONE);
     GRPC_CLOSURE_INIT(&w->my_closure, on_external_watch_complete_locked, w,
                       grpc_combiner_scheduler(w->chand->combiner));
-    w->chand->request_router->NotifyOnConnectivityStateChange(w->state,
-                                                              &w->my_closure);
+    grpc_connectivity_state_notify_on_state_change(&w->chand->state_tracker,
+                                                   w->state, &w->my_closure);
   } else {
     GPR_ASSERT(w->watcher_timer_init == nullptr);
     found = lookup_external_connectivity_watcher(w->chand, w->on_complete);
     if (found) {
       GPR_ASSERT(found->on_complete == w->on_complete);
-      found->chand->request_router->NotifyOnConnectivityStateChange(
-          nullptr, &found->my_closure);
+      grpc_connectivity_state_notify_on_state_change(
+          &found->chand->state_tracker, nullptr, &found->my_closure);
     }
     grpc_polling_entity_del_from_pollset_set(&w->pollent,
                                              w->chand->interested_parties);

src/core/ext/filters/client_channel/http_connect_handshaker.cc

@@ -33,151 +33,160 @@
 #include "src/core/lib/channel/handshaker_registry.h"
 #include "src/core/lib/gpr/env.h"
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/mutex_lock.h"
 #include "src/core/lib/http/format_request.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/uri/uri_parser.h"
 
-typedef struct http_connect_handshaker {
-  // Base class.  Must be first.
-  grpc_handshaker base;
+namespace grpc_core {
 
-  gpr_refcount refcount;
-  gpr_mu mu;
+namespace {
 
-  bool shutdown;
+class HttpConnectHandshaker : public Handshaker {
+ public:
+  HttpConnectHandshaker();
+  void Shutdown(grpc_error* why) override;
+  void DoHandshake(grpc_tcp_server_acceptor* acceptor,
+                   grpc_closure* on_handshake_done,
+                   HandshakerArgs* args) override;
+  const char* name() const override { return "http_connect"; }
+
+ private:
+  virtual ~HttpConnectHandshaker();
+  void CleanupArgsForFailureLocked();
+  void HandshakeFailedLocked(grpc_error* error);
+  static void OnWriteDone(void* arg, grpc_error* error);
+  static void OnReadDone(void* arg, grpc_error* error);
+
+  gpr_mu mu_;
+
+  bool is_shutdown_ = false;
   // Endpoint and read buffer to destroy after a shutdown.
-  grpc_endpoint* endpoint_to_destroy;
-  grpc_slice_buffer* read_buffer_to_destroy;
+  grpc_endpoint* endpoint_to_destroy_ = nullptr;
+  grpc_slice_buffer* read_buffer_to_destroy_ = nullptr;
 
   // State saved while performing the handshake.
-  grpc_handshaker_args* args;
-  grpc_closure* on_handshake_done;
+  HandshakerArgs* args_ = nullptr;
+  grpc_closure* on_handshake_done_ = nullptr;
 
   // Objects for processing the HTTP CONNECT request and response.
-  grpc_slice_buffer write_buffer;
-  grpc_closure request_done_closure;
-  grpc_closure response_read_closure;
-  grpc_http_parser http_parser;
-  grpc_http_response http_response;
-} http_connect_handshaker;
+  grpc_slice_buffer write_buffer_;
+  grpc_closure request_done_closure_;
+  grpc_closure response_read_closure_;
+  grpc_http_parser http_parser_;
+  grpc_http_response http_response_;
+};
 
-// Unref and clean up handshaker.
-static void http_connect_handshaker_unref(http_connect_handshaker* handshaker) {
-  if (gpr_unref(&handshaker->refcount)) {
-    gpr_mu_destroy(&handshaker->mu);
-    if (handshaker->endpoint_to_destroy != nullptr) {
-      grpc_endpoint_destroy(handshaker->endpoint_to_destroy);
-    }
-    if (handshaker->read_buffer_to_destroy != nullptr) {
-      grpc_slice_buffer_destroy_internal(handshaker->read_buffer_to_destroy);
-      gpr_free(handshaker->read_buffer_to_destroy);
-    }
-    grpc_slice_buffer_destroy_internal(&handshaker->write_buffer);
-    grpc_http_parser_destroy(&handshaker->http_parser);
-    grpc_http_response_destroy(&handshaker->http_response);
-    gpr_free(handshaker);
+HttpConnectHandshaker::~HttpConnectHandshaker() {
+  gpr_mu_destroy(&mu_);
+  if (endpoint_to_destroy_ != nullptr) {
+    grpc_endpoint_destroy(endpoint_to_destroy_);
+  }
+  if (read_buffer_to_destroy_ != nullptr) {
+    grpc_slice_buffer_destroy_internal(read_buffer_to_destroy_);
+    gpr_free(read_buffer_to_destroy_);
   }
+  grpc_slice_buffer_destroy_internal(&write_buffer_);
+  grpc_http_parser_destroy(&http_parser_);
+  grpc_http_response_destroy(&http_response_);
 }
 
 // Set args fields to nullptr, saving the endpoint and read buffer for
 // later destruction.
-static void cleanup_args_for_failure_locked(
-    http_connect_handshaker* handshaker) {
-  handshaker->endpoint_to_destroy = handshaker->args->endpoint;
-  handshaker->args->endpoint = nullptr;
-  handshaker->read_buffer_to_destroy = handshaker->args->read_buffer;
-  handshaker->args->read_buffer = nullptr;
-  grpc_channel_args_destroy(handshaker->args->args);
-  handshaker->args->args = nullptr;
+void HttpConnectHandshaker::CleanupArgsForFailureLocked() {
+  endpoint_to_destroy_ = args_->endpoint;
+  args_->endpoint = nullptr;
+  read_buffer_to_destroy_ = args_->read_buffer;
+  args_->read_buffer = nullptr;
+  grpc_channel_args_destroy(args_->args);
+  args_->args = nullptr;
 }
 
 // If the handshake failed or we're shutting down, clean up and invoke the
 // callback with the error.
-static void handshake_failed_locked(http_connect_handshaker* handshaker,
-                                    grpc_error* error) {
+void HttpConnectHandshaker::HandshakeFailedLocked(grpc_error* error) {
   if (error == GRPC_ERROR_NONE) {
     // If we were shut down after an endpoint operation succeeded but
     // before the endpoint callback was invoked, we need to generate our
     // own error.
     error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Handshaker shutdown");
   }
-  if (!handshaker->shutdown) {
+  if (!is_shutdown_) {
     // TODO(ctiller): It is currently necessary to shutdown endpoints
     // before destroying them, even if we know that there are no
     // pending read/write callbacks.  This should be fixed, at which
     // point this can be removed.
-    grpc_endpoint_shutdown(handshaker->args->endpoint, GRPC_ERROR_REF(error));
+    grpc_endpoint_shutdown(args_->endpoint, GRPC_ERROR_REF(error));
     // Not shutting down, so the handshake failed.  Clean up before
     // invoking the callback.
-    cleanup_args_for_failure_locked(handshaker);
+    CleanupArgsForFailureLocked();
     // Set shutdown to true so that subsequent calls to
     // http_connect_handshaker_shutdown() do nothing.
-    handshaker->shutdown = true;
+    is_shutdown_ = true;
   }
   // Invoke callback.
-  GRPC_CLOSURE_SCHED(handshaker->on_handshake_done, error);
+  GRPC_CLOSURE_SCHED(on_handshake_done_, error);
 }
 
 // Callback invoked when finished writing HTTP CONNECT request.
-static void on_write_done(void* arg, grpc_error* error) {
-  http_connect_handshaker* handshaker =
-      static_cast<http_connect_handshaker*>(arg);
-  gpr_mu_lock(&handshaker->mu);
-  if (error != GRPC_ERROR_NONE || handshaker->shutdown) {
+void HttpConnectHandshaker::OnWriteDone(void* arg, grpc_error* error) {
+  auto* handshaker = static_cast<HttpConnectHandshaker*>(arg);
+  gpr_mu_lock(&handshaker->mu_);
+  if (error != GRPC_ERROR_NONE || handshaker->is_shutdown_) {
     // If the write failed or we're shutting down, clean up and invoke the
     // callback with the error.
-    handshake_failed_locked(handshaker, GRPC_ERROR_REF(error));
-    gpr_mu_unlock(&handshaker->mu);
-    http_connect_handshaker_unref(handshaker);
+    handshaker->HandshakeFailedLocked(GRPC_ERROR_REF(error));
+    gpr_mu_unlock(&handshaker->mu_);
+    handshaker->Unref();
   } else {
     // Otherwise, read the response.
     // The read callback inherits our ref to the handshaker.
-    grpc_endpoint_read(handshaker->args->endpoint,
-                       handshaker->args->read_buffer,
-                       &handshaker->response_read_closure);
-    gpr_mu_unlock(&handshaker->mu);
+    grpc_endpoint_read(handshaker->args_->endpoint,
+                       handshaker->args_->read_buffer,
+                       &handshaker->response_read_closure_);
+    gpr_mu_unlock(&handshaker->mu_);
   }
 }
 
 // Callback invoked for reading HTTP CONNECT response.
-static void on_read_done(void* arg, grpc_error* error) {
-  http_connect_handshaker* handshaker =
-      static_cast<http_connect_handshaker*>(arg);
-  gpr_mu_lock(&handshaker->mu);
-  if (error != GRPC_ERROR_NONE || handshaker->shutdown) {
+void HttpConnectHandshaker::OnReadDone(void* arg, grpc_error* error) {
+  auto* handshaker = static_cast<HttpConnectHandshaker*>(arg);
+
+  gpr_mu_lock(&handshaker->mu_);
+  if (error != GRPC_ERROR_NONE || handshaker->is_shutdown_) {
     // If the read failed or we're shutting down, clean up and invoke the
     // callback with the error.
-    handshake_failed_locked(handshaker, GRPC_ERROR_REF(error));
+    handshaker->HandshakeFailedLocked(GRPC_ERROR_REF(error));
     goto done;
   }
   // Add buffer to parser.
-  for (size_t i = 0; i < handshaker->args->read_buffer->count; ++i) {
-    if (GRPC_SLICE_LENGTH(handshaker->args->read_buffer->slices[i]) > 0) {
+  for (size_t i = 0; i < handshaker->args_->read_buffer->count; ++i) {
+    if (GRPC_SLICE_LENGTH(handshaker->args_->read_buffer->slices[i]) > 0) {
       size_t body_start_offset = 0;
-      error = grpc_http_parser_parse(&handshaker->http_parser,
-                                     handshaker->args->read_buffer->slices[i],
+      error = grpc_http_parser_parse(&handshaker->http_parser_,
+                                     handshaker->args_->read_buffer->slices[i],
                                      &body_start_offset);
       if (error != GRPC_ERROR_NONE) {
-        handshake_failed_locked(handshaker, error);
+        handshaker->HandshakeFailedLocked(error);
         goto done;
       }
-      if (handshaker->http_parser.state == GRPC_HTTP_BODY) {
+      if (handshaker->http_parser_.state == GRPC_HTTP_BODY) {
         // Remove the data we've already read from the read buffer,
         // leaving only the leftover bytes (if any).
         grpc_slice_buffer tmp_buffer;
         grpc_slice_buffer_init(&tmp_buffer);
         if (body_start_offset <
-            GRPC_SLICE_LENGTH(handshaker->args->read_buffer->slices[i])) {
+            GRPC_SLICE_LENGTH(handshaker->args_->read_buffer->slices[i])) {
           grpc_slice_buffer_add(
               &tmp_buffer,
-              grpc_slice_split_tail(&handshaker->args->read_buffer->slices[i],
+              grpc_slice_split_tail(&handshaker->args_->read_buffer->slices[i],
                                     body_start_offset));
         }
         grpc_slice_buffer_addn(&tmp_buffer,
-                               &handshaker->args->read_buffer->slices[i + 1],
-                               handshaker->args->read_buffer->count - i - 1);
-        grpc_slice_buffer_swap(handshaker->args->read_buffer, &tmp_buffer);
+                               &handshaker->args_->read_buffer->slices[i + 1],
+                               handshaker->args_->read_buffer->count - i - 1);
+        grpc_slice_buffer_swap(handshaker->args_->read_buffer, &tmp_buffer);
         grpc_slice_buffer_destroy_internal(&tmp_buffer);
         break;
       }
@@ -194,64 +203,53 @@ static void on_read_done(void* arg, grpc_error* error) {
   // need to fix the HTTP parser to understand when the body is
   // complete (e.g., handling chunked transfer encoding or looking
   // at the Content-Length: header).
-  if (handshaker->http_parser.state != GRPC_HTTP_BODY) {
-    grpc_slice_buffer_reset_and_unref_internal(handshaker->args->read_buffer);
-    grpc_endpoint_read(handshaker->args->endpoint,
-                       handshaker->args->read_buffer,
-                       &handshaker->response_read_closure);
-    gpr_mu_unlock(&handshaker->mu);
+  if (handshaker->http_parser_.state != GRPC_HTTP_BODY) {
+    grpc_slice_buffer_reset_and_unref_internal(handshaker->args_->read_buffer);
+    grpc_endpoint_read(handshaker->args_->endpoint,
+                       handshaker->args_->read_buffer,
+                       &handshaker->response_read_closure_);
+    gpr_mu_unlock(&handshaker->mu_);
     return;
   }
   // Make sure we got a 2xx response.
-  if (handshaker->http_response.status < 200 ||
-      handshaker->http_response.status >= 300) {
+  if (handshaker->http_response_.status < 200 ||
+      handshaker->http_response_.status >= 300) {
     char* msg;
     gpr_asprintf(&msg, "HTTP proxy returned response code %d",
-                 handshaker->http_response.status);
+                 handshaker->http_response_.status);
     error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg);
     gpr_free(msg);
-    handshake_failed_locked(handshaker, error);
+    handshaker->HandshakeFailedLocked(error);
     goto done;
   }
   // Success.  Invoke handshake-done callback.
-  GRPC_CLOSURE_SCHED(handshaker->on_handshake_done, error);
+  GRPC_CLOSURE_SCHED(handshaker->on_handshake_done_, error);
 done:
   // Set shutdown to true so that subsequent calls to
   // http_connect_handshaker_shutdown() do nothing.
-  handshaker->shutdown = true;
-  gpr_mu_unlock(&handshaker->mu);
-  http_connect_handshaker_unref(handshaker);
+  handshaker->is_shutdown_ = true;
+  gpr_mu_unlock(&handshaker->mu_);
+  handshaker->Unref();
 }
 
 //
 // Public handshaker methods
 //
 
-static void http_connect_handshaker_destroy(grpc_handshaker* handshaker_in) {
-  http_connect_handshaker* handshaker =
-      reinterpret_cast<http_connect_handshaker*>(handshaker_in);
-  http_connect_handshaker_unref(handshaker);
-}
-
-static void http_connect_handshaker_shutdown(grpc_handshaker* handshaker_in,
-                                             grpc_error* why) {
-  http_connect_handshaker* handshaker =
-      reinterpret_cast<http_connect_handshaker*>(handshaker_in);
-  gpr_mu_lock(&handshaker->mu);
-  if (!handshaker->shutdown) {
-    handshaker->shutdown = true;
-    grpc_endpoint_shutdown(handshaker->args->endpoint, GRPC_ERROR_REF(why));
-    cleanup_args_for_failure_locked(handshaker);
+void HttpConnectHandshaker::Shutdown(grpc_error* why) {
+  gpr_mu_lock(&mu_);
+  if (!is_shutdown_) {
+    is_shutdown_ = true;
+    grpc_endpoint_shutdown(args_->endpoint, GRPC_ERROR_REF(why));
+    CleanupArgsForFailureLocked();
   }
-  gpr_mu_unlock(&handshaker->mu);
+  gpr_mu_unlock(&mu_);
   GRPC_ERROR_UNREF(why);
 }
 
-static void http_connect_handshaker_do_handshake(
-    grpc_handshaker* handshaker_in, grpc_tcp_server_acceptor* acceptor,
-    grpc_closure* on_handshake_done, grpc_handshaker_args* args) {
-  http_connect_handshaker* handshaker =
-      reinterpret_cast<http_connect_handshaker*>(handshaker_in);
+void HttpConnectHandshaker::DoHandshake(grpc_tcp_server_acceptor* acceptor,
+                                        grpc_closure* on_handshake_done,
+                                        HandshakerArgs* args) {
   // Check for HTTP CONNECT channel arg.
   // If not found, invoke on_handshake_done without doing anything.
   const grpc_arg* arg =
@@ -260,9 +258,9 @@ static void http_connect_handshaker_do_handshake(
   if (server_name == nullptr) {
     // Set shutdown to true so that subsequent calls to
     // http_connect_handshaker_shutdown() do nothing.
-    gpr_mu_lock(&handshaker->mu);
-    handshaker->shutdown = true;
-    gpr_mu_unlock(&handshaker->mu);
+    gpr_mu_lock(&mu_);
+    is_shutdown_ = true;
+    gpr_mu_unlock(&mu_);
     GRPC_CLOSURE_SCHED(on_handshake_done, GRPC_ERROR_NONE);
     return;
   }
@@ -280,6 +278,7 @@ static void http_connect_handshaker_do_handshake(
         gpr_malloc(sizeof(grpc_http_header) * num_header_strings));
     for (size_t i = 0; i < num_header_strings; ++i) {
       char* sep = strchr(header_strings[i], ':');
+
       if (sep == nullptr) {
         gpr_log(GPR_ERROR, "skipping unparseable HTTP CONNECT header: %s",
                 header_strings[i]);
@@ -292,9 +291,9 @@ static void http_connect_handshaker_do_handshake(
     }
   }
   // Save state in the handshaker object.
-  gpr_mu_lock(&handshaker->mu);
-  handshaker->args = args;
-  handshaker->on_handshake_done = on_handshake_done;
+  MutexLock lock(&mu_);
+  args_ = args;
+  on_handshake_done_ = on_handshake_done;
   // Log connection via proxy.
   char* proxy_name = grpc_endpoint_get_peer(args->endpoint);
   gpr_log(GPR_INFO, "Connecting to server %s via HTTP proxy %s", server_name,
@@ -302,15 +301,18 @@ static void http_connect_handshaker_do_handshake(
   gpr_free(proxy_name);
   // Construct HTTP CONNECT request.
   grpc_httpcli_request request;
-  memset(&request, 0, sizeof(request));
   request.host = server_name;
+  request.ssl_host_override = nullptr;
   request.http.method = (char*)"CONNECT";
   request.http.path = server_name;
+  request.http.version = GRPC_HTTP_HTTP10;  // Set by OnReadDone
   request.http.hdrs = headers;
   request.http.hdr_count = num_headers;
+  request.http.body_length = 0;
+  request.http.body = nullptr;
   request.handshaker = &grpc_httpcli_plaintext;
   grpc_slice request_slice = grpc_httpcli_format_connect_request(&request);
-  grpc_slice_buffer_add(&handshaker->write_buffer, request_slice);
+  grpc_slice_buffer_add(&write_buffer_, request_slice);
   // Clean up.
   gpr_free(headers);
   for (size_t i = 0; i < num_header_strings; ++i) {
@@ -318,54 +320,42 @@ static void http_connect_handshaker_do_handshake(
   }
   gpr_free(header_strings);
   // Take a new ref to be held by the write callback.
-  gpr_ref(&handshaker->refcount);
-  grpc_endpoint_write(args->endpoint, &handshaker->write_buffer,
-                      &handshaker->request_done_closure, nullptr);
-  gpr_mu_unlock(&handshaker->mu);
+  Ref().release();
+  grpc_endpoint_write(args->endpoint, &write_buffer_, &request_done_closure_,
+                      nullptr);
 }
 
-static const grpc_handshaker_vtable http_connect_handshaker_vtable = {
-    http_connect_handshaker_destroy, http_connect_handshaker_shutdown,
-    http_connect_handshaker_do_handshake, "http_connect"};
-
-static grpc_handshaker* grpc_http_connect_handshaker_create() {
-  http_connect_handshaker* handshaker =
-      static_cast<http_connect_handshaker*>(gpr_malloc(sizeof(*handshaker)));
-  memset(handshaker, 0, sizeof(*handshaker));
-  grpc_handshaker_init(&http_connect_handshaker_vtable, &handshaker->base);
-  gpr_mu_init(&handshaker->mu);
-  gpr_ref_init(&handshaker->refcount, 1);
-  grpc_slice_buffer_init(&handshaker->write_buffer);
-  GRPC_CLOSURE_INIT(&handshaker->request_done_closure, on_write_done,
-                    handshaker, grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&handshaker->response_read_closure, on_read_done,
-                    handshaker, grpc_schedule_on_exec_ctx);
-  grpc_http_parser_init(&handshaker->http_parser, GRPC_HTTP_RESPONSE,
-                        &handshaker->http_response);
-  return &handshaker->base;
+HttpConnectHandshaker::HttpConnectHandshaker() {
+  gpr_mu_init(&mu_);
+  grpc_slice_buffer_init(&write_buffer_);
+  GRPC_CLOSURE_INIT(&request_done_closure_, &HttpConnectHandshaker::OnWriteDone,
+                    this, grpc_schedule_on_exec_ctx);
+  GRPC_CLOSURE_INIT(&response_read_closure_, &HttpConnectHandshaker::OnReadDone,
+                    this, grpc_schedule_on_exec_ctx);
+  grpc_http_parser_init(&http_parser_, GRPC_HTTP_RESPONSE, &http_response_);
 }
 
 //
 // handshaker factory
 //
 
-static void handshaker_factory_add_handshakers(
-    grpc_handshaker_factory* factory, const grpc_channel_args* args,
-    grpc_pollset_set* interested_parties,
-    grpc_handshake_manager* handshake_mgr) {
-  grpc_handshake_manager_add(handshake_mgr,
-                             grpc_http_connect_handshaker_create());
-}
-
-static void handshaker_factory_destroy(grpc_handshaker_factory* factory) {}
+class HttpConnectHandshakerFactory : public HandshakerFactory {
+ public:
+  void AddHandshakers(const grpc_channel_args* args,
+                      grpc_pollset_set* interested_parties,
+                      HandshakeManager* handshake_mgr) override {
+    handshake_mgr->Add(MakeRefCounted<HttpConnectHandshaker>());
+  }
+  ~HttpConnectHandshakerFactory() override = default;
+};
 
-static const grpc_handshaker_factory_vtable handshaker_factory_vtable = {
-    handshaker_factory_add_handshakers, handshaker_factory_destroy};
+}  // namespace
 
-static grpc_handshaker_factory handshaker_factory = {
-    &handshaker_factory_vtable};
+}  // namespace grpc_core
 
 void grpc_http_connect_register_handshaker_factory() {
-  grpc_handshaker_factory_register(true /* at_start */, HANDSHAKER_CLIENT,
-                                   &handshaker_factory);
+  using namespace grpc_core;
+  HandshakerRegistry::RegisterHandshakerFactory(
+      true /* at_start */, HANDSHAKER_CLIENT,
+      UniquePtr<HandshakerFactory>(New<HttpConnectHandshakerFactory>()));
 }

src/core/ext/filters/client_channel/lb_policy.cc

@@ -54,35 +54,15 @@ grpc_json* LoadBalancingPolicy::ParseLoadBalancingConfig(
   return nullptr;
 }
 
-LoadBalancingPolicy::LoadBalancingPolicy(Args args)
-    : InternallyRefCounted(&grpc_trace_lb_policy_refcount),
+LoadBalancingPolicy::LoadBalancingPolicy(Args args, intptr_t initial_refcount)
+    : InternallyRefCounted(&grpc_trace_lb_policy_refcount, initial_refcount),
       combiner_(GRPC_COMBINER_REF(args.combiner, "lb_policy")),
-      client_channel_factory_(args.client_channel_factory),
-      subchannel_pool_(std::move(args.subchannel_pool)),
       interested_parties_(grpc_pollset_set_create()),
-      request_reresolution_(nullptr) {}
+      channel_control_helper_(std::move(args.channel_control_helper)) {}
 
 LoadBalancingPolicy::~LoadBalancingPolicy() {
   grpc_pollset_set_destroy(interested_parties_);
   GRPC_COMBINER_UNREF(combiner_, "lb_policy");
 }
 
-void LoadBalancingPolicy::TryReresolutionLocked(
-    grpc_core::TraceFlag* grpc_lb_trace, grpc_error* error) {
-  if (request_reresolution_ != nullptr) {
-    GRPC_CLOSURE_SCHED(request_reresolution_, error);
-    request_reresolution_ = nullptr;
-    if (grpc_lb_trace->enabled()) {
-      gpr_log(GPR_INFO,
-              "%s %p: scheduling re-resolution closure with error=%s.",
-              grpc_lb_trace->name(), this, grpc_error_string(error));
-    }
-  } else {
-    if (grpc_lb_trace->enabled()) {
-      gpr_log(GPR_INFO, "%s %p: no available re-resolution closure.",
-              grpc_lb_trace->name(), this);
-    }
-  }
-}
-
 }  // namespace grpc_core

src/core/ext/filters/client_channel/lb_policy.h

@@ -24,7 +24,6 @@
 #include "src/core/ext/filters/client_channel/client_channel_channelz.h"
 #include "src/core/ext/filters/client_channel/client_channel_factory.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
-#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h"
 #include "src/core/lib/gprpp/abstract.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
@@ -43,61 +42,194 @@ namespace grpc_core {
 ///
 /// Any I/O done by the LB policy should be done under the pollset_set
 /// returned by \a interested_parties().
+// TODO(roth): Once we move to EventManager-based polling, remove the
+// interested_parties() hooks from the API.
 class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
  public:
-  struct Args {
-    /// The combiner under which all LB policy calls will be run.
-    /// Policy does NOT take ownership of the reference to the combiner.
-    // TODO(roth): Once we have a C++-like interface for combiners, this
-    // API should change to take a smart pointer that does pass ownership
-    // of a reference.
-    grpc_combiner* combiner = nullptr;
-    /// Used to create channels and subchannels.
-    grpc_client_channel_factory* client_channel_factory = nullptr;
-    /// Subchannel pool.
-    RefCountedPtr<SubchannelPoolInterface> subchannel_pool;
-    /// Channel args from the resolver.
-    /// Note that the LB policy gets the set of addresses from the
-    /// GRPC_ARG_SERVER_ADDRESS_LIST channel arg.
-    grpc_channel_args* args = nullptr;
-    /// Load balancing config from the resolver.
-    grpc_json* lb_config = nullptr;
-  };
-
   /// State used for an LB pick.
   struct PickState {
     /// Initial metadata associated with the picking call.
+    /// This is both an input and output parameter; the LB policy may
+    /// use metadata here to influence its routing decision, and it may
+    /// add new metadata here to be sent with the call to the chosen backend.
     grpc_metadata_batch* initial_metadata = nullptr;
-    /// Pointer to bitmask used for selective cancelling. See
-    /// \a CancelMatchingPicksLocked() and \a GRPC_INITIAL_METADATA_* in
-    /// grpc_types.h.
-    uint32_t* initial_metadata_flags = nullptr;
     /// Storage for LB token in \a initial_metadata, or nullptr if not used.
+    // TODO(roth): Remove this from the API.  Maybe have the LB policy
+    // allocate this on the arena instead?
     grpc_linked_mdelem lb_token_mdelem_storage;
-    /// Closure to run when pick is complete, if not completed synchronously.
-    /// If null, pick will fail if a result is not available synchronously.
-    grpc_closure* on_complete = nullptr;
-    // Callback set by lb policy to be notified of trailing metadata.
-    // The callback must be scheduled on grpc_schedule_on_exec_ctx.
+    /// Callback set by lb policy to be notified of trailing metadata.
+    /// The callback must be scheduled on grpc_schedule_on_exec_ctx.
     grpc_closure* recv_trailing_metadata_ready = nullptr;
-    // The address that will be set to point to the original
-    // recv_trailing_metadata_ready callback, to be invoked by the LB
-    // policy's recv_trailing_metadata_ready callback when complete.
-    // Must be non-null if recv_trailing_metadata_ready is non-null.
+    /// The address that will be set to point to the original
+    /// recv_trailing_metadata_ready callback, to be invoked by the LB
+    /// policy's recv_trailing_metadata_ready callback when complete.
+    /// Must be non-null if recv_trailing_metadata_ready is non-null.
     grpc_closure** original_recv_trailing_metadata_ready = nullptr;
-    // If this is not nullptr, then the client channel will point it to the
-    // call's trailing metadata before invoking recv_trailing_metadata_ready.
-    // If this is nullptr, then the callback will still be called.
-    // The lb does not have ownership of the metadata.
+    /// If this is not nullptr, then the client channel will point it to the
+    /// call's trailing metadata before invoking recv_trailing_metadata_ready.
+    /// If this is nullptr, then the callback will still be called.
+    /// The lb does not have ownership of the metadata.
     grpc_metadata_batch** recv_trailing_metadata = nullptr;
     /// Will be set to the selected subchannel, or nullptr on failure or when
     /// the LB policy decides to drop the call.
     RefCountedPtr<ConnectedSubchannel> connected_subchannel;
     /// Will be populated with context to pass to the subchannel call, if
     /// needed.
+    // TODO(roth): Remove this from the API, especially since it's not
+    // working properly anyway (see https://github.com/grpc/grpc/issues/15927).
     grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT] = {};
-    /// Next pointer.  For internal use by LB policy.
-    PickState* next = nullptr;
+  };
+
+  /// A picker is the object used to actual perform picks.
+  ///
+  /// Pickers are intended to encapsulate all of the state and logic
+  /// needed on the data plane (i.e., to actually process picks for
+  /// individual RPCs sent on the channel) while excluding all of the
+  /// state and logic needed on the control plane (i.e., resolver
+  /// updates, connectivity state notifications, etc); the latter should
+  /// live in the LB policy object itself.
+  ///
+  /// Currently, pickers are always accessed from within the
+  /// client_channel combiner, so they do not have to be thread-safe.
+  // TODO(roth): In a subsequent PR, split the data plane work (i.e.,
+  // the interaction with the picker) and the control plane work (i.e.,
+  // the interaction with the LB policy) into two different
+  // synchronization mechanisms, to avoid lock contention between the two.
+  class SubchannelPicker {
+   public:
+    enum PickResult {
+      // Pick complete.  If connected_subchannel is non-null, client channel
+      // can immediately proceed with the call on connected_subchannel;
+      // otherwise, call should be dropped.
+      PICK_COMPLETE,
+      // Pick cannot be completed until something changes on the control
+      // plane.  Client channel will queue the pick and try again the
+      // next time the picker is updated.
+      PICK_QUEUE,
+      // LB policy is in transient failure.  If the pick is wait_for_ready,
+      // client channel will wait for the next picker and try again;
+      // otherwise, the call will be failed immediately (although it may
+      // be retried if the client channel is configured to do so).
+      // The Pick() method will set its error parameter if this value is
+      // returned.
+      PICK_TRANSIENT_FAILURE,
+    };
+
+    SubchannelPicker() = default;
+    virtual ~SubchannelPicker() = default;
+
+    virtual PickResult Pick(PickState* pick, grpc_error** error) GRPC_ABSTRACT;
+
+    GRPC_ABSTRACT_BASE_CLASS
+  };
+
+  // A picker that returns PICK_QUEUE for all picks.
+  // Also calls the parent LB policy's ExitIdleLocked() method when the
+  // first pick is seen.
+  class QueuePicker : public SubchannelPicker {
+   public:
+    explicit QueuePicker(RefCountedPtr<LoadBalancingPolicy> parent)
+        : parent_(std::move(parent)) {}
+
+    PickResult Pick(PickState* pick, grpc_error** error) override {
+      // We invoke the parent's ExitIdleLocked() via a closure instead
+      // of doing it directly here, for two reasons:
+      // 1. ExitIdleLocked() may cause the policy's state to change and
+      //    a new picker to be delivered to the channel.  If that new
+      //    picker is delivered before ExitIdleLocked() returns, then by
+      //    the time this function returns, the pick will already have
+      //    been processed, and we'll be trying to re-process the same
+      //    pick again, leading to a crash.
+      // 2. In a subsequent PR, we will split the data plane and control
+      //    plane synchronization into separate combiners, at which
+      //    point this will need to hop from the data plane combiner into
+      //    the control plane combiner.
+      if (!exit_idle_called_) {
+        exit_idle_called_ = true;
+        parent_->Ref().release();  // ref held by closure.
+        GRPC_CLOSURE_SCHED(
+            GRPC_CLOSURE_CREATE(&CallExitIdle, parent_.get(),
+                                grpc_combiner_scheduler(parent_->combiner())),
+            GRPC_ERROR_NONE);
+      }
+      return PICK_QUEUE;
+    }
+
+   private:
+    static void CallExitIdle(void* arg, grpc_error* error) {
+      LoadBalancingPolicy* parent = static_cast<LoadBalancingPolicy*>(arg);
+      parent->ExitIdleLocked();
+      parent->Unref();
+    }
+
+    RefCountedPtr<LoadBalancingPolicy> parent_;
+    bool exit_idle_called_ = false;
+  };
+
+  // A picker that returns PICK_TRANSIENT_FAILURE for all picks.
+  class TransientFailurePicker : public SubchannelPicker {
+   public:
+    explicit TransientFailurePicker(grpc_error* error) : error_(error) {}
+    ~TransientFailurePicker() { GRPC_ERROR_UNREF(error_); }
+
+    PickResult Pick(PickState* pick, grpc_error** error) override {
+      *error = GRPC_ERROR_REF(error_);
+      return PICK_TRANSIENT_FAILURE;
+    }
+
+   private:
+    grpc_error* error_;
+  };
+
+  /// A proxy object used by the LB policy to communicate with the client
+  /// channel.
+  class ChannelControlHelper {
+   public:
+    ChannelControlHelper() = default;
+    virtual ~ChannelControlHelper() = default;
+
+    /// Creates a new subchannel with the specified channel args.
+    virtual Subchannel* CreateSubchannel(const grpc_channel_args& args)
+        GRPC_ABSTRACT;
+
+    /// Creates a channel with the specified target, type, and channel args.
+    virtual grpc_channel* CreateChannel(
+        const char* target, grpc_client_channel_type type,
+        const grpc_channel_args& args) GRPC_ABSTRACT;
+
+    /// Sets the connectivity state and returns a new picker to be used
+    /// by the client channel.
+    virtual void UpdateState(grpc_connectivity_state state,
+                             grpc_error* state_error,
+                             UniquePtr<SubchannelPicker> picker) {
+      std::move(picker);  // Suppress clang-tidy complaint.
+      // The rest of this is copied from the GRPC_ABSTRACT macro.
+      gpr_log(GPR_ERROR, "Function marked GRPC_ABSTRACT was not implemented");
+      GPR_ASSERT(false);
+    }
+
+    /// Requests that the resolver re-resolve.
+    virtual void RequestReresolution() GRPC_ABSTRACT;
+
+    GRPC_ABSTRACT_BASE_CLASS
+  };
+
+  /// Args used to instantiate an LB policy.
+  struct Args {
+    /// The combiner under which all LB policy calls will be run.
+    /// Policy does NOT take ownership of the reference to the combiner.
+    // TODO(roth): Once we have a C++-like interface for combiners, this
+    // API should change to take a smart pointer that does pass ownership
+    // of a reference.
+    grpc_combiner* combiner = nullptr;
+    /// Channel control helper.
+    UniquePtr<ChannelControlHelper> channel_control_helper;
+    /// Channel args from the resolver.
+    /// Note that the LB policy gets the set of addresses from the
+    /// GRPC_ARG_SERVER_ADDRESS_LIST channel arg.
+    const grpc_channel_args* args = nullptr;
+    /// Load balancing config from the resolver.
+    grpc_json* lb_config = nullptr;
   };
 
   // Not copyable nor movable.
@@ -113,48 +245,6 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
   virtual void UpdateLocked(const grpc_channel_args& args,
                             grpc_json* lb_config) GRPC_ABSTRACT;
 
-  /// Finds an appropriate subchannel for a call, based on data in \a pick.
-  /// \a pick must remain alive until the pick is complete.
-  ///
-  /// If a result is known immediately, returns true, setting \a *error
-  /// upon failure.  Otherwise, \a pick->on_complete will be invoked once
-  /// the pick is complete with its error argument set to indicate success
-  /// or failure.
-  ///
-  /// If \a pick->on_complete is null and no result is known immediately,
-  /// a synchronous failure will be returned (i.e., \a *error will be
-  /// set and true will be returned).
-  virtual bool PickLocked(PickState* pick, grpc_error** error) GRPC_ABSTRACT;
-
-  /// Cancels \a pick.
-  /// The \a on_complete callback of the pending pick will be invoked with
-  /// \a pick->connected_subchannel set to null.
-  virtual void CancelPickLocked(PickState* pick,
-                                grpc_error* error) GRPC_ABSTRACT;
-
-  /// Cancels all pending picks for which their \a initial_metadata_flags (as
-  /// given in the call to \a PickLocked()) matches
-  /// \a initial_metadata_flags_eq when ANDed with
-  /// \a initial_metadata_flags_mask.
-  virtual void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                         uint32_t initial_metadata_flags_eq,
-                                         grpc_error* error) GRPC_ABSTRACT;
-
-  /// Requests a notification when the connectivity state of the policy
-  /// changes from \a *state.  When that happens, sets \a *state to the
-  /// new state and schedules \a closure.
-  virtual void NotifyOnStateChangeLocked(grpc_connectivity_state* state,
-                                         grpc_closure* closure) GRPC_ABSTRACT;
-
-  /// Returns the policy's current connectivity state.  Sets \a error to
-  /// the associated error, if any.
-  virtual grpc_connectivity_state CheckConnectivityLocked(
-      grpc_error** connectivity_error) GRPC_ABSTRACT;
-
-  /// Hands off pending picks to \a new_policy.
-  virtual void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy)
-      GRPC_ABSTRACT;
-
   /// Tries to enter a READY connectivity state.
   /// TODO(roth): As part of restructuring how we handle IDLE state,
   /// consider whether this method is still needed.
@@ -183,18 +273,11 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
   /// given the JSON node of a LoadBalancingConfig array.
   static grpc_json* ParseLoadBalancingConfig(const grpc_json* lb_config_array);
 
-  /// Sets the re-resolution closure to \a request_reresolution.
-  void SetReresolutionClosureLocked(grpc_closure* request_reresolution) {
-    GPR_ASSERT(request_reresolution_ == nullptr);
-    request_reresolution_ = request_reresolution;
-  }
-
   grpc_pollset_set* interested_parties() const { return interested_parties_; }
 
-  // Callers that need their own reference can call the returned
-  // object's Ref() method.
-  SubchannelPoolInterface* subchannel_pool() const {
-    return subchannel_pool_.get();
+  void set_channelz_node(
+      RefCountedPtr<channelz::ClientChannelNode> channelz_node) {
+    channelz_node_ = std::move(channelz_node);
   }
 
   GRPC_ABSTRACT_BASE_CLASS
@@ -202,12 +285,18 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
  protected:
   GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE
 
-  explicit LoadBalancingPolicy(Args args);
+  explicit LoadBalancingPolicy(Args args, intptr_t initial_refcount = 1);
   virtual ~LoadBalancingPolicy();
 
   grpc_combiner* combiner() const { return combiner_; }
-  grpc_client_channel_factory* client_channel_factory() const {
-    return client_channel_factory_;
+
+  // Note: This will return null after ShutdownLocked() has been called.
+  ChannelControlHelper* channel_control_helper() const {
+    return channel_control_helper_.get();
+  }
+
+  channelz::ClientChannelNode* channelz_node() const {
+    return channelz_node_.get();
   }
 
   /// Shuts down the policy.  Any pending picks that have not been
@@ -215,27 +304,22 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
   /// failed.
   virtual void ShutdownLocked() GRPC_ABSTRACT;
 
-  /// Tries to request a re-resolution.
-  void TryReresolutionLocked(grpc_core::TraceFlag* grpc_lb_trace,
-                             grpc_error* error);
-
  private:
   static void ShutdownAndUnrefLocked(void* arg, grpc_error* ignored) {
     LoadBalancingPolicy* policy = static_cast<LoadBalancingPolicy*>(arg);
     policy->ShutdownLocked();
+    policy->channel_control_helper_.reset();
     policy->Unref();
   }
 
   /// Combiner under which LB policy actions take place.
   grpc_combiner* combiner_;
-  /// Client channel factory, used to create channels and subchannels.
-  grpc_client_channel_factory* client_channel_factory_;
-  /// Subchannel pool.
-  RefCountedPtr<SubchannelPoolInterface> subchannel_pool_;
   /// Owned pointer to interested parties in load balancing decisions.
   grpc_pollset_set* interested_parties_;
-  /// Callback to force a re-resolution.
-  grpc_closure* request_reresolution_;
+  /// Channel control helper.
+  UniquePtr<ChannelControlHelper> channel_control_helper_;
+  /// Channelz node.
+  RefCountedPtr<channelz::ClientChannelNode> channelz_node_;
 };
 
 }  // namespace grpc_core

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -74,7 +74,6 @@
 #include <grpc/support/time.h>
 
 #include "src/core/ext/filters/client_channel/client_channel.h"
-#include "src/core/ext/filters/client_channel/client_channel_factory.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h"
 #include "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h"
@@ -131,16 +130,6 @@ class GrpcLb : public LoadBalancingPolicy {
 
   void UpdateLocked(const grpc_channel_args& args,
                     grpc_json* lb_config) override;
-  bool PickLocked(PickState* pick, grpc_error** error) override;
-  void CancelPickLocked(PickState* pick, grpc_error* error) override;
-  void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                 uint32_t initial_metadata_flags_eq,
-                                 grpc_error* error) override;
-  void NotifyOnStateChangeLocked(grpc_connectivity_state* state,
-                                 grpc_closure* closure) override;
-  grpc_connectivity_state CheckConnectivityLocked(
-      grpc_error** connectivity_error) override;
-  void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override;
   void ExitIdleLocked() override;
   void ResetBackoffLocked() override;
   void FillChildRefsForChannelz(
@@ -148,31 +137,6 @@ class GrpcLb : public LoadBalancingPolicy {
       channelz::ChildRefsList* child_channels) override;
 
  private:
-  /// Linked list of pending pick requests. It stores all information needed to
-  /// eventually call (Round Robin's) pick() on them. They mainly stay pending
-  /// waiting for the RR policy to be created.
-  ///
-  /// Note that when a pick is sent to the RR policy, we inject our own
-  /// on_complete callback, so that we can intercept the result before
-  /// invoking the original on_complete callback.  This allows us to set the
-  /// LB token metadata and add client_stats to the call context.
-  /// See \a pending_pick_complete() for details.
-  struct PendingPick {
-    // The grpclb instance that created the wrapping. This instance is not
-    // owned; reference counts are untouched. It's used only for logging
-    // purposes.
-    GrpcLb* grpclb_policy;
-    // The original pick.
-    PickState* pick;
-    // Our on_complete closure and the original one.
-    grpc_closure on_complete;
-    grpc_closure* original_on_complete;
-    // Stats for client-side load reporting.
-    RefCountedPtr<GrpcLbClientStats> client_stats;
-    // Next pending pick.
-    PendingPick* next = nullptr;
-  };
-
   /// Contains a call to the LB server and all the data related to the call.
   class BalancerCallState : public InternallyRefCounted<BalancerCallState> {
    public:
@@ -248,6 +212,80 @@ class GrpcLb : public LoadBalancingPolicy {
     grpc_closure client_load_report_closure_;
   };
 
+  class Serverlist : public RefCounted<Serverlist> {
+   public:
+    // Takes ownership of serverlist.
+    explicit Serverlist(grpc_grpclb_serverlist* serverlist)
+        : serverlist_(serverlist) {}
+
+    ~Serverlist() { grpc_grpclb_destroy_serverlist(serverlist_); }
+
+    bool operator==(const Serverlist& other) const;
+
+    const grpc_grpclb_serverlist* serverlist() const { return serverlist_; }
+
+    // Returns a text representation suitable for logging.
+    UniquePtr<char> AsText() const;
+
+    // Extracts all non-drop entries into a ServerAddressList.
+    ServerAddressList GetServerAddressList() const;
+
+    // Returns true if the serverlist contains at least one drop entry and
+    // no backend address entries.
+    bool ContainsAllDropEntries() const;
+
+    // Returns the LB token to use for a drop, or null if the call
+    // should not be dropped.
+    // Intended to be called from picker, so calls will be externally
+    // synchronized.
+    const char* ShouldDrop();
+
+   private:
+    grpc_grpclb_serverlist* serverlist_;
+    size_t drop_index_ = 0;
+  };
+
+  class Picker : public SubchannelPicker {
+   public:
+    Picker(GrpcLb* parent, RefCountedPtr<Serverlist> serverlist,
+           UniquePtr<SubchannelPicker> child_picker,
+           RefCountedPtr<GrpcLbClientStats> client_stats)
+        : parent_(parent),
+          serverlist_(std::move(serverlist)),
+          child_picker_(std::move(child_picker)),
+          client_stats_(std::move(client_stats)) {}
+
+    PickResult Pick(PickState* pick, grpc_error** error) override;
+
+   private:
+    // Storing the address for logging, but not holding a ref.
+    // DO NOT DEFERENCE!
+    GrpcLb* parent_;
+
+    // Serverlist to be used for determining drops.
+    RefCountedPtr<Serverlist> serverlist_;
+
+    UniquePtr<SubchannelPicker> child_picker_;
+    RefCountedPtr<GrpcLbClientStats> client_stats_;
+  };
+
+  class Helper : public ChannelControlHelper {
+   public:
+    explicit Helper(RefCountedPtr<GrpcLb> parent)
+        : parent_(std::move(parent)) {}
+
+    Subchannel* CreateSubchannel(const grpc_channel_args& args) override;
+    grpc_channel* CreateChannel(const char* target,
+                                grpc_client_channel_type type,
+                                const grpc_channel_args& args) override;
+    void UpdateState(grpc_connectivity_state state, grpc_error* state_error,
+                     UniquePtr<SubchannelPicker> picker) override;
+    void RequestReresolution() override;
+
+   private:
+    RefCountedPtr<GrpcLb> parent_;
+  };
+
   ~GrpcLb();
 
   void ShutdownLocked() override;
@@ -264,24 +302,10 @@ class GrpcLb : public LoadBalancingPolicy {
   static void OnBalancerChannelConnectivityChangedLocked(void* arg,
                                                          grpc_error* error);
 
-  // Pending pick methods.
-  static void PendingPickSetMetadataAndContext(PendingPick* pp);
-  PendingPick* PendingPickCreate(PickState* pick);
-  void AddPendingPick(PendingPick* pp);
-  static void OnPendingPickComplete(void* arg, grpc_error* error);
-
   // Methods for dealing with the RR policy.
   void CreateOrUpdateRoundRobinPolicyLocked();
   grpc_channel_args* CreateRoundRobinPolicyArgsLocked();
   void CreateRoundRobinPolicyLocked(Args args);
-  bool PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp,
-                                      grpc_error** error);
-  void UpdateConnectivityStateFromRoundRobinPolicyLocked(
-      grpc_error* rr_state_error);
-  static void OnRoundRobinConnectivityChangedLocked(void* arg,
-                                                    grpc_error* error);
-  static void OnRoundRobinRequestReresolutionLocked(void* arg,
-                                                    grpc_error* error);
 
   // Who the client is trying to communicate with.
   const char* server_name_ = nullptr;
@@ -292,7 +316,6 @@ class GrpcLb : public LoadBalancingPolicy {
   // Internal state.
   bool started_picking_ = false;
   bool shutting_down_ = false;
-  grpc_connectivity_state_tracker state_tracker_;
 
   // The channel for communicating with the LB server.
   grpc_channel* lb_channel_ = nullptr;
@@ -321,11 +344,7 @@ class GrpcLb : public LoadBalancingPolicy {
 
   // The deserialized response from the balancer. May be nullptr until one
   // such response has arrived.
-  grpc_grpclb_serverlist* serverlist_ = nullptr;
-  // Index into serverlist for next pick.
-  // If the server at this index is a drop, we return a drop.
-  // Otherwise, we delegate to the RR policy.
-  size_t serverlist_index_ = 0;
+  RefCountedPtr<Serverlist> serverlist_;
 
   // Timeout in milliseconds for before using fallback backend addresses.
   // 0 means not using fallback.
@@ -337,20 +356,65 @@ class GrpcLb : public LoadBalancingPolicy {
   grpc_timer lb_fallback_timer_;
   grpc_closure lb_on_fallback_;
 
-  // Pending picks that are waiting on the RR policy's connectivity.
-  PendingPick* pending_picks_ = nullptr;
-
   // The RR policy to use for the backends.
   OrphanablePtr<LoadBalancingPolicy> rr_policy_;
-  grpc_connectivity_state rr_connectivity_state_;
-  grpc_closure on_rr_connectivity_changed_;
-  grpc_closure on_rr_request_reresolution_;
 };
 
 //
-// serverlist parsing code
+// GrpcLb::Serverlist
 //
 
+bool GrpcLb::Serverlist::operator==(const Serverlist& other) const {
+  return grpc_grpclb_serverlist_equals(serverlist_, other.serverlist_);
+}
+
+void ParseServer(const grpc_grpclb_server* server,
+                 grpc_resolved_address* addr) {
+  memset(addr, 0, sizeof(*addr));
+  if (server->drop) return;
+  const uint16_t netorder_port = grpc_htons((uint16_t)server->port);
+  /* the addresses are given in binary format (a in(6)_addr struct) in
+   * server->ip_address.bytes. */
+  const grpc_grpclb_ip_address* ip = &server->ip_address;
+  if (ip->size == 4) {
+    addr->len = static_cast<socklen_t>(sizeof(grpc_sockaddr_in));
+    grpc_sockaddr_in* addr4 = reinterpret_cast<grpc_sockaddr_in*>(&addr->addr);
+    addr4->sin_family = GRPC_AF_INET;
+    memcpy(&addr4->sin_addr, ip->bytes, ip->size);
+    addr4->sin_port = netorder_port;
+  } else if (ip->size == 16) {
+    addr->len = static_cast<socklen_t>(sizeof(grpc_sockaddr_in6));
+    grpc_sockaddr_in6* addr6 = (grpc_sockaddr_in6*)&addr->addr;
+    addr6->sin6_family = GRPC_AF_INET6;
+    memcpy(&addr6->sin6_addr, ip->bytes, ip->size);
+    addr6->sin6_port = netorder_port;
+  }
+}
+
+UniquePtr<char> GrpcLb::Serverlist::AsText() const {
+  gpr_strvec entries;
+  gpr_strvec_init(&entries);
+  for (size_t i = 0; i < serverlist_->num_servers; ++i) {
+    const auto* server = serverlist_->servers[i];
+    char* ipport;
+    if (server->drop) {
+      ipport = gpr_strdup("(drop)");
+    } else {
+      grpc_resolved_address addr;
+      ParseServer(server, &addr);
+      grpc_sockaddr_to_string(&ipport, &addr, false);
+    }
+    char* entry;
+    gpr_asprintf(&entry, "  %" PRIuPTR ": %s token=%s\n", i, ipport,
+                 server->load_balance_token);
+    gpr_free(ipport);
+    gpr_strvec_add(&entries, entry);
+  }
+  UniquePtr<char> result(gpr_strvec_flatten(&entries, nullptr));
+  gpr_strvec_destroy(&entries);
+  return result;
+}
+
 // vtable for LB token channel arg.
 void* lb_token_copy(void* token) {
   return token == nullptr
@@ -393,35 +457,12 @@ bool IsServerValid(const grpc_grpclb_server* server, size_t idx, bool log) {
   return true;
 }
 
-void ParseServer(const grpc_grpclb_server* server,
-                 grpc_resolved_address* addr) {
-  memset(addr, 0, sizeof(*addr));
-  if (server->drop) return;
-  const uint16_t netorder_port = grpc_htons((uint16_t)server->port);
-  /* the addresses are given in binary format (a in(6)_addr struct) in
-   * server->ip_address.bytes. */
-  const grpc_grpclb_ip_address* ip = &server->ip_address;
-  if (ip->size == 4) {
-    addr->len = static_cast<socklen_t>(sizeof(grpc_sockaddr_in));
-    grpc_sockaddr_in* addr4 = reinterpret_cast<grpc_sockaddr_in*>(&addr->addr);
-    addr4->sin_family = GRPC_AF_INET;
-    memcpy(&addr4->sin_addr, ip->bytes, ip->size);
-    addr4->sin_port = netorder_port;
-  } else if (ip->size == 16) {
-    addr->len = static_cast<socklen_t>(sizeof(grpc_sockaddr_in6));
-    grpc_sockaddr_in6* addr6 = (grpc_sockaddr_in6*)&addr->addr;
-    addr6->sin6_family = GRPC_AF_INET6;
-    memcpy(&addr6->sin6_addr, ip->bytes, ip->size);
-    addr6->sin6_port = netorder_port;
-  }
-}
-
-// Returns addresses extracted from \a serverlist.
-ServerAddressList ProcessServerlist(const grpc_grpclb_serverlist* serverlist) {
+// Returns addresses extracted from the serverlist.
+ServerAddressList GrpcLb::Serverlist::GetServerAddressList() const {
   ServerAddressList addresses;
-  for (size_t i = 0; i < serverlist->num_servers; ++i) {
-    const grpc_grpclb_server* server = serverlist->servers[i];
-    if (!IsServerValid(serverlist->servers[i], i, false)) continue;
+  for (size_t i = 0; i < serverlist_->num_servers; ++i) {
+    const grpc_grpclb_server* server = serverlist_->servers[i];
+    if (!IsServerValid(serverlist_->servers[i], i, false)) continue;
     // Address processing.
     grpc_resolved_address addr;
     ParseServer(server, &addr);
@@ -456,6 +497,176 @@ ServerAddressList ProcessServerlist(const grpc_grpclb_serverlist* serverlist) {
   return addresses;
 }
 
+bool GrpcLb::Serverlist::ContainsAllDropEntries() const {
+  if (serverlist_->num_servers == 0) return false;
+  for (size_t i = 0; i < serverlist_->num_servers; ++i) {
+    if (!serverlist_->servers[i]->drop) return false;
+  }
+  return true;
+}
+
+const char* GrpcLb::Serverlist::ShouldDrop() {
+  if (serverlist_->num_servers == 0) return nullptr;
+  grpc_grpclb_server* server = serverlist_->servers[drop_index_];
+  drop_index_ = (drop_index_ + 1) % serverlist_->num_servers;
+  return server->drop ? server->load_balance_token : nullptr;
+}
+
+//
+// GrpcLb::Picker
+//
+
+// Adds lb_token of selected subchannel (address) to the call's initial
+// metadata.
+grpc_error* AddLbTokenToInitialMetadata(
+    grpc_mdelem lb_token, grpc_linked_mdelem* lb_token_mdelem_storage,
+    grpc_metadata_batch* initial_metadata) {
+  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
+  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
+  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
+                                      lb_token);
+}
+
+// Destroy function used when embedding client stats in call context.
+void DestroyClientStats(void* arg) {
+  static_cast<GrpcLbClientStats*>(arg)->Unref();
+}
+
+GrpcLb::Picker::PickResult GrpcLb::Picker::Pick(PickState* pick,
+                                                grpc_error** error) {
+  // Check if we should drop the call.
+  const char* drop_token = serverlist_->ShouldDrop();
+  if (drop_token != nullptr) {
+    // Update client load reporting stats to indicate the number of
+    // dropped calls.  Note that we have to do this here instead of in
+    // the client_load_reporting filter, because we do not create a
+    // subchannel call (and therefore no client_load_reporting filter)
+    // for dropped calls.
+    if (client_stats_ != nullptr) {
+      client_stats_->AddCallDroppedLocked(drop_token);
+    }
+    return PICK_COMPLETE;
+  }
+  // Forward pick to child policy.
+  PickResult result = child_picker_->Pick(pick, error);
+  // If pick succeeded, add LB token to initial metadata.
+  if (result == PickResult::PICK_COMPLETE &&
+      pick->connected_subchannel != nullptr) {
+    const grpc_arg* arg = grpc_channel_args_find(
+        pick->connected_subchannel->args(), GRPC_ARG_GRPCLB_ADDRESS_LB_TOKEN);
+    if (arg == nullptr) {
+      gpr_log(GPR_ERROR,
+              "[grpclb %p picker %p] No LB token for connected subchannel "
+              "pick %p",
+              parent_, this, pick);
+      abort();
+    }
+    grpc_mdelem lb_token = {reinterpret_cast<uintptr_t>(arg->value.pointer.p)};
+    AddLbTokenToInitialMetadata(GRPC_MDELEM_REF(lb_token),
+                                &pick->lb_token_mdelem_storage,
+                                pick->initial_metadata);
+    // Pass on client stats via context. Passes ownership of the reference.
+    if (client_stats_ != nullptr) {
+      pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
+          client_stats_->Ref().release();
+      pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
+          DestroyClientStats;
+    }
+  }
+  return result;
+}
+
+//
+// GrpcLb::Helper
+//
+
+Subchannel* GrpcLb::Helper::CreateSubchannel(const grpc_channel_args& args) {
+  if (parent_->shutting_down_) return nullptr;
+  return parent_->channel_control_helper()->CreateSubchannel(args);
+}
+
+grpc_channel* GrpcLb::Helper::CreateChannel(const char* target,
+                                            grpc_client_channel_type type,
+                                            const grpc_channel_args& args) {
+  if (parent_->shutting_down_) return nullptr;
+  return parent_->channel_control_helper()->CreateChannel(target, type, args);
+}
+
+void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
+                                 grpc_error* state_error,
+                                 UniquePtr<SubchannelPicker> picker) {
+  if (parent_->shutting_down_) {
+    GRPC_ERROR_UNREF(state_error);
+    return;
+  }
+  // There are three cases to consider here:
+  // 1. We're in fallback mode.  In this case, we're always going to use
+  //    RR's result, so we pass its picker through as-is.
+  // 2. The serverlist contains only drop entries.  In this case, we
+  //    want to use our own picker so that we can return the drops.
+  // 3. Not in fallback mode and serverlist is not all drops (i.e., it
+  //    may be empty or contain at least one backend address).  There are
+  //    two sub-cases:
+  //    a. RR is reporting state READY.  In this case, we wrap RR's
+  //       picker in our own, so that we can handle drops and LB token
+  //       metadata for each pick.
+  //    b. RR is reporting a state other than READY.  In this case, we
+  //       don't want to use our own picker, because we don't want to
+  //       process drops for picks that yield a QUEUE result; this would
+  //       result in dropping too many calls, since we will see the
+  //       queued picks multiple times, and we'd consider each one a
+  //       separate call for the drop calculation.
+  //
+  // Cases 1 and 3b: return picker from RR as-is.
+  if (parent_->serverlist_ == nullptr ||
+      (!parent_->serverlist_->ContainsAllDropEntries() &&
+       state != GRPC_CHANNEL_READY)) {
+    if (grpc_lb_glb_trace.enabled()) {
+      gpr_log(GPR_INFO,
+              "[grpclb %p helper %p] state=%s passing RR picker %p as-is",
+              parent_.get(), this, grpc_connectivity_state_name(state),
+              picker.get());
+    }
+    parent_->channel_control_helper()->UpdateState(state, state_error,
+                                                   std::move(picker));
+    return;
+  }
+  // Cases 2 and 3a: wrap picker from RR in our own picker.
+  if (grpc_lb_glb_trace.enabled()) {
+    gpr_log(GPR_INFO, "[grpclb %p helper %p] state=%s wrapping RR picker %p",
+            parent_.get(), this, grpc_connectivity_state_name(state),
+            picker.get());
+  }
+  RefCountedPtr<GrpcLbClientStats> client_stats;
+  if (parent_->lb_calld_ != nullptr &&
+      parent_->lb_calld_->client_stats() != nullptr) {
+    client_stats = parent_->lb_calld_->client_stats()->Ref();
+  }
+  parent_->channel_control_helper()->UpdateState(
+      state, state_error,
+      UniquePtr<SubchannelPicker>(
+          New<Picker>(parent_.get(), parent_->serverlist_, std::move(picker),
+                      std::move(client_stats))));
+}
+
+void GrpcLb::Helper::RequestReresolution() {
+  if (parent_->shutting_down_) return;
+  if (grpc_lb_glb_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[grpclb %p] Re-resolution requested from the internal RR policy "
+            "(%p).",
+            parent_.get(), parent_->rr_policy_.get());
+  }
+  // If we are talking to a balancer, we expect to get updated addresses
+  // from the balancer, so we can ignore the re-resolution request from
+  // the RR policy. Otherwise, pass the re-resolution request up to the
+  // channel.
+  if (parent_->lb_calld_ == nullptr ||
+      !parent_->lb_calld_->seen_initial_response()) {
+    parent_->channel_control_helper()->RequestReresolution();
+  }
+}
+
 //
 // GrpcLb::BalancerCallState
 //
@@ -754,27 +965,20 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
                   response_slice)) != nullptr) {
     // Have seen initial response, look for serverlist.
     GPR_ASSERT(lb_calld->lb_call_ != nullptr);
+    auto serverlist_wrapper = MakeRefCounted<Serverlist>(serverlist);
     if (grpc_lb_glb_trace.enabled()) {
+      UniquePtr<char> serverlist_text = serverlist_wrapper->AsText();
       gpr_log(GPR_INFO,
               "[grpclb %p] lb_calld=%p: Serverlist with %" PRIuPTR
-              " servers received",
-              grpclb_policy, lb_calld, serverlist->num_servers);
-      for (size_t i = 0; i < serverlist->num_servers; ++i) {
-        grpc_resolved_address addr;
-        ParseServer(serverlist->servers[i], &addr);
-        char* ipport;
-        grpc_sockaddr_to_string(&ipport, &addr, false);
-        gpr_log(GPR_INFO,
-                "[grpclb %p] lb_calld=%p: Serverlist[%" PRIuPTR "]: %s",
-                grpclb_policy, lb_calld, i, ipport);
-        gpr_free(ipport);
-      }
+              " servers received:\n%s",
+              grpclb_policy, lb_calld, serverlist->num_servers,
+              serverlist_text.get());
     }
     // Start sending client load report only after we start using the
     // serverlist returned from the current LB call.
     if (lb_calld->client_stats_report_interval_ > 0 &&
         lb_calld->client_stats_ == nullptr) {
-      lb_calld->client_stats_.reset(New<GrpcLbClientStats>());
+      lb_calld->client_stats_ = MakeRefCounted<GrpcLbClientStats>();
       // TODO(roth): We currently track this ref manually.  Once the
       // ClosureRef API is ready, we should pass the RefCountedPtr<> along
       // with the callback.
@@ -783,19 +987,16 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
       lb_calld->ScheduleNextClientLoadReportLocked();
     }
     // Check if the serverlist differs from the previous one.
-    if (grpc_grpclb_serverlist_equals(grpclb_policy->serverlist_, serverlist)) {
+    if (grpclb_policy->serverlist_ != nullptr &&
+        *grpclb_policy->serverlist_ == *serverlist_wrapper) {
       if (grpc_lb_glb_trace.enabled()) {
         gpr_log(GPR_INFO,
                 "[grpclb %p] lb_calld=%p: Incoming server list identical to "
                 "current, ignoring.",
                 grpclb_policy, lb_calld);
       }
-      grpc_grpclb_destroy_serverlist(serverlist);
     } else {  // New serverlist.
-      if (grpclb_policy->serverlist_ != nullptr) {
-        // Dispose of the old serverlist.
-        grpc_grpclb_destroy_serverlist(grpclb_policy->serverlist_);
-      } else {
+      if (grpclb_policy->serverlist_ == nullptr) {
         // Dispose of the fallback.
         grpclb_policy->fallback_backend_addresses_.reset();
         if (grpclb_policy->fallback_timer_callback_pending_) {
@@ -805,8 +1006,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
       // Update the serverlist in the GrpcLb instance. This serverlist
       // instance will be destroyed either upon the next update or when the
       // GrpcLb instance is destroyed.
-      grpclb_policy->serverlist_ = serverlist;
-      grpclb_policy->serverlist_index_ = 0;
+      grpclb_policy->serverlist_ = std::move(serverlist_wrapper);
       grpclb_policy->CreateOrUpdateRoundRobinPolicyLocked();
     }
   } else {
@@ -853,13 +1053,13 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
             lb_calld->lb_call_, grpc_error_string(error));
     gpr_free(status_details);
   }
-  grpclb_policy->TryReresolutionLocked(&grpc_lb_glb_trace, GRPC_ERROR_NONE);
   // If this lb_calld is still in use, this call ended because of a failure so
   // we want to retry connecting. Otherwise, we have deliberately ended this
   // call and no further action is required.
   if (lb_calld == grpclb_policy->lb_calld_.get()) {
     grpclb_policy->lb_calld_.reset();
     GPR_ASSERT(!grpclb_policy->shutting_down_);
+    grpclb_policy->channel_control_helper()->RequestReresolution();
     if (lb_calld->seen_initial_response_) {
       // If we lose connection to the LB server, reset the backoff and restart
       // the LB call immediately.
@@ -917,6 +1117,9 @@ grpc_channel_args* BuildBalancerChannelArgs(
       // LB policy name, since we want to use the default (pick_first) in
       // the LB channel.
       GRPC_ARG_LB_POLICY_NAME,
+      // Strip out the service config, since we don't want the LB policy
+      // config specified for the parent channel to affect the LB channel.
+      GRPC_ARG_SERVICE_CONFIG,
       // The channel arg for the server URI, since that will be different for
       // the LB channel than for the parent channel.  The client channel
       // factory will re-add this arg with the right value.
@@ -928,7 +1131,7 @@ grpc_channel_args* BuildBalancerChannelArgs(
       // resolver will have is_balancer=false, whereas our own addresses have
       // is_balancer=true.  We need the LB channel to return addresses with
       // is_balancer=false so that it does not wind up recursively using the
-      // grpclb LB policy, as per the special case logic in client_channel.c.
+      // grpclb LB policy.
       GRPC_ARG_SERVER_ADDRESS_LIST,
       // The fake resolver response generator, because we are replacing it
       // with the one from the grpclb policy, used to propagate updates to
@@ -988,13 +1191,6 @@ GrpcLb::GrpcLb(LoadBalancingPolicy::Args args)
   GRPC_CLOSURE_INIT(&lb_channel_on_connectivity_changed_,
                     &GrpcLb::OnBalancerChannelConnectivityChangedLocked, this,
                     grpc_combiner_scheduler(args.combiner));
-  GRPC_CLOSURE_INIT(&on_rr_connectivity_changed_,
-                    &GrpcLb::OnRoundRobinConnectivityChangedLocked, this,
-                    grpc_combiner_scheduler(args.combiner));
-  GRPC_CLOSURE_INIT(&on_rr_request_reresolution_,
-                    &GrpcLb::OnRoundRobinRequestReresolutionLocked, this,
-                    grpc_combiner_scheduler(args.combiner));
-  grpc_connectivity_state_init(&state_tracker_, GRPC_CHANNEL_IDLE, "grpclb");
   // Record server name.
   const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI);
   const char* server_uri = grpc_channel_arg_get_string(arg);
@@ -1017,20 +1213,18 @@ GrpcLb::GrpcLb(LoadBalancingPolicy::Args args)
       arg, {GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX});
   // Process channel args.
   ProcessChannelArgsLocked(*args.args);
+  // Initialize channel with a picker that will start us connecting.
+  channel_control_helper()->UpdateState(
+      GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
+      UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
 }
 
 GrpcLb::~GrpcLb() {
-  GPR_ASSERT(pending_picks_ == nullptr);
   gpr_free((void*)server_name_);
   grpc_channel_args_destroy(args_);
-  grpc_connectivity_state_destroy(&state_tracker_);
-  if (serverlist_ != nullptr) {
-    grpc_grpclb_destroy_serverlist(serverlist_);
-  }
 }
 
 void GrpcLb::ShutdownLocked() {
-  grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   shutting_down_ = true;
   lb_calld_.reset();
   if (retry_timer_callback_pending_) {
@@ -1040,7 +1234,6 @@ void GrpcLb::ShutdownLocked() {
     grpc_timer_cancel(&lb_fallback_timer_);
   }
   rr_policy_.reset();
-  TryReresolutionLocked(&grpc_lb_glb_trace, GRPC_ERROR_CANCELLED);
   // We destroy the LB channel here instead of in our destructor because
   // destroying the channel triggers a last callback to
   // OnBalancerChannelConnectivityChangedLocked(), and we need to be
@@ -1050,109 +1243,12 @@ void GrpcLb::ShutdownLocked() {
     lb_channel_ = nullptr;
     gpr_atm_no_barrier_store(&lb_channel_uuid_, 0);
   }
-  grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_SHUTDOWN,
-                              GRPC_ERROR_REF(error), "grpclb_shutdown");
-  // Clear pending picks.
-  PendingPick* pp;
-  while ((pp = pending_picks_) != nullptr) {
-    pending_picks_ = pp->next;
-    pp->pick->connected_subchannel.reset();
-    // Note: pp is deleted in this callback.
-    GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
-  }
-  GRPC_ERROR_UNREF(error);
 }
 
 //
 // public methods
 //
 
-void GrpcLb::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) {
-  PendingPick* pp;
-  while ((pp = pending_picks_) != nullptr) {
-    pending_picks_ = pp->next;
-    pp->pick->on_complete = pp->original_on_complete;
-    grpc_error* error = GRPC_ERROR_NONE;
-    if (new_policy->PickLocked(pp->pick, &error)) {
-      // Synchronous return; schedule closure.
-      GRPC_CLOSURE_SCHED(pp->pick->on_complete, error);
-    }
-    Delete(pp);
-  }
-}
-
-// Cancel a specific pending pick.
-//
-// A grpclb pick progresses as follows:
-// - If there's a Round Robin policy (rr_policy_) available, it'll be
-//   handed over to the RR policy (in CreateRoundRobinPolicyLocked()). From
-//   that point onwards, it'll be RR's responsibility. For cancellations, that
-//   implies the pick needs also be cancelled by the RR instance.
-// - Otherwise, without an RR instance, picks stay pending at this policy's
-//   level (grpclb), inside the pending_picks_ list. To cancel these,
-//   we invoke the completion closure and set the pick's connected
-//   subchannel to nullptr right here.
-void GrpcLb::CancelPickLocked(PickState* pick, grpc_error* error) {
-  PendingPick* pp = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pp != nullptr) {
-    PendingPick* next = pp->next;
-    if (pp->pick == pick) {
-      pick->connected_subchannel.reset();
-      // Note: pp is deleted in this callback.
-      GRPC_CLOSURE_SCHED(&pp->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pp->next = pending_picks_;
-      pending_picks_ = pp;
-    }
-    pp = next;
-  }
-  if (rr_policy_ != nullptr) {
-    rr_policy_->CancelPickLocked(pick, GRPC_ERROR_REF(error));
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
-// Cancel all pending picks.
-//
-// A grpclb pick progresses as follows:
-// - If there's a Round Robin policy (rr_policy_) available, it'll be
-//   handed over to the RR policy (in CreateRoundRobinPolicyLocked()). From
-//   that point onwards, it'll be RR's responsibility. For cancellations, that
-//   implies the pick needs also be cancelled by the RR instance.
-// - Otherwise, without an RR instance, picks stay pending at this policy's
-//   level (grpclb), inside the pending_picks_ list. To cancel these,
-//   we invoke the completion closure and set the pick's connected
-//   subchannel to nullptr right here.
-void GrpcLb::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                       uint32_t initial_metadata_flags_eq,
-                                       grpc_error* error) {
-  PendingPick* pp = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pp != nullptr) {
-    PendingPick* next = pp->next;
-    if ((*pp->pick->initial_metadata_flags & initial_metadata_flags_mask) ==
-        initial_metadata_flags_eq) {
-      // Note: pp is deleted in this callback.
-      GRPC_CLOSURE_SCHED(&pp->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pp->next = pending_picks_;
-      pending_picks_ = pp;
-    }
-    pp = next;
-  }
-  if (rr_policy_ != nullptr) {
-    rr_policy_->CancelMatchingPicksLocked(initial_metadata_flags_mask,
-                                          initial_metadata_flags_eq,
-                                          GRPC_ERROR_REF(error));
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
 void GrpcLb::ExitIdleLocked() {
   if (!started_picking_) {
     StartPickingLocked();
@@ -1168,37 +1264,6 @@ void GrpcLb::ResetBackoffLocked() {
   }
 }
 
-bool GrpcLb::PickLocked(PickState* pick, grpc_error** error) {
-  PendingPick* pp = PendingPickCreate(pick);
-  bool pick_done = false;
-  if (rr_policy_ != nullptr) {
-    if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", this,
-              rr_policy_.get());
-    }
-    pick_done =
-        PickFromRoundRobinPolicyLocked(false /* force_async */, pp, error);
-  } else {  // rr_policy_ == NULL
-    if (pick->on_complete == nullptr) {
-      *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "No pick result available but synchronous result required.");
-      pick_done = true;
-    } else {
-      if (grpc_lb_glb_trace.enabled()) {
-        gpr_log(GPR_INFO,
-                "[grpclb %p] No RR policy. Adding to grpclb's pending picks",
-                this);
-      }
-      AddPendingPick(pp);
-      if (!started_picking_) {
-        StartPickingLocked();
-      }
-      pick_done = false;
-    }
-  }
-  return pick_done;
-}
-
 void GrpcLb::FillChildRefsForChannelz(
     channelz::ChildRefsList* child_subchannels,
     channelz::ChildRefsList* child_channels) {
@@ -1212,17 +1277,6 @@ void GrpcLb::FillChildRefsForChannelz(
   }
 }
 
-grpc_connectivity_state GrpcLb::CheckConnectivityLocked(
-    grpc_error** connectivity_error) {
-  return grpc_connectivity_state_get(&state_tracker_, connectivity_error);
-}
-
-void GrpcLb::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
-                                       grpc_closure* notify) {
-  grpc_connectivity_state_notify_on_state_change(&state_tracker_, current,
-                                                 notify);
-}
-
 // Returns the backend addresses extracted from the given addresses.
 UniquePtr<ServerAddressList> ExtractBackendAddresses(
     const ServerAddressList& addresses) {
@@ -1268,9 +1322,8 @@ void GrpcLb::ProcessChannelArgsLocked(const grpc_channel_args& args) {
   if (lb_channel_ == nullptr) {
     char* uri_str;
     gpr_asprintf(&uri_str, "fake:///%s", server_name_);
-    lb_channel_ = grpc_client_channel_factory_create_channel(
-        client_channel_factory(), uri_str,
-        GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, lb_channel_args);
+    lb_channel_ = channel_control_helper()->CreateChannel(
+        uri_str, GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, *lb_channel_args);
     GPR_ASSERT(lb_channel_ != nullptr);
     grpc_core::channelz::ChannelNode* channel_node =
         grpc_channel_get_channelz_node(lb_channel_);
@@ -1451,143 +1504,10 @@ void GrpcLb::OnBalancerChannelConnectivityChangedLocked(void* arg,
   }
 }
 
-//
-// PendingPick
-//
-
-// Adds lb_token of selected subchannel (address) to the call's initial
-// metadata.
-grpc_error* AddLbTokenToInitialMetadata(
-    grpc_mdelem lb_token, grpc_linked_mdelem* lb_token_mdelem_storage,
-    grpc_metadata_batch* initial_metadata) {
-  GPR_ASSERT(lb_token_mdelem_storage != nullptr);
-  GPR_ASSERT(!GRPC_MDISNULL(lb_token));
-  return grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage,
-                                      lb_token);
-}
-
-// Destroy function used when embedding client stats in call context.
-void DestroyClientStats(void* arg) {
-  static_cast<GrpcLbClientStats*>(arg)->Unref();
-}
-
-void GrpcLb::PendingPickSetMetadataAndContext(PendingPick* pp) {
-  // If connected_subchannel is nullptr, no pick has been made by the RR
-  // policy (e.g., all addresses failed to connect). There won't be any
-  // LB token available.
-  if (pp->pick->connected_subchannel != nullptr) {
-    const grpc_arg* arg =
-        grpc_channel_args_find(pp->pick->connected_subchannel->args(),
-                               GRPC_ARG_GRPCLB_ADDRESS_LB_TOKEN);
-    if (arg != nullptr) {
-      grpc_mdelem lb_token = {
-          reinterpret_cast<uintptr_t>(arg->value.pointer.p)};
-      AddLbTokenToInitialMetadata(GRPC_MDELEM_REF(lb_token),
-                                  &pp->pick->lb_token_mdelem_storage,
-                                  pp->pick->initial_metadata);
-    } else {
-      gpr_log(GPR_ERROR,
-              "[grpclb %p] No LB token for connected subchannel pick %p",
-              pp->grpclb_policy, pp->pick);
-      abort();
-    }
-    // Pass on client stats via context. Passes ownership of the reference.
-    if (pp->client_stats != nullptr) {
-      pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
-          pp->client_stats.release();
-      pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
-          DestroyClientStats;
-    }
-  } else {
-    pp->client_stats.reset();
-  }
-}
-
-/* The \a on_complete closure passed as part of the pick requires keeping a
- * reference to its associated round robin instance. We wrap this closure in
- * order to unref the round robin instance upon its invocation */
-void GrpcLb::OnPendingPickComplete(void* arg, grpc_error* error) {
-  PendingPick* pp = static_cast<PendingPick*>(arg);
-  PendingPickSetMetadataAndContext(pp);
-  GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_REF(error));
-  Delete(pp);
-}
-
-GrpcLb::PendingPick* GrpcLb::PendingPickCreate(PickState* pick) {
-  PendingPick* pp = New<PendingPick>();
-  pp->grpclb_policy = this;
-  pp->pick = pick;
-  GRPC_CLOSURE_INIT(&pp->on_complete, &GrpcLb::OnPendingPickComplete, pp,
-                    grpc_schedule_on_exec_ctx);
-  pp->original_on_complete = pick->on_complete;
-  pick->on_complete = &pp->on_complete;
-  return pp;
-}
-
-void GrpcLb::AddPendingPick(PendingPick* pp) {
-  pp->next = pending_picks_;
-  pending_picks_ = pp;
-}
-
 //
 // code for interacting with the RR policy
 //
 
-// Performs a pick over \a rr_policy_. Given that a pick can return
-// immediately (ignoring its completion callback), we need to perform the
-// cleanups this callback would otherwise be responsible for.
-// If \a force_async is true, then we will manually schedule the
-// completion callback even if the pick is available immediately.
-bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp,
-                                            grpc_error** error) {
-  // Check for drops if we are not using fallback backend addresses.
-  if (serverlist_ != nullptr && serverlist_->num_servers > 0) {
-    // Look at the index into the serverlist to see if we should drop this call.
-    grpc_grpclb_server* server = serverlist_->servers[serverlist_index_++];
-    if (serverlist_index_ == serverlist_->num_servers) {
-      serverlist_index_ = 0;  // Wrap-around.
-    }
-    if (server->drop) {
-      // Update client load reporting stats to indicate the number of
-      // dropped calls.  Note that we have to do this here instead of in
-      // the client_load_reporting filter, because we do not create a
-      // subchannel call (and therefore no client_load_reporting filter)
-      // for dropped calls.
-      if (lb_calld_ != nullptr && lb_calld_->client_stats() != nullptr) {
-        lb_calld_->client_stats()->AddCallDroppedLocked(
-            server->load_balance_token);
-      }
-      if (force_async) {
-        GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE);
-        Delete(pp);
-        return false;
-      }
-      Delete(pp);
-      return true;
-    }
-  }
-  // Set client_stats.
-  if (lb_calld_ != nullptr && lb_calld_->client_stats() != nullptr) {
-    pp->client_stats = lb_calld_->client_stats()->Ref();
-  }
-  // Pick via the RR policy.
-  bool pick_done = rr_policy_->PickLocked(pp->pick, error);
-  if (pick_done) {
-    PendingPickSetMetadataAndContext(pp);
-    if (force_async) {
-      GRPC_CLOSURE_SCHED(pp->original_on_complete, *error);
-      *error = GRPC_ERROR_NONE;
-      pick_done = false;
-    }
-    Delete(pp);
-  }
-  // else, the pending pick will be registered and taken care of by the
-  // pending pick list inside the RR policy.  Eventually,
-  // OnPendingPickComplete() will be called, which will (among other
-  // things) add the LB token to the call's initial metadata.
-  return pick_done;
-}
-
 void GrpcLb::CreateRoundRobinPolicyLocked(Args args) {
   GPR_ASSERT(rr_policy_ == nullptr);
   rr_policy_ = LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(
@@ -1601,40 +1521,12 @@ void GrpcLb::CreateRoundRobinPolicyLocked(Args args) {
     gpr_log(GPR_INFO, "[grpclb %p] Created new RR policy %p", this,
             rr_policy_.get());
   }
-  // TODO(roth): We currently track this ref manually.  Once the new
-  // ClosureRef API is done, pass the RefCountedPtr<> along with the closure.
-  auto self = Ref(DEBUG_LOCATION, "on_rr_reresolution_requested");
-  self.release();
-  rr_policy_->SetReresolutionClosureLocked(&on_rr_request_reresolution_);
-  grpc_error* rr_state_error = nullptr;
-  rr_connectivity_state_ = rr_policy_->CheckConnectivityLocked(&rr_state_error);
-  // Connectivity state is a function of the RR policy updated/created.
-  UpdateConnectivityStateFromRoundRobinPolicyLocked(rr_state_error);
   // Add the gRPC LB's interested_parties pollset_set to that of the newly
   // created RR policy. This will make the RR policy progress upon activity on
   // gRPC LB, which in turn is tied to the application's call.
   grpc_pollset_set_add_pollset_set(rr_policy_->interested_parties(),
                                    interested_parties());
-  // Subscribe to changes to the connectivity of the new RR.
-  // TODO(roth): We currently track this ref manually.  Once the new
-  // ClosureRef API is done, pass the RefCountedPtr<> along with the closure.
-  self = Ref(DEBUG_LOCATION, "on_rr_connectivity_changed");
-  self.release();
-  rr_policy_->NotifyOnStateChangeLocked(&rr_connectivity_state_,
-                                        &on_rr_connectivity_changed_);
   rr_policy_->ExitIdleLocked();
-  // Send pending picks to RR policy.
-  PendingPick* pp;
-  while ((pp = pending_picks_)) {
-    pending_picks_ = pp->next;
-    if (grpc_lb_glb_trace.enabled()) {
-      gpr_log(GPR_INFO,
-              "[grpclb %p] Pending pick about to (async) PICK from RR %p", this,
-              rr_policy_.get());
-    }
-    grpc_error* error = GRPC_ERROR_NONE;
-    PickFromRoundRobinPolicyLocked(true /* force_async */, pp, &error);
-  }
 }
 
 grpc_channel_args* GrpcLb::CreateRoundRobinPolicyArgsLocked() {
@@ -1642,7 +1534,7 @@ grpc_channel_args* GrpcLb::CreateRoundRobinPolicyArgsLocked() {
   ServerAddressList* addresses = &tmp_addresses;
   bool is_backend_from_grpclb_load_balancer = false;
   if (serverlist_ != nullptr) {
-    tmp_addresses = ProcessServerlist(serverlist_);
+    tmp_addresses = serverlist_->GetServerAddressList();
     is_backend_from_grpclb_load_balancer = true;
   } else {
     // If CreateOrUpdateRoundRobinPolicyLocked() is invoked when we haven't
@@ -1691,110 +1583,14 @@ void GrpcLb::CreateOrUpdateRoundRobinPolicyLocked() {
   } else {
     LoadBalancingPolicy::Args lb_policy_args;
     lb_policy_args.combiner = combiner();
-    lb_policy_args.client_channel_factory = client_channel_factory();
     lb_policy_args.args = args;
-    lb_policy_args.subchannel_pool = subchannel_pool()->Ref();
+    lb_policy_args.channel_control_helper =
+        UniquePtr<ChannelControlHelper>(New<Helper>(Ref()));
     CreateRoundRobinPolicyLocked(std::move(lb_policy_args));
   }
   grpc_channel_args_destroy(args);
 }
 
-void GrpcLb::OnRoundRobinRequestReresolutionLocked(void* arg,
-                                                   grpc_error* error) {
-  GrpcLb* grpclb_policy = static_cast<GrpcLb*>(arg);
-  if (grpclb_policy->shutting_down_ || error != GRPC_ERROR_NONE) {
-    grpclb_policy->Unref(DEBUG_LOCATION, "on_rr_reresolution_requested");
-    return;
-  }
-  if (grpc_lb_glb_trace.enabled()) {
-    gpr_log(
-        GPR_INFO,
-        "[grpclb %p] Re-resolution requested from the internal RR policy (%p).",
-        grpclb_policy, grpclb_policy->rr_policy_.get());
-  }
-  // If we are talking to a balancer, we expect to get updated addresses form
-  // the balancer, so we can ignore the re-resolution request from the RR
-  // policy. Otherwise, handle the re-resolution request using the
-  // grpclb policy's original re-resolution closure.
-  if (grpclb_policy->lb_calld_ == nullptr ||
-      !grpclb_policy->lb_calld_->seen_initial_response()) {
-    grpclb_policy->TryReresolutionLocked(&grpc_lb_glb_trace, GRPC_ERROR_NONE);
-  }
-  // Give back the wrapper closure to the RR policy.
-  grpclb_policy->rr_policy_->SetReresolutionClosureLocked(
-      &grpclb_policy->on_rr_request_reresolution_);
-}
-
-void GrpcLb::UpdateConnectivityStateFromRoundRobinPolicyLocked(
-    grpc_error* rr_state_error) {
-  const grpc_connectivity_state curr_glb_state =
-      grpc_connectivity_state_check(&state_tracker_);
-  /* The new connectivity status is a function of the previous one and the new
-   * input coming from the status of the RR policy.
-   *
-   *  current state (grpclb's)
-   *  |
-   *  v  || I  |  C  |  R  |  TF  |  SD  |  <- new state (RR's)
-   *  ===++====+=====+=====+======+======+
-   *   I || I  |  C  |  R  | [I]  | [I]  |
-   *  ---++----+-----+-----+------+------+
-   *   C || I  |  C  |  R  | [C]  | [C]  |
-   *  ---++----+-----+-----+------+------+
-   *   R || I  |  C  |  R  | [R]  | [R]  |
-   *  ---++----+-----+-----+------+------+
-   *  TF || I  |  C  |  R  | [TF] | [TF] |
-   *  ---++----+-----+-----+------+------+
-   *  SD || NA |  NA |  NA |  NA  |  NA  | (*)
-   *  ---++----+-----+-----+------+------+
-   *
-   * A [STATE] indicates that the old RR policy is kept. In those cases, STATE
-   * is the current state of grpclb, which is left untouched.
-   *
-   *  In summary, if the new state is TRANSIENT_FAILURE or SHUTDOWN, stick to
-   *  the previous RR instance.
-   *
-   *  Note that the status is never updated to SHUTDOWN as a result of calling
-   *  this function. Only glb_shutdown() has the power to set that state.
-   *
-   *  (*) This function mustn't be called during shutting down. */
-  GPR_ASSERT(curr_glb_state != GRPC_CHANNEL_SHUTDOWN);
-  switch (rr_connectivity_state_) {
-    case GRPC_CHANNEL_TRANSIENT_FAILURE:
-    case GRPC_CHANNEL_SHUTDOWN:
-      GPR_ASSERT(rr_state_error != GRPC_ERROR_NONE);
-      break;
-    case GRPC_CHANNEL_IDLE:
-    case GRPC_CHANNEL_CONNECTING:
-    case GRPC_CHANNEL_READY:
-      GPR_ASSERT(rr_state_error == GRPC_ERROR_NONE);
-  }
-  if (grpc_lb_glb_trace.enabled()) {
-    gpr_log(
-        GPR_INFO,
-        "[grpclb %p] Setting grpclb's state to %s from new RR policy %p state.",
-        this, grpc_connectivity_state_name(rr_connectivity_state_),
-        rr_policy_.get());
-  }
-  grpc_connectivity_state_set(&state_tracker_, rr_connectivity_state_,
-                              rr_state_error,
-                              "update_lb_connectivity_status_locked");
-}
-
-void GrpcLb::OnRoundRobinConnectivityChangedLocked(void* arg,
-                                                   grpc_error* error) {
-  GrpcLb* grpclb_policy = static_cast<GrpcLb*>(arg);
-  if (grpclb_policy->shutting_down_) {
-    grpclb_policy->Unref(DEBUG_LOCATION, "on_rr_connectivity_changed");
-    return;
-  }
-  grpclb_policy->UpdateConnectivityStateFromRoundRobinPolicyLocked(
-      GRPC_ERROR_REF(error));
-  // Resubscribe. Reuse the "on_rr_connectivity_changed" ref.
-  grpclb_policy->rr_policy_->NotifyOnStateChangeLocked(
-      &grpclb_policy->rr_connectivity_state_,
-      &grpclb_policy->on_rr_connectivity_changed_);
-}
-
 //
 // factory
 //

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc

@@ -43,7 +43,7 @@ void GrpcLbClientStats::AddCallFinished(
   }
 }
 
-void GrpcLbClientStats::AddCallDroppedLocked(char* token) {
+void GrpcLbClientStats::AddCallDroppedLocked(const char* token) {
   // Increment num_calls_started and num_calls_finished.
   gpr_atm_full_fetch_add(&num_calls_started_, (gpr_atm)1);
   gpr_atm_full_fetch_add(&num_calls_finished_, (gpr_atm)1);

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h

@@ -48,7 +48,7 @@ class GrpcLbClientStats : public RefCounted<GrpcLbClientStats> {
                        bool finished_known_received);
 
   // This method is not thread-safe; caller must synchronize.
-  void AddCallDroppedLocked(char* token);
+  void AddCallDroppedLocked(const char* token);
 
   // This method is not thread-safe; caller must synchronize.
   void GetLocked(int64_t* num_calls_started, int64_t* num_calls_finished,

src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc

@@ -52,16 +52,6 @@ class PickFirst : public LoadBalancingPolicy {
 
   void UpdateLocked(const grpc_channel_args& args,
                     grpc_json* lb_config) override;
-  bool PickLocked(PickState* pick, grpc_error** error) override;
-  void CancelPickLocked(PickState* pick, grpc_error* error) override;
-  void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                 uint32_t initial_metadata_flags_eq,
-                                 grpc_error* error) override;
-  void NotifyOnStateChangeLocked(grpc_connectivity_state* state,
-                                 grpc_closure* closure) override;
-  grpc_connectivity_state CheckConnectivityLocked(
-      grpc_error** connectivity_error) override;
-  void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override;
   void ExitIdleLocked() override;
   void ResetBackoffLocked() override;
   void FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels,
@@ -99,10 +89,9 @@ class PickFirst : public LoadBalancingPolicy {
     PickFirstSubchannelList(PickFirst* policy, TraceFlag* tracer,
                             const ServerAddressList& addresses,
                             grpc_combiner* combiner,
-                            grpc_client_channel_factory* client_channel_factory,
                             const grpc_channel_args& args)
         : SubchannelList(policy, tracer, addresses, combiner,
-                         client_channel_factory, args) {
+                         policy->channel_control_helper(), args) {
       // Need to maintain a ref to the LB policy as long as we maintain
       // any references to subchannels, since the subchannels'
       // pollset_sets will include the LB policy's pollset_set.
@@ -115,6 +104,20 @@ class PickFirst : public LoadBalancingPolicy {
     }
   };
 
+  class Picker : public SubchannelPicker {
+   public:
+    explicit Picker(RefCountedPtr<ConnectedSubchannel> connected_subchannel)
+        : connected_subchannel_(std::move(connected_subchannel)) {}
+
+    PickResult Pick(PickState* pick, grpc_error** error) override {
+      pick->connected_subchannel = connected_subchannel_;
+      return PICK_COMPLETE;
+    }
+
+   private:
+    RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
+  };
+
   // Helper class to ensure that any function that modifies the child refs
   // data structures will update the channelz snapshot data structures before
   // returning.
@@ -142,10 +145,6 @@ class PickFirst : public LoadBalancingPolicy {
   bool started_picking_ = false;
   // Are we shut down?
   bool shutdown_ = false;
-  // List of picks that are waiting on connectivity.
-  PickState* pending_picks_ = nullptr;
-  // Our connectivity state tracker.
-  grpc_connectivity_state_tracker state_tracker_;
 
   /// Lock and data used to capture snapshots of this channels child
   /// channels and subchannels. This data is consumed by channelz.
@@ -155,13 +154,15 @@ class PickFirst : public LoadBalancingPolicy {
 };
 
 PickFirst::PickFirst(Args args) : LoadBalancingPolicy(std::move(args)) {
-  GPR_ASSERT(args.client_channel_factory != nullptr);
   gpr_mu_init(&child_refs_mu_);
-  grpc_connectivity_state_init(&state_tracker_, GRPC_CHANNEL_IDLE,
-                               "pick_first");
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_INFO, "Pick First %p created.", this);
   }
+  // Initialize channel with a picker that will start us connecting upon
+  // the first pick.
+  channel_control_helper()->UpdateState(
+      GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
+      UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
   UpdateLocked(*args.args, args.lb_config);
 }
 
@@ -172,81 +173,16 @@ PickFirst::~PickFirst() {
   gpr_mu_destroy(&child_refs_mu_);
   GPR_ASSERT(subchannel_list_ == nullptr);
   GPR_ASSERT(latest_pending_subchannel_list_ == nullptr);
-  GPR_ASSERT(pending_picks_ == nullptr);
-  grpc_connectivity_state_destroy(&state_tracker_);
-}
-
-void PickFirst::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) {
-  PickState* pick;
-  while ((pick = pending_picks_) != nullptr) {
-    pending_picks_ = pick->next;
-    grpc_error* error = GRPC_ERROR_NONE;
-    if (new_policy->PickLocked(pick, &error)) {
-      // Synchronous return, schedule closure.
-      GRPC_CLOSURE_SCHED(pick->on_complete, error);
-    }
-  }
 }
 
 void PickFirst::ShutdownLocked() {
   AutoChildRefsUpdater guard(this);
-  grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_INFO, "Pick First %p Shutting down", this);
   }
   shutdown_ = true;
-  PickState* pick;
-  while ((pick = pending_picks_) != nullptr) {
-    pending_picks_ = pick->next;
-    pick->connected_subchannel.reset();
-    GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
-  }
-  grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_SHUTDOWN,
-                              GRPC_ERROR_REF(error), "shutdown");
   subchannel_list_.reset();
   latest_pending_subchannel_list_.reset();
-  TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_CANCELLED);
-  GRPC_ERROR_UNREF(error);
-}
-
-void PickFirst::CancelPickLocked(PickState* pick, grpc_error* error) {
-  PickState* pp = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pp != nullptr) {
-    PickState* next = pp->next;
-    if (pp == pick) {
-      pick->connected_subchannel.reset();
-      GRPC_CLOSURE_SCHED(pick->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pp->next = pending_picks_;
-      pending_picks_ = pp;
-    }
-    pp = next;
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
-void PickFirst::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                          uint32_t initial_metadata_flags_eq,
-                                          grpc_error* error) {
-  PickState* pick = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pick != nullptr) {
-    PickState* next = pick->next;
-    if ((*pick->initial_metadata_flags & initial_metadata_flags_mask) ==
-        initial_metadata_flags_eq) {
-      GRPC_CLOSURE_SCHED(pick->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pick->next = pending_picks_;
-      pending_picks_ = pick;
-    }
-    pick = next;
-  }
-  GRPC_ERROR_UNREF(error);
 }
 
 void PickFirst::StartPickingLocked() {
@@ -270,36 +206,6 @@ void PickFirst::ResetBackoffLocked() {
   }
 }
 
-bool PickFirst::PickLocked(PickState* pick, grpc_error** error) {
-  // If we have a selected subchannel already, return synchronously.
-  if (selected_ != nullptr) {
-    pick->connected_subchannel = selected_->connected_subchannel()->Ref();
-    return true;
-  }
-  // No subchannel selected yet, so handle asynchronously.
-  if (pick->on_complete == nullptr) {
-    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-        "No pick result available but synchronous result required.");
-    return true;
-  }
-  pick->next = pending_picks_;
-  pending_picks_ = pick;
-  if (!started_picking_) {
-    StartPickingLocked();
-  }
-  return false;
-}
-
-grpc_connectivity_state PickFirst::CheckConnectivityLocked(grpc_error** error) {
-  return grpc_connectivity_state_get(&state_tracker_, error);
-}
-
-void PickFirst::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
-                                          grpc_closure* notify) {
-  grpc_connectivity_state_notify_on_state_change(&state_tracker_, current,
-                                                 notify);
-}
-
 void PickFirst::FillChildRefsForChannelz(
     channelz::ChildRefsList* child_subchannels_to_fill,
     channelz::ChildRefsList* ignored) {
@@ -341,10 +247,11 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args,
   if (addresses == nullptr) {
     if (subchannel_list_ == nullptr) {
       // If we don't have a current subchannel list, go into TRANSIENT FAILURE.
-      grpc_connectivity_state_set(
-          &state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing update in args"),
-          "pf_update_missing");
+      grpc_error* error =
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing update in args");
+      channel_control_helper()->UpdateState(
+          GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+          UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
     } else {
       // otherwise, keep using the current subchannel list (ignore this update).
       gpr_log(GPR_ERROR,
@@ -364,18 +271,17 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args,
   grpc_channel_args* new_args =
       grpc_channel_args_copy_and_add(&args, &new_arg, 1);
   auto subchannel_list = MakeOrphanable<PickFirstSubchannelList>(
-      this, &grpc_lb_pick_first_trace, *addresses, combiner(),
-      client_channel_factory(), *new_args);
+      this, &grpc_lb_pick_first_trace, *addresses, combiner(), *new_args);
   grpc_channel_args_destroy(new_args);
   if (subchannel_list->num_subchannels() == 0) {
     // Empty update or no valid subchannels. Unsubscribe from all current
     // subchannels and put the channel in TRANSIENT_FAILURE.
-    grpc_connectivity_state_set(
-        &state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
-        "pf_update_empty");
     subchannel_list_ = std::move(subchannel_list);  // Empty list.
     selected_ = nullptr;
+    grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update");
+    channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+        UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
     return;
   }
   // If one of the subchannels in the new list is already in state
@@ -453,7 +359,8 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
   if (p->selected_ == this) {
     if (grpc_lb_pick_first_trace.enabled()) {
       gpr_log(GPR_INFO,
-              "Pick First %p connectivity changed for selected subchannel", p);
+              "Pick First %p selected subchannel connectivity changed to %s", p,
+              grpc_connectivity_state_name(connectivity_state));
     }
     // If the new state is anything other than READY and there is a
     // pending update, switch to the pending update.
@@ -469,14 +376,12 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
       p->selected_ = nullptr;
       StopConnectivityWatchLocked();
       p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
-      grpc_connectivity_state_set(
-          &p->state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-          error != GRPC_ERROR_NONE
-              ? GRPC_ERROR_REF(error)
-              : GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-                    "selected subchannel not ready; switching to pending "
-                    "update"),
-          "selected_not_ready+switch_to_update");
+      grpc_error* new_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+          "selected subchannel not ready; switching to pending update", &error,
+          1);
+      p->channel_control_helper()->UpdateState(
+          GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(new_error),
+          UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(new_error)));
     } else {
       if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
         // If the selected subchannel goes bad, request a re-resolution. We also
@@ -484,17 +389,28 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
         // is that if the new state is TRANSIENT_FAILURE due to a GOAWAY
         // reception we don't want to connect to the re-resolved backends until
         // we leave the IDLE state.
-        grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_IDLE,
-                                    GRPC_ERROR_NONE,
-                                    "selected_changed+reresolve");
         p->started_picking_ = false;
-        p->TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_NONE);
+        p->channel_control_helper()->RequestReresolution();
         // In transient failure. Rely on re-resolution to recover.
         p->selected_ = nullptr;
         StopConnectivityWatchLocked();
+        p->channel_control_helper()->UpdateState(
+            GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
+            UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
       } else {
-        grpc_connectivity_state_set(&p->state_tracker_, connectivity_state,
-                                    GRPC_ERROR_REF(error), "selected_changed");
+        // This is unlikely but can happen when a subchannel has been asked
+        // to reconnect by a different channel and this channel has dropped
+        // some connectivity state notifications.
+        if (connectivity_state == GRPC_CHANNEL_READY) {
+          p->channel_control_helper()->UpdateState(
+              GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
+              UniquePtr<SubchannelPicker>(
+                  New<Picker>(connected_subchannel()->Ref())));
+        } else {  // CONNECTING
+          p->channel_control_helper()->UpdateState(
+              connectivity_state, GRPC_ERROR_REF(error),
+              UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
+        }
         // Renew notification.
         RenewConnectivityWatchLocked();
       }
@@ -527,10 +443,14 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
       // Case 1: Only set state to TRANSIENT_FAILURE if we've tried
       // all subchannels.
       if (sd->Index() == 0 && subchannel_list() == p->subchannel_list_.get()) {
-        p->TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_NONE);
-        grpc_connectivity_state_set(
-            &p->state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-            GRPC_ERROR_REF(error), "exhausted_subchannels");
+        p->channel_control_helper()->RequestReresolution();
+        grpc_error* new_error =
+            GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                "failed to connect to all addresses", &error, 1);
+        p->channel_control_helper()->UpdateState(
+            GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(new_error),
+            UniquePtr<SubchannelPicker>(
+                New<TransientFailurePicker>(new_error)));
       }
       sd->CheckConnectivityStateAndStartWatchingLocked();
       break;
@@ -539,9 +459,9 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
     case GRPC_CHANNEL_IDLE: {
       // Only update connectivity state in case 1.
       if (subchannel_list() == p->subchannel_list_.get()) {
-        grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_CONNECTING,
-                                    GRPC_ERROR_REF(error),
-                                    "connecting_changed");
+        p->channel_control_helper()->UpdateState(
+            GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
+            UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
       }
       // Renew notification.
       RenewConnectivityWatchLocked();
@@ -578,23 +498,13 @@ void PickFirst::PickFirstSubchannelData::ProcessUnselectedReadyLocked() {
     p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
   }
   // Cases 1 and 2.
-  grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_READY,
-                              GRPC_ERROR_NONE, "subchannel_ready");
   p->selected_ = this;
+  p->channel_control_helper()->UpdateState(
+      GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
+      UniquePtr<SubchannelPicker>(New<Picker>(connected_subchannel()->Ref())));
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", p, subchannel());
   }
-  // Update any calls that were waiting for a pick.
-  PickState* pick;
-  while ((pick = p->pending_picks_)) {
-    p->pending_picks_ = pick->next;
-    pick->connected_subchannel = p->selected_->connected_subchannel()->Ref();
-    if (grpc_lb_pick_first_trace.enabled()) {
-      gpr_log(GPR_INFO, "Servicing pending pick with selected subchannel %p",
-              p->selected_->subchannel());
-    }
-    GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
-  }
 }
 
 void PickFirst::PickFirstSubchannelData::

src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -26,6 +26,7 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stdlib.h>
 #include <string.h>
 
 #include <grpc/support/alloc.h>
@@ -62,16 +63,6 @@ class RoundRobin : public LoadBalancingPolicy {
 
   void UpdateLocked(const grpc_channel_args& args,
                     grpc_json* lb_config) override;
-  bool PickLocked(PickState* pick, grpc_error** error) override;
-  void CancelPickLocked(PickState* pick, grpc_error* error) override;
-  void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                 uint32_t initial_metadata_flags_eq,
-                                 grpc_error* error) override;
-  void NotifyOnStateChangeLocked(grpc_connectivity_state* state,
-                                 grpc_closure* closure) override;
-  grpc_connectivity_state CheckConnectivityLocked(
-      grpc_error** connectivity_error) override;
-  void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override;
   void ExitIdleLocked() override;
   void ResetBackoffLocked() override;
   void FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels,
@@ -117,14 +108,12 @@ class RoundRobin : public LoadBalancingPolicy {
       : public SubchannelList<RoundRobinSubchannelList,
                               RoundRobinSubchannelData> {
    public:
-    RoundRobinSubchannelList(
-        RoundRobin* policy, TraceFlag* tracer,
-        const ServerAddressList& addresses, grpc_combiner* combiner,
-        grpc_client_channel_factory* client_channel_factory,
-        const grpc_channel_args& args)
+    RoundRobinSubchannelList(RoundRobin* policy, TraceFlag* tracer,
+                             const ServerAddressList& addresses,
+                             grpc_combiner* combiner,
+                             const grpc_channel_args& args)
         : SubchannelList(policy, tracer, addresses, combiner,
-                         client_channel_factory, args),
-          last_ready_index_(num_subchannels() - 1) {
+                         policy->channel_control_helper(), args) {
       // Need to maintain a ref to the LB policy as long as we maintain
       // any references to subchannels, since the subchannels'
       // pollset_sets will include the LB policy's pollset_set.
@@ -157,15 +146,25 @@ class RoundRobin : public LoadBalancingPolicy {
     // subchannels in each state.
     void UpdateRoundRobinStateFromSubchannelStateCountsLocked();
 
-    size_t GetNextReadySubchannelIndexLocked();
-    void UpdateLastReadySubchannelIndexLocked(size_t last_ready_index);
-
    private:
     size_t num_ready_ = 0;
     size_t num_connecting_ = 0;
     size_t num_transient_failure_ = 0;
     grpc_error* last_transient_failure_error_ = GRPC_ERROR_NONE;
-    size_t last_ready_index_;  // Index into list of last pick.
+  };
+
+  class Picker : public SubchannelPicker {
+   public:
+    Picker(RoundRobin* parent, RoundRobinSubchannelList* subchannel_list);
+
+    PickResult Pick(PickState* pick, grpc_error** error) override;
+
+   private:
+    // Using pointer value only, no ref held -- do not dereference!
+    RoundRobin* parent_;
+
+    size_t last_picked_index_;
+    InlinedVector<RefCountedPtr<ConnectedSubchannel>, 10> subchannels_;
   };
 
   // Helper class to ensure that any function that modifies the child refs
@@ -183,8 +182,6 @@ class RoundRobin : public LoadBalancingPolicy {
   void ShutdownLocked() override;
 
   void StartPickingLocked();
-  bool DoPickLocked(PickState* pick);
-  void DrainPendingPicksLocked();
   void UpdateChildRefsLocked();
 
   /** list of subchannels */
@@ -199,10 +196,6 @@ class RoundRobin : public LoadBalancingPolicy {
   bool started_picking_ = false;
   /** are we shutting down? */
   bool shutdown_ = false;
-  /** List of picks that are waiting on connectivity */
-  PickState* pending_picks_ = nullptr;
-  /** our connectivity state tracker */
-  grpc_connectivity_state_tracker state_tracker_;
   /// Lock and data used to capture snapshots of this channel's child
   /// channels and subchannels. This data is consumed by channelz.
   gpr_mu child_refs_mu_;
@@ -210,16 +203,62 @@ class RoundRobin : public LoadBalancingPolicy {
   channelz::ChildRefsList child_channels_;
 };
 
+//
+// RoundRobin::Picker
+//
+
+RoundRobin::Picker::Picker(RoundRobin* parent,
+                           RoundRobinSubchannelList* subchannel_list)
+    : parent_(parent) {
+  for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
+    auto* connected_subchannel =
+        subchannel_list->subchannel(i)->connected_subchannel();
+    if (connected_subchannel != nullptr) {
+      subchannels_.push_back(connected_subchannel->Ref());
+    }
+  }
+  // For discussion on why we generate a random starting index for
+  // the picker, see https://github.com/grpc/grpc-go/issues/2580.
+  // TODO(roth): rand(3) is not thread-safe.  This should be replaced with
+  // something better as part of https://github.com/grpc/grpc/issues/17891.
+  last_picked_index_ = rand() % subchannels_.size();
+  if (grpc_lb_round_robin_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[RR %p picker %p] created picker from subchannel_list=%p "
+            "with %" PRIuPTR " READY subchannels; last_picked_index_=%" PRIuPTR,
+            parent_, this, subchannel_list, subchannels_.size(),
+            last_picked_index_);
+  }
+}
+
+RoundRobin::Picker::PickResult RoundRobin::Picker::Pick(PickState* pick,
+                                                        grpc_error** error) {
+  last_picked_index_ = (last_picked_index_ + 1) % subchannels_.size();
+  if (grpc_lb_round_robin_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[RR %p picker %p] returning index %" PRIuPTR
+            ", connected_subchannel=%p",
+            parent_, this, last_picked_index_,
+            subchannels_[last_picked_index_].get());
+  }
+  pick->connected_subchannel = subchannels_[last_picked_index_];
+  return PICK_COMPLETE;
+}
+
+//
+// RoundRobin
+//
+
 RoundRobin::RoundRobin(Args args) : LoadBalancingPolicy(std::move(args)) {
-  GPR_ASSERT(args.client_channel_factory != nullptr);
   gpr_mu_init(&child_refs_mu_);
-  grpc_connectivity_state_init(&state_tracker_, GRPC_CHANNEL_IDLE,
-                               "round_robin");
-  UpdateLocked(*args.args, args.lb_config);
   if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO, "[RR %p] Created with %" PRIuPTR " subchannels", this,
-            subchannel_list_->num_subchannels());
+    gpr_log(GPR_INFO, "[RR %p] Created", this);
   }
+  // Initialize channel with a picker that will start us connecting.
+  channel_control_helper()->UpdateState(
+      GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
+      UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
+  UpdateLocked(*args.args, args.lb_config);
 }
 
 RoundRobin::~RoundRobin() {
@@ -229,82 +268,16 @@ RoundRobin::~RoundRobin() {
   gpr_mu_destroy(&child_refs_mu_);
   GPR_ASSERT(subchannel_list_ == nullptr);
   GPR_ASSERT(latest_pending_subchannel_list_ == nullptr);
-  GPR_ASSERT(pending_picks_ == nullptr);
-  grpc_connectivity_state_destroy(&state_tracker_);
-}
-
-void RoundRobin::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) {
-  PickState* pick;
-  while ((pick = pending_picks_) != nullptr) {
-    pending_picks_ = pick->next;
-    grpc_error* error = GRPC_ERROR_NONE;
-    if (new_policy->PickLocked(pick, &error)) {
-      // Synchronous return, schedule closure.
-      GRPC_CLOSURE_SCHED(pick->on_complete, error);
-    }
-  }
 }
 
 void RoundRobin::ShutdownLocked() {
   AutoChildRefsUpdater guard(this);
-  grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   if (grpc_lb_round_robin_trace.enabled()) {
     gpr_log(GPR_INFO, "[RR %p] Shutting down", this);
   }
   shutdown_ = true;
-  PickState* pick;
-  while ((pick = pending_picks_) != nullptr) {
-    pending_picks_ = pick->next;
-    pick->connected_subchannel.reset();
-    GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_REF(error));
-  }
-  grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_SHUTDOWN,
-                              GRPC_ERROR_REF(error), "rr_shutdown");
   subchannel_list_.reset();
   latest_pending_subchannel_list_.reset();
-  TryReresolutionLocked(&grpc_lb_round_robin_trace, GRPC_ERROR_CANCELLED);
-  GRPC_ERROR_UNREF(error);
-}
-
-void RoundRobin::CancelPickLocked(PickState* pick, grpc_error* error) {
-  PickState* pp = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pp != nullptr) {
-    PickState* next = pp->next;
-    if (pp == pick) {
-      pick->connected_subchannel.reset();
-      GRPC_CLOSURE_SCHED(pick->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pp->next = pending_picks_;
-      pending_picks_ = pp;
-    }
-    pp = next;
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
-void RoundRobin::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                           uint32_t initial_metadata_flags_eq,
-                                           grpc_error* error) {
-  PickState* pick = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pick != nullptr) {
-    PickState* next = pick->next;
-    if ((*pick->initial_metadata_flags & initial_metadata_flags_mask) ==
-        initial_metadata_flags_eq) {
-      pick->connected_subchannel.reset();
-      GRPC_CLOSURE_SCHED(pick->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pick->next = pending_picks_;
-      pending_picks_ = pick;
-    }
-    pick = next;
-  }
-  GRPC_ERROR_UNREF(error);
 }
 
 void RoundRobin::StartPickingLocked() {
@@ -325,60 +298,6 @@ void RoundRobin::ResetBackoffLocked() {
   }
 }
 
-bool RoundRobin::DoPickLocked(PickState* pick) {
-  const size_t next_ready_index =
-      subchannel_list_->GetNextReadySubchannelIndexLocked();
-  if (next_ready_index < subchannel_list_->num_subchannels()) {
-    /* readily available, report right away */
-    RoundRobinSubchannelData* sd =
-        subchannel_list_->subchannel(next_ready_index);
-    GPR_ASSERT(sd->connected_subchannel() != nullptr);
-    pick->connected_subchannel = sd->connected_subchannel()->Ref();
-    if (grpc_lb_round_robin_trace.enabled()) {
-      gpr_log(GPR_INFO,
-              "[RR %p] Picked target <-- Subchannel %p (connected %p) (sl %p, "
-              "index %" PRIuPTR ")",
-              this, sd->subchannel(), pick->connected_subchannel.get(),
-              sd->subchannel_list(), next_ready_index);
-    }
-    /* only advance the last picked pointer if the selection was used */
-    subchannel_list_->UpdateLastReadySubchannelIndexLocked(next_ready_index);
-    return true;
-  }
-  return false;
-}
-
-void RoundRobin::DrainPendingPicksLocked() {
-  PickState* pick;
-  while ((pick = pending_picks_)) {
-    pending_picks_ = pick->next;
-    GPR_ASSERT(DoPickLocked(pick));
-    GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE);
-  }
-}
-
-bool RoundRobin::PickLocked(PickState* pick, grpc_error** error) {
-  if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", this, shutdown_);
-  }
-  GPR_ASSERT(!shutdown_);
-  if (subchannel_list_ != nullptr) {
-    if (DoPickLocked(pick)) return true;
-  }
-  if (pick->on_complete == nullptr) {
-    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-        "No pick result available but synchronous result required.");
-    return true;
-  }
-  /* no pick currently available. Save for later in list of pending picks */
-  pick->next = pending_picks_;
-  pending_picks_ = pick;
-  if (!started_picking_) {
-    StartPickingLocked();
-  }
-  return false;
-}
-
 void RoundRobin::FillChildRefsForChannelz(
     channelz::ChildRefsList* child_subchannels_to_fill,
     channelz::ChildRefsList* ignored) {
@@ -462,8 +381,8 @@ void RoundRobin::RoundRobinSubchannelList::UpdateStateCountersLocked(
   last_transient_failure_error_ = transient_failure_error;
 }
 
-// Sets the RR policy's connectivity state based on the current
-// subchannel list.
+// Sets the RR policy's connectivity state and generates a new picker based
+// on the current subchannel list.
 void RoundRobin::RoundRobinSubchannelList::
     MaybeUpdateRoundRobinConnectivityStateLocked() {
   RoundRobin* p = static_cast<RoundRobin*>(policy());
@@ -485,18 +404,21 @@ void RoundRobin::RoundRobinSubchannelList::
    */
   if (num_ready_ > 0) {
     /* 1) READY */
-    grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_READY,
-                                GRPC_ERROR_NONE, "rr_ready");
+    p->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
+        UniquePtr<SubchannelPicker>(New<Picker>(p, this)));
   } else if (num_connecting_ > 0) {
     /* 2) CONNECTING */
-    grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_CONNECTING,
-                                GRPC_ERROR_NONE, "rr_connecting");
+    p->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
+        UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
   } else if (num_transient_failure_ == num_subchannels()) {
     /* 3) TRANSIENT_FAILURE */
-    grpc_connectivity_state_set(&p->state_tracker_,
-                                GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                GRPC_ERROR_REF(last_transient_failure_error_),
-                                "rr_exhausted_subchannels");
+    p->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_TRANSIENT_FAILURE,
+        GRPC_ERROR_REF(last_transient_failure_error_),
+        UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(
+            GRPC_ERROR_REF(last_transient_failure_error_))));
   }
 }
 
@@ -525,8 +447,6 @@ void RoundRobin::RoundRobinSubchannelList::
       }
       p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
     }
-    // Drain pending picks.
-    p->DrainPendingPicksLocked();
   }
   // Update the RR policy's connectivity state if needed.
   MaybeUpdateRoundRobinConnectivityStateLocked();
@@ -566,7 +486,7 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
               "Requesting re-resolution",
               p, subchannel());
     }
-    p->TryReresolutionLocked(&grpc_lb_round_robin_trace, GRPC_ERROR_NONE);
+    p->channel_control_helper()->RequestReresolution();
   }
   // Update state counters.
   UpdateConnectivityStateLocked(connectivity_state, error);
@@ -575,73 +495,6 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
   RenewConnectivityWatchLocked();
 }
 
-/** Returns the index into p->subchannel_list->subchannels of the next
- * subchannel in READY state, or p->subchannel_list->num_subchannels if no
- * subchannel is READY.
- *
- * Note that this function does *not* update p->last_ready_subchannel_index.
- * The caller must do that if it returns a pick. */
-size_t
-RoundRobin::RoundRobinSubchannelList::GetNextReadySubchannelIndexLocked() {
-  if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO,
-            "[RR %p] getting next ready subchannel (out of %" PRIuPTR
-            "), last_ready_index=%" PRIuPTR,
-            policy(), num_subchannels(), last_ready_index_);
-  }
-  for (size_t i = 0; i < num_subchannels(); ++i) {
-    const size_t index = (i + last_ready_index_ + 1) % num_subchannels();
-    if (grpc_lb_round_robin_trace.enabled()) {
-      gpr_log(
-          GPR_INFO,
-          "[RR %p] checking subchannel %p, subchannel_list %p, index %" PRIuPTR
-          ": state=%s",
-          policy(), subchannel(index)->subchannel(), this, index,
-          grpc_connectivity_state_name(
-              subchannel(index)->connectivity_state()));
-    }
-    if (subchannel(index)->connectivity_state() == GRPC_CHANNEL_READY) {
-      if (grpc_lb_round_robin_trace.enabled()) {
-        gpr_log(GPR_INFO,
-                "[RR %p] found next ready subchannel (%p) at index %" PRIuPTR
-                " of subchannel_list %p",
-                policy(), subchannel(index)->subchannel(), index, this);
-      }
-      return index;
-    }
-  }
-  if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO, "[RR %p] no subchannels in ready state", this);
-  }
-  return num_subchannels();
-}
-
-// Sets last_ready_index_ to last_ready_index.
-void RoundRobin::RoundRobinSubchannelList::UpdateLastReadySubchannelIndexLocked(
-    size_t last_ready_index) {
-  GPR_ASSERT(last_ready_index < num_subchannels());
-  last_ready_index_ = last_ready_index;
-  if (grpc_lb_round_robin_trace.enabled()) {
-    gpr_log(GPR_INFO,
-            "[RR %p] setting last_ready_subchannel_index=%" PRIuPTR
-            " (SC %p, CSC %p)",
-            policy(), last_ready_index,
-            subchannel(last_ready_index)->subchannel(),
-            subchannel(last_ready_index)->connected_subchannel());
-  }
-}
-
-grpc_connectivity_state RoundRobin::CheckConnectivityLocked(
-    grpc_error** error) {
-  return grpc_connectivity_state_get(&state_tracker_, error);
-}
-
-void RoundRobin::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
-                                           grpc_closure* notify) {
-  grpc_connectivity_state_notify_on_state_change(&state_tracker_, current,
-                                                 notify);
-}
-
 void RoundRobin::UpdateLocked(const grpc_channel_args& args,
                               grpc_json* lb_config) {
   AutoChildRefsUpdater guard(this);
@@ -651,10 +504,11 @@ void RoundRobin::UpdateLocked(const grpc_channel_args& args,
     // If we don't have a current subchannel list, go into TRANSIENT_FAILURE.
     // Otherwise, keep using the current subchannel list (ignore this update).
     if (subchannel_list_ == nullptr) {
-      grpc_connectivity_state_set(
-          &state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing update in args"),
-          "rr_update_missing");
+      grpc_error* error =
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Missing update in args");
+      channel_control_helper()->UpdateState(
+          GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+          UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
     }
     return;
   }
@@ -671,17 +525,16 @@ void RoundRobin::UpdateLocked(const grpc_channel_args& args,
     }
   }
   latest_pending_subchannel_list_ = MakeOrphanable<RoundRobinSubchannelList>(
-      this, &grpc_lb_round_robin_trace, *addresses, combiner(),
-      client_channel_factory(), args);
+      this, &grpc_lb_round_robin_trace, *addresses, combiner(), args);
   // If we haven't started picking yet or the new list is empty,
   // immediately promote the new list to the current list.
   if (!started_picking_ ||
       latest_pending_subchannel_list_->num_subchannels() == 0) {
     if (latest_pending_subchannel_list_->num_subchannels() == 0) {
-      grpc_connectivity_state_set(
-          &state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
-          "rr_update_empty");
+      grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update");
+      channel_control_helper()->UpdateState(
+          GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+          UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
     }
     subchannel_list_ = std::move(latest_pending_subchannel_list_);
   } else {

src/core/ext/filters/client_channel/lb_policy/subchannel_list.h

@@ -232,7 +232,7 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
  protected:
   SubchannelList(LoadBalancingPolicy* policy, TraceFlag* tracer,
                  const ServerAddressList& addresses, grpc_combiner* combiner,
-                 grpc_client_channel_factory* client_channel_factory,
+                 LoadBalancingPolicy::ChannelControlHelper* helper,
                  const grpc_channel_args& args);
 
   virtual ~SubchannelList();
@@ -486,7 +486,7 @@ template <typename SubchannelListType, typename SubchannelDataType>
 SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
     LoadBalancingPolicy* policy, TraceFlag* tracer,
     const ServerAddressList& addresses, grpc_combiner* combiner,
-    grpc_client_channel_factory* client_channel_factory,
+    LoadBalancingPolicy::ChannelControlHelper* helper,
     const grpc_channel_args& args)
     : InternallyRefCounted<SubchannelListType>(tracer),
       policy_(policy),
@@ -509,12 +509,8 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
                                          GRPC_ARG_INHIBIT_HEALTH_CHECKING};
   // Create a subchannel for each address.
   for (size_t i = 0; i < addresses.size(); i++) {
-    // If there were any balancer addresses, we would have chosen grpclb
-    // policy, which does not use a SubchannelList.
     GPR_ASSERT(!addresses[i].IsBalancer());
-    InlinedVector<grpc_arg, 4> args_to_add;
-    args_to_add.emplace_back(
-        SubchannelPoolInterface::CreateChannelArg(policy_->subchannel_pool()));
+    InlinedVector<grpc_arg, 3> args_to_add;
     const size_t subchannel_address_arg_index = args_to_add.size();
     args_to_add.emplace_back(
         Subchannel::CreateSubchannelAddressArg(&addresses[i].address()));
@@ -527,8 +523,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
         &args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove),
         args_to_add.data(), args_to_add.size());
     gpr_free(args_to_add[subchannel_address_arg_index].value.string);
-    Subchannel* subchannel = grpc_client_channel_factory_create_subchannel(
-        client_channel_factory, new_args);
+    Subchannel* subchannel = helper->CreateSubchannel(*new_args);
     grpc_channel_args_destroy(new_args);
     if (subchannel == nullptr) {
       // Subchannel could not be created.

src/core/ext/filters/client_channel/lb_policy/xds/xds.cc

@@ -70,7 +70,6 @@
 #include <grpc/support/time.h>
 
 #include "src/core/ext/filters/client_channel/client_channel.h"
-#include "src/core/ext/filters/client_channel/client_channel_factory.h"
 #include "src/core/ext/filters/client_channel/lb_policy/xds/xds.h"
 #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h"
 #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h"
@@ -125,16 +124,6 @@ class XdsLb : public LoadBalancingPolicy {
 
   void UpdateLocked(const grpc_channel_args& args,
                     grpc_json* lb_config) override;
-  bool PickLocked(PickState* pick, grpc_error** error) override;
-  void CancelPickLocked(PickState* pick, grpc_error* error) override;
-  void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                 uint32_t initial_metadata_flags_eq,
-                                 grpc_error* error) override;
-  void NotifyOnStateChangeLocked(grpc_connectivity_state* state,
-                                 grpc_closure* closure) override;
-  grpc_connectivity_state CheckConnectivityLocked(
-      grpc_error** connectivity_error) override;
-  void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override;
   void ExitIdleLocked() override;
   void ResetBackoffLocked() override;
   void FillChildRefsForChannelz(
@@ -142,31 +131,6 @@ class XdsLb : public LoadBalancingPolicy {
       channelz::ChildRefsList* child_channels) override;
 
  private:
-  /// Linked list of pending pick requests. It stores all information needed to
-  /// eventually call pick() on them. They mainly stay pending waiting for the
-  /// child policy to be created.
-  ///
-  /// Note that when a pick is sent to the child policy, we inject our own
-  /// on_complete callback, so that we can intercept the result before
-  /// invoking the original on_complete callback.  This allows us to set the
-  /// LB token metadata and add client_stats to the call context.
-  /// See \a pending_pick_complete() for details.
-  struct PendingPick {
-    // The xds lb instance that created the wrapping. This instance is not
-    // owned; reference counts are untouched. It's used only for logging
-    // purposes.
-    XdsLb* xdslb_policy;
-    // The original pick.
-    PickState* pick;
-    // Our on_complete closure and the original one.
-    grpc_closure on_complete;
-    grpc_closure* original_on_complete;
-    // Stats for client-side load reporting.
-    RefCountedPtr<XdsLbClientStats> client_stats;
-    // Next pending pick.
-    PendingPick* next = nullptr;
-  };
-
   /// Contains a call to the LB server and all the data related to the call.
   class BalancerCallState : public InternallyRefCounted<BalancerCallState> {
    public:
@@ -241,6 +205,36 @@ class XdsLb : public LoadBalancingPolicy {
     grpc_closure client_load_report_closure_;
   };
 
+  class Picker : public SubchannelPicker {
+   public:
+    Picker(UniquePtr<SubchannelPicker> child_picker,
+           RefCountedPtr<XdsLbClientStats> client_stats)
+        : child_picker_(std::move(child_picker)),
+          client_stats_(std::move(client_stats)) {}
+
+    PickResult Pick(PickState* pick, grpc_error** error) override;
+
+   private:
+    UniquePtr<SubchannelPicker> child_picker_;
+    RefCountedPtr<XdsLbClientStats> client_stats_;
+  };
+
+  class Helper : public ChannelControlHelper {
+   public:
+    explicit Helper(RefCountedPtr<XdsLb> parent) : parent_(std::move(parent)) {}
+
+    Subchannel* CreateSubchannel(const grpc_channel_args& args) override;
+    grpc_channel* CreateChannel(const char* target,
+                                grpc_client_channel_type type,
+                                const grpc_channel_args& args) override;
+    void UpdateState(grpc_connectivity_state state, grpc_error* state_error,
+                     UniquePtr<SubchannelPicker> picker) override;
+    void RequestReresolution() override;
+
+   private:
+    RefCountedPtr<XdsLb> parent_;
+  };
+
   ~XdsLb();
 
   void ShutdownLocked() override;
@@ -263,24 +257,10 @@ class XdsLb : public LoadBalancingPolicy {
   static void OnBalancerChannelConnectivityChangedLocked(void* arg,
                                                          grpc_error* error);
 
-  // Pending pick methods.
-  static void PendingPickCleanup(PendingPick* pp);
-  PendingPick* PendingPickCreate(PickState* pick);
-  void AddPendingPick(PendingPick* pp);
-  static void OnPendingPickComplete(void* arg, grpc_error* error);
-
   // Methods for dealing with the child policy.
   void CreateOrUpdateChildPolicyLocked();
   grpc_channel_args* CreateChildPolicyArgsLocked();
   void CreateChildPolicyLocked(const char* name, Args args);
-  bool PickFromChildPolicyLocked(bool force_async, PendingPick* pp,
-                                 grpc_error** error);
-  void UpdateConnectivityStateFromChildPolicyLocked(
-      grpc_error* child_state_error);
-  static void OnChildPolicyConnectivityChangedLocked(void* arg,
-                                                     grpc_error* error);
-  static void OnChildPolicyRequestReresolutionLocked(void* arg,
-                                                     grpc_error* error);
 
   // Who the client is trying to communicate with.
   const char* server_name_ = nullptr;
@@ -294,7 +274,6 @@ class XdsLb : public LoadBalancingPolicy {
   // Internal state.
   bool started_picking_ = false;
   bool shutting_down_ = false;
-  grpc_connectivity_state_tracker state_tracker_;
 
   // The channel for communicating with the LB server.
   grpc_channel* lb_channel_ = nullptr;
@@ -337,17 +316,91 @@ class XdsLb : public LoadBalancingPolicy {
   grpc_timer lb_fallback_timer_;
   grpc_closure lb_on_fallback_;
 
-  // Pending picks that are waiting on the xDS policy's connectivity.
-  PendingPick* pending_picks_ = nullptr;
-
   // The policy to use for the backends.
   OrphanablePtr<LoadBalancingPolicy> child_policy_;
   UniquePtr<char> child_policy_json_string_;
-  grpc_connectivity_state child_connectivity_state_;
-  grpc_closure on_child_connectivity_changed_;
-  grpc_closure on_child_request_reresolution_;
 };
 
+//
+// XdsLb::Picker
+//
+
+// Destroy function used when embedding client stats in call context.
+void DestroyClientStats(void* arg) {
+  static_cast<XdsLbClientStats*>(arg)->Unref();
+}
+
+XdsLb::Picker::PickResult XdsLb::Picker::Pick(PickState* pick,
+                                              grpc_error** error) {
+  // TODO(roth): Add support for drop handling.
+  // Forward pick to child policy.
+  PickResult result = child_picker_->Pick(pick, error);
+  // If pick succeeded, add client stats.
+  if (result == PickResult::PICK_COMPLETE &&
+      pick->connected_subchannel != nullptr && client_stats_ != nullptr) {
+    pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
+        client_stats_->Ref().release();
+    pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
+        DestroyClientStats;
+  }
+  return result;
+}
+
+//
+// XdsLb::Helper
+//
+
+Subchannel* XdsLb::Helper::CreateSubchannel(const grpc_channel_args& args) {
+  if (parent_->shutting_down_) return nullptr;
+  return parent_->channel_control_helper()->CreateSubchannel(args);
+}
+
+grpc_channel* XdsLb::Helper::CreateChannel(const char* target,
+                                           grpc_client_channel_type type,
+                                           const grpc_channel_args& args) {
+  if (parent_->shutting_down_) return nullptr;
+  return parent_->channel_control_helper()->CreateChannel(target, type, args);
+}
+
+void XdsLb::Helper::UpdateState(grpc_connectivity_state state,
+                                grpc_error* state_error,
+                                UniquePtr<SubchannelPicker> picker) {
+  if (parent_->shutting_down_) {
+    GRPC_ERROR_UNREF(state_error);
+    return;
+  }
+  // TODO(juanlishen): When in fallback mode, pass the child picker
+  // through without wrapping it.  (Or maybe use a different helper for
+  // the fallback policy?)
+  RefCountedPtr<XdsLbClientStats> client_stats;
+  if (parent_->lb_calld_ != nullptr &&
+      parent_->lb_calld_->client_stats() != nullptr) {
+    client_stats = parent_->lb_calld_->client_stats()->Ref();
+  }
+  parent_->channel_control_helper()->UpdateState(
+      state, state_error,
+      UniquePtr<SubchannelPicker>(
+          New<Picker>(std::move(picker), std::move(client_stats))));
+}
+
+void XdsLb::Helper::RequestReresolution() {
+  if (parent_->shutting_down_) return;
+  if (grpc_lb_xds_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[xdslb %p] Re-resolution requested from the internal RR policy "
+            "(%p).",
+            parent_.get(), parent_->child_policy_.get());
+  }
+  // If we are talking to a balancer, we expect to get updated addresses
+  // from the balancer, so we can ignore the re-resolution request from
+  // the RR policy. Otherwise, pass the re-resolution request up to the
+  // channel.
+  if (parent_->lb_calld_ == nullptr ||
+      !parent_->lb_calld_->seen_initial_response()) {
+    parent_->channel_control_helper()->RequestReresolution();
+  }
+}
+
 //
 // serverlist parsing code
 //
@@ -709,7 +762,7 @@ void XdsLb::BalancerCallState::OnBalancerMessageReceivedLocked(
       // serverlist returned from the current LB call.
       if (lb_calld->client_stats_report_interval_ > 0 &&
           lb_calld->client_stats_ == nullptr) {
-        lb_calld->client_stats_.reset(New<XdsLbClientStats>());
+        lb_calld->client_stats_ = MakeRefCounted<XdsLbClientStats>();
         // TODO(roth): We currently track this ref manually.  Once the
         // ClosureRef API is ready, we should pass the RefCountedPtr<> along
         // with the callback.
@@ -792,13 +845,13 @@ void XdsLb::BalancerCallState::OnBalancerStatusReceivedLocked(
             lb_calld->lb_call_, grpc_error_string(error));
     gpr_free(status_details);
   }
-  xdslb_policy->TryReresolutionLocked(&grpc_lb_xds_trace, GRPC_ERROR_NONE);
   // If this lb_calld is still in use, this call ended because of a failure so
   // we want to retry connecting. Otherwise, we have deliberately ended this
   // call and no further action is required.
   if (lb_calld == xdslb_policy->lb_calld_.get()) {
     xdslb_policy->lb_calld_.reset();
     GPR_ASSERT(!xdslb_policy->shutting_down_);
+    xdslb_policy->channel_control_helper()->RequestReresolution();
     if (lb_calld->seen_initial_response_) {
       // If we lose connection to the LB server, reset the backoff and restart
       // the LB call immediately.
@@ -919,13 +972,6 @@ XdsLb::XdsLb(LoadBalancingPolicy::Args args)
   GRPC_CLOSURE_INIT(&lb_channel_on_connectivity_changed_,
                     &XdsLb::OnBalancerChannelConnectivityChangedLocked, this,
                     grpc_combiner_scheduler(args.combiner));
-  GRPC_CLOSURE_INIT(&on_child_connectivity_changed_,
-                    &XdsLb::OnChildPolicyConnectivityChangedLocked, this,
-                    grpc_combiner_scheduler(args.combiner));
-  GRPC_CLOSURE_INIT(&on_child_request_reresolution_,
-                    &XdsLb::OnChildPolicyRequestReresolutionLocked, this,
-                    grpc_combiner_scheduler(args.combiner));
-  grpc_connectivity_state_init(&state_tracker_, GRPC_CHANNEL_IDLE, "xds");
   // Record server name.
   const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI);
   const char* server_uri = grpc_channel_arg_get_string(arg);
@@ -950,21 +996,22 @@ XdsLb::XdsLb(LoadBalancingPolicy::Args args)
   ParseLbConfig(args.lb_config);
   // Process channel args.
   ProcessChannelArgsLocked(*args.args);
+  // Initialize channel with a picker that will start us connecting.
+  channel_control_helper()->UpdateState(
+      GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
+      UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
 }
 
 XdsLb::~XdsLb() {
-  GPR_ASSERT(pending_picks_ == nullptr);
   gpr_mu_destroy(&lb_channel_mu_);
   gpr_free((void*)server_name_);
   grpc_channel_args_destroy(args_);
-  grpc_connectivity_state_destroy(&state_tracker_);
   if (serverlist_ != nullptr) {
     xds_grpclb_destroy_serverlist(serverlist_);
   }
 }
 
 void XdsLb::ShutdownLocked() {
-  grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Channel shutdown");
   shutting_down_ = true;
   lb_calld_.reset();
   if (retry_timer_callback_pending_) {
@@ -974,7 +1021,6 @@ void XdsLb::ShutdownLocked() {
     grpc_timer_cancel(&lb_fallback_timer_);
   }
   child_policy_.reset();
-  TryReresolutionLocked(&grpc_lb_xds_trace, GRPC_ERROR_CANCELLED);
   // We destroy the LB channel here instead of in our destructor because
   // destroying the channel triggers a last callback to
   // OnBalancerChannelConnectivityChangedLocked(), and we need to be
@@ -985,109 +1031,12 @@ void XdsLb::ShutdownLocked() {
     lb_channel_ = nullptr;
     gpr_mu_unlock(&lb_channel_mu_);
   }
-  grpc_connectivity_state_set(&state_tracker_, GRPC_CHANNEL_SHUTDOWN,
-                              GRPC_ERROR_REF(error), "xds_shutdown");
-  // Clear pending picks.
-  PendingPick* pp;
-  while ((pp = pending_picks_) != nullptr) {
-    pending_picks_ = pp->next;
-    pp->pick->connected_subchannel.reset();
-    // Note: pp is deleted in this callback.
-    GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error));
-  }
-  GRPC_ERROR_UNREF(error);
 }
 
 //
 // public methods
 //
 
-void XdsLb::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) {
-  PendingPick* pp;
-  while ((pp = pending_picks_) != nullptr) {
-    pending_picks_ = pp->next;
-    pp->pick->on_complete = pp->original_on_complete;
-    grpc_error* error = GRPC_ERROR_NONE;
-    if (new_policy->PickLocked(pp->pick, &error)) {
-      // Synchronous return; schedule closure.
-      GRPC_CLOSURE_SCHED(pp->pick->on_complete, error);
-    }
-    Delete(pp);
-  }
-}
-
-// Cancel a specific pending pick.
-//
-// A pick progresses as follows:
-// - If there's a child policy available, it'll be handed over to child policy
-//   (in CreateChildPolicyLocked()). From that point onwards, it'll be the
-//   child policy's responsibility. For cancellations, that implies the pick
-//   needs to be also cancelled by the child policy instance.
-// - Otherwise, without a child policy instance, picks stay pending at this
-//   policy's level (xds), inside the pending_picks_ list. To cancel these,
-//   we invoke the completion closure and set the pick's connected
-//   subchannel to nullptr right here.
-void XdsLb::CancelPickLocked(PickState* pick, grpc_error* error) {
-  PendingPick* pp = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pp != nullptr) {
-    PendingPick* next = pp->next;
-    if (pp->pick == pick) {
-      pick->connected_subchannel.reset();
-      // Note: pp is deleted in this callback.
-      GRPC_CLOSURE_SCHED(&pp->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pp->next = pending_picks_;
-      pending_picks_ = pp;
-    }
-    pp = next;
-  }
-  if (child_policy_ != nullptr) {
-    child_policy_->CancelPickLocked(pick, GRPC_ERROR_REF(error));
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
-// Cancel all pending picks.
-//
-// A pick progresses as follows:
-// - If there's a child policy available, it'll be handed over to child policy
-//   (in CreateChildPolicyLocked()). From that point onwards, it'll be the
-//   child policy's responsibility. For cancellations, that implies the pick
-//   needs to be also cancelled by the child policy instance.
-// - Otherwise, without a child policy instance, picks stay pending at this
-//   policy's level (xds), inside the pending_picks_ list. To cancel these,
-//   we invoke the completion closure and set the pick's connected
-//   subchannel to nullptr right here.
-void XdsLb::CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask,
-                                      uint32_t initial_metadata_flags_eq,
-                                      grpc_error* error) {
-  PendingPick* pp = pending_picks_;
-  pending_picks_ = nullptr;
-  while (pp != nullptr) {
-    PendingPick* next = pp->next;
-    if ((*pp->pick->initial_metadata_flags & initial_metadata_flags_mask) ==
-        initial_metadata_flags_eq) {
-      // Note: pp is deleted in this callback.
-      GRPC_CLOSURE_SCHED(&pp->on_complete,
-                         GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                             "Pick Cancelled", &error, 1));
-    } else {
-      pp->next = pending_picks_;
-      pending_picks_ = pp;
-    }
-    pp = next;
-  }
-  if (child_policy_ != nullptr) {
-    child_policy_->CancelMatchingPicksLocked(initial_metadata_flags_mask,
-                                             initial_metadata_flags_eq,
-                                             GRPC_ERROR_REF(error));
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
 void XdsLb::ExitIdleLocked() {
   if (!started_picking_) {
     StartPickingLocked();
@@ -1103,36 +1052,6 @@ void XdsLb::ResetBackoffLocked() {
   }
 }
 
-bool XdsLb::PickLocked(PickState* pick, grpc_error** error) {
-  PendingPick* pp = PendingPickCreate(pick);
-  bool pick_done = false;
-  if (child_policy_ != nullptr) {
-    if (grpc_lb_xds_trace.enabled()) {
-      gpr_log(GPR_INFO, "[xdslb %p] about to PICK from policy %p", this,
-              child_policy_.get());
-    }
-    pick_done = PickFromChildPolicyLocked(false /* force_async */, pp, error);
-  } else {  // child_policy_ == NULL
-    if (pick->on_complete == nullptr) {
-      *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
-          "No pick result available but synchronous result required.");
-      pick_done = true;
-    } else {
-      if (grpc_lb_xds_trace.enabled()) {
-        gpr_log(GPR_INFO,
-                "[xdslb %p] No child policy. Adding to xds's pending picks",
-                this);
-      }
-      AddPendingPick(pp);
-      if (!started_picking_) {
-        StartPickingLocked();
-      }
-      pick_done = false;
-    }
-  }
-  return pick_done;
-}
-
 void XdsLb::FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels,
                                      channelz::ChildRefsList* child_channels) {
   // delegate to the child_policy_ to fill the children subchannels.
@@ -1147,17 +1066,6 @@ void XdsLb::FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels,
   }
 }
 
-grpc_connectivity_state XdsLb::CheckConnectivityLocked(
-    grpc_error** connectivity_error) {
-  return grpc_connectivity_state_get(&state_tracker_, connectivity_error);
-}
-
-void XdsLb::NotifyOnStateChangeLocked(grpc_connectivity_state* current,
-                                      grpc_closure* closure) {
-  grpc_connectivity_state_notify_on_state_change(&state_tracker_, current,
-                                                 closure);
-}
-
 void XdsLb::ProcessChannelArgsLocked(const grpc_channel_args& args) {
   const ServerAddressList* addresses = FindServerAddressListChannelArg(&args);
   if (addresses == nullptr) {
@@ -1185,9 +1093,8 @@ void XdsLb::ProcessChannelArgsLocked(const grpc_channel_args& args) {
     char* uri_str;
     gpr_asprintf(&uri_str, "fake:///%s", server_name_);
     gpr_mu_lock(&lb_channel_mu_);
-    lb_channel_ = grpc_client_channel_factory_create_channel(
-        client_channel_factory(), uri_str,
-        GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, lb_channel_args);
+    lb_channel_ = channel_control_helper()->CreateChannel(
+        uri_str, GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, *lb_channel_args);
     gpr_mu_unlock(&lb_channel_mu_);
     GPR_ASSERT(lb_channel_ != nullptr);
     gpr_free(uri_str);
@@ -1402,90 +1309,10 @@ void XdsLb::OnBalancerChannelConnectivityChangedLocked(void* arg,
   }
 }
 
-//
-// PendingPick
-//
-
-// Destroy function used when embedding client stats in call context.
-void DestroyClientStats(void* arg) {
-  static_cast<XdsLbClientStats*>(arg)->Unref();
-}
-
-void XdsLb::PendingPickCleanup(PendingPick* pp) {
-  // If connected_subchannel is nullptr, no pick has been made by the
-  // child policy (e.g., all addresses failed to connect).
-  if (pp->pick->connected_subchannel != nullptr) {
-    // Pass on client stats via context. Passes ownership of the reference.
-    if (pp->client_stats != nullptr) {
-      pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].value =
-          pp->client_stats.release();
-      pp->pick->subchannel_call_context[GRPC_GRPCLB_CLIENT_STATS].destroy =
-          DestroyClientStats;
-    }
-  } else {
-    pp->client_stats.reset();
-  }
-}
-
-/* The \a on_complete closure passed as part of the pick requires keeping a
- * reference to its associated child policy instance. We wrap this closure in
- * order to unref the child policy instance upon its invocation */
-void XdsLb::OnPendingPickComplete(void* arg, grpc_error* error) {
-  PendingPick* pp = static_cast<PendingPick*>(arg);
-  PendingPickCleanup(pp);
-  GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_REF(error));
-  Delete(pp);
-}
-
-XdsLb::PendingPick* XdsLb::PendingPickCreate(PickState* pick) {
-  PendingPick* pp = New<PendingPick>();
-  pp->xdslb_policy = this;
-  pp->pick = pick;
-  GRPC_CLOSURE_INIT(&pp->on_complete, &XdsLb::OnPendingPickComplete, pp,
-                    grpc_schedule_on_exec_ctx);
-  pp->original_on_complete = pick->on_complete;
-  pick->on_complete = &pp->on_complete;
-  return pp;
-}
-
-void XdsLb::AddPendingPick(PendingPick* pp) {
-  pp->next = pending_picks_;
-  pending_picks_ = pp;
-}
-
 //
 // code for interacting with the child policy
 //
 
-// Performs a pick over \a child_policy_. Given that a pick can return
-// immediately (ignoring its completion callback), we need to perform the
-// cleanups this callback would otherwise be responsible for.
-// If \a force_async is true, then we will manually schedule the
-// completion callback even if the pick is available immediately.
-bool XdsLb::PickFromChildPolicyLocked(bool force_async, PendingPick* pp,
-                                      grpc_error** error) {
-  // Set client_stats.
-  if (lb_calld_ != nullptr && lb_calld_->client_stats() != nullptr) {
-    pp->client_stats = lb_calld_->client_stats()->Ref();
-  }
-  // Pick via the child policy.
-  bool pick_done = child_policy_->PickLocked(pp->pick, error);
-  if (pick_done) {
-    PendingPickCleanup(pp);
-    if (force_async) {
-      GRPC_CLOSURE_SCHED(pp->original_on_complete, *error);
-      *error = GRPC_ERROR_NONE;
-      pick_done = false;
-    }
-    Delete(pp);
-  }
-  // else, the pending pick will be registered and taken care of by the
-  // pending pick list inside the child policy.  Eventually,
-  // OnPendingPickComplete() will be called, which will (among other
-  // things) add the LB token to the call's initial metadata.
-  return pick_done;
-}
-
 void XdsLb::CreateChildPolicyLocked(const char* name, Args args) {
   GPR_ASSERT(child_policy_ == nullptr);
   child_policy_ = LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(
@@ -1494,42 +1321,12 @@ void XdsLb::CreateChildPolicyLocked(const char* name, Args args) {
     gpr_log(GPR_ERROR, "[xdslb %p] Failure creating a child policy", this);
     return;
   }
-  // TODO(roth): We currently track this ref manually.  Once the new
-  // ClosureRef API is done, pass the RefCountedPtr<> along with the closure.
-  auto self = Ref(DEBUG_LOCATION, "on_child_reresolution_requested");
-  self.release();
-  child_policy_->SetReresolutionClosureLocked(&on_child_request_reresolution_);
-  grpc_error* child_state_error = nullptr;
-  child_connectivity_state_ =
-      child_policy_->CheckConnectivityLocked(&child_state_error);
-  // Connectivity state is a function of the child policy updated/created.
-  UpdateConnectivityStateFromChildPolicyLocked(child_state_error);
   // Add the xDS's interested_parties pollset_set to that of the newly created
   // child policy. This will make the child policy progress upon activity on
   // xDS LB, which in turn is tied to the application's call.
   grpc_pollset_set_add_pollset_set(child_policy_->interested_parties(),
                                    interested_parties());
-  // Subscribe to changes to the connectivity of the new child policy.
-  // TODO(roth): We currently track this ref manually.  Once the new
-  // ClosureRef API is done, pass the RefCountedPtr<> along with the closure.
-  self = Ref(DEBUG_LOCATION, "on_child_connectivity_changed");
-  self.release();
-  child_policy_->NotifyOnStateChangeLocked(&child_connectivity_state_,
-                                           &on_child_connectivity_changed_);
   child_policy_->ExitIdleLocked();
-  // Send pending picks to child policy.
-  PendingPick* pp;
-  while ((pp = pending_picks_)) {
-    pending_picks_ = pp->next;
-    if (grpc_lb_xds_trace.enabled()) {
-      gpr_log(
-          GPR_INFO,
-          "[xdslb %p] Pending pick about to (async) PICK from child policy %p",
-          this, child_policy_.get());
-    }
-    grpc_error* error = GRPC_ERROR_NONE;
-    PickFromChildPolicyLocked(true /* force_async */, pp, &error);
-  }
 }
 
 grpc_channel_args* XdsLb::CreateChildPolicyArgsLocked() {
@@ -1587,9 +1384,9 @@ void XdsLb::CreateOrUpdateChildPolicyLocked() {
   } else {
     LoadBalancingPolicy::Args lb_policy_args;
     lb_policy_args.combiner = combiner();
-    lb_policy_args.client_channel_factory = client_channel_factory();
-    lb_policy_args.subchannel_pool = subchannel_pool()->Ref();
     lb_policy_args.args = args;
+    lb_policy_args.channel_control_helper =
+        UniquePtr<ChannelControlHelper>(New<Helper>(Ref()));
     lb_policy_args.lb_config = child_policy_config;
     CreateChildPolicyLocked(child_policy_name, std::move(lb_policy_args));
     if (grpc_lb_xds_trace.enabled()) {
@@ -1601,102 +1398,6 @@ void XdsLb::CreateOrUpdateChildPolicyLocked() {
   grpc_json_destroy(child_policy_json);
 }
 
-void XdsLb::OnChildPolicyRequestReresolutionLocked(void* arg,
-                                                   grpc_error* error) {
-  XdsLb* xdslb_policy = static_cast<XdsLb*>(arg);
-  if (xdslb_policy->shutting_down_ || error != GRPC_ERROR_NONE) {
-    xdslb_policy->Unref(DEBUG_LOCATION, "on_child_reresolution_requested");
-    return;
-  }
-  if (grpc_lb_xds_trace.enabled()) {
-    gpr_log(GPR_INFO,
-            "[xdslb %p] Re-resolution requested from child policy "
-            "(%p).",
-            xdslb_policy, xdslb_policy->child_policy_.get());
-  }
-  // If we are talking to a balancer, we expect to get updated addresses form
-  // the balancer, so we can ignore the re-resolution request from the child
-  // policy.
-  // Otherwise, handle the re-resolution request using the xds policy's
-  // original re-resolution closure.
-  if (xdslb_policy->lb_calld_ == nullptr ||
-      !xdslb_policy->lb_calld_->seen_initial_response()) {
-    xdslb_policy->TryReresolutionLocked(&grpc_lb_xds_trace, GRPC_ERROR_NONE);
-  }
-  // Give back the wrapper closure to the child policy.
-  xdslb_policy->child_policy_->SetReresolutionClosureLocked(
-      &xdslb_policy->on_child_request_reresolution_);
-}
-
-void XdsLb::UpdateConnectivityStateFromChildPolicyLocked(
-    grpc_error* child_state_error) {
-  const grpc_connectivity_state curr_glb_state =
-      grpc_connectivity_state_check(&state_tracker_);
-  /* The new connectivity status is a function of the previous one and the new
-   * input coming from the status of the child policy.
-   *
-   *  current state (xds's)
-   *  |
-   *  v  || I  |  C  |  R  |  TF  |  SD  |  <- new state (child policy's)
-   *  ===++====+=====+=====+======+======+
-   *   I || I  |  C  |  R  | [I]  | [I]  |
-   *  ---++----+-----+-----+------+------+
-   *   C || I  |  C  |  R  | [C]  | [C]  |
-   *  ---++----+-----+-----+------+------+
-   *   R || I  |  C  |  R  | [R]  | [R]  |
-   *  ---++----+-----+-----+------+------+
-   *  TF || I  |  C  |  R  | [TF] | [TF] |
-   *  ---++----+-----+-----+------+------+
-   *  SD || NA |  NA |  NA |  NA  |  NA  | (*)
-   *  ---++----+-----+-----+------+------+
-   *
-   * A [STATE] indicates that the old child policy is kept. In those cases,
-   * STATE is the current state of xds, which is left untouched.
-   *
-   *  In summary, if the new state is TRANSIENT_FAILURE or SHUTDOWN, stick to
-   *  the previous child policy instance.
-   *
-   *  Note that the status is never updated to SHUTDOWN as a result of calling
-   *  this function. Only glb_shutdown() has the power to set that state.
-   *
-   *  (*) This function mustn't be called during shutting down. */
-  GPR_ASSERT(curr_glb_state != GRPC_CHANNEL_SHUTDOWN);
-  switch (child_connectivity_state_) {
-    case GRPC_CHANNEL_TRANSIENT_FAILURE:
-    case GRPC_CHANNEL_SHUTDOWN:
-      GPR_ASSERT(child_state_error != GRPC_ERROR_NONE);
-      break;
-    case GRPC_CHANNEL_IDLE:
-    case GRPC_CHANNEL_CONNECTING:
-    case GRPC_CHANNEL_READY:
-      GPR_ASSERT(child_state_error == GRPC_ERROR_NONE);
-  }
-  if (grpc_lb_xds_trace.enabled()) {
-    gpr_log(GPR_INFO,
-            "[xdslb %p] Setting xds's state to %s from child policy %p state.",
-            this, grpc_connectivity_state_name(child_connectivity_state_),
-            child_policy_.get());
-  }
-  grpc_connectivity_state_set(&state_tracker_, child_connectivity_state_,
-                              child_state_error,
-                              "update_lb_connectivity_status_locked");
-}
-
-void XdsLb::OnChildPolicyConnectivityChangedLocked(void* arg,
-                                                   grpc_error* error) {
-  XdsLb* xdslb_policy = static_cast<XdsLb*>(arg);
-  if (xdslb_policy->shutting_down_) {
-    xdslb_policy->Unref(DEBUG_LOCATION, "on_child_connectivity_changed");
-    return;
-  }
-  xdslb_policy->UpdateConnectivityStateFromChildPolicyLocked(
-      GRPC_ERROR_REF(error));
-  // Resubscribe. Reuse the "on_child_connectivity_changed" ref.
-  xdslb_policy->child_policy_->NotifyOnStateChangeLocked(
-      &xdslb_policy->child_connectivity_state_,
-      &xdslb_policy->on_child_connectivity_changed_);
-}
-
 //
 // factory
 //

src/core/ext/filters/client_channel/request_routing.cc

@@ -1,946 +0,0 @@
-/*
- *
- * Copyright 2015 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <grpc/support/port_platform.h>
-
-#include "src/core/ext/filters/client_channel/request_routing.h"
-
-#include <inttypes.h>
-#include <limits.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <string.h>
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/sync.h>
-
-#include "src/core/ext/filters/client_channel/backup_poller.h"
-#include "src/core/ext/filters/client_channel/global_subchannel_pool.h"
-#include "src/core/ext/filters/client_channel/http_connect_handshaker.h"
-#include "src/core/ext/filters/client_channel/lb_policy_registry.h"
-#include "src/core/ext/filters/client_channel/local_subchannel_pool.h"
-#include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
-#include "src/core/ext/filters/client_channel/resolver_registry.h"
-#include "src/core/ext/filters/client_channel/retry_throttle.h"
-#include "src/core/ext/filters/client_channel/server_address.h"
-#include "src/core/ext/filters/client_channel/subchannel.h"
-#include "src/core/ext/filters/deadline/deadline_filter.h"
-#include "src/core/lib/backoff/backoff.h"
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/connected_channel.h"
-#include "src/core/lib/channel/status_util.h"
-#include "src/core/lib/gpr/string.h"
-#include "src/core/lib/gprpp/inlined_vector.h"
-#include "src/core/lib/gprpp/manual_constructor.h"
-#include "src/core/lib/iomgr/combiner.h"
-#include "src/core/lib/iomgr/iomgr.h"
-#include "src/core/lib/iomgr/polling_entity.h"
-#include "src/core/lib/profiling/timers.h"
-#include "src/core/lib/slice/slice_internal.h"
-#include "src/core/lib/slice/slice_string_helpers.h"
-#include "src/core/lib/surface/channel.h"
-#include "src/core/lib/transport/connectivity_state.h"
-#include "src/core/lib/transport/error_utils.h"
-#include "src/core/lib/transport/metadata.h"
-#include "src/core/lib/transport/metadata_batch.h"
-#include "src/core/lib/transport/service_config.h"
-#include "src/core/lib/transport/static_metadata.h"
-#include "src/core/lib/transport/status_metadata.h"
-
-namespace grpc_core {
-
-//
-// RequestRouter::Request::ResolverResultWaiter
-//
-
-// Handles waiting for a resolver result.
-// Used only for the first call on an idle channel.
-class RequestRouter::Request::ResolverResultWaiter {
- public:
-  explicit ResolverResultWaiter(Request* request)
-      : request_router_(request->request_router_),
-        request_(request),
-        tracer_enabled_(request_router_->tracer_->enabled()) {
-    if (tracer_enabled_) {
-      gpr_log(GPR_INFO,
-              "request_router=%p request=%p: deferring pick pending resolver "
-              "result",
-              request_router_, request);
-    }
-    // Add closure to be run when a resolver result is available.
-    GRPC_CLOSURE_INIT(&done_closure_, &DoneLocked, this,
-                      grpc_combiner_scheduler(request_router_->combiner_));
-    AddToWaitingList();
-    // Set cancellation closure, so that we abort if the call is cancelled.
-    GRPC_CLOSURE_INIT(&cancel_closure_, &CancelLocked, this,
-                      grpc_combiner_scheduler(request_router_->combiner_));
-    grpc_call_combiner_set_notify_on_cancel(request->call_combiner_,
-                                            &cancel_closure_);
-  }
-
- private:
-  // Adds done_closure_ to
-  // request_router_->waiting_for_resolver_result_closures_.
-  void AddToWaitingList() {
-    grpc_closure_list_append(
-        &request_router_->waiting_for_resolver_result_closures_, &done_closure_,
-        GRPC_ERROR_NONE);
-  }
-
-  // Invoked when a resolver result is available.
-  static void DoneLocked(void* arg, grpc_error* error) {
-    ResolverResultWaiter* self = static_cast<ResolverResultWaiter*>(arg);
-    RequestRouter* request_router = self->request_router_;
-    // If CancelLocked() has already run, delete ourselves without doing
-    // anything.  Note that the call stack may have already been destroyed,
-    // so it's not safe to access anything in state_.
-    if (GPR_UNLIKELY(self->finished_)) {
-      if (self->tracer_enabled_) {
-        gpr_log(GPR_INFO,
-                "request_router=%p: call cancelled before resolver result",
-                request_router);
-      }
-      Delete(self);
-      return;
-    }
-    // Otherwise, process the resolver result.
-    Request* request = self->request_;
-    if (GPR_UNLIKELY(error != GRPC_ERROR_NONE)) {
-      if (self->tracer_enabled_) {
-        gpr_log(GPR_INFO,
-                "request_router=%p request=%p: resolver failed to return data",
-                request_router, request);
-      }
-      GRPC_CLOSURE_RUN(request->on_route_done_, GRPC_ERROR_REF(error));
-    } else if (GPR_UNLIKELY(request_router->resolver_ == nullptr)) {
-      // Shutting down.
-      if (self->tracer_enabled_) {
-        gpr_log(GPR_INFO, "request_router=%p request=%p: resolver disconnected",
-                request_router, request);
-      }
-      GRPC_CLOSURE_RUN(request->on_route_done_,
-                       GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected"));
-    } else if (GPR_UNLIKELY(request_router->lb_policy_ == nullptr)) {
-      // Transient resolver failure.
-      // If call has wait_for_ready=true, try again; otherwise, fail.
-      if (*request->pick_.initial_metadata_flags &
-          GRPC_INITIAL_METADATA_WAIT_FOR_READY) {
-        if (self->tracer_enabled_) {
-          gpr_log(GPR_INFO,
-                  "request_router=%p request=%p: resolver returned but no LB "
-                  "policy; wait_for_ready=true; trying again",
-                  request_router, request);
-        }
-        // Re-add ourselves to the waiting list.
-        self->AddToWaitingList();
-        // Return early so that we don't set finished_ to true below.
-        return;
-      } else {
-        if (self->tracer_enabled_) {
-          gpr_log(GPR_INFO,
-                  "request_router=%p request=%p: resolver returned but no LB "
-                  "policy; wait_for_ready=false; failing",
-                  request_router, request);
-        }
-        GRPC_CLOSURE_RUN(
-            request->on_route_done_,
-            grpc_error_set_int(
-                GRPC_ERROR_CREATE_FROM_STATIC_STRING("Name resolution failure"),
-                GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE));
-      }
-    } else {
-      if (self->tracer_enabled_) {
-        gpr_log(GPR_INFO,
-                "request_router=%p request=%p: resolver returned, doing LB "
-                "pick",
-                request_router, request);
-      }
-      request->ProcessServiceConfigAndStartLbPickLocked();
-    }
-    self->finished_ = true;
-  }
-
-  // Invoked when the call is cancelled.
-  // Note: This runs under the client_channel combiner, but will NOT be
-  // holding the call combiner.
-  static void CancelLocked(void* arg, grpc_error* error) {
-    ResolverResultWaiter* self = static_cast<ResolverResultWaiter*>(arg);
-    RequestRouter* request_router = self->request_router_;
-    // If DoneLocked() has already run, delete ourselves without doing anything.
-    if (self->finished_) {
-      Delete(self);
-      return;
-    }
-    Request* request = self->request_;
-    // If we are being cancelled, immediately invoke on_route_done_
-    // to propagate the error back to the caller.
-    if (error != GRPC_ERROR_NONE) {
-      if (self->tracer_enabled_) {
-        gpr_log(GPR_INFO,
-                "request_router=%p request=%p: cancelling call waiting for "
-                "name resolution",
-                request_router, request);
-      }
-      // Note: Although we are not in the call combiner here, we are
-      // basically stealing the call combiner from the pending pick, so
-      // it's safe to run on_route_done_ here -- we are essentially
-      // calling it here instead of calling it in DoneLocked().
-      GRPC_CLOSURE_RUN(request->on_route_done_,
-                       GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                           "Pick cancelled", &error, 1));
-    }
-    self->finished_ = true;
-  }
-
-  RequestRouter* request_router_;
-  Request* request_;
-  const bool tracer_enabled_;
-  grpc_closure done_closure_;
-  grpc_closure cancel_closure_;
-  bool finished_ = false;
-};
-
-//
-// RequestRouter::Request::AsyncPickCanceller
-//
-
-// Handles the call combiner cancellation callback for an async LB pick.
-class RequestRouter::Request::AsyncPickCanceller {
- public:
-  explicit AsyncPickCanceller(Request* request)
-      : request_router_(request->request_router_),
-        request_(request),
-        tracer_enabled_(request_router_->tracer_->enabled()) {
-    GRPC_CALL_STACK_REF(request->owning_call_, "pick_callback_cancel");
-    // Set cancellation closure, so that we abort if the call is cancelled.
-    GRPC_CLOSURE_INIT(&cancel_closure_, &CancelLocked, this,
-                      grpc_combiner_scheduler(request_router_->combiner_));
-    grpc_call_combiner_set_notify_on_cancel(request->call_combiner_,
-                                            &cancel_closure_);
-  }
-
-  void MarkFinishedLocked() {
-    finished_ = true;
-    GRPC_CALL_STACK_UNREF(request_->owning_call_, "pick_callback_cancel");
-  }
-
- private:
-  // Invoked when the call is cancelled.
-  // Note: This runs under the client_channel combiner, but will NOT be
-  // holding the call combiner.
-  static void CancelLocked(void* arg, grpc_error* error) {
-    AsyncPickCanceller* self = static_cast<AsyncPickCanceller*>(arg);
-    Request* request = self->request_;
-    RequestRouter* request_router = self->request_router_;
-    if (!self->finished_) {
-      // Note: request_router->lb_policy_ may have changed since we started our
-      // pick, in which case we will be cancelling the pick on a policy other
-      // than the one we started it on.  However, this will just be a no-op.
-      if (error != GRPC_ERROR_NONE && request_router->lb_policy_ != nullptr) {
-        if (self->tracer_enabled_) {
-          gpr_log(GPR_INFO,
-                  "request_router=%p request=%p: cancelling pick from LB "
-                  "policy %p",
-                  request_router, request, request_router->lb_policy_.get());
-        }
-        request_router->lb_policy_->CancelPickLocked(&request->pick_,
-                                                     GRPC_ERROR_REF(error));
-      }
-      request->pick_canceller_ = nullptr;
-      GRPC_CALL_STACK_UNREF(request->owning_call_, "pick_callback_cancel");
-    }
-    Delete(self);
-  }
-
-  RequestRouter* request_router_;
-  Request* request_;
-  const bool tracer_enabled_;
-  grpc_closure cancel_closure_;
-  bool finished_ = false;
-};
-
-//
-// RequestRouter::Request
-//
-
-RequestRouter::Request::Request(grpc_call_stack* owning_call,
-                                grpc_call_combiner* call_combiner,
-                                grpc_polling_entity* pollent,
-                                grpc_metadata_batch* send_initial_metadata,
-                                uint32_t* send_initial_metadata_flags,
-                                ApplyServiceConfigCallback apply_service_config,
-                                void* apply_service_config_user_data,
-                                grpc_closure* on_route_done)
-    : owning_call_(owning_call),
-      call_combiner_(call_combiner),
-      pollent_(pollent),
-      apply_service_config_(apply_service_config),
-      apply_service_config_user_data_(apply_service_config_user_data),
-      on_route_done_(on_route_done) {
-  pick_.initial_metadata = send_initial_metadata;
-  pick_.initial_metadata_flags = send_initial_metadata_flags;
-}
-
-RequestRouter::Request::~Request() {
-  if (pick_.connected_subchannel != nullptr) {
-    pick_.connected_subchannel.reset();
-  }
-  for (size_t i = 0; i < GRPC_CONTEXT_COUNT; ++i) {
-    if (pick_.subchannel_call_context[i].destroy != nullptr) {
-      pick_.subchannel_call_context[i].destroy(
-          pick_.subchannel_call_context[i].value);
-    }
-  }
-}
-
-// Invoked once resolver results are available.
-void RequestRouter::Request::ProcessServiceConfigAndStartLbPickLocked() {
-  // Get service config data if needed.
-  if (!apply_service_config_(apply_service_config_user_data_)) return;
-  // Start LB pick.
-  StartLbPickLocked();
-}
-
-void RequestRouter::Request::MaybeAddCallToInterestedPartiesLocked() {
-  if (!pollent_added_to_interested_parties_) {
-    pollent_added_to_interested_parties_ = true;
-    grpc_polling_entity_add_to_pollset_set(
-        pollent_, request_router_->interested_parties_);
-  }
-}
-
-void RequestRouter::Request::MaybeRemoveCallFromInterestedPartiesLocked() {
-  if (pollent_added_to_interested_parties_) {
-    pollent_added_to_interested_parties_ = false;
-    grpc_polling_entity_del_from_pollset_set(
-        pollent_, request_router_->interested_parties_);
-  }
-}
-
-// Starts a pick on the LB policy.
-void RequestRouter::Request::StartLbPickLocked() {
-  if (request_router_->tracer_->enabled()) {
-    gpr_log(GPR_INFO,
-            "request_router=%p request=%p: starting pick on lb_policy=%p",
-            request_router_, this, request_router_->lb_policy_.get());
-  }
-  GRPC_CLOSURE_INIT(&on_pick_done_, &LbPickDoneLocked, this,
-                    grpc_combiner_scheduler(request_router_->combiner_));
-  pick_.on_complete = &on_pick_done_;
-  GRPC_CALL_STACK_REF(owning_call_, "pick_callback");
-  grpc_error* error = GRPC_ERROR_NONE;
-  const bool pick_done =
-      request_router_->lb_policy_->PickLocked(&pick_, &error);
-  if (pick_done) {
-    // Pick completed synchronously.
-    if (request_router_->tracer_->enabled()) {
-      gpr_log(GPR_INFO,
-              "request_router=%p request=%p: pick completed synchronously",
-              request_router_, this);
-    }
-    GRPC_CLOSURE_RUN(on_route_done_, error);
-    GRPC_CALL_STACK_UNREF(owning_call_, "pick_callback");
-  } else {
-    // Pick will be returned asynchronously.
-    // Add the request's polling entity to the request_router's
-    // interested_parties, so that the I/O of the LB policy can be done
-    // under it.  It will be removed in LbPickDoneLocked().
-    MaybeAddCallToInterestedPartiesLocked();
-    // Request notification on call cancellation.
-    // We allocate a separate object to track cancellation, since the
-    // cancellation closure might still be pending when we need to reuse
-    // the memory in which this Request object is stored for a subsequent
-    // retry attempt.
-    pick_canceller_ = New<AsyncPickCanceller>(this);
-  }
-}
-
-// Callback invoked by LoadBalancingPolicy::PickLocked() for async picks.
-// Unrefs the LB policy and invokes on_route_done_.
-void RequestRouter::Request::LbPickDoneLocked(void* arg, grpc_error* error) {
-  Request* self = static_cast<Request*>(arg);
-  RequestRouter* request_router = self->request_router_;
-  if (request_router->tracer_->enabled()) {
-    gpr_log(GPR_INFO,
-            "request_router=%p request=%p: pick completed asynchronously",
-            request_router, self);
-  }
-  self->MaybeRemoveCallFromInterestedPartiesLocked();
-  if (self->pick_canceller_ != nullptr) {
-    self->pick_canceller_->MarkFinishedLocked();
-  }
-  GRPC_CLOSURE_RUN(self->on_route_done_, GRPC_ERROR_REF(error));
-  GRPC_CALL_STACK_UNREF(self->owning_call_, "pick_callback");
-}
-
-//
-// RequestRouter::LbConnectivityWatcher
-//
-
-class RequestRouter::LbConnectivityWatcher {
- public:
-  LbConnectivityWatcher(RequestRouter* request_router,
-                        grpc_connectivity_state state,
-                        LoadBalancingPolicy* lb_policy,
-                        grpc_channel_stack* owning_stack,
-                        grpc_combiner* combiner)
-      : request_router_(request_router),
-        state_(state),
-        lb_policy_(lb_policy),
-        owning_stack_(owning_stack) {
-    GRPC_CHANNEL_STACK_REF(owning_stack_, "LbConnectivityWatcher");
-    GRPC_CLOSURE_INIT(&on_changed_, &OnLbPolicyStateChangedLocked, this,
-                      grpc_combiner_scheduler(combiner));
-    lb_policy_->NotifyOnStateChangeLocked(&state_, &on_changed_);
-  }
-
-  ~LbConnectivityWatcher() {
-    GRPC_CHANNEL_STACK_UNREF(owning_stack_, "LbConnectivityWatcher");
-  }
-
- private:
-  static void OnLbPolicyStateChangedLocked(void* arg, grpc_error* error) {
-    LbConnectivityWatcher* self = static_cast<LbConnectivityWatcher*>(arg);
-    // If the notification is not for the current policy, we're stale,
-    // so delete ourselves.
-    if (self->lb_policy_ != self->request_router_->lb_policy_.get()) {
-      Delete(self);
-      return;
-    }
-    // Otherwise, process notification.
-    if (self->request_router_->tracer_->enabled()) {
-      gpr_log(GPR_INFO, "request_router=%p: lb_policy=%p state changed to %s",
-              self->request_router_, self->lb_policy_,
-              grpc_connectivity_state_name(self->state_));
-    }
-    self->request_router_->SetConnectivityStateLocked(
-        self->state_, GRPC_ERROR_REF(error), "lb_changed");
-    // If shutting down, terminate watch.
-    if (self->state_ == GRPC_CHANNEL_SHUTDOWN) {
-      Delete(self);
-      return;
-    }
-    // Renew watch.
-    self->lb_policy_->NotifyOnStateChangeLocked(&self->state_,
-                                                &self->on_changed_);
-  }
-
-  RequestRouter* request_router_;
-  grpc_connectivity_state state_;
-  // LB policy address. No ref held, so not safe to dereference unless
-  // it happens to match request_router->lb_policy_.
-  LoadBalancingPolicy* lb_policy_;
-  grpc_channel_stack* owning_stack_;
-  grpc_closure on_changed_;
-};
-
-//
-// RequestRounter::ReresolutionRequestHandler
-//
-
-class RequestRouter::ReresolutionRequestHandler {
- public:
-  ReresolutionRequestHandler(RequestRouter* request_router,
-                             LoadBalancingPolicy* lb_policy,
-                             grpc_channel_stack* owning_stack,
-                             grpc_combiner* combiner)
-      : request_router_(request_router),
-        lb_policy_(lb_policy),
-        owning_stack_(owning_stack) {
-    GRPC_CHANNEL_STACK_REF(owning_stack_, "ReresolutionRequestHandler");
-    GRPC_CLOSURE_INIT(&closure_, &OnRequestReresolutionLocked, this,
-                      grpc_combiner_scheduler(combiner));
-    lb_policy_->SetReresolutionClosureLocked(&closure_);
-  }
-
- private:
-  static void OnRequestReresolutionLocked(void* arg, grpc_error* error) {
-    ReresolutionRequestHandler* self =
-        static_cast<ReresolutionRequestHandler*>(arg);
-    RequestRouter* request_router = self->request_router_;
-    // If this invocation is for a stale LB policy, treat it as an LB shutdown
-    // signal.
-    if (self->lb_policy_ != request_router->lb_policy_.get() ||
-        error != GRPC_ERROR_NONE || request_router->resolver_ == nullptr) {
-      GRPC_CHANNEL_STACK_UNREF(request_router->owning_stack_,
-                               "ReresolutionRequestHandler");
-      Delete(self);
-      return;
-    }
-    if (request_router->tracer_->enabled()) {
-      gpr_log(GPR_INFO, "request_router=%p: started name re-resolving",
-              request_router);
-    }
-    request_router->resolver_->RequestReresolutionLocked();
-    // Give back the closure to the LB policy.
-    self->lb_policy_->SetReresolutionClosureLocked(&self->closure_);
-  }
-
-  RequestRouter* request_router_;
-  // LB policy address. No ref held, so not safe to dereference unless
-  // it happens to match request_router->lb_policy_.
-  LoadBalancingPolicy* lb_policy_;
-  grpc_channel_stack* owning_stack_;
-  grpc_closure closure_;
-};
-
-//
-// RequestRouter
-//
-
-RequestRouter::RequestRouter(
-    grpc_channel_stack* owning_stack, grpc_combiner* combiner,
-    grpc_client_channel_factory* client_channel_factory,
-    grpc_pollset_set* interested_parties, TraceFlag* tracer,
-    ProcessResolverResultCallback process_resolver_result,
-    void* process_resolver_result_user_data, const char* target_uri,
-    const grpc_channel_args* args, grpc_error** error)
-    : owning_stack_(owning_stack),
-      combiner_(combiner),
-      client_channel_factory_(client_channel_factory),
-      interested_parties_(interested_parties),
-      tracer_(tracer),
-      process_resolver_result_(process_resolver_result),
-      process_resolver_result_user_data_(process_resolver_result_user_data) {
-  // Get subchannel pool.
-  const grpc_arg* arg =
-      grpc_channel_args_find(args, GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL);
-  if (grpc_channel_arg_get_bool(arg, false)) {
-    subchannel_pool_ = MakeRefCounted<LocalSubchannelPool>();
-  } else {
-    subchannel_pool_ = GlobalSubchannelPool::instance();
-  }
-  GRPC_CLOSURE_INIT(&on_resolver_result_changed_,
-                    &RequestRouter::OnResolverResultChangedLocked, this,
-                    grpc_combiner_scheduler(combiner));
-  grpc_connectivity_state_init(&state_tracker_, GRPC_CHANNEL_IDLE,
-                               "request_router");
-  grpc_channel_args* new_args = nullptr;
-  if (process_resolver_result == nullptr) {
-    grpc_arg arg = grpc_channel_arg_integer_create(
-        const_cast<char*>(GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION), 0);
-    new_args = grpc_channel_args_copy_and_add(args, &arg, 1);
-  }
-  resolver_ = ResolverRegistry::CreateResolver(
-      target_uri, (new_args == nullptr ? args : new_args), interested_parties_,
-      combiner_);
-  grpc_channel_args_destroy(new_args);
-  if (resolver_ == nullptr) {
-    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("resolver creation failed");
-  }
-}
-
-RequestRouter::~RequestRouter() {
-  if (resolver_ != nullptr) {
-    // The only way we can get here is if we never started resolving,
-    // because we take a ref to the channel stack when we start
-    // resolving and do not release it until the resolver callback is
-    // invoked after the resolver shuts down.
-    resolver_.reset();
-  }
-  if (lb_policy_ != nullptr) {
-    grpc_pollset_set_del_pollset_set(lb_policy_->interested_parties(),
-                                     interested_parties_);
-    lb_policy_.reset();
-  }
-  if (client_channel_factory_ != nullptr) {
-    grpc_client_channel_factory_unref(client_channel_factory_);
-  }
-  grpc_connectivity_state_destroy(&state_tracker_);
-}
-
-namespace {
-
-const char* GetChannelConnectivityStateChangeString(
-    grpc_connectivity_state state) {
-  switch (state) {
-    case GRPC_CHANNEL_IDLE:
-      return "Channel state change to IDLE";
-    case GRPC_CHANNEL_CONNECTING:
-      return "Channel state change to CONNECTING";
-    case GRPC_CHANNEL_READY:
-      return "Channel state change to READY";
-    case GRPC_CHANNEL_TRANSIENT_FAILURE:
-      return "Channel state change to TRANSIENT_FAILURE";
-    case GRPC_CHANNEL_SHUTDOWN:
-      return "Channel state change to SHUTDOWN";
-  }
-  GPR_UNREACHABLE_CODE(return "UNKNOWN");
-}
-
-}  // namespace
-
-void RequestRouter::SetConnectivityStateLocked(grpc_connectivity_state state,
-                                               grpc_error* error,
-                                               const char* reason) {
-  if (lb_policy_ != nullptr) {
-    if (state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-      // Cancel picks with wait_for_ready=false.
-      lb_policy_->CancelMatchingPicksLocked(
-          /* mask= */ GRPC_INITIAL_METADATA_WAIT_FOR_READY,
-          /* check= */ 0, GRPC_ERROR_REF(error));
-    } else if (state == GRPC_CHANNEL_SHUTDOWN) {
-      // Cancel all picks.
-      lb_policy_->CancelMatchingPicksLocked(/* mask= */ 0, /* check= */ 0,
-                                            GRPC_ERROR_REF(error));
-    }
-  }
-  if (tracer_->enabled()) {
-    gpr_log(GPR_INFO, "request_router=%p: setting connectivity state to %s",
-            this, grpc_connectivity_state_name(state));
-  }
-  if (channelz_node_ != nullptr) {
-    channelz_node_->AddTraceEvent(
-        channelz::ChannelTrace::Severity::Info,
-        grpc_slice_from_static_string(
-            GetChannelConnectivityStateChangeString(state)));
-  }
-  grpc_connectivity_state_set(&state_tracker_, state, error, reason);
-}
-
-void RequestRouter::StartResolvingLocked() {
-  if (tracer_->enabled()) {
-    gpr_log(GPR_INFO, "request_router=%p: starting name resolution", this);
-  }
-  GPR_ASSERT(!started_resolving_);
-  started_resolving_ = true;
-  GRPC_CHANNEL_STACK_REF(owning_stack_, "resolver");
-  resolver_->NextLocked(&resolver_result_, &on_resolver_result_changed_);
-}
-
-// Invoked from the resolver NextLocked() callback when the resolver
-// is shutting down.
-void RequestRouter::OnResolverShutdownLocked(grpc_error* error) {
-  if (tracer_->enabled()) {
-    gpr_log(GPR_INFO, "request_router=%p: shutting down", this);
-  }
-  if (lb_policy_ != nullptr) {
-    if (tracer_->enabled()) {
-      gpr_log(GPR_INFO, "request_router=%p: shutting down lb_policy=%p", this,
-              lb_policy_.get());
-    }
-    grpc_pollset_set_del_pollset_set(lb_policy_->interested_parties(),
-                                     interested_parties_);
-    lb_policy_.reset();
-  }
-  if (resolver_ != nullptr) {
-    // This should never happen; it can only be triggered by a resolver
-    // implementation spotaneously deciding to report shutdown without
-    // being orphaned.  This code is included just to be defensive.
-    if (tracer_->enabled()) {
-      gpr_log(GPR_INFO,
-              "request_router=%p: spontaneous shutdown from resolver %p", this,
-              resolver_.get());
-    }
-    resolver_.reset();
-    SetConnectivityStateLocked(GRPC_CHANNEL_SHUTDOWN,
-                               GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                                   "Resolver spontaneous shutdown", &error, 1),
-                               "resolver_spontaneous_shutdown");
-  }
-  grpc_closure_list_fail_all(&waiting_for_resolver_result_closures_,
-                             GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                                 "Channel disconnected", &error, 1));
-  GRPC_CLOSURE_LIST_SCHED(&waiting_for_resolver_result_closures_);
-  GRPC_CHANNEL_STACK_UNREF(owning_stack_, "resolver");
-  grpc_channel_args_destroy(resolver_result_);
-  resolver_result_ = nullptr;
-  GRPC_ERROR_UNREF(error);
-}
-
-// Creates a new LB policy, replacing any previous one.
-// If the new policy is created successfully, sets *connectivity_state and
-// *connectivity_error to its initial connectivity state; otherwise,
-// leaves them unchanged.
-void RequestRouter::CreateNewLbPolicyLocked(
-    const char* lb_policy_name, grpc_json* lb_config,
-    grpc_connectivity_state* connectivity_state,
-    grpc_error** connectivity_error, TraceStringVector* trace_strings) {
-  LoadBalancingPolicy::Args lb_policy_args;
-  lb_policy_args.combiner = combiner_;
-  lb_policy_args.client_channel_factory = client_channel_factory_;
-  lb_policy_args.subchannel_pool = subchannel_pool_;
-  lb_policy_args.args = resolver_result_;
-  lb_policy_args.lb_config = lb_config;
-  OrphanablePtr<LoadBalancingPolicy> new_lb_policy =
-      LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(lb_policy_name,
-                                                             lb_policy_args);
-  if (GPR_UNLIKELY(new_lb_policy == nullptr)) {
-    gpr_log(GPR_ERROR, "could not create LB policy \"%s\"", lb_policy_name);
-    if (channelz_node_ != nullptr) {
-      char* str;
-      gpr_asprintf(&str, "Could not create LB policy \'%s\'", lb_policy_name);
-      trace_strings->push_back(str);
-    }
-  } else {
-    if (tracer_->enabled()) {
-      gpr_log(GPR_INFO, "request_router=%p: created new LB policy \"%s\" (%p)",
-              this, lb_policy_name, new_lb_policy.get());
-    }
-    if (channelz_node_ != nullptr) {
-      char* str;
-      gpr_asprintf(&str, "Created new LB policy \'%s\'", lb_policy_name);
-      trace_strings->push_back(str);
-    }
-    // Swap out the LB policy and update the fds in interested_parties_.
-    if (lb_policy_ != nullptr) {
-      if (tracer_->enabled()) {
-        gpr_log(GPR_INFO, "request_router=%p: shutting down lb_policy=%p", this,
-                lb_policy_.get());
-      }
-      grpc_pollset_set_del_pollset_set(lb_policy_->interested_parties(),
-                                       interested_parties_);
-      lb_policy_->HandOffPendingPicksLocked(new_lb_policy.get());
-    }
-    lb_policy_ = std::move(new_lb_policy);
-    grpc_pollset_set_add_pollset_set(lb_policy_->interested_parties(),
-                                     interested_parties_);
-    // Create re-resolution request handler for the new LB policy.  It
-    // will delete itself when no longer needed.
-    New<ReresolutionRequestHandler>(this, lb_policy_.get(), owning_stack_,
-                                    combiner_);
-    // Get the new LB policy's initial connectivity state and start a
-    // connectivity watch.
-    GRPC_ERROR_UNREF(*connectivity_error);
-    *connectivity_state =
-        lb_policy_->CheckConnectivityLocked(connectivity_error);
-    if (exit_idle_when_lb_policy_arrives_) {
-      lb_policy_->ExitIdleLocked();
-      exit_idle_when_lb_policy_arrives_ = false;
-    }
-    // Create new watcher.  It will delete itself when done.
-    New<LbConnectivityWatcher>(this, *connectivity_state, lb_policy_.get(),
-                               owning_stack_, combiner_);
-  }
-}
-
-void RequestRouter::MaybeAddTraceMessagesForAddressChangesLocked(
-    TraceStringVector* trace_strings) {
-  const ServerAddressList* addresses =
-      FindServerAddressListChannelArg(resolver_result_);
-  const bool resolution_contains_addresses =
-      addresses != nullptr && addresses->size() > 0;
-  if (!resolution_contains_addresses &&
-      previous_resolution_contained_addresses_) {
-    trace_strings->push_back(gpr_strdup("Address list became empty"));
-  } else if (resolution_contains_addresses &&
-             !previous_resolution_contained_addresses_) {
-    trace_strings->push_back(gpr_strdup("Address list became non-empty"));
-  }
-  previous_resolution_contained_addresses_ = resolution_contains_addresses;
-}
-
-void RequestRouter::ConcatenateAndAddChannelTraceLocked(
-    TraceStringVector* trace_strings) const {
-  if (!trace_strings->empty()) {
-    gpr_strvec v;
-    gpr_strvec_init(&v);
-    gpr_strvec_add(&v, gpr_strdup("Resolution event: "));
-    bool is_first = 1;
-    for (size_t i = 0; i < trace_strings->size(); ++i) {
-      if (!is_first) gpr_strvec_add(&v, gpr_strdup(", "));
-      is_first = false;
-      gpr_strvec_add(&v, (*trace_strings)[i]);
-    }
-    char* flat;
-    size_t flat_len = 0;
-    flat = gpr_strvec_flatten(&v, &flat_len);
-    channelz_node_->AddTraceEvent(channelz::ChannelTrace::Severity::Info,
-                                  grpc_slice_new(flat, flat_len, gpr_free));
-    gpr_strvec_destroy(&v);
-  }
-}
-
-// Callback invoked when a resolver result is available.
-void RequestRouter::OnResolverResultChangedLocked(void* arg,
-                                                  grpc_error* error) {
-  RequestRouter* self = static_cast<RequestRouter*>(arg);
-  if (self->tracer_->enabled()) {
-    const char* disposition =
-        self->resolver_result_ != nullptr
-            ? ""
-            : (error == GRPC_ERROR_NONE ? " (transient error)"
-                                        : " (resolver shutdown)");
-    gpr_log(GPR_INFO,
-            "request_router=%p: got resolver result: resolver_result=%p "
-            "error=%s%s",
-            self, self->resolver_result_, grpc_error_string(error),
-            disposition);
-  }
-  // Handle shutdown.
-  if (error != GRPC_ERROR_NONE || self->resolver_ == nullptr) {
-    self->OnResolverShutdownLocked(GRPC_ERROR_REF(error));
-    return;
-  }
-  // Data used to set the channel's connectivity state.
-  bool set_connectivity_state = true;
-  // We only want to trace the address resolution in the follow cases:
-  // (a) Address resolution resulted in service config change.
-  // (b) Address resolution that causes number of backends to go from
-  //     zero to non-zero.
-  // (c) Address resolution that causes number of backends to go from
-  //     non-zero to zero.
-  // (d) Address resolution that causes a new LB policy to be created.
-  //
-  // we track a list of strings to eventually be concatenated and traced.
-  TraceStringVector trace_strings;
-  grpc_connectivity_state connectivity_state = GRPC_CHANNEL_TRANSIENT_FAILURE;
-  grpc_error* connectivity_error =
-      GRPC_ERROR_CREATE_FROM_STATIC_STRING("No load balancing policy");
-  // resolver_result_ will be null in the case of a transient
-  // resolution error.  In that case, we don't have any new result to
-  // process, which means that we keep using the previous result (if any).
-  if (self->resolver_result_ == nullptr) {
-    if (self->tracer_->enabled()) {
-      gpr_log(GPR_INFO, "request_router=%p: resolver transient failure", self);
-    }
-    // Don't override connectivity state if we already have an LB policy.
-    if (self->lb_policy_ != nullptr) set_connectivity_state = false;
-  } else {
-    // Parse the resolver result.
-    const char* lb_policy_name = nullptr;
-    grpc_json* lb_policy_config = nullptr;
-    const bool service_config_changed = self->process_resolver_result_(
-        self->process_resolver_result_user_data_, *self->resolver_result_,
-        &lb_policy_name, &lb_policy_config);
-    GPR_ASSERT(lb_policy_name != nullptr);
-    // Check to see if we're already using the right LB policy.
-    const bool lb_policy_name_changed =
-        self->lb_policy_ == nullptr ||
-        strcmp(self->lb_policy_->name(), lb_policy_name) != 0;
-    if (self->lb_policy_ != nullptr && !lb_policy_name_changed) {
-      // Continue using the same LB policy.  Update with new addresses.
-      if (self->tracer_->enabled()) {
-        gpr_log(GPR_INFO,
-                "request_router=%p: updating existing LB policy \"%s\" (%p)",
-                self, lb_policy_name, self->lb_policy_.get());
-      }
-      self->lb_policy_->UpdateLocked(*self->resolver_result_, lb_policy_config);
-      // No need to set the channel's connectivity state; the existing
-      // watch on the LB policy will take care of that.
-      set_connectivity_state = false;
-    } else {
-      // Instantiate new LB policy.
-      self->CreateNewLbPolicyLocked(lb_policy_name, lb_policy_config,
-                                    &connectivity_state, &connectivity_error,
-                                    &trace_strings);
-    }
-    // Add channel trace event.
-    if (self->channelz_node_ != nullptr) {
-      if (service_config_changed) {
-        // TODO(ncteisen): might be worth somehow including a snippet of the
-        // config in the trace, at the risk of bloating the trace logs.
-        trace_strings.push_back(gpr_strdup("Service config changed"));
-      }
-      self->MaybeAddTraceMessagesForAddressChangesLocked(&trace_strings);
-      self->ConcatenateAndAddChannelTraceLocked(&trace_strings);
-    }
-    // Clean up.
-    grpc_channel_args_destroy(self->resolver_result_);
-    self->resolver_result_ = nullptr;
-  }
-  // Set the channel's connectivity state if needed.
-  if (set_connectivity_state) {
-    self->SetConnectivityStateLocked(connectivity_state, connectivity_error,
-                                     "resolver_result");
-  } else {
-    GRPC_ERROR_UNREF(connectivity_error);
-  }
-  // Invoke closures that were waiting for results and renew the watch.
-  GRPC_CLOSURE_LIST_SCHED(&self->waiting_for_resolver_result_closures_);
-  self->resolver_->NextLocked(&self->resolver_result_,
-                              &self->on_resolver_result_changed_);
-}
-
-void RequestRouter::RouteCallLocked(Request* request) {
-  GPR_ASSERT(request->pick_.connected_subchannel == nullptr);
-  request->request_router_ = this;
-  if (lb_policy_ != nullptr) {
-    // We already have resolver results, so process the service config
-    // and start an LB pick.
-    request->ProcessServiceConfigAndStartLbPickLocked();
-  } else if (resolver_ == nullptr) {
-    GRPC_CLOSURE_RUN(request->on_route_done_,
-                     GRPC_ERROR_CREATE_FROM_STATIC_STRING("Disconnected"));
-  } else {
-    // We do not yet have an LB policy, so wait for a resolver result.
-    if (!started_resolving_) {
-      StartResolvingLocked();
-    }
-    // Create a new waiter, which will delete itself when done.
-    New<Request::ResolverResultWaiter>(request);
-    // Add the request's polling entity to the request_router's
-    // interested_parties, so that the I/O of the resolver can be done
-    // under it.  It will be removed in LbPickDoneLocked().
-    request->MaybeAddCallToInterestedPartiesLocked();
-  }
-}
-
-void RequestRouter::ShutdownLocked(grpc_error* error) {
-  if (resolver_ != nullptr) {
-    SetConnectivityStateLocked(GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error),
-                               "disconnect");
-    resolver_.reset();
-    if (!started_resolving_) {
-      grpc_closure_list_fail_all(&waiting_for_resolver_result_closures_,
-                                 GRPC_ERROR_REF(error));
-      GRPC_CLOSURE_LIST_SCHED(&waiting_for_resolver_result_closures_);
-    }
-    if (lb_policy_ != nullptr) {
-      grpc_pollset_set_del_pollset_set(lb_policy_->interested_parties(),
-                                       interested_parties_);
-      lb_policy_.reset();
-    }
-  }
-  GRPC_ERROR_UNREF(error);
-}
-
-grpc_connectivity_state RequestRouter::GetConnectivityState() {
-  return grpc_connectivity_state_check(&state_tracker_);
-}
-
-void RequestRouter::NotifyOnConnectivityStateChange(
-    grpc_connectivity_state* state, grpc_closure* closure) {
-  grpc_connectivity_state_notify_on_state_change(&state_tracker_, state,
-                                                 closure);
-}
-
-void RequestRouter::ExitIdleLocked() {
-  if (lb_policy_ != nullptr) {
-    lb_policy_->ExitIdleLocked();
-  } else {
-    exit_idle_when_lb_policy_arrives_ = true;
-    if (!started_resolving_ && resolver_ != nullptr) {
-      StartResolvingLocked();
-    }
-  }
-}
-
-void RequestRouter::ResetConnectionBackoffLocked() {
-  if (resolver_ != nullptr) {
-    resolver_->ResetBackoffLocked();
-    resolver_->RequestReresolutionLocked();
-  }
-  if (lb_policy_ != nullptr) {
-    lb_policy_->ResetBackoffLocked();
-  }
-}
-
-}  // namespace grpc_core

src/core/ext/filters/client_channel/request_routing.h

@@ -1,181 +0,0 @@
-/*
- *
- * Copyright 2018 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_REQUEST_ROUTING_H
-#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_REQUEST_ROUTING_H
-
-#include <grpc/support/port_platform.h>
-
-#include "src/core/ext/filters/client_channel/client_channel_channelz.h"
-#include "src/core/ext/filters/client_channel/client_channel_factory.h"
-#include "src/core/ext/filters/client_channel/lb_policy.h"
-#include "src/core/ext/filters/client_channel/resolver.h"
-#include "src/core/ext/filters/client_channel/subchannel_pool_interface.h"
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/debug/trace.h"
-#include "src/core/lib/gprpp/inlined_vector.h"
-#include "src/core/lib/gprpp/orphanable.h"
-#include "src/core/lib/iomgr/call_combiner.h"
-#include "src/core/lib/iomgr/closure.h"
-#include "src/core/lib/iomgr/polling_entity.h"
-#include "src/core/lib/iomgr/pollset_set.h"
-#include "src/core/lib/transport/connectivity_state.h"
-#include "src/core/lib/transport/metadata_batch.h"
-
-namespace grpc_core {
-
-class RequestRouter {
- public:
-  class Request {
-   public:
-    // Synchronous callback that applies the service config to a call.
-    // Returns false if the call should be failed.
-    typedef bool (*ApplyServiceConfigCallback)(void* user_data);
-
-    Request(grpc_call_stack* owning_call, grpc_call_combiner* call_combiner,
-            grpc_polling_entity* pollent,
-            grpc_metadata_batch* send_initial_metadata,
-            uint32_t* send_initial_metadata_flags,
-            ApplyServiceConfigCallback apply_service_config,
-            void* apply_service_config_user_data, grpc_closure* on_route_done);
-
-    ~Request();
-
-    // TODO(roth): It seems a bit ugly to expose this member in a
-    // non-const way.  Find a better API to avoid this.
-    LoadBalancingPolicy::PickState* pick() { return &pick_; }
-
-   private:
-    friend class RequestRouter;
-
-    class ResolverResultWaiter;
-    class AsyncPickCanceller;
-
-    void ProcessServiceConfigAndStartLbPickLocked();
-    void StartLbPickLocked();
-    static void LbPickDoneLocked(void* arg, grpc_error* error);
-
-    void MaybeAddCallToInterestedPartiesLocked();
-    void MaybeRemoveCallFromInterestedPartiesLocked();
-
-    // Populated by caller.
-    grpc_call_stack* owning_call_;
-    grpc_call_combiner* call_combiner_;
-    grpc_polling_entity* pollent_;
-    ApplyServiceConfigCallback apply_service_config_;
-    void* apply_service_config_user_data_;
-    grpc_closure* on_route_done_;
-    LoadBalancingPolicy::PickState pick_;
-
-    // Internal state.
-    RequestRouter* request_router_ = nullptr;
-    bool pollent_added_to_interested_parties_ = false;
-    grpc_closure on_pick_done_;
-    AsyncPickCanceller* pick_canceller_ = nullptr;
-  };
-
-  // Synchronous callback that takes the service config JSON string and
-  // LB policy name.
-  // Returns true if the service config has changed since the last result.
-  typedef bool (*ProcessResolverResultCallback)(void* user_data,
-                                                const grpc_channel_args& args,
-                                                const char** lb_policy_name,
-                                                grpc_json** lb_policy_config);
-
-  RequestRouter(grpc_channel_stack* owning_stack, grpc_combiner* combiner,
-                grpc_client_channel_factory* client_channel_factory,
-                grpc_pollset_set* interested_parties, TraceFlag* tracer,
-                ProcessResolverResultCallback process_resolver_result,
-                void* process_resolver_result_user_data, const char* target_uri,
-                const grpc_channel_args* args, grpc_error** error);
-
-  ~RequestRouter();
-
-  void set_channelz_node(channelz::ClientChannelNode* channelz_node) {
-    channelz_node_ = channelz_node;
-  }
-
-  void RouteCallLocked(Request* request);
-
-  // TODO(roth): Add methods to cancel picks.
-
-  void ShutdownLocked(grpc_error* error);
-
-  void ExitIdleLocked();
-  void ResetConnectionBackoffLocked();
-
-  grpc_connectivity_state GetConnectivityState();
-  void NotifyOnConnectivityStateChange(grpc_connectivity_state* state,
-                                       grpc_closure* closure);
-
-  LoadBalancingPolicy* lb_policy() const { return lb_policy_.get(); }
-
- private:
-  using TraceStringVector = InlinedVector<char*, 3>;
-
-  class ReresolutionRequestHandler;
-  class LbConnectivityWatcher;
-
-  void StartResolvingLocked();
-  void OnResolverShutdownLocked(grpc_error* error);
-  void CreateNewLbPolicyLocked(const char* lb_policy_name, grpc_json* lb_config,
-                               grpc_connectivity_state* connectivity_state,
-                               grpc_error** connectivity_error,
-                               TraceStringVector* trace_strings);
-  void MaybeAddTraceMessagesForAddressChangesLocked(
-      TraceStringVector* trace_strings);
-  void ConcatenateAndAddChannelTraceLocked(
-      TraceStringVector* trace_strings) const;
-  static void OnResolverResultChangedLocked(void* arg, grpc_error* error);
-
-  void SetConnectivityStateLocked(grpc_connectivity_state state,
-                                  grpc_error* error, const char* reason);
-
-  // Passed in from caller at construction time.
-  grpc_channel_stack* owning_stack_;
-  grpc_combiner* combiner_;
-  grpc_client_channel_factory* client_channel_factory_;
-  grpc_pollset_set* interested_parties_;
-  TraceFlag* tracer_;
-
-  channelz::ClientChannelNode* channelz_node_ = nullptr;
-
-  // Resolver and associated state.
-  OrphanablePtr<Resolver> resolver_;
-  ProcessResolverResultCallback process_resolver_result_;
-  void* process_resolver_result_user_data_;
-  bool started_resolving_ = false;
-  grpc_channel_args* resolver_result_ = nullptr;
-  bool previous_resolution_contained_addresses_ = false;
-  grpc_closure_list waiting_for_resolver_result_closures_;
-  grpc_closure on_resolver_result_changed_;
-
-  // LB policy and associated state.
-  OrphanablePtr<LoadBalancingPolicy> lb_policy_;
-  bool exit_idle_when_lb_policy_arrives_ = false;
-
-  // Subchannel pool to pass to LB policy.
-  RefCountedPtr<SubchannelPoolInterface> subchannel_pool_;
-
-  grpc_connectivity_state_tracker state_tracker_;
-};
-
-}  // namespace grpc_core
-
-#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_REQUEST_ROUTING_H */

src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc

@@ -151,7 +151,7 @@ AresDnsResolver::AresDnsResolver(const ResolverArgs& args)
   // Disable service config option
   const grpc_arg* arg = grpc_channel_args_find(
       channel_args_, GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION);
-  request_service_config_ = !grpc_channel_arg_get_bool(arg, false);
+  request_service_config_ = !grpc_channel_arg_get_bool(arg, true);
   // Min time b/t resolutions option
   arg = grpc_channel_args_find(channel_args_,
                                GRPC_ARG_DNS_MIN_TIME_BETWEEN_RESOLUTIONS_MS);

src/core/ext/filters/client_channel/resolving_lb_policy.cc

@@ -0,0 +1,460 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/filters/client_channel/resolving_lb_policy.h"
+
+#include <inttypes.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+#include "src/core/ext/filters/client_channel/backup_poller.h"
+#include "src/core/ext/filters/client_channel/http_connect_handshaker.h"
+#include "src/core/ext/filters/client_channel/lb_policy_registry.h"
+#include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
+#include "src/core/ext/filters/client_channel/resolver_registry.h"
+#include "src/core/ext/filters/client_channel/retry_throttle.h"
+#include "src/core/ext/filters/client_channel/server_address.h"
+#include "src/core/ext/filters/client_channel/subchannel.h"
+#include "src/core/ext/filters/deadline/deadline_filter.h"
+#include "src/core/lib/backoff/backoff.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/channel/status_util.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
+#include "src/core/lib/iomgr/combiner.h"
+#include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/profiling/timers.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/slice/slice_string_helpers.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/error_utils.h"
+#include "src/core/lib/transport/metadata.h"
+#include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/lib/transport/service_config.h"
+#include "src/core/lib/transport/static_metadata.h"
+#include "src/core/lib/transport/status_metadata.h"
+
+namespace grpc_core {
+
+//
+// ResolvingLoadBalancingPolicy::ResolvingControlHelper
+//
+
+class ResolvingLoadBalancingPolicy::ResolvingControlHelper
+    : public LoadBalancingPolicy::ChannelControlHelper {
+ public:
+  explicit ResolvingControlHelper(
+      RefCountedPtr<ResolvingLoadBalancingPolicy> parent)
+      : parent_(std::move(parent)) {}
+
+  Subchannel* CreateSubchannel(const grpc_channel_args& args) override {
+    if (parent_->resolver_ == nullptr) return nullptr;  // Shutting down.
+    return parent_->channel_control_helper()->CreateSubchannel(args);
+  }
+
+  grpc_channel* CreateChannel(const char* target, grpc_client_channel_type type,
+                              const grpc_channel_args& args) override {
+    if (parent_->resolver_ == nullptr) return nullptr;  // Shutting down.
+    return parent_->channel_control_helper()->CreateChannel(target, type, args);
+  }
+
+  void UpdateState(grpc_connectivity_state state, grpc_error* state_error,
+                   UniquePtr<SubchannelPicker> picker) override {
+    if (parent_->resolver_ == nullptr) {
+      // shutting down.
+      GRPC_ERROR_UNREF(state_error);
+      return;
+    }
+    parent_->channel_control_helper()->UpdateState(state, state_error,
+                                                   std::move(picker));
+  }
+
+  void RequestReresolution() override {
+    if (parent_->tracer_->enabled()) {
+      gpr_log(GPR_INFO, "resolving_lb=%p: started name re-resolving",
+              parent_.get());
+    }
+    if (parent_->resolver_ != nullptr) {
+      parent_->resolver_->RequestReresolutionLocked();
+    }
+  }
+
+ private:
+  RefCountedPtr<ResolvingLoadBalancingPolicy> parent_;
+};
+
+//
+// ResolvingLoadBalancingPolicy
+//
+
+ResolvingLoadBalancingPolicy::ResolvingLoadBalancingPolicy(
+    Args args, TraceFlag* tracer, UniquePtr<char> target_uri,
+    UniquePtr<char> child_policy_name, grpc_json* child_lb_config,
+    grpc_error** error)
+    : LoadBalancingPolicy(std::move(args)),
+      tracer_(tracer),
+      target_uri_(std::move(target_uri)),
+      child_policy_name_(std::move(child_policy_name)),
+      child_lb_config_str_(grpc_json_dump_to_string(child_lb_config, 0)),
+      child_lb_config_(grpc_json_parse_string(child_lb_config_str_.get())) {
+  GPR_ASSERT(child_policy_name_ != nullptr);
+  // Don't fetch service config, since this ctor is for use in nested LB
+  // policies, not at the top level, and we only fetch the service
+  // config at the top level.
+  grpc_arg arg = grpc_channel_arg_integer_create(
+      const_cast<char*>(GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION), 0);
+  grpc_channel_args* new_args =
+      grpc_channel_args_copy_and_add(args.args, &arg, 1);
+  *error = Init(*new_args);
+  grpc_channel_args_destroy(new_args);
+}
+
+ResolvingLoadBalancingPolicy::ResolvingLoadBalancingPolicy(
+    Args args, TraceFlag* tracer, UniquePtr<char> target_uri,
+    ProcessResolverResultCallback process_resolver_result,
+    void* process_resolver_result_user_data, grpc_error** error)
+    : LoadBalancingPolicy(std::move(args)),
+      tracer_(tracer),
+      target_uri_(std::move(target_uri)),
+      process_resolver_result_(process_resolver_result),
+      process_resolver_result_user_data_(process_resolver_result_user_data) {
+  GPR_ASSERT(process_resolver_result != nullptr);
+  *error = Init(*args.args);
+}
+
+grpc_error* ResolvingLoadBalancingPolicy::Init(const grpc_channel_args& args) {
+  GRPC_CLOSURE_INIT(
+      &on_resolver_result_changed_,
+      &ResolvingLoadBalancingPolicy::OnResolverResultChangedLocked, this,
+      grpc_combiner_scheduler(combiner()));
+  resolver_ = ResolverRegistry::CreateResolver(
+      target_uri_.get(), &args, interested_parties(), combiner());
+  if (resolver_ == nullptr) {
+    return GRPC_ERROR_CREATE_FROM_STATIC_STRING("resolver creation failed");
+  }
+  // Return our picker to the channel.
+  channel_control_helper()->UpdateState(
+      GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
+      UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
+  return GRPC_ERROR_NONE;
+}
+
+ResolvingLoadBalancingPolicy::~ResolvingLoadBalancingPolicy() {
+  GPR_ASSERT(resolver_ == nullptr);
+  GPR_ASSERT(lb_policy_ == nullptr);
+  grpc_json_destroy(child_lb_config_);
+}
+
+void ResolvingLoadBalancingPolicy::ShutdownLocked() {
+  if (resolver_ != nullptr) {
+    resolver_.reset();
+    if (lb_policy_ != nullptr) {
+      grpc_pollset_set_del_pollset_set(lb_policy_->interested_parties(),
+                                       interested_parties());
+      lb_policy_.reset();
+    }
+  }
+}
+
+void ResolvingLoadBalancingPolicy::ExitIdleLocked() {
+  if (lb_policy_ != nullptr) {
+    lb_policy_->ExitIdleLocked();
+  } else {
+    if (!started_resolving_ && resolver_ != nullptr) {
+      StartResolvingLocked();
+    }
+  }
+}
+
+void ResolvingLoadBalancingPolicy::ResetBackoffLocked() {
+  if (resolver_ != nullptr) {
+    resolver_->ResetBackoffLocked();
+    resolver_->RequestReresolutionLocked();
+  }
+  if (lb_policy_ != nullptr) {
+    lb_policy_->ResetBackoffLocked();
+  }
+}
+
+void ResolvingLoadBalancingPolicy::FillChildRefsForChannelz(
+    channelz::ChildRefsList* child_subchannels,
+    channelz::ChildRefsList* child_channels) {
+  if (lb_policy_ != nullptr) {
+    lb_policy_->FillChildRefsForChannelz(child_subchannels, child_channels);
+  }
+}
+
+void ResolvingLoadBalancingPolicy::StartResolvingLocked() {
+  if (tracer_->enabled()) {
+    gpr_log(GPR_INFO, "resolving_lb=%p: starting name resolution", this);
+  }
+  GPR_ASSERT(!started_resolving_);
+  started_resolving_ = true;
+  Ref().release();
+  resolver_->NextLocked(&resolver_result_, &on_resolver_result_changed_);
+}
+
+// Invoked from the resolver NextLocked() callback when the resolver
+// is shutting down.
+void ResolvingLoadBalancingPolicy::OnResolverShutdownLocked(grpc_error* error) {
+  if (tracer_->enabled()) {
+    gpr_log(GPR_INFO, "resolving_lb=%p: shutting down", this);
+  }
+  if (lb_policy_ != nullptr) {
+    if (tracer_->enabled()) {
+      gpr_log(GPR_INFO, "resolving_lb=%p: shutting down lb_policy=%p", this,
+              lb_policy_.get());
+    }
+    grpc_pollset_set_del_pollset_set(lb_policy_->interested_parties(),
+                                     interested_parties());
+    lb_policy_.reset();
+  }
+  if (resolver_ != nullptr) {
+    // This should never happen; it can only be triggered by a resolver
+    // implementation spotaneously deciding to report shutdown without
+    // being orphaned.  This code is included just to be defensive.
+    if (tracer_->enabled()) {
+      gpr_log(GPR_INFO,
+              "resolving_lb=%p: spontaneous shutdown from resolver %p", this,
+              resolver_.get());
+    }
+    resolver_.reset();
+    grpc_error* error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+        "Resolver spontaneous shutdown", &error, 1);
+    channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error),
+        UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
+  }
+  grpc_channel_args_destroy(resolver_result_);
+  resolver_result_ = nullptr;
+  GRPC_ERROR_UNREF(error);
+  Unref();
+}
+
+// Creates a new LB policy, replacing any previous one.
+// Updates trace_strings to indicate what was done.
+void ResolvingLoadBalancingPolicy::CreateNewLbPolicyLocked(
+    const char* lb_policy_name, grpc_json* lb_config,
+    TraceStringVector* trace_strings) {
+  LoadBalancingPolicy::Args lb_policy_args;
+  lb_policy_args.combiner = combiner();
+  lb_policy_args.channel_control_helper =
+      UniquePtr<ChannelControlHelper>(New<ResolvingControlHelper>(Ref()));
+  lb_policy_args.args = resolver_result_;
+  lb_policy_args.lb_config = lb_config;
+  OrphanablePtr<LoadBalancingPolicy> new_lb_policy =
+      LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(
+          lb_policy_name, std::move(lb_policy_args));
+  if (GPR_UNLIKELY(new_lb_policy == nullptr)) {
+    gpr_log(GPR_ERROR, "could not create LB policy \"%s\"", lb_policy_name);
+    if (channelz_node() != nullptr) {
+      char* str;
+      gpr_asprintf(&str, "Could not create LB policy \"%s\"", lb_policy_name);
+      trace_strings->push_back(str);
+    }
+  } else {
+    if (tracer_->enabled()) {
+      gpr_log(GPR_INFO, "resolving_lb=%p: created new LB policy \"%s\" (%p)",
+              this, lb_policy_name, new_lb_policy.get());
+    }
+    if (channelz_node() != nullptr) {
+      char* str;
+      gpr_asprintf(&str, "Created new LB policy \"%s\"", lb_policy_name);
+      trace_strings->push_back(str);
+    }
+    // Propagate channelz node.
+    auto* channelz = channelz_node();
+    if (channelz != nullptr) {
+      new_lb_policy->set_channelz_node(channelz->Ref());
+    }
+    // Swap out the LB policy and update the fds in interested_parties_.
+    if (lb_policy_ != nullptr) {
+      if (tracer_->enabled()) {
+        gpr_log(GPR_INFO, "resolving_lb=%p: shutting down lb_policy=%p", this,
+                lb_policy_.get());
+      }
+      grpc_pollset_set_del_pollset_set(lb_policy_->interested_parties(),
+                                       interested_parties());
+    }
+    lb_policy_ = std::move(new_lb_policy);
+    grpc_pollset_set_add_pollset_set(lb_policy_->interested_parties(),
+                                     interested_parties());
+    lb_policy_->ExitIdleLocked();
+  }
+}
+
+void ResolvingLoadBalancingPolicy::MaybeAddTraceMessagesForAddressChangesLocked(
+    TraceStringVector* trace_strings) {
+  const ServerAddressList* addresses =
+      FindServerAddressListChannelArg(resolver_result_);
+  const bool resolution_contains_addresses =
+      addresses != nullptr && addresses->size() > 0;
+  if (!resolution_contains_addresses &&
+      previous_resolution_contained_addresses_) {
+    trace_strings->push_back(gpr_strdup("Address list became empty"));
+  } else if (resolution_contains_addresses &&
+             !previous_resolution_contained_addresses_) {
+    trace_strings->push_back(gpr_strdup("Address list became non-empty"));
+  }
+  previous_resolution_contained_addresses_ = resolution_contains_addresses;
+}
+
+void ResolvingLoadBalancingPolicy::ConcatenateAndAddChannelTraceLocked(
+    TraceStringVector* trace_strings) const {
+  if (!trace_strings->empty()) {
+    gpr_strvec v;
+    gpr_strvec_init(&v);
+    gpr_strvec_add(&v, gpr_strdup("Resolution event: "));
+    bool is_first = 1;
+    for (size_t i = 0; i < trace_strings->size(); ++i) {
+      if (!is_first) gpr_strvec_add(&v, gpr_strdup(", "));
+      is_first = false;
+      gpr_strvec_add(&v, (*trace_strings)[i]);
+    }
+    char* flat;
+    size_t flat_len = 0;
+    flat = gpr_strvec_flatten(&v, &flat_len);
+    channelz_node()->AddTraceEvent(channelz::ChannelTrace::Severity::Info,
+                                   grpc_slice_new(flat, flat_len, gpr_free));
+    gpr_strvec_destroy(&v);
+  }
+}
+
+// Callback invoked when a resolver result is available.
+void ResolvingLoadBalancingPolicy::OnResolverResultChangedLocked(
+    void* arg, grpc_error* error) {
+  auto* self = static_cast<ResolvingLoadBalancingPolicy*>(arg);
+  if (self->tracer_->enabled()) {
+    const char* disposition =
+        self->resolver_result_ != nullptr
+            ? ""
+            : (error == GRPC_ERROR_NONE ? " (transient error)"
+                                        : " (resolver shutdown)");
+    gpr_log(GPR_INFO,
+            "resolving_lb=%p: got resolver result: resolver_result=%p "
+            "error=%s%s",
+            self, self->resolver_result_, grpc_error_string(error),
+            disposition);
+  }
+  // Handle shutdown.
+  if (error != GRPC_ERROR_NONE || self->resolver_ == nullptr) {
+    self->OnResolverShutdownLocked(GRPC_ERROR_REF(error));
+    return;
+  }
+  // We only want to trace the address resolution in the follow cases:
+  // (a) Address resolution resulted in service config change.
+  // (b) Address resolution that causes number of backends to go from
+  //     zero to non-zero.
+  // (c) Address resolution that causes number of backends to go from
+  //     non-zero to zero.
+  // (d) Address resolution that causes a new LB policy to be created.
+  //
+  // we track a list of strings to eventually be concatenated and traced.
+  TraceStringVector trace_strings;
+  // resolver_result_ will be null in the case of a transient
+  // resolution error.  In that case, we don't have any new result to
+  // process, which means that we keep using the previous result (if any).
+  if (self->resolver_result_ == nullptr) {
+    if (self->tracer_->enabled()) {
+      gpr_log(GPR_INFO, "resolving_lb=%p: resolver transient failure", self);
+    }
+    // If we already have an LB policy from a previous resolution
+    // result, then we continue to let it set the connectivity state.
+    // Otherwise, we go into TRANSIENT_FAILURE.
+    if (self->lb_policy_ == nullptr) {
+      // TODO(roth): When we change the resolver API to be able to
+      // return transient errors in a cleaner way, we should make it the
+      // resolver's responsibility to attach a status to the error,
+      // rather than doing it centrally here.
+      grpc_error* state_error = grpc_error_set_int(
+          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+              "Resolver transient failure", &error, 1),
+          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+      self->channel_control_helper()->UpdateState(
+          GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(state_error),
+          UniquePtr<SubchannelPicker>(
+              New<TransientFailurePicker>(state_error)));
+    }
+  } else {
+    // Parse the resolver result.
+    const char* lb_policy_name = nullptr;
+    grpc_json* lb_policy_config = nullptr;
+    bool service_config_changed = false;
+    if (self->process_resolver_result_ != nullptr) {
+      service_config_changed = self->process_resolver_result_(
+          self->process_resolver_result_user_data_, *self->resolver_result_,
+          &lb_policy_name, &lb_policy_config);
+    } else {
+      lb_policy_name = self->child_policy_name_.get();
+      lb_policy_config = self->child_lb_config_;
+    }
+    GPR_ASSERT(lb_policy_name != nullptr);
+    // Check to see if we're already using the right LB policy.
+    const bool lb_policy_name_changed =
+        self->lb_policy_ == nullptr ||
+        strcmp(self->lb_policy_->name(), lb_policy_name) != 0;
+    if (self->lb_policy_ != nullptr && !lb_policy_name_changed) {
+      // Continue using the same LB policy.  Update with new addresses.
+      if (self->tracer_->enabled()) {
+        gpr_log(GPR_INFO,
+                "resolving_lb=%p: updating existing LB policy \"%s\" (%p)",
+                self, lb_policy_name, self->lb_policy_.get());
+      }
+      self->lb_policy_->UpdateLocked(*self->resolver_result_, lb_policy_config);
+    } else {
+      // Instantiate new LB policy.
+      if (self->tracer_->enabled()) {
+        gpr_log(GPR_INFO, "resolving_lb=%p: creating new LB policy \"%s\"",
+                self, lb_policy_name);
+      }
+      self->CreateNewLbPolicyLocked(lb_policy_name, lb_policy_config,
+                                    &trace_strings);
+    }
+    // Add channel trace event.
+    if (self->channelz_node() != nullptr) {
+      if (service_config_changed) {
+        // TODO(ncteisen): might be worth somehow including a snippet of the
+        // config in the trace, at the risk of bloating the trace logs.
+        trace_strings.push_back(gpr_strdup("Service config changed"));
+      }
+      self->MaybeAddTraceMessagesForAddressChangesLocked(&trace_strings);
+      self->ConcatenateAndAddChannelTraceLocked(&trace_strings);
+    }
+    // Clean up.
+    grpc_channel_args_destroy(self->resolver_result_);
+    self->resolver_result_ = nullptr;
+  }
+  // Renew resolver callback.
+  self->resolver_->NextLocked(&self->resolver_result_,
+                              &self->on_resolver_result_changed_);
+}
+
+}  // namespace grpc_core

src/core/ext/filters/client_channel/resolving_lb_policy.h

@@ -0,0 +1,137 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVING_LB_POLICY_H
+#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVING_LB_POLICY_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/filters/client_channel/client_channel_channelz.h"
+#include "src/core/ext/filters/client_channel/lb_policy.h"
+#include "src/core/ext/filters/client_channel/resolver.h"
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/debug/trace.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/orphanable.h"
+#include "src/core/lib/iomgr/call_combiner.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/polling_entity.h"
+#include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/metadata_batch.h"
+
+namespace grpc_core {
+
+// An LB policy that wraps a resolver and a child LB policy to make use
+// of the addresses returned by the resolver.
+//
+// When used in the client_channel code, the resolver will attempt to
+// fetch the service config, and the child LB policy name and config
+// will be determined based on the service config.
+//
+// When used in an LB policy implementation that needs to do another
+// round of resolution before creating a child policy, the resolver does
+// not fetch the service config, and the caller must pre-determine the
+// child LB policy and config to use.
+class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy {
+ public:
+  // If error is set when this returns, then construction failed, and
+  // the caller may not use the new object.
+  ResolvingLoadBalancingPolicy(Args args, TraceFlag* tracer,
+                               UniquePtr<char> target_uri,
+                               UniquePtr<char> child_policy_name,
+                               grpc_json* child_lb_config, grpc_error** error);
+
+  // Private ctor, to be used by client_channel only!
+  //
+  // Synchronous callback that takes the resolver result and sets
+  // lb_policy_name and lb_policy_config to point to the right data.
+  // Returns true if the service config has changed since the last result.
+  typedef bool (*ProcessResolverResultCallback)(void* user_data,
+                                                const grpc_channel_args& args,
+                                                const char** lb_policy_name,
+                                                grpc_json** lb_policy_config);
+  // If error is set when this returns, then construction failed, and
+  // the caller may not use the new object.
+  ResolvingLoadBalancingPolicy(
+      Args args, TraceFlag* tracer, UniquePtr<char> target_uri,
+      ProcessResolverResultCallback process_resolver_result,
+      void* process_resolver_result_user_data, grpc_error** error);
+
+  virtual const char* name() const override { return "resolving_lb"; }
+
+  // No-op -- should never get updates from the channel.
+  // TODO(roth): Need to support updating child LB policy's config.
+  // For xds policy, will also need to support updating config
+  // independently of args from resolver, since they will be coming from
+  // different places.  Maybe change LB policy API to support that?
+  void UpdateLocked(const grpc_channel_args& args,
+                    grpc_json* lb_config) override {}
+
+  void ExitIdleLocked() override;
+
+  void ResetBackoffLocked() override;
+
+  void FillChildRefsForChannelz(
+      channelz::ChildRefsList* child_subchannels,
+      channelz::ChildRefsList* child_channels) override;
+
+ private:
+  using TraceStringVector = InlinedVector<char*, 3>;
+
+  class ResolvingControlHelper;
+
+  ~ResolvingLoadBalancingPolicy();
+
+  grpc_error* Init(const grpc_channel_args& args);
+  void ShutdownLocked() override;
+
+  void StartResolvingLocked();
+  void OnResolverShutdownLocked(grpc_error* error);
+  void CreateNewLbPolicyLocked(const char* lb_policy_name, grpc_json* lb_config,
+                               TraceStringVector* trace_strings);
+  void MaybeAddTraceMessagesForAddressChangesLocked(
+      TraceStringVector* trace_strings);
+  void ConcatenateAndAddChannelTraceLocked(
+      TraceStringVector* trace_strings) const;
+  static void OnResolverResultChangedLocked(void* arg, grpc_error* error);
+
+  // Passed in from caller at construction time.
+  TraceFlag* tracer_;
+  UniquePtr<char> target_uri_;
+  ProcessResolverResultCallback process_resolver_result_ = nullptr;
+  void* process_resolver_result_user_data_ = nullptr;
+  UniquePtr<char> child_policy_name_;
+  UniquePtr<char> child_lb_config_str_;
+  grpc_json* child_lb_config_ = nullptr;
+
+  // Resolver and associated state.
+  OrphanablePtr<Resolver> resolver_;
+  bool started_resolving_ = false;
+  grpc_channel_args* resolver_result_ = nullptr;
+  bool previous_resolution_contained_addresses_ = false;
+  grpc_closure on_resolver_result_changed_;
+
+  // Child LB policy and associated state.
+  OrphanablePtr<LoadBalancingPolicy> lb_policy_;
+};
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVING_LB_POLICY_H */

src/core/ext/filters/client_channel/subchannel.cc

@@ -116,21 +116,6 @@ void ConnectedSubchannel::Ping(grpc_closure* on_initiate,
   elem->filter->start_transport_op(elem, op);
 }
 
-namespace {
-
-void SubchannelCallDestroy(void* arg, grpc_error* error) {
-  GPR_TIMER_SCOPE("subchannel_call_destroy", 0);
-  SubchannelCall* call = static_cast<SubchannelCall*>(arg);
-  grpc_closure* after_call_stack_destroy = call->after_call_stack_destroy();
-  call->~SubchannelCall();
-  // This should be the last step to destroy the subchannel call, because
-  // call->after_call_stack_destroy(), if not null, will free the call arena.
-  grpc_call_stack_destroy(SUBCHANNEL_CALL_TO_CALL_STACK(call), nullptr,
-                          after_call_stack_destroy);
-}
-
-}  // namespace
-
 RefCountedPtr<SubchannelCall> ConnectedSubchannel::CreateCall(
     const CallArgs& args, grpc_error** error) {
   const size_t allocation_size =
@@ -149,7 +134,7 @@ RefCountedPtr<SubchannelCall> ConnectedSubchannel::CreateCall(
       args.arena,        /* arena */
       args.call_combiner /* call_combiner */
   };
-  *error = grpc_call_stack_init(channel_stack_, 1, SubchannelCallDestroy,
+  *error = grpc_call_stack_init(channel_stack_, 1, SubchannelCall::Destroy,
                                 call.get(), &call_args);
   if (GPR_UNLIKELY(*error != GRPC_ERROR_NONE)) {
     const char* error_string = grpc_error_string(*error);
@@ -226,6 +211,25 @@ void SubchannelCall::Unref(const DebugLocation& location, const char* reason) {
   GRPC_CALL_STACK_UNREF(SUBCHANNEL_CALL_TO_CALL_STACK(this), reason);
 }
 
+void SubchannelCall::Destroy(void* arg, grpc_error* error) {
+  GPR_TIMER_SCOPE("subchannel_call_destroy", 0);
+  SubchannelCall* self = static_cast<SubchannelCall*>(arg);
+  // Keep some members before destroying the subchannel call.
+  grpc_closure* after_call_stack_destroy = self->after_call_stack_destroy_;
+  RefCountedPtr<ConnectedSubchannel> connected_subchannel =
+      std::move(self->connected_subchannel_);
+  // Destroy the subchannel call.
+  self->~SubchannelCall();
+  // Destroy the call stack. This should be after destroying the subchannel
+  // call, because call->after_call_stack_destroy(), if not null, will free the
+  // call arena.
+  grpc_call_stack_destroy(SUBCHANNEL_CALL_TO_CALL_STACK(self), nullptr,
+                          after_call_stack_destroy);
+  // Automatically reset connected_subchannel. This should be after destroying
+  // the call stack, because destroying call stack needs access to the channel
+  // stack.
+}
+
 void SubchannelCall::MaybeInterceptRecvTrailingMetadata(
     grpc_transport_stream_op_batch* batch) {
   // only intercept payloads with recv trailing.
@@ -952,22 +956,17 @@ void Subchannel::OnConnectingFinished(void* arg, grpc_error* error) {
   } else if (c->disconnected_) {
     GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
   } else {
-    c->SetConnectivityStateLocked(
-        GRPC_CHANNEL_TRANSIENT_FAILURE,
-        grpc_error_set_int(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                               "Connect Failed", &error, 1),
-                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
-        "connect_failed");
-    grpc_connectivity_state_set(
-        &c->state_and_health_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE,
-        grpc_error_set_int(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                               "Connect Failed", &error, 1),
-                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE),
-        "connect_failed");
-
     const char* errmsg = grpc_error_string(error);
     gpr_log(GPR_INFO, "Connect failed: %s", errmsg);
-
+    error =
+        grpc_error_set_int(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+                               "Connect Failed", &error, 1),
+                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+    c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
+                                  GRPC_ERROR_REF(error), "connect_failed");
+    grpc_connectivity_state_set(&c->state_and_health_tracker_,
+                                GRPC_CHANNEL_TRANSIENT_FAILURE, error,
+                                "connect_failed");
     c->MaybeStartConnectingLocked();
     GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
   }

src/core/ext/filters/client_channel/subchannel.h

@@ -131,10 +131,6 @@ class SubchannelCall {
   // Returns the call stack of the subchannel call.
   grpc_call_stack* GetCallStack();
 
-  grpc_closure* after_call_stack_destroy() const {
-    return after_call_stack_destroy_;
-  }
-
   // Sets the 'then_schedule_closure' argument for call stack destruction.
   // Must be called once per call.
   void SetAfterCallStackDestroy(grpc_closure* closure);
@@ -148,6 +144,8 @@ class SubchannelCall {
   void Unref();
   void Unref(const DebugLocation& location, const char* reason);
 
+  static void Destroy(void* arg, grpc_error* error);
+
  private:
   // Allow RefCountedPtr<> to access IncrementRefCount().
   template <typename T>

src/core/ext/transport/chttp2/client/chttp2_connector.cc

@@ -55,7 +55,7 @@ typedef struct {
 
   grpc_closure connected;
 
-  grpc_handshake_manager* handshake_mgr;
+  grpc_core::RefCountedPtr<grpc_core::HandshakeManager> handshake_mgr;
 } chttp2_connector;
 
 static void chttp2_connector_ref(grpc_connector* con) {
@@ -79,7 +79,7 @@ static void chttp2_connector_shutdown(grpc_connector* con, grpc_error* why) {
   gpr_mu_lock(&c->mu);
   c->shutdown = true;
   if (c->handshake_mgr != nullptr) {
-    grpc_handshake_manager_shutdown(c->handshake_mgr, GRPC_ERROR_REF(why));
+    c->handshake_mgr->Shutdown(GRPC_ERROR_REF(why));
   }
   // If handshaking is not yet in progress, shutdown the endpoint.
   // Otherwise, the handshaker will do this for us.
@@ -91,7 +91,7 @@ static void chttp2_connector_shutdown(grpc_connector* con, grpc_error* why) {
 }
 
 static void on_handshake_done(void* arg, grpc_error* error) {
-  grpc_handshaker_args* args = static_cast<grpc_handshaker_args*>(arg);
+  auto* args = static_cast<grpc_core::HandshakerArgs*>(arg);
   chttp2_connector* c = static_cast<chttp2_connector*>(args->user_data);
   gpr_mu_lock(&c->mu);
   if (error != GRPC_ERROR_NONE || c->shutdown) {
@@ -152,20 +152,20 @@ static void on_handshake_done(void* arg, grpc_error* error) {
   grpc_closure* notify = c->notify;
   c->notify = nullptr;
   GRPC_CLOSURE_SCHED(notify, error);
-  grpc_handshake_manager_destroy(c->handshake_mgr);
-  c->handshake_mgr = nullptr;
+  c->handshake_mgr.reset();
   gpr_mu_unlock(&c->mu);
   chttp2_connector_unref(reinterpret_cast<grpc_connector*>(c));
 }
 
 static void start_handshake_locked(chttp2_connector* c) {
-  c->handshake_mgr = grpc_handshake_manager_create();
-  grpc_handshakers_add(HANDSHAKER_CLIENT, c->args.channel_args,
-                       c->args.interested_parties, c->handshake_mgr);
+  c->handshake_mgr = grpc_core::MakeRefCounted<grpc_core::HandshakeManager>();
+  grpc_core::HandshakerRegistry::AddHandshakers(
+      grpc_core::HANDSHAKER_CLIENT, c->args.channel_args,
+      c->args.interested_parties, c->handshake_mgr.get());
   grpc_endpoint_add_to_pollset_set(c->endpoint, c->args.interested_parties);
-  grpc_handshake_manager_do_handshake(
-      c->handshake_mgr, c->endpoint, c->args.channel_args, c->args.deadline,
-      nullptr /* acceptor */, on_handshake_done, c);
+  c->handshake_mgr->DoHandshake(c->endpoint, c->args.channel_args,
+                                c->args.deadline, nullptr /* acceptor */,
+                                on_handshake_done, c);
   c->endpoint = nullptr;  // Endpoint handed off to handshake manager.
 }
 

src/core/ext/transport/chttp2/server/chttp2_server.cc

@@ -54,7 +54,7 @@ typedef struct {
   bool shutdown;
   grpc_closure tcp_server_shutdown_complete;
   grpc_closure* server_destroy_listener_done;
-  grpc_handshake_manager* pending_handshake_mgrs;
+  grpc_core::HandshakeManager* pending_handshake_mgrs;
   grpc_core::RefCountedPtr<grpc_core::channelz::ListenSocketNode>
       channelz_listen_socket;
 } server_state;
@@ -64,7 +64,7 @@ typedef struct {
   server_state* svr_state;
   grpc_pollset* accepting_pollset;
   grpc_tcp_server_acceptor* acceptor;
-  grpc_handshake_manager* handshake_mgr;
+  grpc_core::RefCountedPtr<grpc_core::HandshakeManager> handshake_mgr;
   // State for enforcing handshake timeout on receiving HTTP/2 settings.
   grpc_chttp2_transport* transport;
   grpc_millis deadline;
@@ -112,7 +112,7 @@ static void on_receive_settings(void* arg, grpc_error* error) {
 }
 
 static void on_handshake_done(void* arg, grpc_error* error) {
-  grpc_handshaker_args* args = static_cast<grpc_handshaker_args*>(arg);
+  auto* args = static_cast<grpc_core::HandshakerArgs*>(arg);
   server_connection_state* connection_state =
       static_cast<server_connection_state*>(args->user_data);
   gpr_mu_lock(&connection_state->svr_state->mu);
@@ -175,11 +175,10 @@ static void on_handshake_done(void* arg, grpc_error* error) {
       }
     }
   }
-  grpc_handshake_manager_pending_list_remove(
-      &connection_state->svr_state->pending_handshake_mgrs,
-      connection_state->handshake_mgr);
+  connection_state->handshake_mgr->RemoveFromPendingMgrList(
+      &connection_state->svr_state->pending_handshake_mgrs);
   gpr_mu_unlock(&connection_state->svr_state->mu);
-  grpc_handshake_manager_destroy(connection_state->handshake_mgr);
+  connection_state->handshake_mgr.reset();
   gpr_free(connection_state->acceptor);
   grpc_tcp_server_unref(connection_state->svr_state->tcp_server);
   server_connection_state_unref(connection_state);
@@ -211,9 +210,8 @@ static void on_accept(void* arg, grpc_endpoint* tcp,
     gpr_free(acceptor);
     return;
   }
-  grpc_handshake_manager* handshake_mgr = grpc_handshake_manager_create();
-  grpc_handshake_manager_pending_list_add(&state->pending_handshake_mgrs,
-                                          handshake_mgr);
+  auto handshake_mgr = grpc_core::MakeRefCounted<grpc_core::HandshakeManager>();
+  handshake_mgr->AddToPendingMgrList(&state->pending_handshake_mgrs);
   grpc_tcp_server_ref(state->tcp_server);
   gpr_mu_unlock(&state->mu);
   server_connection_state* connection_state =
@@ -227,19 +225,19 @@ static void on_accept(void* arg, grpc_endpoint* tcp,
   connection_state->interested_parties = grpc_pollset_set_create();
   grpc_pollset_set_add_pollset(connection_state->interested_parties,
                                connection_state->accepting_pollset);
-  grpc_handshakers_add(HANDSHAKER_SERVER, state->args,
-                       connection_state->interested_parties,
-                       connection_state->handshake_mgr);
+  grpc_core::HandshakerRegistry::AddHandshakers(
+      grpc_core::HANDSHAKER_SERVER, state->args,
+      connection_state->interested_parties,
+      connection_state->handshake_mgr.get());
   const grpc_arg* timeout_arg =
       grpc_channel_args_find(state->args, GRPC_ARG_SERVER_HANDSHAKE_TIMEOUT_MS);
   connection_state->deadline =
       grpc_core::ExecCtx::Get()->Now() +
       grpc_channel_arg_get_integer(timeout_arg,
                                    {120 * GPR_MS_PER_SEC, 1, INT_MAX});
-  grpc_handshake_manager_do_handshake(connection_state->handshake_mgr, tcp,
-                                      state->args, connection_state->deadline,
-                                      acceptor, on_handshake_done,
-                                      connection_state);
+  connection_state->handshake_mgr->DoHandshake(
+      tcp, state->args, connection_state->deadline, acceptor, on_handshake_done,
+      connection_state);
 }
 
 /* Server callback: start listening on our ports */
@@ -260,8 +258,9 @@ static void tcp_server_shutdown_complete(void* arg, grpc_error* error) {
   gpr_mu_lock(&state->mu);
   grpc_closure* destroy_done = state->server_destroy_listener_done;
   GPR_ASSERT(state->shutdown);
-  grpc_handshake_manager_pending_list_shutdown_all(
-      state->pending_handshake_mgrs, GRPC_ERROR_REF(error));
+  if (state->pending_handshake_mgrs != nullptr) {
+    state->pending_handshake_mgrs->ShutdownAllPending(GRPC_ERROR_REF(error));
+  }
   state->channelz_listen_socket.reset();
   gpr_mu_unlock(&state->mu);
   // Flush queued work before destroying handshaker factory, since that

src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -1119,9 +1119,6 @@ static void queue_setting_update(grpc_chttp2_transport* t,
 void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t,
                                      uint32_t goaway_error,
                                      grpc_slice goaway_text) {
-  // GRPC_CHTTP2_IF_TRACING(
-  //     gpr_log(GPR_INFO, "got goaway [%d]: %s", goaway_error, msg));
-
   // Discard the error from a previous goaway frame (if any)
   if (t->goaway_error != GRPC_ERROR_NONE) {
     GRPC_ERROR_UNREF(t->goaway_error);
@@ -1132,6 +1129,10 @@ void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t,
           GRPC_ERROR_INT_HTTP2_ERROR, static_cast<intptr_t>(goaway_error)),
       GRPC_ERROR_STR_RAW_BYTES, goaway_text);
 
+  /* We want to log this irrespective of whether http tracing is enabled */
+  gpr_log(GPR_INFO, "%s: Got goaway [%d] err=%s", t->peer_string, goaway_error,
+          grpc_error_string(t->goaway_error));
+
   /* When a client receives a GOAWAY with error code ENHANCE_YOUR_CALM and debug
    * data equal to "too_many_pings", it should log the occurrence at a log level
    * that is enabled by default and double the configured KEEPALIVE_TIME used
@@ -1774,6 +1775,9 @@ void grpc_chttp2_ack_ping(grpc_chttp2_transport* t, uint64_t id) {
 }
 
 static void send_goaway(grpc_chttp2_transport* t, grpc_error* error) {
+  /* We want to log this irrespective of whether http tracing is enabled */
+  gpr_log(GPR_INFO, "%s: Sending goaway err=%s", t->peer_string,
+          grpc_error_string(error));
   t->sent_goaway_state = GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED;
   grpc_http2_error_code http_error;
   grpc_slice slice;
@@ -2554,6 +2558,10 @@ static void read_action_locked(void* tp, grpc_error* error) {
   } else if (t->closed_with_error == GRPC_ERROR_NONE) {
     keep_reading = true;
     GRPC_CHTTP2_REF_TRANSPORT(t, "keep_reading");
+    /* Since we have read a byte, reset the keepalive timer */
+    if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING) {
+      grpc_timer_cancel(&t->keepalive_ping_timer);
+    }
   }
   grpc_slice_buffer_reset_and_unref_internal(&t->read_buffer);
 
@@ -2723,6 +2731,9 @@ static void start_keepalive_ping_locked(void* arg, grpc_error* error) {
   if (t->channelz_socket != nullptr) {
     t->channelz_socket->RecordKeepaliveSent();
   }
+  if (grpc_http_trace.enabled()) {
+    gpr_log(GPR_INFO, "%s: Start keepalive ping", t->peer_string);
+  }
   GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive watchdog");
   grpc_timer_init(&t->keepalive_watchdog_timer,
                   grpc_core::ExecCtx::Get()->Now() + t->keepalive_timeout,
@@ -2733,6 +2744,9 @@ static void finish_keepalive_ping_locked(void* arg, grpc_error* error) {
   grpc_chttp2_transport* t = static_cast<grpc_chttp2_transport*>(arg);
   if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_PINGING) {
     if (error == GRPC_ERROR_NONE) {
+      if (grpc_http_trace.enabled()) {
+        gpr_log(GPR_INFO, "%s: Finish keepalive ping", t->peer_string);
+      }
       t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING;
       grpc_timer_cancel(&t->keepalive_watchdog_timer);
       GRPC_CHTTP2_REF_TRANSPORT(t, "init keepalive ping");
@@ -2748,6 +2762,8 @@ static void keepalive_watchdog_fired_locked(void* arg, grpc_error* error) {
   grpc_chttp2_transport* t = static_cast<grpc_chttp2_transport*>(arg);
   if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_PINGING) {
     if (error == GRPC_ERROR_NONE) {
+      gpr_log(GPR_ERROR, "%s: Keepalive watchdog fired. Closing transport.",
+              t->peer_string);
       t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DYING;
       close_transport_locked(
           t, grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(

src/core/ext/transport/cronet/transport/cronet_transport.cc

@@ -441,6 +441,7 @@ static void convert_cronet_array_to_metadata(
 */
 static void on_failed(bidirectional_stream* stream, int net_error) {
   gpr_log(GPR_ERROR, "on_failed(%p, %d)", stream, net_error);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
 
   stream_obj* s = static_cast<stream_obj*>(stream->annotation);
@@ -467,6 +468,7 @@ static void on_failed(bidirectional_stream* stream, int net_error) {
 */
 static void on_canceled(bidirectional_stream* stream) {
   CRONET_LOG(GPR_DEBUG, "on_canceled(%p)", stream);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
 
   stream_obj* s = static_cast<stream_obj*>(stream->annotation);
@@ -493,6 +495,7 @@ static void on_canceled(bidirectional_stream* stream) {
 */
 static void on_succeeded(bidirectional_stream* stream) {
   CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
 
   stream_obj* s = static_cast<stream_obj*>(stream->annotation);
@@ -511,6 +514,7 @@ static void on_succeeded(bidirectional_stream* stream) {
 */
 static void on_stream_ready(bidirectional_stream* stream) {
   CRONET_LOG(GPR_DEBUG, "W: on_stream_ready(%p)", stream);
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   stream_obj* s = static_cast<stream_obj*>(stream->annotation);
   grpc_cronet_transport* t = s->curr_ct;
@@ -541,6 +545,7 @@ static void on_response_headers_received(
     bidirectional_stream* stream,
     const bidirectional_stream_header_array* headers,
     const char* negotiated_protocol) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   CRONET_LOG(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream,
              headers, negotiated_protocol);
@@ -580,6 +585,7 @@ static void on_response_headers_received(
   Cronet callback
 */
 static void on_write_completed(bidirectional_stream* stream, const char* data) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   stream_obj* s = static_cast<stream_obj*>(stream->annotation);
   CRONET_LOG(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data);
@@ -598,6 +604,7 @@ static void on_write_completed(bidirectional_stream* stream, const char* data) {
 */
 static void on_read_completed(bidirectional_stream* stream, char* data,
                               int count) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   stream_obj* s = static_cast<stream_obj*>(stream->annotation);
   CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data,
@@ -640,6 +647,7 @@ static void on_read_completed(bidirectional_stream* stream, char* data,
 static void on_response_trailers_received(
     bidirectional_stream* stream,
     const bidirectional_stream_header_array* trailers) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   CRONET_LOG(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream,
              trailers);

src/core/lib/channel/handshaker.cc

@@ -30,302 +30,229 @@
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/slice/slice_internal.h"
 
-grpc_core::TraceFlag grpc_handshaker_trace(false, "handshaker");
+namespace grpc_core {
 
-//
-// grpc_handshaker
-//
+TraceFlag grpc_handshaker_trace(false, "handshaker");
 
-void grpc_handshaker_init(const grpc_handshaker_vtable* vtable,
-                          grpc_handshaker* handshaker) {
-  handshaker->vtable = vtable;
-}
-
-void grpc_handshaker_destroy(grpc_handshaker* handshaker) {
-  handshaker->vtable->destroy(handshaker);
-}
-
-void grpc_handshaker_shutdown(grpc_handshaker* handshaker, grpc_error* why) {
-  handshaker->vtable->shutdown(handshaker, why);
-}
-
-void grpc_handshaker_do_handshake(grpc_handshaker* handshaker,
-                                  grpc_tcp_server_acceptor* acceptor,
-                                  grpc_closure* on_handshake_done,
-                                  grpc_handshaker_args* args) {
-  handshaker->vtable->do_handshake(handshaker, acceptor, on_handshake_done,
-                                   args);
-}
+namespace {
 
-const char* grpc_handshaker_name(grpc_handshaker* handshaker) {
-  return handshaker->vtable->name;
+char* HandshakerArgsString(HandshakerArgs* args) {
+  char* args_str = grpc_channel_args_string(args->args);
+  size_t num_args = args->args != nullptr ? args->args->num_args : 0;
+  size_t read_buffer_length =
+      args->read_buffer != nullptr ? args->read_buffer->length : 0;
+  char* str;
+  gpr_asprintf(&str,
+               "{endpoint=%p, args=%p {size=%" PRIuPTR
+               ": %s}, read_buffer=%p (length=%" PRIuPTR "), exit_early=%d}",
+               args->endpoint, args->args, num_args, args_str,
+               args->read_buffer, read_buffer_length, args->exit_early);
+  gpr_free(args_str);
+  return str;
 }
 
-//
-// grpc_handshake_manager
-//
+}  // namespace
 
-struct grpc_handshake_manager {
-  gpr_mu mu;
-  gpr_refcount refs;
-  bool shutdown;
-  // An array of handshakers added via grpc_handshake_manager_add().
-  size_t count;
-  grpc_handshaker** handshakers;
-  // The index of the handshaker to invoke next and closure to invoke it.
-  size_t index;
-  grpc_closure call_next_handshaker;
-  // The acceptor to call the handshakers with.
-  grpc_tcp_server_acceptor* acceptor;
-  // Deadline timer across all handshakers.
-  grpc_timer deadline_timer;
-  grpc_closure on_timeout;
-  // The final callback and user_data to invoke after the last handshaker.
-  grpc_closure on_handshake_done;
-  void* user_data;
-  // Handshaker args.
-  grpc_handshaker_args args;
-  // Links to the previous and next managers in a list of all pending handshakes
-  // Used at server side only.
-  grpc_handshake_manager* prev;
-  grpc_handshake_manager* next;
-};
+HandshakeManager::HandshakeManager() { gpr_mu_init(&mu_); }
 
-grpc_handshake_manager* grpc_handshake_manager_create() {
-  grpc_handshake_manager* mgr = static_cast<grpc_handshake_manager*>(
-      gpr_zalloc(sizeof(grpc_handshake_manager)));
-  gpr_mu_init(&mgr->mu);
-  gpr_ref_init(&mgr->refs, 1);
-  return mgr;
-}
-
-void grpc_handshake_manager_pending_list_add(grpc_handshake_manager** head,
-                                             grpc_handshake_manager* mgr) {
-  GPR_ASSERT(mgr->prev == nullptr);
-  GPR_ASSERT(mgr->next == nullptr);
-  mgr->next = *head;
+/// Add \a mgr to the server side list of all pending handshake managers, the
+/// list starts with \a *head.
+// Not thread-safe. Caller needs to synchronize.
+void HandshakeManager::AddToPendingMgrList(HandshakeManager** head) {
+  GPR_ASSERT(prev_ == nullptr);
+  GPR_ASSERT(next_ == nullptr);
+  next_ = *head;
   if (*head) {
-    (*head)->prev = mgr;
+    (*head)->prev_ = this;
   }
-  *head = mgr;
+  *head = this;
 }
 
-void grpc_handshake_manager_pending_list_remove(grpc_handshake_manager** head,
-                                                grpc_handshake_manager* mgr) {
-  if (mgr->next != nullptr) {
-    mgr->next->prev = mgr->prev;
+/// Remove \a mgr from the server side list of all pending handshake managers.
+// Not thread-safe. Caller needs to synchronize.
+void HandshakeManager::RemoveFromPendingMgrList(HandshakeManager** head) {
+  if (next_ != nullptr) {
+    next_->prev_ = prev_;
   }
-  if (mgr->prev != nullptr) {
-    mgr->prev->next = mgr->next;
+  if (prev_ != nullptr) {
+    prev_->next_ = next_;
   } else {
-    GPR_ASSERT(*head == mgr);
-    *head = mgr->next;
+    GPR_ASSERT(*head == this);
+    *head = next_;
   }
 }
 
-void grpc_handshake_manager_pending_list_shutdown_all(
-    grpc_handshake_manager* head, grpc_error* why) {
+/// Shutdown all pending handshake managers starting at head on the server
+/// side. Not thread-safe. Caller needs to synchronize.
+void HandshakeManager::ShutdownAllPending(grpc_error* why) {
+  auto* head = this;
   while (head != nullptr) {
-    grpc_handshake_manager_shutdown(head, GRPC_ERROR_REF(why));
-    head = head->next;
+    head->Shutdown(GRPC_ERROR_REF(why));
+    head = head->next_;
   }
   GRPC_ERROR_UNREF(why);
 }
 
-static bool is_power_of_2(size_t n) { return (n & (n - 1)) == 0; }
-
-void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
-                                grpc_handshaker* handshaker) {
+void HandshakeManager::Add(RefCountedPtr<Handshaker> handshaker) {
   if (grpc_handshaker_trace.enabled()) {
     gpr_log(
         GPR_INFO,
         "handshake_manager %p: adding handshaker %s [%p] at index %" PRIuPTR,
-        mgr, grpc_handshaker_name(handshaker), handshaker, mgr->count);
-  }
-  gpr_mu_lock(&mgr->mu);
-  // To avoid allocating memory for each handshaker we add, we double
-  // the number of elements every time we need more.
-  size_t realloc_count = 0;
-  if (mgr->count == 0) {
-    realloc_count = 2;
-  } else if (mgr->count >= 2 && is_power_of_2(mgr->count)) {
-    realloc_count = mgr->count * 2;
-  }
-  if (realloc_count > 0) {
-    mgr->handshakers = static_cast<grpc_handshaker**>(gpr_realloc(
-        mgr->handshakers, realloc_count * sizeof(grpc_handshaker*)));
+        this, handshaker->name(), handshaker.get(), handshakers_.size());
   }
-  mgr->handshakers[mgr->count++] = handshaker;
-  gpr_mu_unlock(&mgr->mu);
+  MutexLock lock(&mu_);
+  handshakers_.push_back(std::move(handshaker));
 }
 
-static void grpc_handshake_manager_unref(grpc_handshake_manager* mgr) {
-  if (gpr_unref(&mgr->refs)) {
-    for (size_t i = 0; i < mgr->count; ++i) {
-      grpc_handshaker_destroy(mgr->handshakers[i]);
-    }
-    gpr_free(mgr->handshakers);
-    gpr_mu_destroy(&mgr->mu);
-    gpr_free(mgr);
-  }
-}
-
-void grpc_handshake_manager_destroy(grpc_handshake_manager* mgr) {
-  grpc_handshake_manager_unref(mgr);
+HandshakeManager::~HandshakeManager() {
+  handshakers_.clear();
+  gpr_mu_destroy(&mu_);
 }
 
-void grpc_handshake_manager_shutdown(grpc_handshake_manager* mgr,
-                                     grpc_error* why) {
-  gpr_mu_lock(&mgr->mu);
-  // Shutdown the handshaker that's currently in progress, if any.
-  if (!mgr->shutdown && mgr->index > 0) {
-    mgr->shutdown = true;
-    grpc_handshaker_shutdown(mgr->handshakers[mgr->index - 1],
-                             GRPC_ERROR_REF(why));
+void HandshakeManager::Shutdown(grpc_error* why) {
+  {
+    MutexLock lock(&mu_);
+    // Shutdown the handshaker that's currently in progress, if any.
+    if (!is_shutdown_ && index_ > 0) {
+      is_shutdown_ = true;
+      handshakers_[index_ - 1]->Shutdown(GRPC_ERROR_REF(why));
+    }
   }
-  gpr_mu_unlock(&mgr->mu);
   GRPC_ERROR_UNREF(why);
 }
 
-static char* handshaker_args_string(grpc_handshaker_args* args) {
-  char* args_str = grpc_channel_args_string(args->args);
-  size_t num_args = args->args != nullptr ? args->args->num_args : 0;
-  size_t read_buffer_length =
-      args->read_buffer != nullptr ? args->read_buffer->length : 0;
-  char* str;
-  gpr_asprintf(&str,
-               "{endpoint=%p, args=%p {size=%" PRIuPTR
-               ": %s}, read_buffer=%p (length=%" PRIuPTR "), exit_early=%d}",
-               args->endpoint, args->args, num_args, args_str,
-               args->read_buffer, read_buffer_length, args->exit_early);
-  gpr_free(args_str);
-  return str;
-}
-
 // Helper function to call either the next handshaker or the
 // on_handshake_done callback.
 // Returns true if we've scheduled the on_handshake_done callback.
-static bool call_next_handshaker_locked(grpc_handshake_manager* mgr,
-                                        grpc_error* error) {
+bool HandshakeManager::CallNextHandshakerLocked(grpc_error* error) {
   if (grpc_handshaker_trace.enabled()) {
-    char* args_str = handshaker_args_string(&mgr->args);
+    char* args_str = HandshakerArgsString(&args_);
     gpr_log(GPR_INFO,
             "handshake_manager %p: error=%s shutdown=%d index=%" PRIuPTR
             ", args=%s",
-            mgr, grpc_error_string(error), mgr->shutdown, mgr->index, args_str);
+            this, grpc_error_string(error), is_shutdown_, index_, args_str);
     gpr_free(args_str);
   }
-  GPR_ASSERT(mgr->index <= mgr->count);
+  GPR_ASSERT(index_ <= handshakers_.size());
   // If we got an error or we've been shut down or we're exiting early or
   // we've finished the last handshaker, invoke the on_handshake_done
   // callback.  Otherwise, call the next handshaker.
-  if (error != GRPC_ERROR_NONE || mgr->shutdown || mgr->args.exit_early ||
-      mgr->index == mgr->count) {
-    if (error == GRPC_ERROR_NONE && mgr->shutdown) {
+  if (error != GRPC_ERROR_NONE || is_shutdown_ || args_.exit_early ||
+      index_ == handshakers_.size()) {
+    if (error == GRPC_ERROR_NONE && is_shutdown_) {
       error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("handshaker shutdown");
       // It is possible that the endpoint has already been destroyed by
       // a shutdown call while this callback was sitting on the ExecCtx
       // with no error.
-      if (mgr->args.endpoint != nullptr) {
+      if (args_.endpoint != nullptr) {
         // TODO(roth): It is currently necessary to shutdown endpoints
         // before destroying then, even when we know that there are no
         // pending read/write callbacks.  This should be fixed, at which
         // point this can be removed.
-        grpc_endpoint_shutdown(mgr->args.endpoint, GRPC_ERROR_REF(error));
-        grpc_endpoint_destroy(mgr->args.endpoint);
-        mgr->args.endpoint = nullptr;
-        grpc_channel_args_destroy(mgr->args.args);
-        mgr->args.args = nullptr;
-        grpc_slice_buffer_destroy_internal(mgr->args.read_buffer);
-        gpr_free(mgr->args.read_buffer);
-        mgr->args.read_buffer = nullptr;
+        grpc_endpoint_shutdown(args_.endpoint, GRPC_ERROR_REF(error));
+        grpc_endpoint_destroy(args_.endpoint);
+        args_.endpoint = nullptr;
+        grpc_channel_args_destroy(args_.args);
+        args_.args = nullptr;
+        grpc_slice_buffer_destroy_internal(args_.read_buffer);
+        gpr_free(args_.read_buffer);
+        args_.read_buffer = nullptr;
       }
     }
     if (grpc_handshaker_trace.enabled()) {
       gpr_log(GPR_INFO,
               "handshake_manager %p: handshaking complete -- scheduling "
               "on_handshake_done with error=%s",
-              mgr, grpc_error_string(error));
+              this, grpc_error_string(error));
     }
     // Cancel deadline timer, since we're invoking the on_handshake_done
     // callback now.
-    grpc_timer_cancel(&mgr->deadline_timer);
-    GRPC_CLOSURE_SCHED(&mgr->on_handshake_done, error);
-    mgr->shutdown = true;
+    grpc_timer_cancel(&deadline_timer_);
+    GRPC_CLOSURE_SCHED(&on_handshake_done_, error);
+    is_shutdown_ = true;
   } else {
+    auto handshaker = handshakers_[index_];
     if (grpc_handshaker_trace.enabled()) {
       gpr_log(
           GPR_INFO,
           "handshake_manager %p: calling handshaker %s [%p] at index %" PRIuPTR,
-          mgr, grpc_handshaker_name(mgr->handshakers[mgr->index]),
-          mgr->handshakers[mgr->index], mgr->index);
+          this, handshaker->name(), handshaker.get(), index_);
     }
-    grpc_handshaker_do_handshake(mgr->handshakers[mgr->index], mgr->acceptor,
-                                 &mgr->call_next_handshaker, &mgr->args);
+    handshaker->DoHandshake(acceptor_, &call_next_handshaker_, &args_);
   }
-  ++mgr->index;
-  return mgr->shutdown;
+  ++index_;
+  return is_shutdown_;
 }
 
-// A function used as the handshaker-done callback when chaining
-// handshakers together.
-static void call_next_handshaker(void* arg, grpc_error* error) {
-  grpc_handshake_manager* mgr = static_cast<grpc_handshake_manager*>(arg);
-  gpr_mu_lock(&mgr->mu);
-  bool done = call_next_handshaker_locked(mgr, GRPC_ERROR_REF(error));
-  gpr_mu_unlock(&mgr->mu);
+void HandshakeManager::CallNextHandshakerFn(void* arg, grpc_error* error) {
+  auto* mgr = static_cast<HandshakeManager*>(arg);
+  bool done;
+  {
+    MutexLock lock(&mgr->mu_);
+    done = mgr->CallNextHandshakerLocked(GRPC_ERROR_REF(error));
+  }
   // If we're invoked the final callback, we won't be coming back
   // to this function, so we can release our reference to the
   // handshake manager.
   if (done) {
-    grpc_handshake_manager_unref(mgr);
+    mgr->Unref();
   }
 }
 
-// Callback invoked when deadline is exceeded.
-static void on_timeout(void* arg, grpc_error* error) {
-  grpc_handshake_manager* mgr = static_cast<grpc_handshake_manager*>(arg);
-  if (error == GRPC_ERROR_NONE) {  // Timer fired, rather than being cancelled.
-    grpc_handshake_manager_shutdown(
-        mgr, GRPC_ERROR_CREATE_FROM_STATIC_STRING("Handshake timed out"));
+void HandshakeManager::OnTimeoutFn(void* arg, grpc_error* error) {
+  auto* mgr = static_cast<HandshakeManager*>(arg);
+  if (error == GRPC_ERROR_NONE) {  // Timer fired, rather than being cancelled
+    mgr->Shutdown(GRPC_ERROR_CREATE_FROM_STATIC_STRING("Handshake timed out"));
   }
-  grpc_handshake_manager_unref(mgr);
+  mgr->Unref();
 }
 
-void grpc_handshake_manager_do_handshake(grpc_handshake_manager* mgr,
-                                         grpc_endpoint* endpoint,
-                                         const grpc_channel_args* channel_args,
-                                         grpc_millis deadline,
-                                         grpc_tcp_server_acceptor* acceptor,
-                                         grpc_iomgr_cb_func on_handshake_done,
-                                         void* user_data) {
-  gpr_mu_lock(&mgr->mu);
-  GPR_ASSERT(mgr->index == 0);
-  GPR_ASSERT(!mgr->shutdown);
-  // Construct handshaker args.  These will be passed through all
-  // handshakers and eventually be freed by the on_handshake_done callback.
-  mgr->args.endpoint = endpoint;
-  mgr->args.args = grpc_channel_args_copy(channel_args);
-  mgr->args.user_data = user_data;
-  mgr->args.read_buffer = static_cast<grpc_slice_buffer*>(
-      gpr_malloc(sizeof(*mgr->args.read_buffer)));
-  grpc_slice_buffer_init(mgr->args.read_buffer);
-  // Initialize state needed for calling handshakers.
-  mgr->acceptor = acceptor;
-  GRPC_CLOSURE_INIT(&mgr->call_next_handshaker, call_next_handshaker, mgr,
-                    grpc_schedule_on_exec_ctx);
-  GRPC_CLOSURE_INIT(&mgr->on_handshake_done, on_handshake_done, &mgr->args,
-                    grpc_schedule_on_exec_ctx);
-  // Start deadline timer, which owns a ref.
-  gpr_ref(&mgr->refs);
-  GRPC_CLOSURE_INIT(&mgr->on_timeout, on_timeout, mgr,
-                    grpc_schedule_on_exec_ctx);
-  grpc_timer_init(&mgr->deadline_timer, deadline, &mgr->on_timeout);
-  // Start first handshaker, which also owns a ref.
-  gpr_ref(&mgr->refs);
-  bool done = call_next_handshaker_locked(mgr, GRPC_ERROR_NONE);
-  gpr_mu_unlock(&mgr->mu);
+void HandshakeManager::DoHandshake(grpc_endpoint* endpoint,
+                                   const grpc_channel_args* channel_args,
+                                   grpc_millis deadline,
+                                   grpc_tcp_server_acceptor* acceptor,
+                                   grpc_iomgr_cb_func on_handshake_done,
+                                   void* user_data) {
+  bool done;
+  {
+    MutexLock lock(&mu_);
+    GPR_ASSERT(index_ == 0);
+    GPR_ASSERT(!is_shutdown_);
+    // Construct handshaker args.  These will be passed through all
+    // handshakers and eventually be freed by the on_handshake_done callback.
+    args_.endpoint = endpoint;
+    args_.args = grpc_channel_args_copy(channel_args);
+    args_.user_data = user_data;
+    args_.read_buffer =
+        static_cast<grpc_slice_buffer*>(gpr_malloc(sizeof(*args_.read_buffer)));
+    grpc_slice_buffer_init(args_.read_buffer);
+    // Initialize state needed for calling handshakers.
+    acceptor_ = acceptor;
+    GRPC_CLOSURE_INIT(&call_next_handshaker_,
+                      &HandshakeManager::CallNextHandshakerFn, this,
+                      grpc_schedule_on_exec_ctx);
+    GRPC_CLOSURE_INIT(&on_handshake_done_, on_handshake_done, &args_,
+                      grpc_schedule_on_exec_ctx);
+    // Start deadline timer, which owns a ref.
+    Ref().release();
+    GRPC_CLOSURE_INIT(&on_timeout_, &HandshakeManager::OnTimeoutFn, this,
+                      grpc_schedule_on_exec_ctx);
+    grpc_timer_init(&deadline_timer_, deadline, &on_timeout_);
+    // Start first handshaker, which also owns a ref.
+    Ref().release();
+    done = CallNextHandshakerLocked(GRPC_ERROR_NONE);
+  }
   if (done) {
-    grpc_handshake_manager_unref(mgr);
+    Unref();
   }
 }
+
+}  // namespace grpc_core
+
+void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
+                                grpc_handshaker* handshaker) {
+  // This is a transition method to aid the API change for handshakers.
+  using namespace grpc_core;
+  RefCountedPtr<Handshaker> refd_hs(static_cast<Handshaker*>(handshaker));
+  mgr->Add(refd_hs);
+}

src/core/lib/channel/handshaker.h

@@ -21,12 +21,21 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <grpc/support/string_util.h>
+
 #include <grpc/impl/codegen/grpc_types.h>
 
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/mutex_lock.h"
+#include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/tcp_server.h"
+#include "src/core/lib/iomgr/timer.h"
+
+namespace grpc_core {
 
 /// Handshakers are used to perform initial handshakes on a connection
 /// before the client sends the initial request.  Some examples of what
@@ -35,12 +44,6 @@
 ///
 /// In general, handshakers should be used via a handshake manager.
 
-///
-/// grpc_handshaker
-///
-
-typedef struct grpc_handshaker grpc_handshaker;
-
 /// Arguments passed through handshakers and to the on_handshake_done callback.
 ///
 /// For handshakers, all members are input/output parameters; for
@@ -55,115 +58,121 @@ typedef struct grpc_handshaker grpc_handshaker;
 ///
 /// For the on_handshake_done callback, all members are input arguments,
 /// which the callback takes ownership of.
-typedef struct {
-  grpc_endpoint* endpoint;
-  grpc_channel_args* args;
-  grpc_slice_buffer* read_buffer;
+struct HandshakerArgs {
+  grpc_endpoint* endpoint = nullptr;
+  grpc_channel_args* args = nullptr;
+  grpc_slice_buffer* read_buffer = nullptr;
   // A handshaker may set this to true before invoking on_handshake_done
   // to indicate that subsequent handshakers should be skipped.
-  bool exit_early;
+  bool exit_early = false;
   // User data passed through the handshake manager.  Not used by
   // individual handshakers.
-  void* user_data;
-} grpc_handshaker_args;
+  void* user_data = nullptr;
+};
 
-typedef struct {
-  /// Destroys the handshaker.
-  void (*destroy)(grpc_handshaker* handshaker);
+///
+/// Handshaker
+///
 
-  /// Shuts down the handshaker (e.g., to clean up when the operation is
-  /// aborted in the middle).
-  void (*shutdown)(grpc_handshaker* handshaker, grpc_error* why);
-
-  /// Performs handshaking, modifying \a args as needed (e.g., to
-  /// replace \a endpoint with a wrapped endpoint).
-  /// When finished, invokes \a on_handshake_done.
-  /// \a acceptor will be NULL for client-side handshakers.
-  void (*do_handshake)(grpc_handshaker* handshaker,
-                       grpc_tcp_server_acceptor* acceptor,
-                       grpc_closure* on_handshake_done,
-                       grpc_handshaker_args* args);
-
-  /// The name of the handshaker, for debugging purposes.
-  const char* name;
-} grpc_handshaker_vtable;
-
-/// Base struct.  To subclass, make this the first member of the
-/// implementation struct.
-struct grpc_handshaker {
-  const grpc_handshaker_vtable* vtable;
+class Handshaker : public RefCounted<Handshaker> {
+ public:
+  virtual ~Handshaker() = default;
+  virtual void Shutdown(grpc_error* why) GRPC_ABSTRACT;
+  virtual void DoHandshake(grpc_tcp_server_acceptor* acceptor,
+                           grpc_closure* on_handshake_done,
+                           HandshakerArgs* args) GRPC_ABSTRACT;
+  virtual const char* name() const GRPC_ABSTRACT;
+  GRPC_ABSTRACT_BASE_CLASS
 };
 
-/// Called by concrete implementations to initialize the base struct.
-void grpc_handshaker_init(const grpc_handshaker_vtable* vtable,
-                          grpc_handshaker* handshaker);
+//
+// HandshakeManager
+//
 
-void grpc_handshaker_destroy(grpc_handshaker* handshaker);
-void grpc_handshaker_shutdown(grpc_handshaker* handshaker, grpc_error* why);
-void grpc_handshaker_do_handshake(grpc_handshaker* handshaker,
-                                  grpc_tcp_server_acceptor* acceptor,
-                                  grpc_closure* on_handshake_done,
-                                  grpc_handshaker_args* args);
-const char* grpc_handshaker_name(grpc_handshaker* handshaker);
+class HandshakeManager : public RefCounted<HandshakeManager> {
+ public:
+  HandshakeManager();
+  ~HandshakeManager();
 
-///
-/// grpc_handshake_manager
-///
+  /// Add \a mgr to the server side list of all pending handshake managers, the
+  /// list starts with \a *head.
+  // Not thread-safe. Caller needs to synchronize.
+  void AddToPendingMgrList(HandshakeManager** head);
+
+  /// Remove \a mgr from the server side list of all pending handshake managers.
+  // Not thread-safe. Caller needs to synchronize.
+  void RemoveFromPendingMgrList(HandshakeManager** head);
 
-typedef struct grpc_handshake_manager grpc_handshake_manager;
+  /// Shutdown all pending handshake managers starting at head on the server
+  /// side. Not thread-safe. Caller needs to synchronize.
+  void ShutdownAllPending(grpc_error* why);
 
-/// Creates a new handshake manager.  Caller takes ownership.
-grpc_handshake_manager* grpc_handshake_manager_create();
+  /// Adds a handshaker to the handshake manager.
+  /// Takes ownership of \a handshaker.
+  void Add(RefCountedPtr<Handshaker> handshaker);
 
-/// Adds a handshaker to the handshake manager.
-/// Takes ownership of \a handshaker.
+  /// Shuts down the handshake manager (e.g., to clean up when the operation is
+  /// aborted in the middle).
+  void Shutdown(grpc_error* why);
+
+  /// Invokes handshakers in the order they were added.
+  /// Takes ownership of \a endpoint, and then passes that ownership to
+  /// the \a on_handshake_done callback.
+  /// Does NOT take ownership of \a channel_args.  Instead, makes a copy before
+  /// invoking the first handshaker.
+  /// \a acceptor will be nullptr for client-side handshakers.
+  ///
+  /// When done, invokes \a on_handshake_done with a HandshakerArgs
+  /// object as its argument.  If the callback is invoked with error !=
+  /// GRPC_ERROR_NONE, then handshaking failed and the handshaker has done
+  /// the necessary clean-up.  Otherwise, the callback takes ownership of
+  /// the arguments.
+  void DoHandshake(grpc_endpoint* endpoint,
+                   const grpc_channel_args* channel_args, grpc_millis deadline,
+                   grpc_tcp_server_acceptor* acceptor,
+                   grpc_iomgr_cb_func on_handshake_done, void* user_data);
+
+ private:
+  bool CallNextHandshakerLocked(grpc_error* error);
+
+  // A function used as the handshaker-done callback when chaining
+  // handshakers together.
+  static void CallNextHandshakerFn(void* arg, grpc_error* error);
+
+  // Callback invoked when deadline is exceeded.
+  static void OnTimeoutFn(void* arg, grpc_error* error);
+
+  static const size_t HANDSHAKERS_INIT_SIZE = 2;
+
+  gpr_mu mu_;
+  bool is_shutdown_ = false;
+  // An array of handshakers added via grpc_handshake_manager_add().
+  InlinedVector<RefCountedPtr<Handshaker>, HANDSHAKERS_INIT_SIZE> handshakers_;
+  // The index of the handshaker to invoke next and closure to invoke it.
+  size_t index_ = 0;
+  grpc_closure call_next_handshaker_;
+  // The acceptor to call the handshakers with.
+  grpc_tcp_server_acceptor* acceptor_;
+  // Deadline timer across all handshakers.
+  grpc_timer deadline_timer_;
+  grpc_closure on_timeout_;
+  // The final callback and user_data to invoke after the last handshaker.
+  grpc_closure on_handshake_done_;
+  // Handshaker args.
+  HandshakerArgs args_;
+  // Links to the previous and next managers in a list of all pending handshakes
+  // Used at server side only.
+  HandshakeManager* prev_ = nullptr;
+  HandshakeManager* next_ = nullptr;
+};
+
+}  // namespace grpc_core
+
+// TODO(arjunroy): These are transitional to account for the new handshaker API
+// and will eventually be removed entirely.
+typedef grpc_core::HandshakeManager grpc_handshake_manager;
+typedef grpc_core::Handshaker grpc_handshaker;
 void grpc_handshake_manager_add(grpc_handshake_manager* mgr,
                                 grpc_handshaker* handshaker);
 
-/// Destroys the handshake manager.
-void grpc_handshake_manager_destroy(grpc_handshake_manager* mgr);
-
-/// Shuts down the handshake manager (e.g., to clean up when the operation is
-/// aborted in the middle).
-/// The caller must still call grpc_handshake_manager_destroy() after
-/// calling this function.
-void grpc_handshake_manager_shutdown(grpc_handshake_manager* mgr,
-                                     grpc_error* why);
-
-/// Invokes handshakers in the order they were added.
-/// Takes ownership of \a endpoint, and then passes that ownership to
-/// the \a on_handshake_done callback.
-/// Does NOT take ownership of \a channel_args.  Instead, makes a copy before
-/// invoking the first handshaker.
-/// \a acceptor will be nullptr for client-side handshakers.
-///
-/// When done, invokes \a on_handshake_done with a grpc_handshaker_args
-/// object as its argument.  If the callback is invoked with error !=
-/// GRPC_ERROR_NONE, then handshaking failed and the handshaker has done
-/// the necessary clean-up.  Otherwise, the callback takes ownership of
-/// the arguments.
-void grpc_handshake_manager_do_handshake(grpc_handshake_manager* mgr,
-                                         grpc_endpoint* endpoint,
-                                         const grpc_channel_args* channel_args,
-                                         grpc_millis deadline,
-                                         grpc_tcp_server_acceptor* acceptor,
-                                         grpc_iomgr_cb_func on_handshake_done,
-                                         void* user_data);
-
-/// Add \a mgr to the server side list of all pending handshake managers, the
-/// list starts with \a *head.
-// Not thread-safe. Caller needs to synchronize.
-void grpc_handshake_manager_pending_list_add(grpc_handshake_manager** head,
-                                             grpc_handshake_manager* mgr);
-
-/// Remove \a mgr from the server side list of all pending handshake managers.
-// Not thread-safe. Caller needs to synchronize.
-void grpc_handshake_manager_pending_list_remove(grpc_handshake_manager** head,
-                                                grpc_handshake_manager* mgr);
-
-/// Shutdown all pending handshake managers on the server side.
-// Not thread-safe. Caller needs to synchronize.
-void grpc_handshake_manager_pending_list_shutdown_all(
-    grpc_handshake_manager* head, grpc_error* why);
-
 #endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_H */

src/core/lib/channel/handshaker_factory.cc

@@ -1,42 +0,0 @@
-/*
- *
- * Copyright 2016 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <grpc/support/port_platform.h>
-
-#include "src/core/lib/channel/handshaker_factory.h"
-
-#include <grpc/support/log.h>
-
-void grpc_handshaker_factory_add_handshakers(
-    grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args,
-    grpc_pollset_set* interested_parties,
-    grpc_handshake_manager* handshake_mgr) {
-  if (handshaker_factory != nullptr) {
-    GPR_ASSERT(handshaker_factory->vtable != nullptr);
-    handshaker_factory->vtable->add_handshakers(
-        handshaker_factory, args, interested_parties, handshake_mgr);
-  }
-}
-
-void grpc_handshaker_factory_destroy(
-    grpc_handshaker_factory* handshaker_factory) {
-  if (handshaker_factory != nullptr) {
-    GPR_ASSERT(handshaker_factory->vtable != nullptr);
-    handshaker_factory->vtable->destroy(handshaker_factory);
-  }
-}

src/core/lib/channel/handshaker_factory.h

@@ -27,26 +27,18 @@
 
 // A handshaker factory is used to create handshakers.
 
-typedef struct grpc_handshaker_factory grpc_handshaker_factory;
-
-typedef struct {
-  void (*add_handshakers)(grpc_handshaker_factory* handshaker_factory,
-                          const grpc_channel_args* args,
-                          grpc_pollset_set* interested_parties,
-                          grpc_handshake_manager* handshake_mgr);
-  void (*destroy)(grpc_handshaker_factory* handshaker_factory);
-} grpc_handshaker_factory_vtable;
-
-struct grpc_handshaker_factory {
-  const grpc_handshaker_factory_vtable* vtable;
-};
+namespace grpc_core {
+
+class HandshakerFactory {
+ public:
+  virtual void AddHandshakers(const grpc_channel_args* args,
+                              grpc_pollset_set* interested_parties,
+                              HandshakeManager* handshake_mgr) GRPC_ABSTRACT;
+  virtual ~HandshakerFactory() = default;
 
-void grpc_handshaker_factory_add_handshakers(
-    grpc_handshaker_factory* handshaker_factory, const grpc_channel_args* args,
-    grpc_pollset_set* interested_parties,
-    grpc_handshake_manager* handshake_mgr);
+  GRPC_ABSTRACT_BASE_CLASS
+};
 
-void grpc_handshaker_factory_destroy(
-    grpc_handshaker_factory* handshaker_factory);
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_FACTORY_H */

src/core/lib/channel/handshaker_registry.cc

@@ -19,8 +19,11 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/channel/handshaker_registry.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
+#include "src/core/lib/gprpp/memory.h"
 
 #include <string.h>
+#include <algorithm>
 
 #include <grpc/support/alloc.h>
 
@@ -28,74 +31,83 @@
 // grpc_handshaker_factory_list
 //
 
-typedef struct {
-  grpc_handshaker_factory** list;
-  size_t num_factories;
-} grpc_handshaker_factory_list;
-
-static void grpc_handshaker_factory_list_register(
-    grpc_handshaker_factory_list* list, bool at_start,
-    grpc_handshaker_factory* factory) {
-  list->list = static_cast<grpc_handshaker_factory**>(gpr_realloc(
-      list->list,
-      (list->num_factories + 1) * sizeof(grpc_handshaker_factory*)));
-  if (at_start) {
-    memmove(list->list + 1, list->list,
-            sizeof(grpc_handshaker_factory*) * list->num_factories);
-    list->list[0] = factory;
-  } else {
-    list->list[list->num_factories] = factory;
-  }
-  ++list->num_factories;
-}
+namespace grpc_core {
+
+namespace {
+
+class HandshakerFactoryList {
+ public:
+  void Register(bool at_start, UniquePtr<HandshakerFactory> factory);
+  void AddHandshakers(const grpc_channel_args* args,
+                      grpc_pollset_set* interested_parties,
+                      HandshakeManager* handshake_mgr);
+
+ private:
+  InlinedVector<UniquePtr<HandshakerFactory>, 2> factories_;
+};
 
-static void grpc_handshaker_factory_list_add_handshakers(
-    grpc_handshaker_factory_list* list, const grpc_channel_args* args,
-    grpc_pollset_set* interested_parties,
-    grpc_handshake_manager* handshake_mgr) {
-  for (size_t i = 0; i < list->num_factories; ++i) {
-    grpc_handshaker_factory_add_handshakers(list->list[i], args,
-                                            interested_parties, handshake_mgr);
+HandshakerFactoryList* g_handshaker_factory_lists = nullptr;
+
+}  // namespace
+
+void HandshakerFactoryList::Register(bool at_start,
+                                     UniquePtr<HandshakerFactory> factory) {
+  factories_.push_back(std::move(factory));
+  if (at_start) {
+    auto* end = &factories_[factories_.size() - 1];
+    std::rotate(&factories_[0], end, end + 1);
   }
 }
 
-static void grpc_handshaker_factory_list_destroy(
-    grpc_handshaker_factory_list* list) {
-  for (size_t i = 0; i < list->num_factories; ++i) {
-    grpc_handshaker_factory_destroy(list->list[i]);
+void HandshakerFactoryList::AddHandshakers(const grpc_channel_args* args,
+                                           grpc_pollset_set* interested_parties,
+                                           HandshakeManager* handshake_mgr) {
+  for (size_t idx = 0; idx < factories_.size(); ++idx) {
+    auto& handshaker_factory = factories_[idx];
+    handshaker_factory->AddHandshakers(args, interested_parties, handshake_mgr);
   }
-  gpr_free(list->list);
 }
 
 //
 // plugin
 //
 
-static grpc_handshaker_factory_list
-    g_handshaker_factory_lists[NUM_HANDSHAKER_TYPES];
-
-void grpc_handshaker_factory_registry_init() {
-  memset(g_handshaker_factory_lists, 0, sizeof(g_handshaker_factory_lists));
+void HandshakerRegistry::Init() {
+  GPR_ASSERT(g_handshaker_factory_lists == nullptr);
+  g_handshaker_factory_lists = static_cast<HandshakerFactoryList*>(
+      gpr_malloc(sizeof(*g_handshaker_factory_lists) * NUM_HANDSHAKER_TYPES));
+  GPR_ASSERT(g_handshaker_factory_lists != nullptr);
+  for (auto idx = 0; idx < NUM_HANDSHAKER_TYPES; ++idx) {
+    auto factory_list = g_handshaker_factory_lists + idx;
+    new (factory_list) HandshakerFactoryList();
+  }
 }
 
-void grpc_handshaker_factory_registry_shutdown() {
-  for (size_t i = 0; i < NUM_HANDSHAKER_TYPES; ++i) {
-    grpc_handshaker_factory_list_destroy(&g_handshaker_factory_lists[i]);
+void HandshakerRegistry::Shutdown() {
+  GPR_ASSERT(g_handshaker_factory_lists != nullptr);
+  for (auto idx = 0; idx < NUM_HANDSHAKER_TYPES; ++idx) {
+    auto factory_list = g_handshaker_factory_lists + idx;
+    factory_list->~HandshakerFactoryList();
   }
+  gpr_free(g_handshaker_factory_lists);
+  g_handshaker_factory_lists = nullptr;
 }
 
-void grpc_handshaker_factory_register(bool at_start,
-                                      grpc_handshaker_type handshaker_type,
-                                      grpc_handshaker_factory* factory) {
-  grpc_handshaker_factory_list_register(
-      &g_handshaker_factory_lists[handshaker_type], at_start, factory);
+void HandshakerRegistry::RegisterHandshakerFactory(
+    bool at_start, HandshakerType handshaker_type,
+    UniquePtr<HandshakerFactory> factory) {
+  GPR_ASSERT(g_handshaker_factory_lists != nullptr);
+  auto& factory_list = g_handshaker_factory_lists[handshaker_type];
+  factory_list.Register(at_start, std::move(factory));
 }
 
-void grpc_handshakers_add(grpc_handshaker_type handshaker_type,
-                          const grpc_channel_args* args,
-                          grpc_pollset_set* interested_parties,
-                          grpc_handshake_manager* handshake_mgr) {
-  grpc_handshaker_factory_list_add_handshakers(
-      &g_handshaker_factory_lists[handshaker_type], args, interested_parties,
-      handshake_mgr);
+void HandshakerRegistry::AddHandshakers(HandshakerType handshaker_type,
+                                        const grpc_channel_args* args,
+                                        grpc_pollset_set* interested_parties,
+                                        HandshakeManager* handshake_mgr) {
+  GPR_ASSERT(g_handshaker_factory_lists != nullptr);
+  auto& factory_list = g_handshaker_factory_lists[handshaker_type];
+  factory_list.AddHandshakers(args, interested_parties, handshake_mgr);
 }
+
+}  // namespace grpc_core

src/core/lib/channel/handshaker_registry.h

@@ -25,25 +25,30 @@
 
 #include "src/core/lib/channel/handshaker_factory.h"
 
+namespace grpc_core {
+
 typedef enum {
   HANDSHAKER_CLIENT = 0,
   HANDSHAKER_SERVER,
   NUM_HANDSHAKER_TYPES,  // Must be last.
-} grpc_handshaker_type;
-
-void grpc_handshaker_factory_registry_init();
-void grpc_handshaker_factory_registry_shutdown();
-
-/// Registers a new handshaker factory.  Takes ownership.
-/// If \a at_start is true, the new handshaker will be at the beginning of
-/// the list.  Otherwise, it will be added to the end.
-void grpc_handshaker_factory_register(bool at_start,
-                                      grpc_handshaker_type handshaker_type,
-                                      grpc_handshaker_factory* factory);
-
-void grpc_handshakers_add(grpc_handshaker_type handshaker_type,
-                          const grpc_channel_args* args,
-                          grpc_pollset_set* interested_parties,
-                          grpc_handshake_manager* handshake_mgr);
+} HandshakerType;
+
+class HandshakerRegistry {
+ public:
+  /// Registers a new handshaker factory.  Takes ownership.
+  /// If \a at_start is true, the new handshaker will be at the beginning of
+  /// the list.  Otherwise, it will be added to the end.
+  static void RegisterHandshakerFactory(bool at_start,
+                                        HandshakerType handshaker_type,
+                                        UniquePtr<HandshakerFactory> factory);
+  static void AddHandshakers(HandshakerType handshaker_type,
+                             const grpc_channel_args* args,
+                             grpc_pollset_set* interested_parties,
+                             HandshakeManager* handshake_mgr);
+  static void Init();
+  static void Shutdown();
+};
+
+}  // namespace grpc_core
 
 #endif /* GRPC_CORE_LIB_CHANNEL_HANDSHAKER_REGISTRY_H */

src/core/lib/gprpp/orphanable.h

@@ -94,8 +94,9 @@ class InternallyRefCounted : public Orphanable {
   // Note: RefCount tracing is only enabled on debug builds, even when a
   //       TraceFlag is used.
   template <typename TraceFlagT = TraceFlag>
-  explicit InternallyRefCounted(TraceFlagT* trace_flag = nullptr)
-      : refs_(1, trace_flag) {}
+  explicit InternallyRefCounted(TraceFlagT* trace_flag = nullptr,
+                                intptr_t initial_refcount = 1)
+      : refs_(initial_refcount, trace_flag) {}
   virtual ~InternallyRefCounted() = default;
 
   RefCountedPtr<Child> Ref() GRPC_MUST_USE_RESULT {

src/core/lib/gprpp/ref_counted.h

@@ -221,8 +221,9 @@ class RefCounted : public Impl {
   // Note: RefCount tracing is only enabled on debug builds, even when a
   //       TraceFlag is used.
   template <typename TraceFlagT = TraceFlag>
-  explicit RefCounted(TraceFlagT* trace_flag = nullptr)
-      : refs_(1, trace_flag) {}
+  explicit RefCounted(TraceFlagT* trace_flag = nullptr,
+                      intptr_t initial_refcount = 1)
+      : refs_(initial_refcount, trace_flag) {}
 
   // Note: Depending on the Impl used, this dtor can be implicitly virtual.
   ~RefCounted() = default;

src/core/lib/http/httpcli_security_connector.cc

@@ -67,7 +67,7 @@ class grpc_httpcli_ssl_channel_security_connector final
   }
 
   void add_handshakers(grpc_pollset_set* interested_parties,
-                       grpc_handshake_manager* handshake_mgr) override {
+                       grpc_core::HandshakeManager* handshake_mgr) override {
     tsi_handshaker* handshaker = nullptr;
     if (handshaker_factory_ != nullptr) {
       tsi_result result = tsi_ssl_client_handshaker_factory_create_handshaker(
@@ -77,8 +77,7 @@ class grpc_httpcli_ssl_channel_security_connector final
                 tsi_result_to_string(result));
       }
     }
-    grpc_handshake_manager_add(
-        handshake_mgr, grpc_security_handshaker_create(handshaker, this));
+    handshake_mgr->Add(grpc_core::SecurityHandshakerCreate(handshaker, this));
   }
 
   tsi_ssl_client_handshaker_factory* handshaker_factory() const {
@@ -155,11 +154,11 @@ httpcli_ssl_channel_security_connector_create(
 typedef struct {
   void (*func)(void* arg, grpc_endpoint* endpoint);
   void* arg;
-  grpc_handshake_manager* handshake_mgr;
+  grpc_core::RefCountedPtr<grpc_core::HandshakeManager> handshake_mgr;
 } on_done_closure;
 
 static void on_handshake_done(void* arg, grpc_error* error) {
-  grpc_handshaker_args* args = static_cast<grpc_handshaker_args*>(arg);
+  auto* args = static_cast<grpc_core::HandshakerArgs*>(arg);
   on_done_closure* c = static_cast<on_done_closure*>(args->user_data);
   if (error != GRPC_ERROR_NONE) {
     const char* msg = grpc_error_string(error);
@@ -172,14 +171,13 @@ static void on_handshake_done(void* arg, grpc_error* error) {
     gpr_free(args->read_buffer);
     c->func(c->arg, args->endpoint);
   }
-  grpc_handshake_manager_destroy(c->handshake_mgr);
-  gpr_free(c);
+  grpc_core::Delete<on_done_closure>(c);
 }
 
 static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host,
                           grpc_millis deadline,
                           void (*on_done)(void* arg, grpc_endpoint* endpoint)) {
-  on_done_closure* c = static_cast<on_done_closure*>(gpr_malloc(sizeof(*c)));
+  auto* c = grpc_core::New<on_done_closure>();
   const char* pem_root_certs =
       grpc_core::DefaultSslRootStore::GetPemRootCerts();
   const tsi_ssl_root_certs_store* root_store =
@@ -198,12 +196,13 @@ static void ssl_handshake(void* arg, grpc_endpoint* tcp, const char* host,
   GPR_ASSERT(sc != nullptr);
   grpc_arg channel_arg = grpc_security_connector_to_arg(sc.get());
   grpc_channel_args args = {1, &channel_arg};
-  c->handshake_mgr = grpc_handshake_manager_create();
-  grpc_handshakers_add(HANDSHAKER_CLIENT, &args,
-                       nullptr /* interested_parties */, c->handshake_mgr);
-  grpc_handshake_manager_do_handshake(
-      c->handshake_mgr, tcp, nullptr /* channel_args */, deadline,
-      nullptr /* acceptor */, on_handshake_done, c /* user_data */);
+  c->handshake_mgr = grpc_core::MakeRefCounted<grpc_core::HandshakeManager>();
+  grpc_core::HandshakerRegistry::AddHandshakers(
+      grpc_core::HANDSHAKER_CLIENT, &args, /*interested_parties=*/nullptr,
+      c->handshake_mgr.get());
+  c->handshake_mgr->DoHandshake(tcp, /*channel_args=*/nullptr, deadline,
+                                /*acceptor=*/nullptr, on_handshake_done,
+                                /*user_data=*/c);
   sc.reset(DEBUG_LOCATION, "httpcli");
 }
 

src/core/lib/iomgr/buffer_list.cc

@@ -188,8 +188,8 @@ void extract_opt_stats_from_cmsg(ConnectionMetrics* metrics,
 }
 
 static int get_socket_tcp_info(grpc_core::tcp_info* info, int fd) {
-  info->length = sizeof(*info) - sizeof(socklen_t);
   memset(info, 0, sizeof(*info));
+  info->length = sizeof(*info) - sizeof(socklen_t);
   return getsockopt(fd, IPPROTO_TCP, TCP_INFO, info, &(info->length));
 }
 } /* namespace */

src/core/lib/iomgr/cfstream_handle.cc

@@ -52,6 +52,7 @@ CFStreamHandle* CFStreamHandle::CreateStreamHandle(
 void CFStreamHandle::ReadCallback(CFReadStreamRef stream,
                                   CFStreamEventType type,
                                   void* client_callback_info) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   CFStreamHandle* handle = static_cast<CFStreamHandle*>(client_callback_info);
   if (grpc_tcp_trace.enabled()) {
@@ -77,6 +78,7 @@ void CFStreamHandle::ReadCallback(CFReadStreamRef stream,
 void CFStreamHandle::WriteCallback(CFWriteStreamRef stream,
                                    CFStreamEventType type,
                                    void* clientCallBackInfo) {
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
   grpc_core::ExecCtx exec_ctx;
   CFStreamHandle* handle = static_cast<CFStreamHandle*>(clientCallBackInfo);
   if (grpc_tcp_trace.enabled()) {

src/core/lib/iomgr/ev_epollex_linux.cc

@@ -45,6 +45,7 @@
 #include "src/core/lib/gpr/spinlock.h"
 #include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/inlined_vector.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/gprpp/mutex_lock.h"
 #include "src/core/lib/iomgr/block_annotate.h"
@@ -78,18 +79,6 @@ typedef enum { PO_MULTI, PO_FD, PO_EMPTY } pollable_type;
 
 typedef struct pollable pollable;
 
-typedef struct cached_fd {
-  // Set to the grpc_fd's salt value. See 'salt' variable' in grpc_fd for more
-  // details
-  intptr_t salt;
-
-  // The underlying fd
-  int fd;
-
-  // A recency time counter that helps to determine the LRU fd in the cache
-  uint64_t last_used;
-} cached_fd;
-
 /// A pollable is something that can be polled: it has an epoll set to poll on,
 /// and a wakeup fd for kicks
 /// There are three broad types:
@@ -120,33 +109,6 @@ struct pollable {
   int event_cursor;
   int event_count;
   struct epoll_event events[MAX_EPOLL_EVENTS];
-
-  // We may be calling pollable_add_fd() on the same (pollable, fd) multiple
-  // times. To prevent pollable_add_fd() from making multiple sys calls to
-  // epoll_ctl() to add the fd, we maintain a cache of what fds are already
-  // present in the underlying epoll-set.
-  //
-  // Since this is not a correctness issue, we do not need to maintain all the
-  // fds in the cache. Hence we just use an LRU cache of size 'MAX_FDS_IN_CACHE'
-  //
-  // NOTE: An ideal implementation of this should do the following:
-  //  1) Add fds to the cache in pollable_add_fd() function (i.e whenever the fd
-  //     is added to the pollable's epoll set)
-  //  2) Remove the fd from the cache whenever the fd is removed from the
-  //     underlying epoll set (i.e whenever fd_orphan() is called).
-  //
-  // Implementing (2) above (i.e removing fds from cache on fd_orphan) adds a
-  // lot of complexity since an fd can be present in multiple pollables. So our
-  // implementation ONLY DOES (1) and NOT (2).
-  //
-  // The cache_fd.salt variable helps here to maintain correctness (it serves as
-  // an epoch that differentiates one grpc_fd from the other even though both of
-  // them may have the same fd number)
-  //
-  // The following implements LRU-eviction cache of fds in this pollable
-  cached_fd fd_cache[MAX_FDS_IN_CACHE];
-  int fd_cache_size;
-  uint64_t fd_cache_counter;  // Recency timer tick counter
 };
 
 static const char* pollable_type_string(pollable_type t) {
@@ -189,37 +151,86 @@ static void pollable_unref(pollable* p, int line, const char* reason);
  * Fd Declarations
  */
 
-// Monotonically increasing Epoch counter that is assinged to each grpc_fd. See
-// the description of 'salt' variable in 'grpc_fd' for more details
-// TODO: (sreek/kpayson) gpr_atm is intptr_t which may not be wide-enough on
-// 32-bit systems. Change this to int_64 - atleast on 32-bit systems
-static gpr_atm g_fd_salt;
-
 struct grpc_fd {
-  int fd;
+  grpc_fd(int fd, const char* name, bool track_err)
+      : fd(fd), track_err(track_err) {
+    gpr_mu_init(&orphan_mu);
+    gpr_mu_init(&pollable_mu);
+    read_closure.InitEvent();
+    write_closure.InitEvent();
+    error_closure.InitEvent();
+
+    char* fd_name;
+    gpr_asprintf(&fd_name, "%s fd=%d", name, fd);
+    grpc_iomgr_register_object(&iomgr_object, fd_name);
+#ifndef NDEBUG
+    if (grpc_trace_fd_refcount.enabled()) {
+      gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, this, fd_name);
+    }
+#endif
+    gpr_free(fd_name);
+  }
+
+  // This is really the dtor, but the poller threads waking up from
+  // epoll_wait() may access the (read|write|error)_closure after destruction.
+  // Since the object will be added to the free pool, this behavior is
+  // not going to cause issues, except spurious events if the FD is reused
+  // while the race happens.
+  void destroy() {
+    grpc_iomgr_unregister_object(&iomgr_object);
 
-  // Since fd numbers can be reused (after old fds are closed), this serves as
-  // an epoch that uniquely identifies this fd (i.e the pair (salt, fd) is
-  // unique (until the salt counter (i.e g_fd_salt) overflows)
-  intptr_t salt;
+    POLLABLE_UNREF(pollable_obj, "fd_pollable");
+    pollset_fds.clear();
+    gpr_mu_destroy(&pollable_mu);
+    gpr_mu_destroy(&orphan_mu);
+
+    read_closure.DestroyEvent();
+    write_closure.DestroyEvent();
+    error_closure.DestroyEvent();
+
+    invalidate();
+  }
+
+#ifndef NDEBUG
+  /* Since an fd is never really destroyed (i.e gpr_free() is not called), it is
+   * hard-to-debug cases where fd fields are accessed even after calling
+   * fd_destroy(). The following invalidates fd fields to make catching such
+   * errors easier */
+  void invalidate() {
+    fd = -1;
+    gpr_atm_no_barrier_store(&refst, -1);
+    memset(&orphan_mu, -1, sizeof(orphan_mu));
+    memset(&pollable_mu, -1, sizeof(pollable_mu));
+    pollable_obj = nullptr;
+    on_done_closure = nullptr;
+    memset(&iomgr_object, -1, sizeof(iomgr_object));
+    track_err = false;
+  }
+#else
+  void invalidate() {}
+#endif
+
+  int fd;
 
   // refst format:
   //     bit 0    : 1=Active / 0=Orphaned
   //     bits 1-n : refcount
   //  Ref/Unref by two to avoid altering the orphaned bit
-  gpr_atm refst;
+  gpr_atm refst = 1;
 
   gpr_mu orphan_mu;
 
+  // Protects pollable_obj and pollset_fds.
   gpr_mu pollable_mu;
-  pollable* pollable_obj;
+  grpc_core::InlinedVector<int, 1> pollset_fds;  // Used in PO_MULTI.
+  pollable* pollable_obj = nullptr;              // Used in PO_FD.
 
-  grpc_core::ManualConstructor<grpc_core::LockfreeEvent> read_closure;
-  grpc_core::ManualConstructor<grpc_core::LockfreeEvent> write_closure;
-  grpc_core::ManualConstructor<grpc_core::LockfreeEvent> error_closure;
+  grpc_core::LockfreeEvent read_closure;
+  grpc_core::LockfreeEvent write_closure;
+  grpc_core::LockfreeEvent error_closure;
 
-  struct grpc_fd* freelist_next;
-  grpc_closure* on_done_closure;
+  struct grpc_fd* freelist_next = nullptr;
+  grpc_closure* on_done_closure = nullptr;
 
   grpc_iomgr_object iomgr_object;
 
@@ -258,6 +269,7 @@ struct grpc_pollset_worker {
 struct grpc_pollset {
   gpr_mu mu;
   gpr_atm worker_count;
+  gpr_atm active_pollable_type;
   pollable* active_pollable;
   bool kicked_without_poller;
   grpc_closure* shutdown_closure;
@@ -337,39 +349,10 @@ static void ref_by(grpc_fd* fd, int n) {
   GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&fd->refst, n) > 0);
 }
 
-#ifndef NDEBUG
-#define INVALIDATE_FD(fd) invalidate_fd(fd)
-/* Since an fd is never really destroyed (i.e gpr_free() is not called), it is
- * hard to cases where fd fields are accessed even after calling fd_destroy().
- * The following invalidates fd fields to make catching such errors easier */
-static void invalidate_fd(grpc_fd* fd) {
-  fd->fd = -1;
-  fd->salt = -1;
-  gpr_atm_no_barrier_store(&fd->refst, -1);
-  memset(&fd->orphan_mu, -1, sizeof(fd->orphan_mu));
-  memset(&fd->pollable_mu, -1, sizeof(fd->pollable_mu));
-  fd->pollable_obj = nullptr;
-  fd->on_done_closure = nullptr;
-  memset(&fd->iomgr_object, -1, sizeof(fd->iomgr_object));
-  fd->track_err = false;
-}
-#else
-#define INVALIDATE_FD(fd)
-#endif
-
 /* Uninitialize and add to the freelist */
 static void fd_destroy(void* arg, grpc_error* error) {
   grpc_fd* fd = static_cast<grpc_fd*>(arg);
-  grpc_iomgr_unregister_object(&fd->iomgr_object);
-  POLLABLE_UNREF(fd->pollable_obj, "fd_pollable");
-  gpr_mu_destroy(&fd->pollable_mu);
-  gpr_mu_destroy(&fd->orphan_mu);
-
-  fd->read_closure->DestroyEvent();
-  fd->write_closure->DestroyEvent();
-  fd->error_closure->DestroyEvent();
-
-  INVALIDATE_FD(fd);
+  fd->destroy();
 
   /* Add the fd to the freelist */
   gpr_mu_lock(&fd_freelist_mu);
@@ -429,35 +412,9 @@ static grpc_fd* fd_create(int fd, const char* name, bool track_err) {
 
   if (new_fd == nullptr) {
     new_fd = static_cast<grpc_fd*>(gpr_malloc(sizeof(grpc_fd)));
-    new_fd->read_closure.Init();
-    new_fd->write_closure.Init();
-    new_fd->error_closure.Init();
-  }
-
-  new_fd->fd = fd;
-  new_fd->salt = gpr_atm_no_barrier_fetch_add(&g_fd_salt, 1);
-  gpr_atm_rel_store(&new_fd->refst, (gpr_atm)1);
-  gpr_mu_init(&new_fd->orphan_mu);
-  gpr_mu_init(&new_fd->pollable_mu);
-  new_fd->pollable_obj = nullptr;
-  new_fd->read_closure->InitEvent();
-  new_fd->write_closure->InitEvent();
-  new_fd->error_closure->InitEvent();
-  new_fd->freelist_next = nullptr;
-  new_fd->on_done_closure = nullptr;
-
-  char* fd_name;
-  gpr_asprintf(&fd_name, "%s fd=%d", name, fd);
-  grpc_iomgr_register_object(&new_fd->iomgr_object, fd_name);
-#ifndef NDEBUG
-  if (grpc_trace_fd_refcount.enabled()) {
-    gpr_log(GPR_DEBUG, "FD %d %p create %s", fd, new_fd, fd_name);
   }
-#endif
-  gpr_free(fd_name);
 
-  new_fd->track_err = track_err;
-  return new_fd;
+  return new (new_fd) grpc_fd(fd, name, track_err);
 }
 
 static int fd_wrapped_fd(grpc_fd* fd) {
@@ -475,7 +432,6 @@ static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
   // true so that the pollable will no longer access its owner_fd field.
   gpr_mu_lock(&fd->pollable_mu);
   pollable* pollable_obj = fd->pollable_obj;
-  gpr_mu_unlock(&fd->pollable_mu);
 
   if (pollable_obj) {
     gpr_mu_lock(&pollable_obj->owner_orphan_mu);
@@ -487,6 +443,17 @@ static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
   /* If release_fd is not NULL, we should be relinquishing control of the file
      descriptor fd->fd (but we still own the grpc_fd structure). */
   if (release_fd != nullptr) {
+    // Remove the FD from all epolls sets, before releasing it.
+    // Otherwise, we will receive epoll events after we release the FD.
+    epoll_event ev_fd;
+    memset(&ev_fd, 0, sizeof(ev_fd));
+    if (pollable_obj != nullptr) {  // For PO_FD.
+      epoll_ctl(pollable_obj->epfd, EPOLL_CTL_DEL, fd->fd, &ev_fd);
+    }
+    for (size_t i = 0; i < fd->pollset_fds.size(); ++i) {  // For PO_MULTI.
+      const int epfd = fd->pollset_fds[i];
+      epoll_ctl(epfd, EPOLL_CTL_DEL, fd->fd, &ev_fd);
+    }
     *release_fd = fd->fd;
   } else {
     close(fd->fd);
@@ -508,40 +475,58 @@ static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd,
     gpr_mu_unlock(&pollable_obj->owner_orphan_mu);
   }
 
+  gpr_mu_unlock(&fd->pollable_mu);
   gpr_mu_unlock(&fd->orphan_mu);
 
   UNREF_BY(fd, 2, reason); /* Drop the reference */
 }
 
 static bool fd_is_shutdown(grpc_fd* fd) {
-  return fd->read_closure->IsShutdown();
+  return fd->read_closure.IsShutdown();
 }
 
 /* Might be called multiple times */
 static void fd_shutdown(grpc_fd* fd, grpc_error* why) {
-  if (fd->read_closure->SetShutdown(GRPC_ERROR_REF(why))) {
+  if (fd->read_closure.SetShutdown(GRPC_ERROR_REF(why))) {
     if (shutdown(fd->fd, SHUT_RDWR)) {
       if (errno != ENOTCONN) {
         gpr_log(GPR_ERROR, "Error shutting down fd %d. errno: %d",
                 grpc_fd_wrapped_fd(fd), errno);
       }
     }
-    fd->write_closure->SetShutdown(GRPC_ERROR_REF(why));
-    fd->error_closure->SetShutdown(GRPC_ERROR_REF(why));
+    fd->write_closure.SetShutdown(GRPC_ERROR_REF(why));
+    fd->error_closure.SetShutdown(GRPC_ERROR_REF(why));
   }
   GRPC_ERROR_UNREF(why);
 }
 
 static void fd_notify_on_read(grpc_fd* fd, grpc_closure* closure) {
-  fd->read_closure->NotifyOn(closure);
+  fd->read_closure.NotifyOn(closure);
 }
 
 static void fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) {
-  fd->write_closure->NotifyOn(closure);
+  fd->write_closure.NotifyOn(closure);
 }
 
 static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) {
-  fd->error_closure->NotifyOn(closure);
+  fd->error_closure.NotifyOn(closure);
+}
+
+static bool fd_has_pollset(grpc_fd* fd, grpc_pollset* pollset) {
+  const int epfd = pollset->active_pollable->epfd;
+  grpc_core::MutexLock lock(&fd->pollable_mu);
+  for (size_t i = 0; i < fd->pollset_fds.size(); ++i) {
+    if (fd->pollset_fds[i] == epfd) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static void fd_add_pollset(grpc_fd* fd, grpc_pollset* pollset) {
+  const int epfd = pollset->active_pollable->epfd;
+  grpc_core::MutexLock lock(&fd->pollable_mu);
+  fd->pollset_fds.push_back(epfd);
 }
 
 /*******************************************************************************
@@ -594,8 +579,6 @@ static grpc_error* pollable_create(pollable_type type, pollable** p) {
   (*p)->root_worker = nullptr;
   (*p)->event_cursor = 0;
   (*p)->event_count = 0;
-  (*p)->fd_cache_size = 0;
-  (*p)->fd_cache_counter = 0;
   return GRPC_ERROR_NONE;
 }
 
@@ -637,39 +620,6 @@ static grpc_error* pollable_add_fd(pollable* p, grpc_fd* fd) {
   grpc_error* error = GRPC_ERROR_NONE;
   static const char* err_desc = "pollable_add_fd";
   const int epfd = p->epfd;
-  gpr_mu_lock(&p->mu);
-  p->fd_cache_counter++;
-
-  // Handle the case of overflow for our cache counter by
-  // reseting the recency-counter on all cache objects
-  if (p->fd_cache_counter == 0) {
-    for (int i = 0; i < p->fd_cache_size; i++) {
-      p->fd_cache[i].last_used = 0;
-    }
-  }
-
-  int lru_idx = 0;
-  for (int i = 0; i < p->fd_cache_size; i++) {
-    if (p->fd_cache[i].fd == fd->fd && p->fd_cache[i].salt == fd->salt) {
-      GRPC_STATS_INC_POLLSET_FD_CACHE_HITS();
-      p->fd_cache[i].last_used = p->fd_cache_counter;
-      gpr_mu_unlock(&p->mu);
-      return GRPC_ERROR_NONE;
-    } else if (p->fd_cache[i].last_used < p->fd_cache[lru_idx].last_used) {
-      lru_idx = i;
-    }
-  }
-
-  // Add to cache
-  if (p->fd_cache_size < MAX_FDS_IN_CACHE) {
-    lru_idx = p->fd_cache_size;
-    p->fd_cache_size++;
-  }
-  p->fd_cache[lru_idx].fd = fd->fd;
-  p->fd_cache[lru_idx].salt = fd->salt;
-  p->fd_cache[lru_idx].last_used = p->fd_cache_counter;
-  gpr_mu_unlock(&p->mu);
-
   if (grpc_polling_trace.enabled()) {
     gpr_log(GPR_INFO, "add fd %p (%d) to pollable %p", fd, fd->fd, p);
   }
@@ -849,6 +799,7 @@ static grpc_error* pollset_kick_all(grpc_pollset* pollset) {
 static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) {
   gpr_mu_init(&pollset->mu);
   gpr_atm_no_barrier_store(&pollset->worker_count, 0);
+  gpr_atm_no_barrier_store(&pollset->active_pollable_type, PO_EMPTY);
   pollset->active_pollable = POLLABLE_REF(g_empty_pollable, "pollset");
   pollset->kicked_without_poller = false;
   pollset->shutdown_closure = nullptr;
@@ -869,11 +820,11 @@ static int poll_deadline_to_millis_timeout(grpc_millis millis) {
     return static_cast<int>(delta);
 }
 
-static void fd_become_readable(grpc_fd* fd) { fd->read_closure->SetReady(); }
+static void fd_become_readable(grpc_fd* fd) { fd->read_closure.SetReady(); }
 
-static void fd_become_writable(grpc_fd* fd) { fd->write_closure->SetReady(); }
+static void fd_become_writable(grpc_fd* fd) { fd->write_closure.SetReady(); }
 
-static void fd_has_errors(grpc_fd* fd) { fd->error_closure->SetReady(); }
+static void fd_has_errors(grpc_fd* fd) { fd->error_closure.SetReady(); }
 
 /* Get the pollable_obj attached to this fd. If none is attached, create a new
  * pollable object (of type PO_FD), attach it to the fd and return it
@@ -1283,6 +1234,8 @@ static grpc_error* pollset_add_fd_locked(grpc_pollset* pollset, grpc_fd* fd) {
     POLLABLE_UNREF(pollset->active_pollable, "pollset");
     pollset->active_pollable = po_at_start;
   } else {
+    gpr_atm_rel_store(&pollset->active_pollable_type,
+                      pollset->active_pollable->type);
     POLLABLE_UNREF(po_at_start, "pollset_add_fd");
   }
   return error;
@@ -1329,6 +1282,8 @@ static grpc_error* pollset_as_multipollable_locked(grpc_pollset* pollset,
     pollset->active_pollable = po_at_start;
     *pollable_obj = nullptr;
   } else {
+    gpr_atm_rel_store(&pollset->active_pollable_type,
+                      pollset->active_pollable->type);
     *pollable_obj = POLLABLE_REF(pollset->active_pollable, "pollset_set");
     POLLABLE_UNREF(po_at_start, "pollset_as_multipollable");
   }
@@ -1337,9 +1292,23 @@ static grpc_error* pollset_as_multipollable_locked(grpc_pollset* pollset,
 
 static void pollset_add_fd(grpc_pollset* pollset, grpc_fd* fd) {
   GPR_TIMER_SCOPE("pollset_add_fd", 0);
-  gpr_mu_lock(&pollset->mu);
+
+  // We never transition from PO_MULTI to other modes (i.e., PO_FD or PO_EMPTY)
+  // and, thus, it is safe to simply store and check whether the FD has already
+  // been added to the active pollable previously.
+  if (gpr_atm_acq_load(&pollset->active_pollable_type) == PO_MULTI &&
+      fd_has_pollset(fd, pollset)) {
+    return;
+  }
+
+  grpc_core::MutexLock lock(&pollset->mu);
   grpc_error* error = pollset_add_fd_locked(pollset, fd);
-  gpr_mu_unlock(&pollset->mu);
+
+  // If we are in PO_MULTI mode, we should update the pollsets of the FD.
+  if (gpr_atm_no_barrier_load(&pollset->active_pollable_type) == PO_MULTI) {
+    fd_add_pollset(fd, pollset);
+  }
+
   GRPC_LOG_IF_ERROR("pollset_add_fd", error);
 }
 

src/core/lib/iomgr/exec_ctx.h

@@ -49,6 +49,10 @@ typedef struct grpc_combiner grpc_combiner;
    be counted by fork handlers */
 #define GRPC_EXEC_CTX_FLAG_IS_INTERNAL_THREAD 4
 
+/* This application callback exec ctx was initialized by an internal thread, and
+   should not be counted by fork handlers */
+#define GRPC_APP_CALLBACK_EXEC_CTX_FLAG_IS_INTERNAL_THREAD 1
+
 extern grpc_closure_scheduler* grpc_schedule_on_exec_ctx;
 
 gpr_timespec grpc_millis_to_timespec(grpc_millis millis, gpr_clock_type clock);
@@ -58,8 +62,8 @@ grpc_millis grpc_timespec_to_millis_round_up(gpr_timespec timespec);
 namespace grpc_core {
 /** Execution context.
  *  A bag of data that collects information along a callstack.
- *  It is created on the stack at public API entry points, and stored internally
- *  as a thread-local variable.
+ *  It is created on the stack at core entry points (public API or iomgr), and
+ *  stored internally as a thread-local variable.
  *
  *  Generally, to create an exec_ctx instance, add the following line at the top
  *  of the public API entry point or at the start of a thread's work function :
@@ -70,7 +74,7 @@ namespace grpc_core {
  *  grpc_core::ExecCtx::Get()
  *
  *  Specific responsibilities (this may grow in the future):
- *  - track a list of work that needs to be delayed until the top of the
+ *  - track a list of core work that needs to be delayed until the base of the
  *    call stack (this provides a convenient mechanism to run callbacks
  *    without worrying about locking issues)
  *  - provide a decision maker (via IsReadyToFinish) that provides a
@@ -80,10 +84,19 @@ namespace grpc_core {
  *  CONVENTIONS:
  *  - Instance of this must ALWAYS be constructed on the stack, never
  *    heap allocated.
- *  - Exactly one instance of ExecCtx must be created per thread. Instances must
- *    always be called exec_ctx.
  *  - Do not pass exec_ctx as a parameter to a function. Always access it using
  *    grpc_core::ExecCtx::Get().
+ *  - NOTE: In the future, the convention is likely to change to allow only one
+ *          ExecCtx on a thread's stack at the same time. The TODO below
+ *          discusses this plan in more detail.
+ *
+ * TODO(yashykt): Only allow one "active" ExecCtx on a thread at the same time.
+ *                Stage 1: If a new one is created on the stack, it should just
+ *                pass-through to the underlying ExecCtx deeper in the thread's
+ *                stack.
+ *                Stage 2: Assert if a 2nd one is ever created on the stack
+ *                since that implies a core re-entry outside of application
+ *                callbacks.
  */
 class ExecCtx {
  public:
@@ -227,15 +240,61 @@ class ExecCtx {
   ExecCtx* last_exec_ctx_ = Get();
 };
 
+/** Application-callback execution context.
+ *  A bag of data that collects information along a callstack.
+ *  It is created on the stack at core entry points, and stored internally
+ *  as a thread-local variable.
+ *
+ *  There are three key differences between this structure and ExecCtx:
+ *    1. ApplicationCallbackExecCtx builds a list of application-level
+ *       callbacks, but ExecCtx builds a list of internal callbacks to invoke.
+ *    2. ApplicationCallbackExecCtx invokes its callbacks only at destruction;
+ *       there is no explicit Flush method.
+ *    3. If more than one ApplicationCallbackExecCtx is created on the thread's
+ *       stack, only the one closest to the base of the stack is actually
+ *       active and this is the only one that enqueues application callbacks.
+ *       (Unlike ExecCtx, it is not feasible to prevent multiple of these on the
+ *       stack since the executing application callback may itself enter core.
+ *       However, the new one created will just pass callbacks through to the
+ *       base one and those will not be executed until the return to the
+ *       destructor of the base one, preventing unlimited stack growth.)
+ *
+ *  This structure exists because application callbacks may themselves cause a
+ *  core re-entry (e.g., through a public API call) and if that call in turn
+ *  causes another application-callback, there could be arbitrarily growing
+ *  stacks of core re-entries. Instead, any application callbacks instead should
+ *  not be invoked until other core work is done and other application callbacks
+ *  have completed. To accomplish this, any application callback should be
+ *  enqueued using grpc_core::ApplicationCallbackExecCtx::Enqueue .
+ *
+ *  CONVENTIONS:
+ *  - Instances of this must ALWAYS be constructed on the stack, never
+ *    heap allocated.
+ *  - Instances of this are generally constructed before ExecCtx when needed.
+ *    The only exception is for ExecCtx's that are explicitly flushed and
+ *    that survive beyond the scope of the function that can cause application
+ *    callbacks to be invoked (e.g., in the timer thread).
+ *
+ *  Generally, core entry points that may trigger application-level callbacks
+ *  will have the following declarations:
+ *
+ *  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
+ *  grpc_core::ExecCtx exec_ctx;
+ *
+ *  This ordering is important to make sure that the ApplicationCallbackExecCtx
+ *  is destroyed after the ExecCtx (to prevent the re-entry problem described
+ *  above, as well as making sure that ExecCtx core callbacks are invoked first)
+ *
+ */
+
 class ApplicationCallbackExecCtx {
  public:
-  ApplicationCallbackExecCtx() {
-    if (reinterpret_cast<ApplicationCallbackExecCtx*>(
-            gpr_tls_get(&callback_exec_ctx_)) == nullptr) {
-      grpc_core::Fork::IncExecCtxCount();
-      gpr_tls_set(&callback_exec_ctx_, reinterpret_cast<intptr_t>(this));
-    }
-  }
+  /** Default Constructor */
+  ApplicationCallbackExecCtx() { Set(this, flags_); }
+
+  /** Parameterised Constructor */
+  ApplicationCallbackExecCtx(uintptr_t fl) : flags_(fl) { Set(this, flags_); }
+
   ~ApplicationCallbackExecCtx() {
     if (reinterpret_cast<ApplicationCallbackExecCtx*>(
             gpr_tls_get(&callback_exec_ctx_)) == this) {
@@ -248,12 +307,25 @@ class ApplicationCallbackExecCtx {
         (*f->functor_run)(f, f->internal_success);
       }
       gpr_tls_set(&callback_exec_ctx_, reinterpret_cast<intptr_t>(nullptr));
-      grpc_core::Fork::DecExecCtxCount();
+      if (!(GRPC_APP_CALLBACK_EXEC_CTX_FLAG_IS_INTERNAL_THREAD & flags_)) {
+        grpc_core::Fork::DecExecCtxCount();
+      }
     } else {
       GPR_DEBUG_ASSERT(head_ == nullptr);
       GPR_DEBUG_ASSERT(tail_ == nullptr);
     }
   }
+
+  static void Set(ApplicationCallbackExecCtx* exec_ctx, uintptr_t flags) {
+    if (reinterpret_cast<ApplicationCallbackExecCtx*>(
+            gpr_tls_get(&callback_exec_ctx_)) == nullptr) {
+      if (!(GRPC_APP_CALLBACK_EXEC_CTX_FLAG_IS_INTERNAL_THREAD & flags)) {
+        grpc_core::Fork::IncExecCtxCount();
+      }
+      gpr_tls_set(&callback_exec_ctx_, reinterpret_cast<intptr_t>(exec_ctx));
+    }
+  }
+
   static void Enqueue(grpc_experimental_completion_queue_functor* functor,
                       int is_success) {
     functor->internal_success = is_success;
@@ -278,6 +350,7 @@ class ApplicationCallbackExecCtx {
   static void GlobalShutdown(void) { gpr_tls_destroy(&callback_exec_ctx_); }
 
  private:
+  uintptr_t flags_{0u};
   grpc_experimental_completion_queue_functor* head_{nullptr};
   grpc_experimental_completion_queue_functor* tail_{nullptr};
   GPR_TLS_CLASS_DECL(callback_exec_ctx_);

src/core/lib/iomgr/executor.cc

@@ -116,7 +116,8 @@ size_t Executor::RunClosures(const char* executor_name,
   // application-level callbacks. No need to create a new ExecCtx, though,
   // since there already is one and it is flushed (but not destructed) in this
   // function itself.
-  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx(
+      GRPC_APP_CALLBACK_EXEC_CTX_FLAG_IS_INTERNAL_THREAD);
 
   grpc_closure* c = list.head;
   while (c != nullptr) {

src/core/lib/iomgr/fork_posix.cc

@@ -47,11 +47,13 @@ bool registered_handlers = false;
 }  // namespace
 
 void grpc_prefork() {
-  grpc_core::ExecCtx exec_ctx;
   skipped_handler = true;
+  // This  may be called after core shuts down, so verify initialized before
+  // instantiating an ExecCtx.
   if (!grpc_is_initialized()) {
     return;
   }
+  grpc_core::ExecCtx exec_ctx;
   if (!grpc_core::Fork::Enabled()) {
     gpr_log(GPR_ERROR,
             "Fork support not enabled; try running with the "

src/core/lib/iomgr/iomgr.h

@@ -39,6 +39,11 @@ void grpc_iomgr_shutdown();
  * background poller. */
 void grpc_iomgr_shutdown_background_closure();
 
+/* Returns true if polling engine runs in the background, false otherwise.
+ * Currently only 'epollbg' runs in the background.
+ */
+bool grpc_iomgr_run_in_background();
+
 /** Returns true if the caller is a worker thread for any background poller. */
 bool grpc_iomgr_is_any_background_poller_thread();
 

src/core/lib/iomgr/iomgr_posix.cc

@@ -74,4 +74,8 @@ void grpc_set_default_iomgr_platform() {
   grpc_set_iomgr_platform_vtable(&vtable);
 }
 
+bool grpc_iomgr_run_in_background() {
+  return grpc_event_engine_run_in_background();
+}
+
 #endif /* GRPC_POSIX_SOCKET_IOMGR */

src/core/lib/iomgr/iomgr_windows.cc

@@ -92,4 +92,6 @@ void grpc_set_default_iomgr_platform() {
   grpc_set_iomgr_platform_vtable(&vtable);
 }
 
+bool grpc_iomgr_run_in_background() { return false; }
+
 #endif /* GRPC_WINSOCK_SOCKET */

src/core/lib/iomgr/tcp_posix.cc

@@ -715,11 +715,15 @@ static void process_errors(grpc_tcp* tcp) {
     msg.msg_iovlen = 0;
     msg.msg_flags = 0;
 
-    // Allocate aligned space for cmsgs received along with a timestamps
+    /* Allocate enough space so we don't need to keep increasing this as size
+     * of OPT_STATS increase */
+    constexpr size_t cmsg_alloc_space =
+        CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) +
+        CMSG_SPACE(sizeof(sock_extended_err) + sizeof(sockaddr_in)) +
+        CMSG_SPACE(32 * NLA_ALIGN(NLA_HDRLEN + sizeof(uint64_t)));
+    /* Allocate aligned space for cmsgs received along with timestamps */
     union {
-      char rbuf[CMSG_SPACE(sizeof(grpc_core::scm_timestamping)) +
-                CMSG_SPACE(sizeof(sock_extended_err) + sizeof(sockaddr_in)) +
-                CMSG_SPACE(16 * NLA_ALIGN(NLA_HDRLEN + sizeof(uint64_t)))];
+      char rbuf[cmsg_alloc_space];
       struct cmsghdr align;
     } aligned_buf;
     memset(&aligned_buf, 0, sizeof(aligned_buf));
@@ -739,10 +743,8 @@ static void process_errors(grpc_tcp* tcp) {
     if (r == -1) {
       return;
     }
-    if (grpc_tcp_trace.enabled()) {
-      if ((msg.msg_flags & MSG_CTRUNC) != 0) {
-        gpr_log(GPR_INFO, "Error message was truncated.");
-      }
+    if ((msg.msg_flags & MSG_CTRUNC) != 0) {
+      gpr_log(GPR_ERROR, "Error message was truncated.");
     }
 
     if (msg.msg_controllen == 0) {

src/core/lib/iomgr/timer_manager.cc

@@ -110,7 +110,8 @@ static void run_some_timers() {
   // could start seeing application-level callbacks. No need to
   // create a new ExecCtx, though, since there already is one and it is
   // flushed (but not destructed) in this function itself
-  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx;
+  grpc_core::ApplicationCallbackExecCtx callback_exec_ctx(
+      GRPC_APP_CALLBACK_EXEC_CTX_FLAG_IS_INTERNAL_THREAD);
 
   // if there's something to execute...
   gpr_mu_lock(&g_mu);