Merge with head and resolve conflict

BUILD

@@ -74,11 +74,11 @@ config_setting(
 )
 
 # This should be updated along with build.yaml
-g_stands_for = "gandalf"
+g_stands_for = "gale"
 
 core_version = "7.0.0"
 
-version = "1.21.0-dev"
+version = "1.22.0-dev"
 
 GPR_PUBLIC_HDRS = [
     "include/grpc/support/alloc.h",
@@ -218,6 +218,7 @@ GRPCXX_PUBLIC_HDRS = [
     "include/grpcpp/alarm.h",
     "include/grpcpp/alarm_impl.h",
     "include/grpcpp/channel.h",
+    "include/grpcpp/channel_impl.h",
     "include/grpcpp/client_context.h",
     "include/grpcpp/completion_queue.h",
     "include/grpcpp/create_channel.h",
@@ -435,6 +436,7 @@ grpc_cc_library(
         "src/compiler/config.h",
         "src/compiler/cpp_generator.h",
         "src/compiler/cpp_generator_helpers.h",
+        "src/compiler/cpp_plugin.h",
         "src/compiler/csharp_generator.h",
         "src/compiler/csharp_generator_helpers.h",
         "src/compiler/generator_helpers.h",
@@ -997,6 +999,7 @@ grpc_cc_library(
         "src/core/lib/slice/slice_hash_table.h",
         "src/core/lib/slice/slice_internal.h",
         "src/core/lib/slice/slice_string_helpers.h",
+        "src/core/lib/slice/slice_utils.h",
         "src/core/lib/slice/slice_weak_hash_table.h",
         "src/core/lib/surface/api_trace.h",
         "src/core/lib/surface/call.h",
@@ -1561,6 +1564,20 @@ grpc_cc_library(
     ],
 )
 
+grpc_cc_library(
+    name = "grpc_resolver_dns_selection",
+    srcs = [
+        "src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc",
+    ],
+    hdrs = [
+        "src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h",
+    ],
+    language = "c++",
+    deps = [
+        "grpc_base",
+    ],
+)
+
 grpc_cc_library(
     name = "grpc_resolver_dns_native",
     srcs = [
@@ -1570,6 +1587,7 @@ grpc_cc_library(
     deps = [
         "grpc_base",
         "grpc_client_channel",
+        "grpc_resolver_dns_selection",
     ],
 )
 
@@ -1601,6 +1619,7 @@ grpc_cc_library(
     deps = [
         "grpc_base",
         "grpc_client_channel",
+        "grpc_resolver_dns_selection",
     ],
 )
 
@@ -2147,6 +2166,7 @@ grpc_cc_library(
         "include/grpcpp/impl/codegen/client_interceptor.h",
         "include/grpcpp/impl/codegen/client_unary_call.h",
         "include/grpcpp/impl/codegen/completion_queue.h",
+        "include/grpcpp/impl/codegen/completion_queue_impl.h",
         "include/grpcpp/impl/codegen/completion_queue_tag.h",
         "include/grpcpp/impl/codegen/config.h",
         "include/grpcpp/impl/codegen/core_codegen_interface.h",

BUILD.gn

@@ -326,6 +326,8 @@ config("grpc_config") {
         "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.h",
         "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc",
         "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc",
+        "src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc",
+        "src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h",
         "src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc",
         "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc",
         "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h",
@@ -737,6 +739,7 @@ config("grpc_config") {
         "src/core/lib/slice/slice_internal.h",
         "src/core/lib/slice/slice_string_helpers.cc",
         "src/core/lib/slice/slice_string_helpers.h",
+        "src/core/lib/slice/slice_utils.h",
         "src/core/lib/slice/slice_weak_hash_table.h",
         "src/core/lib/surface/api_trace.cc",
         "src/core/lib/surface/api_trace.h",
@@ -1023,6 +1026,7 @@ config("grpc_config") {
         "include/grpcpp/alarm.h",
         "include/grpcpp/alarm_impl.h",
         "include/grpcpp/channel.h",
+        "include/grpcpp/channel_impl.h",
         "include/grpcpp/client_context.h",
         "include/grpcpp/completion_queue.h",
         "include/grpcpp/create_channel.h",
@@ -1054,6 +1058,7 @@ config("grpc_config") {
         "include/grpcpp/impl/codegen/client_interceptor.h",
         "include/grpcpp/impl/codegen/client_unary_call.h",
         "include/grpcpp/impl/codegen/completion_queue.h",
+        "include/grpcpp/impl/codegen/completion_queue_impl.h",
         "include/grpcpp/impl/codegen/completion_queue_tag.h",
         "include/grpcpp/impl/codegen/config.h",
         "include/grpcpp/impl/codegen/config_protobuf.h",
@@ -1279,6 +1284,7 @@ config("grpc_config") {
         "src/core/lib/slice/slice_hash_table.h",
         "src/core/lib/slice/slice_internal.h",
         "src/core/lib/slice/slice_string_helpers.h",
+        "src/core/lib/slice/slice_utils.h",
         "src/core/lib/slice/slice_weak_hash_table.h",
         "src/core/lib/surface/api_trace.h",
         "src/core/lib/surface/call.h",

CMakeLists.txt

@@ -24,7 +24,7 @@
 cmake_minimum_required(VERSION 2.8)
 
 set(PACKAGE_NAME      "grpc")
-set(PACKAGE_VERSION   "1.21.0-dev")
+set(PACKAGE_VERSION   "1.22.0-dev")
 set(PACKAGE_STRING    "${PACKAGE_NAME} ${PACKAGE_VERSION}")
 set(PACKAGE_TARNAME   "${PACKAGE_NAME}-${PACKAGE_VERSION}")
 set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/")
@@ -704,6 +704,7 @@ add_dependencies(buildtests_cxx server_crash_test_client)
 add_dependencies(buildtests_cxx server_early_return_test)
 add_dependencies(buildtests_cxx server_interceptors_end2end_test)
 add_dependencies(buildtests_cxx server_request_call_test)
+add_dependencies(buildtests_cxx service_config_end2end_test)
 add_dependencies(buildtests_cxx service_config_test)
 add_dependencies(buildtests_cxx shutdown_test)
 add_dependencies(buildtests_cxx slice_hash_table_test)
@@ -1302,6 +1303,7 @@ add_library(grpc
   src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc
   src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc
   src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc
+  src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc
   src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
   src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc
   src/core/ext/filters/census/grpc_context.cc
@@ -2698,6 +2700,7 @@ add_library(grpc_unsecure
   src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc
   src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc
   src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc
+  src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc
   src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
   src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc
   src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc
@@ -2906,6 +2909,49 @@ target_link_libraries(test_tcp_server
 )
 
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_library(dns_test_util
+  test/cpp/naming/dns_test_util.cc
+)
+
+if(WIN32 AND MSVC)
+  set_target_properties(dns_test_util PROPERTIES COMPILE_PDB_NAME "dns_test_util"
+    COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
+  )
+  if (gRPC_INSTALL)
+    install(FILES ${CMAKE_CURRENT_BINARY_DIR}/dns_test_util.pdb
+      DESTINATION ${gRPC_INSTALL_LIBDIR} OPTIONAL
+    )
+  endif()
+endif()
+
+
+target_include_directories(dns_test_util
+  PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+target_link_libraries(dns_test_util
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
 endif (gRPC_BUILD_TESTS)
 
 add_library(grpc++
@@ -3041,6 +3087,7 @@ foreach(_hdr
   include/grpcpp/alarm.h
   include/grpcpp/alarm_impl.h
   include/grpcpp/channel.h
+  include/grpcpp/channel_impl.h
   include/grpcpp/client_context.h
   include/grpcpp/completion_queue.h
   include/grpcpp/create_channel.h
@@ -3202,6 +3249,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/client_interceptor.h
   include/grpcpp/impl/codegen/client_unary_call.h
   include/grpcpp/impl/codegen/completion_queue.h
+  include/grpcpp/impl/codegen/completion_queue_impl.h
   include/grpcpp/impl/codegen/completion_queue_tag.h
   include/grpcpp/impl/codegen/config.h
   include/grpcpp/impl/codegen/core_codegen_interface.h
@@ -3656,6 +3704,7 @@ foreach(_hdr
   include/grpcpp/alarm.h
   include/grpcpp/alarm_impl.h
   include/grpcpp/channel.h
+  include/grpcpp/channel_impl.h
   include/grpcpp/client_context.h
   include/grpcpp/completion_queue.h
   include/grpcpp/create_channel.h
@@ -3817,6 +3866,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/client_interceptor.h
   include/grpcpp/impl/codegen/client_unary_call.h
   include/grpcpp/impl/codegen/completion_queue.h
+  include/grpcpp/impl/codegen/completion_queue_impl.h
   include/grpcpp/impl/codegen/completion_queue_tag.h
   include/grpcpp/impl/codegen/config.h
   include/grpcpp/impl/codegen/core_codegen_interface.h
@@ -4251,6 +4301,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/client_interceptor.h
   include/grpcpp/impl/codegen/client_unary_call.h
   include/grpcpp/impl/codegen/completion_queue.h
+  include/grpcpp/impl/codegen/completion_queue_impl.h
   include/grpcpp/impl/codegen/completion_queue_tag.h
   include/grpcpp/impl/codegen/config.h
   include/grpcpp/impl/codegen/core_codegen_interface.h
@@ -4449,6 +4500,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/client_interceptor.h
   include/grpcpp/impl/codegen/client_unary_call.h
   include/grpcpp/impl/codegen/completion_queue.h
+  include/grpcpp/impl/codegen/completion_queue_impl.h
   include/grpcpp/impl/codegen/completion_queue_tag.h
   include/grpcpp/impl/codegen/config.h
   include/grpcpp/impl/codegen/core_codegen_interface.h
@@ -4643,6 +4695,7 @@ foreach(_hdr
   include/grpcpp/alarm.h
   include/grpcpp/alarm_impl.h
   include/grpcpp/channel.h
+  include/grpcpp/channel_impl.h
   include/grpcpp/client_context.h
   include/grpcpp/completion_queue.h
   include/grpcpp/create_channel.h
@@ -4804,6 +4857,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/client_interceptor.h
   include/grpcpp/impl/codegen/client_unary_call.h
   include/grpcpp/impl/codegen/completion_queue.h
+  include/grpcpp/impl/codegen/completion_queue_impl.h
   include/grpcpp/impl/codegen/completion_queue_tag.h
   include/grpcpp/impl/codegen/config.h
   include/grpcpp/impl/codegen/core_codegen_interface.h
@@ -15987,6 +16041,46 @@ target_link_libraries(server_request_call_test
 )
 
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_executable(service_config_end2end_test
+  test/cpp/end2end/service_config_end2end_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(service_config_end2end_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(service_config_end2end_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_util
+  grpc_test_util
+  grpc++
+  grpc
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
@@ -18535,6 +18629,7 @@ target_include_directories(resolver_component_test_unsecure
 target_link_libraries(resolver_component_test_unsecure
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  dns_test_util
   grpc++_test_util_unsecure
   grpc_test_util_unsecure
   grpc++_unsecure
@@ -18576,6 +18671,7 @@ target_include_directories(resolver_component_test
 target_link_libraries(resolver_component_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  dns_test_util
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -18785,6 +18881,7 @@ target_include_directories(cancel_ares_query_test
 target_link_libraries(cancel_ares_query_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  dns_test_util
   grpc++_test_util
   grpc_test_util
   grpc++

Makefile

@@ -460,8 +460,8 @@ Q = @
 endif
 
 CORE_VERSION = 7.0.0
-CPP_VERSION = 1.21.0-dev
-CSHARP_VERSION = 1.21.0-dev
+CPP_VERSION = 1.22.0-dev
+CSHARP_VERSION = 1.22.0-dev
 
 CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES))
 CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS)
@@ -1266,6 +1266,7 @@ server_crash_test_client: $(BINDIR)/$(CONFIG)/server_crash_test_client
 server_early_return_test: $(BINDIR)/$(CONFIG)/server_early_return_test
 server_interceptors_end2end_test: $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test
 server_request_call_test: $(BINDIR)/$(CONFIG)/server_request_call_test
+service_config_end2end_test: $(BINDIR)/$(CONFIG)/service_config_end2end_test
 service_config_test: $(BINDIR)/$(CONFIG)/service_config_test
 shutdown_test: $(BINDIR)/$(CONFIG)/shutdown_test
 slice_hash_table_test: $(BINDIR)/$(CONFIG)/slice_hash_table_test
@@ -1412,9 +1413,9 @@ pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc
 pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc
 
 ifeq ($(EMBED_OPENSSL),true)
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 else
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_core_stats.a $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libhttp2_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 endif
 
 
@@ -1738,6 +1739,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/server_early_return_test \
   $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test \
   $(BINDIR)/$(CONFIG)/server_request_call_test \
+  $(BINDIR)/$(CONFIG)/service_config_end2end_test \
   $(BINDIR)/$(CONFIG)/service_config_test \
   $(BINDIR)/$(CONFIG)/shutdown_test \
   $(BINDIR)/$(CONFIG)/slice_hash_table_test \
@@ -1885,6 +1887,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/server_early_return_test \
   $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test \
   $(BINDIR)/$(CONFIG)/server_request_call_test \
+  $(BINDIR)/$(CONFIG)/service_config_end2end_test \
   $(BINDIR)/$(CONFIG)/service_config_test \
   $(BINDIR)/$(CONFIG)/shutdown_test \
   $(BINDIR)/$(CONFIG)/slice_hash_table_test \
@@ -2407,6 +2410,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test || ( echo test server_interceptors_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing server_request_call_test"
 	$(Q) $(BINDIR)/$(CONFIG)/server_request_call_test || ( echo test server_request_call_test failed ; exit 1 )
+	$(E) "[RUN]     Testing service_config_end2end_test"
+	$(Q) $(BINDIR)/$(CONFIG)/service_config_end2end_test || ( echo test service_config_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing service_config_test"
 	$(Q) $(BINDIR)/$(CONFIG)/service_config_test || ( echo test service_config_test failed ; exit 1 )
 	$(E) "[RUN]     Testing shutdown_test"
@@ -3774,6 +3779,7 @@ LIBGRPC_SRC = \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \
+    src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc \
     src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \
     src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \
     src/core/ext/filters/census/grpc_context.cc \
@@ -5118,6 +5124,7 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \
+    src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc \
     src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \
     src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \
     src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \
@@ -5295,6 +5302,55 @@ endif
 endif
 
 
+LIBDNS_TEST_UTIL_SRC = \
+    test/cpp/naming/dns_test_util.cc \
+
+PUBLIC_HEADERS_CXX += \
+
+LIBDNS_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBDNS_TEST_UTIL_SRC))))
+
+
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure libraries if you don't have OpenSSL.
+
+$(LIBDIR)/$(CONFIG)/libdns_test_util.a: openssl_dep_error
+
+
+else
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
+
+$(LIBDIR)/$(CONFIG)/libdns_test_util.a: protobuf_dep_error
+
+
+else
+
+$(LIBDIR)/$(CONFIG)/libdns_test_util.a: $(ZLIB_DEP) $(OPENSSL_DEP) $(CARES_DEP) $(ADDRESS_SORTING_DEP) $(PROTOBUF_DEP) $(LIBDNS_TEST_UTIL_OBJS) 
+	$(E) "[AR]      Creating $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libdns_test_util.a
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDNS_TEST_UTIL_OBJS) 
+ifeq ($(SYSTEM),Darwin)
+	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libdns_test_util.a
+endif
+
+
+
+
+endif
+
+endif
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(LIBDNS_TEST_UTIL_OBJS:.o=.dep)
+endif
+endif
+
+
 LIBGRPC++_SRC = \
     src/cpp/client/insecure_credentials.cc \
     src/cpp/client/secure_credentials.cc \
@@ -5393,6 +5449,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/alarm.h \
     include/grpcpp/alarm_impl.h \
     include/grpcpp/channel.h \
+    include/grpcpp/channel_impl.h \
     include/grpcpp/client_context.h \
     include/grpcpp/completion_queue.h \
     include/grpcpp/create_channel.h \
@@ -5554,6 +5611,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/client_interceptor.h \
     include/grpcpp/impl/codegen/client_unary_call.h \
     include/grpcpp/impl/codegen/completion_queue.h \
+    include/grpcpp/impl/codegen/completion_queue_impl.h \
     include/grpcpp/impl/codegen/completion_queue_tag.h \
     include/grpcpp/impl/codegen/config.h \
     include/grpcpp/impl/codegen/core_codegen_interface.h \
@@ -6016,6 +6074,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/alarm.h \
     include/grpcpp/alarm_impl.h \
     include/grpcpp/channel.h \
+    include/grpcpp/channel_impl.h \
     include/grpcpp/client_context.h \
     include/grpcpp/completion_queue.h \
     include/grpcpp/create_channel.h \
@@ -6177,6 +6236,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/client_interceptor.h \
     include/grpcpp/impl/codegen/client_unary_call.h \
     include/grpcpp/impl/codegen/completion_queue.h \
+    include/grpcpp/impl/codegen/completion_queue_impl.h \
     include/grpcpp/impl/codegen/completion_queue_tag.h \
     include/grpcpp/impl/codegen/config.h \
     include/grpcpp/impl/codegen/core_codegen_interface.h \
@@ -6583,6 +6643,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/client_interceptor.h \
     include/grpcpp/impl/codegen/client_unary_call.h \
     include/grpcpp/impl/codegen/completion_queue.h \
+    include/grpcpp/impl/codegen/completion_queue_impl.h \
     include/grpcpp/impl/codegen/completion_queue_tag.h \
     include/grpcpp/impl/codegen/config.h \
     include/grpcpp/impl/codegen/core_codegen_interface.h \
@@ -6752,6 +6813,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/client_interceptor.h \
     include/grpcpp/impl/codegen/client_unary_call.h \
     include/grpcpp/impl/codegen/completion_queue.h \
+    include/grpcpp/impl/codegen/completion_queue_impl.h \
     include/grpcpp/impl/codegen/completion_queue_tag.h \
     include/grpcpp/impl/codegen/config.h \
     include/grpcpp/impl/codegen/core_codegen_interface.h \
@@ -6952,6 +7014,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/alarm.h \
     include/grpcpp/alarm_impl.h \
     include/grpcpp/channel.h \
+    include/grpcpp/channel_impl.h \
     include/grpcpp/client_context.h \
     include/grpcpp/completion_queue.h \
     include/grpcpp/create_channel.h \
@@ -7113,6 +7176,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/client_interceptor.h \
     include/grpcpp/impl/codegen/client_unary_call.h \
     include/grpcpp/impl/codegen/completion_queue.h \
+    include/grpcpp/impl/codegen/completion_queue_impl.h \
     include/grpcpp/impl/codegen/completion_queue_tag.h \
     include/grpcpp/impl/codegen/config.h \
     include/grpcpp/impl/codegen/core_codegen_interface.h \
@@ -18952,6 +19016,49 @@ endif
 $(OBJDIR)/$(CONFIG)/test/cpp/server/server_request_call_test.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc
 
 
+SERVICE_CONFIG_END2END_TEST_SRC = \
+    test/cpp/end2end/service_config_end2end_test.cc \
+
+SERVICE_CONFIG_END2END_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SERVICE_CONFIG_END2END_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/service_config_end2end_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+.
+
+$(BINDIR)/$(CONFIG)/service_config_end2end_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/service_config_end2end_test: $(PROTOBUF_DEP) $(SERVICE_CONFIG_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(SERVICE_CONFIG_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/service_config_end2end_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/cpp/end2end/service_config_end2end_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_service_config_end2end_test: $(SERVICE_CONFIG_END2END_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(SERVICE_CONFIG_END2END_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 SERVICE_CONFIG_TEST_SRC = \
     test/core/client_channel/service_config_test.cc \
 
@@ -21334,16 +21441,16 @@ $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/resolver_component_test_unsecure: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure
+	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test_unsecure
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpc_unsecure.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_resolver_component_test_unsecure: $(RESOLVER_COMPONENT_TEST_UNSECURE_OBJS:.o=.dep)
 
@@ -21377,16 +21484,16 @@ $(BINDIR)/$(CONFIG)/resolver_component_test: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/resolver_component_test: $(PROTOBUF_DEP) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test
+	$(Q) $(LDXX) $(LDFLAGS) $(RESOLVER_COMPONENT_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/resolver_component_test
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/naming/resolver_component_test.o:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_resolver_component_test: $(RESOLVER_COMPONENT_TEST_OBJS:.o=.dep)
 
@@ -21592,16 +21699,16 @@ $(BINDIR)/$(CONFIG)/cancel_ares_query_test: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/cancel_ares_query_test: $(PROTOBUF_DEP) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(BINDIR)/$(CONFIG)/cancel_ares_query_test: $(PROTOBUF_DEP) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/cancel_ares_query_test
+	$(Q) $(LDXX) $(LDFLAGS) $(CANCEL_ARES_QUERY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/cancel_ares_query_test
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/naming/cancel_ares_query_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+$(OBJDIR)/$(CONFIG)/test/cpp/naming/cancel_ares_query_test.o:  $(LIBDIR)/$(CONFIG)/libdns_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
 
 deps_cancel_ares_query_test: $(CANCEL_ARES_QUERY_TEST_OBJS:.o=.dep)
 
@@ -22231,6 +22338,7 @@ test/cpp/interop/interop_server.cc: $(OPENSSL_DEP)
 test/cpp/interop/interop_server_bootstrap.cc: $(OPENSSL_DEP)
 test/cpp/interop/server_helper.cc: $(OPENSSL_DEP)
 test/cpp/microbenchmarks/helpers.cc: $(OPENSSL_DEP)
+test/cpp/naming/dns_test_util.cc: $(OPENSSL_DEP)
 test/cpp/qps/benchmark_config.cc: $(OPENSSL_DEP)
 test/cpp/qps/client_async.cc: $(OPENSSL_DEP)
 test/cpp/qps/client_callback.cc: $(OPENSSL_DEP)

bazel/BUILD

@@ -17,15 +17,3 @@ licenses(["notice"])  # Apache v2
 package(default_visibility = ["//:__subpackages__"])
 
 load(":cc_grpc_library.bzl", "cc_grpc_library")
-
-proto_library(
-    name = "well_known_protos_list",
-    srcs = ["@com_google_protobuf//:well_known_protos"],
-)
-
-cc_grpc_library(
-    name = "well_known_protos",
-    srcs = "well_known_protos_list",
-    proto_only = True,
-    deps = [],
-)

bazel/cc_grpc_library.bzl

@@ -1,71 +1,105 @@
 """Generates and compiles C++ grpc stubs from proto_library rules."""
 
 load("//bazel:generate_cc.bzl", "generate_cc")
+load("//bazel:protobuf.bzl", "well_known_proto_libs")
 
-def cc_grpc_library(name, srcs, deps, proto_only, well_known_protos, generate_mocks = False, use_external = False, **kwargs):
-  """Generates C++ grpc classes from a .proto file.
+def cc_grpc_library(
+        name,
+        srcs,
+        deps,
+        proto_only = False,
+        well_known_protos = False,
+        generate_mocks = False,
+        use_external = False,
+        grpc_only = False,
+        **kwargs):
+    """Generates C++ grpc classes for services defined in a proto file.
 
-  Assumes the generated classes will be used in cc_api_version = 2.
+    If grpc_only is True, this rule is compatible with proto_library and
+    cc_proto_library native rules such that it expects proto_library target
+    as srcs argument and generates only grpc library classes, expecting
+    protobuf messages classes library (cc_proto_library target) to be passed in
+    deps argument. By default grpc_only is False which makes this rule to behave
+    in a backwards-compatible mode (trying to generate both proto and grpc
+    classes).
 
-  Arguments:
-      name: name of rule.
-      srcs: a single proto_library, which wraps the .proto files with services.
-      deps: a list of C++ proto_library (or cc_proto_library) which provides
-        the compiled code of any message that the services depend on.
-      well_known_protos: Should this library additionally depend on well known
-        protos
-      use_external: When True the grpc deps are prefixed with //external. This
-        allows grpc to be used as a dependency in other bazel projects.
-      generate_mocks: When True, Google Mock code for client stub is generated.
-      **kwargs: rest of arguments, e.g., compatible_with and visibility.
-  """
-  if len(srcs) > 1:
-    fail("Only one srcs value supported", "srcs")
+    Assumes the generated classes will be used in cc_api_version = 2.
 
-  proto_target = "_" + name + "_only"
-  codegen_target = "_" + name + "_codegen"
-  codegen_grpc_target = "_" + name + "_grpc_codegen"
-  proto_deps = ["_" + dep + "_only" for dep in deps if dep.find(':') == -1]
-  proto_deps += [dep.split(':')[0] + ':' + "_" + dep.split(':')[1] + "_only" for dep in deps if dep.find(':') != -1]
+    Args:
+        name (str): Name of rule.
+        srcs (list): A single .proto file which contains services definitions,
+          or if grpc_only parameter is True, a single proto_library which
+          contains services descriptors.
+        deps (list): A list of C++ proto_library (or cc_proto_library) which
+          provides the compiled code of any message that the services depend on.
+        proto_only (bool): If True, create only C++ proto classes library,
+          avoid creating C++ grpc classes library (expect it in deps).
+          Deprecated, use native cc_proto_library instead. False by default.
+        well_known_protos (bool): Should this library additionally depend on
+          well known protos. Deprecated, the well known protos should be
+          specified as explicit dependencies of the proto_library target
+          (passed in srcs parameter) instead. False by default.
+        generate_mocks (bool): when True, Google Mock code for client stub is
+          generated. False by default.
+        use_external (bool): Not used.
+        grpc_only (bool): if True, generate only grpc library, expecting
+          protobuf messages library (cc_proto_library target) to be passed as
+          deps. False by default (will become True by default eventually).
+        **kwargs: rest of arguments, e.g., compatible_with and visibility
+    """
+    if len(srcs) > 1:
+        fail("Only one srcs value supported", "srcs")
+    if grpc_only and proto_only:
+        fail("A mutualy exclusive configuration is specified: grpc_only = True and proto_only = True")
 
-  native.proto_library(
-      name = proto_target,
-      srcs = srcs,
-      deps = proto_deps,
-      **kwargs
-  )
+    extra_deps = []
+    proto_targets = []
 
-  generate_cc(
-      name = codegen_target,
-      srcs = [proto_target],
-      well_known_protos = well_known_protos,
-      **kwargs
-  )
+    if not grpc_only:
+        proto_target = "_" + name + "_only"
+        cc_proto_target = name if proto_only else "_" + name + "_cc_proto"
 
-  if not proto_only:
-    plugin = "@com_github_grpc_grpc//:grpc_cpp_plugin"
-    generate_cc(
-        name = codegen_grpc_target,
-        srcs = [proto_target],
-        plugin = plugin,
-        well_known_protos = well_known_protos,
-        generate_mocks = generate_mocks,
-        **kwargs
-    )
-    grpc_deps  = ["@com_github_grpc_grpc//:grpc++_codegen_proto",
-                  "//external:protobuf"]
-    native.cc_library(
-        name = name,
-        srcs = [":" + codegen_grpc_target, ":" + codegen_target],
-        hdrs = [":" + codegen_grpc_target, ":" + codegen_target],
-        deps = deps + grpc_deps,
-        **kwargs
-    )
-  else:
-    native.cc_library(
-        name = name,
-        srcs = [":" + codegen_target],
-        hdrs = [":" + codegen_target],
-        deps = deps + ["//external:protobuf"],
-        **kwargs
-    )
+        proto_deps = ["_" + dep + "_only" for dep in deps if dep.find(":") == -1]
+        proto_deps += [dep.split(":")[0] + ":" + "_" + dep.split(":")[1] + "_only" for dep in deps if dep.find(":") != -1]
+        if well_known_protos:
+            proto_deps += well_known_proto_libs()
+
+        native.proto_library(
+            name = proto_target,
+            srcs = srcs,
+            deps = proto_deps,
+            **kwargs
+        )
+
+        native.cc_proto_library(
+            name = cc_proto_target,
+            deps = [":" + proto_target],
+            **kwargs
+        )
+        extra_deps.append(":" + cc_proto_target)
+        proto_targets.append(proto_target)
+    else:
+        if not srcs:
+            fail("srcs cannot be empty", "srcs")
+        proto_targets += srcs
+
+    if not proto_only:
+        codegen_grpc_target = "_" + name + "_grpc_codegen"
+        generate_cc(
+            name = codegen_grpc_target,
+            srcs = proto_targets,
+            plugin = "@com_github_grpc_grpc//:grpc_cpp_plugin",
+            well_known_protos = well_known_protos,
+            generate_mocks = generate_mocks,
+            **kwargs
+        )
+
+        native.cc_library(
+            name = name,
+            srcs = [":" + codegen_grpc_target],
+            hdrs = [":" + codegen_grpc_target],
+            deps = deps +
+                   extra_deps +
+                   ["@com_github_grpc_grpc//:grpc++_codegen_proto"],
+            **kwargs
+        )

bazel/generate_cc.bzl

@@ -18,12 +18,22 @@ _GRPC_PROTO_MOCK_HEADER_FMT = "{}_mock.grpc.pb.h"
 _PROTO_HEADER_FMT = "{}.pb.h"
 _PROTO_SRC_FMT = "{}.pb.cc"
 
-def _strip_package_from_path(label_package, path):
+def _strip_package_from_path(label_package, file):
+    prefix_len = 0
+    if not file.is_source and file.path.startswith(file.root.path):
+        prefix_len = len(file.root.path) + 1
+
+    path = file.path
     if len(label_package) == 0:
         return path
-    if not path.startswith(label_package + "/"):
+    if not path.startswith(label_package + "/", prefix_len):
         fail("'{}' does not lie within '{}'.".format(path, label_package))
-    return path[len(label_package + "/"):]
+    return path[prefix_len + len(label_package + "/"):]
+
+def _get_srcs_file_path(file):
+    if not file.is_source and file.path.startswith(file.root.path):
+        return file.path[len(file.root.path) + 1:]
+    return file.path
 
 def _join_directories(directories):
     massaged_directories = [directory for directory in directories if len(directory) != 0]
@@ -31,7 +41,7 @@ def _join_directories(directories):
 
 def generate_cc_impl(ctx):
     """Implementation of the generate_cc rule."""
-    protos = [f for src in ctx.attr.srcs for f in src.proto.direct_sources]
+    protos = [f for src in ctx.attr.srcs for f in src.proto.check_deps_sources]
     includes = [
         f
         for src in ctx.attr.srcs
@@ -46,14 +56,14 @@ def generate_cc_impl(ctx):
     if ctx.executable.plugin:
         outs += [
             proto_path_to_generated_filename(
-                _strip_package_from_path(label_package, proto.path),
+                _strip_package_from_path(label_package, proto),
                 _GRPC_PROTO_HEADER_FMT,
             )
             for proto in protos
         ]
         outs += [
             proto_path_to_generated_filename(
-                _strip_package_from_path(label_package, proto.path),
+                _strip_package_from_path(label_package, proto),
                 _GRPC_PROTO_SRC_FMT,
             )
             for proto in protos
@@ -61,7 +71,7 @@ def generate_cc_impl(ctx):
         if ctx.attr.generate_mocks:
             outs += [
                 proto_path_to_generated_filename(
-                    _strip_package_from_path(label_package, proto.path),
+                    _strip_package_from_path(label_package, proto),
                     _GRPC_PROTO_MOCK_HEADER_FMT,
                 )
                 for proto in protos
@@ -69,14 +79,14 @@ def generate_cc_impl(ctx):
     else:
         outs += [
             proto_path_to_generated_filename(
-                _strip_package_from_path(label_package, proto.path),
+                _strip_package_from_path(label_package, proto),
                 _PROTO_HEADER_FMT,
             )
             for proto in protos
         ]
         outs += [
             proto_path_to_generated_filename(
-                _strip_package_from_path(label_package, proto.path),
+                _strip_package_from_path(label_package, proto),
                 _PROTO_SRC_FMT,
             )
             for proto in protos
@@ -102,7 +112,7 @@ def generate_cc_impl(ctx):
     # Include the output directory so that protoc puts the generated code in the
     # right directory.
     arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)]
-    arguments += [proto.path for proto in protos]
+    arguments += [_get_srcs_file_path(proto) for proto in protos]
 
     # create a list of well known proto files if the argument is non-None
     well_known_proto_files = []

bazel/protobuf.bzl

@@ -2,6 +2,22 @@
 
 _PROTO_EXTENSION = ".proto"
 
+def well_known_proto_libs():
+    return [
+        "@com_google_protobuf//:any_proto",
+        "@com_google_protobuf//:api_proto",
+        "@com_google_protobuf//:compiler_plugin_proto",
+        "@com_google_protobuf//:descriptor_proto",
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:empty_proto",
+        "@com_google_protobuf//:field_mask_proto",
+        "@com_google_protobuf//:source_context_proto",
+        "@com_google_protobuf//:struct_proto",
+        "@com_google_protobuf//:timestamp_proto",
+        "@com_google_protobuf//:type_proto",
+        "@com_google_protobuf//:wrappers_proto",
+    ]
+
 def get_proto_root(workspace_root):
     """Gets the root protobuf directory.
 
@@ -42,12 +58,16 @@ def proto_path_to_generated_filename(proto_path, fmt_str):
 
 def _get_include_directory(include):
     directory = include.path
-    if directory.startswith("external"):
-        external_separator = directory.find("/")
+    prefix_len = 0
+    if not include.is_source and directory.startswith(include.root.path):
+        prefix_len = len(include.root.path) + 1
+
+    if directory.startswith("external", prefix_len):
+        external_separator = directory.find("/", prefix_len)
         repository_separator = directory.find("/", external_separator + 1)
         return directory[:repository_separator]
     else:
-        return "."
+        return include.root.path if include.root.path else "."
 
 def get_include_protoc_args(includes):
     """Returns protoc args that imports protos relative to their import root.

bazel/python_rules.bzl

@@ -130,21 +130,6 @@ def _generate_py(well_known_protos, **kwargs):
     else:
         __generate_py(**kwargs)
 
-_WELL_KNOWN_PROTO_LIBS = [
-    "@com_google_protobuf//:any_proto",
-    "@com_google_protobuf//:api_proto",
-    "@com_google_protobuf//:compiler_plugin_proto",
-    "@com_google_protobuf//:descriptor_proto",
-    "@com_google_protobuf//:duration_proto",
-    "@com_google_protobuf//:empty_proto",
-    "@com_google_protobuf//:field_mask_proto",
-    "@com_google_protobuf//:source_context_proto",
-    "@com_google_protobuf//:struct_proto",
-    "@com_google_protobuf//:timestamp_proto",
-    "@com_google_protobuf//:type_proto",
-    "@com_google_protobuf//:wrappers_proto",
-]
-
 def py_proto_library(
         name,
         deps,
@@ -167,8 +152,6 @@ def py_proto_library(
     codegen_target = "_{}_codegen".format(name)
     codegen_grpc_target = "_{}_grpc_codegen".format(name)
 
-    well_known_proto_rules = _WELL_KNOWN_PROTO_LIBS if well_known_protos else []
-
     _generate_py(
         name = codegen_target,
         deps = deps,

build.yaml

@@ -13,8 +13,8 @@ settings:
   '#09': Per-language overrides are possible with (eg) ruby_version tag here
   '#10': See the expand_version.py for all the quirks here
   core_version: 7.0.0
-  g_stands_for: gandalf
-  version: 1.21.0-dev
+  g_stands_for: gale
+  version: 1.22.0-dev
 filegroups:
 - name: alts_proto
   headers:
@@ -525,6 +525,7 @@ filegroups:
   - src/core/lib/slice/slice_hash_table.h
   - src/core/lib/slice/slice_internal.h
   - src/core/lib/slice/slice_string_helpers.h
+  - src/core/lib/slice/slice_utils.h
   - src/core/lib/slice/slice_weak_hash_table.h
   - src/core/lib/surface/api_trace.h
   - src/core/lib/surface/call.h
@@ -797,6 +798,7 @@ filegroups:
   uses:
   - grpc_base
   - grpc_client_channel
+  - grpc_resolver_dns_selection
 - name: grpc_resolver_dns_native
   src:
   - src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc
@@ -804,6 +806,14 @@ filegroups:
   uses:
   - grpc_base
   - grpc_client_channel
+  - grpc_resolver_dns_selection
+- name: grpc_resolver_dns_selection
+  headers:
+  - src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h
+  src:
+  - src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc
+  uses:
+  - grpc_base
 - name: grpc_resolver_fake
   headers:
   - src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h
@@ -1254,6 +1264,7 @@ filegroups:
   - include/grpcpp/impl/codegen/client_interceptor.h
   - include/grpcpp/impl/codegen/client_unary_call.h
   - include/grpcpp/impl/codegen/completion_queue.h
+  - include/grpcpp/impl/codegen/completion_queue_impl.h
   - include/grpcpp/impl/codegen/completion_queue_tag.h
   - include/grpcpp/impl/codegen/config.h
   - include/grpcpp/impl/codegen/core_codegen_interface.h
@@ -1351,6 +1362,7 @@ filegroups:
   - include/grpcpp/alarm.h
   - include/grpcpp/alarm_impl.h
   - include/grpcpp/channel.h
+  - include/grpcpp/channel_impl.h
   - include/grpcpp/client_context.h
   - include/grpcpp/completion_queue.h
   - include/grpcpp/create_channel.h
@@ -1679,6 +1691,13 @@ libs:
   - grpc_test_util
   - grpc
   - gpr
+- name: dns_test_util
+  build: private
+  language: c++
+  headers:
+  - test/cpp/naming/dns_test_util.h
+  src:
+  - test/cpp/naming/dns_test_util.cc
 - name: grpc++
   build: all
   language: c++
@@ -5539,6 +5558,18 @@ targets:
   - grpc++_unsecure
   - grpc_unsecure
   - gpr
+- name: service_config_end2end_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/cpp/end2end/service_config_end2end_test.cc
+  deps:
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr
 - name: service_config_test
   gtest: true
   build: test

config.m4

@@ -412,6 +412,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \
     src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \
+    src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc \
     src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \
     src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \
     src/core/ext/filters/census/grpc_context.cc \
@@ -695,6 +696,7 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/pick_first)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/round_robin)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/xds)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns/c_ares)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns/native)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/fake)

config.w32

@@ -387,6 +387,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_libuv_windows.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_posix.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_windows.cc " +
+    "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\dns_resolver_selection.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\native\\dns_resolver.cc " +
     "src\\core\\ext\\filters\\client_channel\\resolver\\sockaddr\\sockaddr_resolver.cc " +
     "src\\core\\ext\\filters\\census\\grpc_context.cc " +

doc/g_stands_for.md

@@ -20,4 +20,5 @@
 - 1.18 'g' stands for ['goose'](https://github.com/grpc/grpc/tree/v1.18.x)
 - 1.19 'g' stands for ['gold'](https://github.com/grpc/grpc/tree/v1.19.x)
 - 1.20 'g' stands for ['godric'](https://github.com/grpc/grpc/tree/v1.20.x)
-- 1.21 'g' stands for ['gandalf'](https://github.com/grpc/grpc/tree/master)
+- 1.21 'g' stands for ['gandalf'](https://github.com/grpc/grpc/tree/v1.21.x)
+- 1.22 'g' stands for ['gale'](https://github.com/grpc/grpc/tree/master)

etc/roots.pem

@@ -4552,3 +4552,149 @@ Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh
 jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw
 3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0=
 -----END CERTIFICATE-----
+
+# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI
+# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI
+# Label: "emSign Root CA - G1"
+# Serial: 235931866688319308814040
+# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac
+# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c
+# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67
+-----BEGIN CERTIFICATE-----
+MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD
+VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU
+ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH
+MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO
+MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv
+Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz
+f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO
+8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq
+d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM
+tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt
+Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB
+o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD
+AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x
+PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM
+wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d
+GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH
+6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby
+RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx
+iN66zB+Afko=
+-----END CERTIFICATE-----
+
+# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI
+# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI
+# Label: "emSign ECC Root CA - G3"
+# Serial: 287880440101571086945156
+# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40
+# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1
+# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b
+-----BEGIN CERTIFICATE-----
+MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG
+EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo
+bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g
+RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ
+TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s
+b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw
+djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0
+WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS
+fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB
+zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
+hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB
+CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD
++JbNR6iC8hZVdyR+EhCVBCyj
+-----END CERTIFICATE-----
+
+# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI
+# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI
+# Label: "emSign Root CA - C1"
+# Serial: 825510296613316004955058
+# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68
+# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01
+# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f
+-----BEGIN CERTIFICATE-----
+MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG
+A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg
+SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw
+MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
+biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v
+dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ
+BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ
+HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH
+3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH
+GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c
+xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1
+aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq
+TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL
+BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87
+/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4
+kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG
+YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT
++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo
+WXzhriKi4gp6D/piq1JM4fHfyr6DDUI=
+-----END CERTIFICATE-----
+
+# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI
+# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI
+# Label: "emSign ECC Root CA - C3"
+# Serial: 582948710642506000014504
+# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5
+# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66
+# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3
+-----BEGIN CERTIFICATE-----
+MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG
+EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx
+IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw
+MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln
+biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND
+IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci
+MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti
+sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O
+BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
+Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c
+3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J
+0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ==
+-----END CERTIFICATE-----
+
+# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post
+# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post
+# Label: "Hongkong Post Root CA 3"
+# Serial: 46170865288971385588281144162979347873371282084
+# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0
+# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02
+# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6
+-----BEGIN CERTIFICATE-----
+MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL
+BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ
+SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n
+a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5
+NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT
+CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u
+Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
+AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO
+dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI
+VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV
+9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY
+2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY
+vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt
+bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb
+x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+
+l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK
+TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj
+Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP
+BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e
+i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw
+DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG
+7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk
+MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr
+gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk
+GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS
+3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm
+Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+
+l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c
+JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP
+L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa
+LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG
+mpv0
+-----END CERTIFICATE-----

examples/BUILD

@@ -17,6 +17,7 @@ licenses(["notice"])  # 3-clause BSD
 package(default_visibility = ["//visibility:public"])
 
 load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
+load("//bazel:cc_grpc_library.bzl", "cc_grpc_library")
 load("//bazel:python_rules.bzl", "py_proto_library")
 
 grpc_proto_library(
@@ -29,11 +30,25 @@ grpc_proto_library(
     srcs = ["protos/hellostreamingworld.proto"],
 )
 
-grpc_proto_library(
-    name = "helloworld",
+# The following three rules demonstrate the usage of the cc_grpc_library rule in
+# in a mode compatible with the native proto_library and cc_proto_library rules.
+proto_library(
+    name = "helloworld_proto",
     srcs = ["protos/helloworld.proto"],
 )
 
+cc_proto_library(
+    name = "helloworld_cc_proto",
+    deps = [":helloworld_proto"],
+)
+
+cc_grpc_library(
+    name = "helloworld_cc_grpc",
+    srcs = [":helloworld_proto"],
+    grpc_only = True,
+    deps = [":helloworld_cc_proto"],
+)
+
 grpc_proto_library(
     name = "route_guide",
     srcs = ["protos/route_guide.proto"],
@@ -59,7 +74,7 @@ cc_binary(
     srcs = ["cpp/helloworld/greeter_client.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -69,7 +84,7 @@ cc_binary(
     srcs = ["cpp/helloworld/greeter_async_client.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -79,7 +94,7 @@ cc_binary(
     srcs = ["cpp/helloworld/greeter_async_client2.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -89,7 +104,7 @@ cc_binary(
     srcs = ["cpp/helloworld/greeter_server.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -99,7 +114,7 @@ cc_binary(
     srcs = ["cpp/helloworld/greeter_async_server.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -109,7 +124,7 @@ cc_binary(
     srcs = ["cpp/metadata/greeter_client.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -119,7 +134,7 @@ cc_binary(
     srcs = ["cpp/metadata/greeter_server.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -129,7 +144,7 @@ cc_binary(
     srcs = ["cpp/load_balancing/greeter_client.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -139,7 +154,7 @@ cc_binary(
     srcs = ["cpp/load_balancing/greeter_server.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -149,7 +164,7 @@ cc_binary(
     srcs = ["cpp/compression/greeter_client.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
@@ -159,15 +174,17 @@ cc_binary(
     srcs = ["cpp/compression/greeter_server.cc"],
     defines = ["BAZEL_BUILD"],
     deps = [
-        ":helloworld",
+        ":helloworld_cc_grpc",
         "//:grpc++",
     ],
 )
 
 cc_binary(
     name = "keyvaluestore_client",
-    srcs = ["cpp/keyvaluestore/caching_interceptor.h",
-            "cpp/keyvaluestore/client.cc"],    
+    srcs = [
+        "cpp/keyvaluestore/caching_interceptor.h",
+        "cpp/keyvaluestore/client.cc",
+    ],
     defines = ["BAZEL_BUILD"],
     deps = [
         ":keyvaluestore",

gRPC-C++.podspec

@@ -23,7 +23,7 @@
 Pod::Spec.new do |s|
   s.name     = 'gRPC-C++'
   # TODO (mxyan): use version that match gRPC version when pod is stabilized
-  # version = '1.21.0-dev'
+  # version = '1.22.0-dev'
   version = '0.0.8-dev'
   s.version  = version
   s.summary  = 'gRPC C++ library'
@@ -31,7 +31,7 @@ Pod::Spec.new do |s|
   s.license  = 'Apache License, Version 2.0'
   s.authors  = { 'The gRPC contributors' => 'grpc-packages@google.com' }
 
-  grpc_version = '1.21.0-dev'
+  grpc_version = '1.22.0-dev'
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
@@ -82,6 +82,7 @@ Pod::Spec.new do |s|
     ss.source_files = 'include/grpcpp/alarm.h',
                       'include/grpcpp/alarm_impl.h',
                       'include/grpcpp/channel.h',
+                      'include/grpcpp/channel_impl.h',
                       'include/grpcpp/client_context.h',
                       'include/grpcpp/completion_queue.h',
                       'include/grpcpp/create_channel.h',
@@ -162,6 +163,7 @@ Pod::Spec.new do |s|
                       'include/grpcpp/impl/codegen/client_interceptor.h',
                       'include/grpcpp/impl/codegen/client_unary_call.h',
                       'include/grpcpp/impl/codegen/completion_queue.h',
+                      'include/grpcpp/impl/codegen/completion_queue_impl.h',
                       'include/grpcpp/impl/codegen/completion_queue_tag.h',
                       'include/grpcpp/impl/codegen/config.h',
                       'include/grpcpp/impl/codegen/core_codegen_interface.h',
@@ -518,6 +520,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/slice/slice_hash_table.h',
                       'src/core/lib/slice/slice_internal.h',
                       'src/core/lib/slice/slice_string_helpers.h',
+                      'src/core/lib/slice/slice_utils.h',
                       'src/core/lib/slice/slice_weak_hash_table.h',
                       'src/core/lib/surface/api_trace.h',
                       'src/core/lib/surface/call.h',
@@ -563,6 +566,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.h',
+                      'src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h',
                       'src/core/ext/filters/max_age/max_age_filter.h',
                       'src/core/ext/filters/message_size/message_size_filter.h',
                       'src/core/ext/filters/http/client_authority_filter.h',
@@ -721,6 +725,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/slice/slice_hash_table.h',
                               'src/core/lib/slice/slice_internal.h',
                               'src/core/lib/slice/slice_string_helpers.h',
+                              'src/core/lib/slice/slice_utils.h',
                               'src/core/lib/slice/slice_weak_hash_table.h',
                               'src/core/lib/surface/api_trace.h',
                               'src/core/lib/surface/call.h',

gRPC-Core.podspec

@@ -22,7 +22,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '1.21.0-dev'
+  version = '1.22.0-dev'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'https://grpc.io'
@@ -494,6 +494,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/slice/slice_hash_table.h',
                       'src/core/lib/slice/slice_internal.h',
                       'src/core/lib/slice/slice_string_helpers.h',
+                      'src/core/lib/slice/slice_utils.h',
                       'src/core/lib/slice/slice_weak_hash_table.h',
                       'src/core/lib/surface/api_trace.h',
                       'src/core/lib/surface/call.h',
@@ -539,6 +540,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.h',
+                      'src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h',
                       'src/core/ext/filters/max_age/max_age_filter.h',
                       'src/core/ext/filters/message_size/message_size_filter.h',
                       'src/core/ext/filters/http/client_authority_filter.h',
@@ -869,6 +871,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc',
                       'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc',
+                      'src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc',
                       'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc',
                       'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc',
                       'src/core/ext/filters/census/grpc_context.cc',
@@ -1145,6 +1148,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/slice/slice_hash_table.h',
                               'src/core/lib/slice/slice_internal.h',
                               'src/core/lib/slice/slice_string_helpers.h',
+                              'src/core/lib/slice/slice_utils.h',
                               'src/core/lib/slice/slice_weak_hash_table.h',
                               'src/core/lib/surface/api_trace.h',
                               'src/core/lib/surface/call.h',
@@ -1190,6 +1194,7 @@ Pod::Spec.new do |s|
                               'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h',
                               'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h',
                               'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.h',
+                              'src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h',
                               'src/core/ext/filters/max_age/max_age_filter.h',
                               'src/core/ext/filters/message_size/message_size_filter.h',
                               'src/core/ext/filters/http/client_authority_filter.h',

gRPC-ProtoRPC.podspec

@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '1.21.0-dev'
+  version = '1.22.0-dev'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'https://grpc.io'

gRPC-RxLibrary.podspec

@@ -21,7 +21,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '1.21.0-dev'
+  version = '1.22.0-dev'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'https://grpc.io'

gRPC.podspec

@@ -20,7 +20,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '1.21.0-dev'
+  version = '1.22.0-dev'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'https://grpc.io'

grpc.gemspec

@@ -428,6 +428,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/slice/slice_hash_table.h )
   s.files += %w( src/core/lib/slice/slice_internal.h )
   s.files += %w( src/core/lib/slice/slice_string_helpers.h )
+  s.files += %w( src/core/lib/slice/slice_utils.h )
   s.files += %w( src/core/lib/slice/slice_weak_hash_table.h )
   s.files += %w( src/core/lib/surface/api_trace.h )
   s.files += %w( src/core/lib/surface/call.h )
@@ -473,6 +474,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.h )
+  s.files += %w( src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h )
   s.files += %w( src/core/ext/filters/max_age/max_age_filter.h )
   s.files += %w( src/core/ext/filters/message_size/message_size_filter.h )
   s.files += %w( src/core/ext/filters/http/client_authority_filter.h )
@@ -806,6 +808,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc )
+  s.files += %w( src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc )
   s.files += %w( src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc )
   s.files += %w( src/core/ext/filters/census/grpc_context.cc )

grpc.gyp

@@ -594,6 +594,7 @@
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc',
+        'src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc',
         'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc',
         'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc',
         'src/core/ext/filters/census/grpc_context.cc',
@@ -1354,6 +1355,7 @@
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc',
         'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc',
+        'src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc',
         'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc',
         'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc',
         'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc',
@@ -1405,6 +1407,15 @@
         'test/core/util/test_tcp_server.cc',
       ],
     },
+    {
+      'target_name': 'dns_test_util',
+      'type': 'static_library',
+      'dependencies': [
+      ],
+      'sources': [
+        'test/cpp/naming/dns_test_util.cc',
+      ],
+    },
     {
       'target_name': 'grpc++',
       'type': 'static_library',

include/grpc/impl/codegen/grpc_types.h

@@ -359,10 +359,12 @@ typedef struct {
  * load balancing policy. Note that this only works with the "ares"
  * DNS resolver, and isn't supported by the "native" DNS resolver. */
 #define GRPC_ARG_DNS_ENABLE_SRV_QUERIES "grpc.dns_enable_srv_queries"
-/** If set, determines the number of milliseconds that the c-ares based
- * DNS resolver will wait on queries before cancelling them. The default value
- * is 10000. Setting this to "0" will disable c-ares query timeouts
- * entirely. */
+/** If set, determines an upper bound on the number of milliseconds that the
+ * c-ares based DNS resolver will wait on queries before cancelling them.
+ * The default value is 120,000. Setting this to "0" will disable the
+ * overall timeout entirely. Note that this doesn't include internal c-ares
+ * timeouts/backoff/retry logic, and so the actual DNS resolution may time out
+ * sooner than the value specified here. */
 #define GRPC_ARG_DNS_ARES_QUERY_TIMEOUT_MS "grpc.dns_ares_query_timeout"
 /** If set, uses a local subchannel pool within the channel. Otherwise, uses the
  * global subchannel pool. */

include/grpcpp/channel.h

@@ -19,21 +19,12 @@
 #ifndef GRPCPP_CHANNEL_H
 #define GRPCPP_CHANNEL_H
 
-#include <memory>
-#include <mutex>
-
-#include <grpc/grpc.h>
-#include <grpcpp/impl/call.h>
-#include <grpcpp/impl/codegen/channel_interface.h>
-#include <grpcpp/impl/codegen/client_interceptor.h>
-#include <grpcpp/impl/codegen/config.h>
-#include <grpcpp/impl/codegen/grpc_library.h>
-#include <grpcpp/impl/codegen/sync.h>
-
-struct grpc_channel;
+#include <grpcpp/channel_impl.h>
 
 namespace grpc {
 
+typedef ::grpc_impl::Channel Channel;
+
 namespace experimental {
 /// Resets the channel's connection backoff.
 /// TODO(roth): Once we see whether this proves useful, either create a gRFC
@@ -41,75 +32,6 @@ namespace experimental {
 void ChannelResetConnectionBackoff(Channel* channel);
 }  // namespace experimental
 
-/// Channels represent a connection to an endpoint. Created by \a CreateChannel.
-class Channel final : public ChannelInterface,
-                      public internal::CallHook,
-                      public std::enable_shared_from_this<Channel>,
-                      private GrpcLibraryCodegen {
- public:
-  ~Channel();
-
-  /// Get the current channel state. If the channel is in IDLE and
-  /// \a try_to_connect is set to true, try to connect.
-  grpc_connectivity_state GetState(bool try_to_connect) override;
-
-  /// Returns the LB policy name, or the empty string if not yet available.
-  grpc::string GetLoadBalancingPolicyName() const;
-
-  /// Returns the service config in JSON form, or the empty string if
-  /// not available.
-  grpc::string GetServiceConfigJSON() const;
-
- private:
-  template <class InputMessage, class OutputMessage>
-  friend class internal::BlockingUnaryCallImpl;
-  friend void experimental::ChannelResetConnectionBackoff(Channel* channel);
-  friend std::shared_ptr<Channel> CreateChannelInternal(
-      const grpc::string& host, grpc_channel* c_channel,
-      std::vector<
-          std::unique_ptr<experimental::ClientInterceptorFactoryInterface>>
-          interceptor_creators);
-  friend class internal::InterceptedChannel;
-  Channel(const grpc::string& host, grpc_channel* c_channel,
-          std::vector<
-              std::unique_ptr<experimental::ClientInterceptorFactoryInterface>>
-              interceptor_creators);
-
-  internal::Call CreateCall(const internal::RpcMethod& method,
-                            ClientContext* context,
-                            CompletionQueue* cq) override;
-  void PerformOpsOnCall(internal::CallOpSetInterface* ops,
-                        internal::Call* call) override;
-  void* RegisterMethod(const char* method) override;
-
-  void NotifyOnStateChangeImpl(grpc_connectivity_state last_observed,
-                               gpr_timespec deadline, CompletionQueue* cq,
-                               void* tag) override;
-  bool WaitForStateChangeImpl(grpc_connectivity_state last_observed,
-                              gpr_timespec deadline) override;
-
-  CompletionQueue* CallbackCQ() override;
-
-  internal::Call CreateCallInternal(const internal::RpcMethod& method,
-                                    ClientContext* context, CompletionQueue* cq,
-                                    size_t interceptor_pos) override;
-
-  const grpc::string host_;
-  grpc_channel* const c_channel_;  // owned
-
-  // mu_ protects callback_cq_ (the per-channel callbackable completion queue)
-  grpc::internal::Mutex mu_;
-
-  // callback_cq_ references the callbackable completion queue associated
-  // with this channel (if any). It is set on the first call to CallbackCQ().
-  // It is _not owned_ by the channel; ownership belongs with its internal
-  // shutdown callback tag (invoked when the CQ is fully shutdown).
-  CompletionQueue* callback_cq_ = nullptr;
-
-  std::vector<std::unique_ptr<experimental::ClientInterceptorFactoryInterface>>
-      interceptor_creators_;
-};
-
 }  // namespace grpc
 
 #endif  // GRPCPP_CHANNEL_H

include/grpcpp/channel_impl.h

@@ -0,0 +1,125 @@
+/*
+ *
+ * Copyright 2015 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCPP_CHANNEL_IMPL_H
+#define GRPCPP_CHANNEL_IMPL_H
+
+#include <memory>
+#include <mutex>
+
+#include <grpc/grpc.h>
+#include <grpcpp/impl/call.h>
+#include <grpcpp/impl/codegen/channel_interface.h>
+#include <grpcpp/impl/codegen/client_interceptor.h>
+#include <grpcpp/impl/codegen/completion_queue.h>
+#include <grpcpp/impl/codegen/config.h>
+#include <grpcpp/impl/codegen/grpc_library.h>
+#include <grpcpp/impl/codegen/sync.h>
+
+struct grpc_channel;
+
+namespace grpc {
+
+std::shared_ptr<::grpc_impl::Channel> CreateChannelInternal(
+    const grpc::string& host, grpc_channel* c_channel,
+    std::vector<
+        std::unique_ptr<experimental::ClientInterceptorFactoryInterface>>
+        interceptor_creators);
+}  // namespace grpc
+namespace grpc_impl {
+
+namespace experimental {
+/// Resets the channel's connection backoff.
+/// TODO(roth): Once we see whether this proves useful, either create a gRFC
+/// and change this to be a method of the Channel class, or remove it.
+void ChannelResetConnectionBackoff(Channel* channel);
+}  // namespace experimental
+
+/// Channels represent a connection to an endpoint. Created by \a CreateChannel.
+class Channel final : public ::grpc::ChannelInterface,
+                      public ::grpc::internal::CallHook,
+                      public std::enable_shared_from_this<Channel>,
+                      private ::grpc::GrpcLibraryCodegen {
+ public:
+  ~Channel();
+
+  /// Get the current channel state. If the channel is in IDLE and
+  /// \a try_to_connect is set to true, try to connect.
+  grpc_connectivity_state GetState(bool try_to_connect) override;
+
+  /// Returns the LB policy name, or the empty string if not yet available.
+  grpc::string GetLoadBalancingPolicyName() const;
+
+  /// Returns the service config in JSON form, or the empty string if
+  /// not available.
+  grpc::string GetServiceConfigJSON() const;
+
+ private:
+  template <class InputMessage, class OutputMessage>
+  friend class ::grpc::internal::BlockingUnaryCallImpl;
+  friend void experimental::ChannelResetConnectionBackoff(Channel* channel);
+  friend std::shared_ptr<Channel> grpc::CreateChannelInternal(
+      const grpc::string& host, grpc_channel* c_channel,
+      std::vector<std::unique_ptr<
+          ::grpc::experimental::ClientInterceptorFactoryInterface>>
+          interceptor_creators);
+  friend class ::grpc::internal::InterceptedChannel;
+  Channel(const grpc::string& host, grpc_channel* c_channel,
+          std::vector<std::unique_ptr<
+              ::grpc::experimental::ClientInterceptorFactoryInterface>>
+              interceptor_creators);
+
+  ::grpc::internal::Call CreateCall(const ::grpc::internal::RpcMethod& method,
+                                    ::grpc::ClientContext* context,
+                                    ::grpc::CompletionQueue* cq) override;
+  void PerformOpsOnCall(::grpc::internal::CallOpSetInterface* ops,
+                        ::grpc::internal::Call* call) override;
+  void* RegisterMethod(const char* method) override;
+
+  void NotifyOnStateChangeImpl(grpc_connectivity_state last_observed,
+                               gpr_timespec deadline,
+                               ::grpc::CompletionQueue* cq, void* tag) override;
+  bool WaitForStateChangeImpl(grpc_connectivity_state last_observed,
+                              gpr_timespec deadline) override;
+
+  ::grpc::CompletionQueue* CallbackCQ() override;
+
+  ::grpc::internal::Call CreateCallInternal(
+      const ::grpc::internal::RpcMethod& method, ::grpc::ClientContext* context,
+      ::grpc::CompletionQueue* cq, size_t interceptor_pos) override;
+
+  const grpc::string host_;
+  grpc_channel* const c_channel_;  // owned
+
+  // mu_ protects callback_cq_ (the per-channel callbackable completion queue)
+  grpc::internal::Mutex mu_;
+
+  // callback_cq_ references the callbackable completion queue associated
+  // with this channel (if any). It is set on the first call to CallbackCQ().
+  // It is _not owned_ by the channel; ownership belongs with its internal
+  // shutdown callback tag (invoked when the CQ is fully shutdown).
+  ::grpc::CompletionQueue* callback_cq_ = nullptr;
+
+  std::vector<
+      std::unique_ptr<::grpc::experimental::ClientInterceptorFactoryInterface>>
+      interceptor_creators_;
+};
+
+}  // namespace grpc_impl
+
+#endif  // GRPCPP_CHANNEL_IMPL_H

include/grpcpp/create_channel_impl.h

@@ -28,7 +28,6 @@
 #include <grpcpp/support/config.h>
 
 namespace grpc_impl {
-
 /// Create a new \a Channel pointing to \a target.
 ///
 /// \param target The URI of the endpoint to connect to.

include/grpcpp/generic/generic_stub_impl.h

@@ -29,12 +29,12 @@
 
 namespace grpc {
 
-class CompletionQueue;
 typedef ClientAsyncReaderWriter<ByteBuffer, ByteBuffer>
     GenericClientAsyncReaderWriter;
 typedef ClientAsyncResponseReader<ByteBuffer> GenericClientAsyncResponseReader;
 }  // namespace grpc
 namespace grpc_impl {
+class CompletionQueue;
 
 /// Generic stubs provide a type-unsafe interface to call gRPC methods
 /// by name.

include/grpcpp/impl/codegen/async_stream.h

@@ -28,8 +28,6 @@
 
 namespace grpc {
 
-class CompletionQueue;
-
 namespace internal {
 /// Common interface for all client side asynchronous streaming.
 class ClientAsyncStreamingInterface {

include/grpcpp/impl/codegen/async_unary_call.h

@@ -29,7 +29,6 @@
 
 namespace grpc {
 
-class CompletionQueue;
 extern CoreCodegenInterface* g_core_codegen_interface;
 
 /// An interface relevant for async client side unary RPCs (which send

include/grpcpp/impl/codegen/call.h

@@ -21,9 +21,11 @@
 #include <grpc/impl/codegen/grpc_types.h>
 #include <grpcpp/impl/codegen/call_hook.h>
 
-namespace grpc {
+namespace grpc_impl {
 class CompletionQueue;
+}
 
+namespace grpc {
 namespace experimental {
 class ClientRpcInfo;
 class ServerRpcInfo;
@@ -41,13 +43,13 @@ class Call final {
         call_(nullptr),
         max_receive_message_size_(-1) {}
   /** call is owned by the caller */
-  Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq)
+  Call(grpc_call* call, CallHook* call_hook, ::grpc_impl::CompletionQueue* cq)
       : call_hook_(call_hook),
         cq_(cq),
         call_(call),
         max_receive_message_size_(-1) {}
 
-  Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq,
+  Call(grpc_call* call, CallHook* call_hook, ::grpc_impl::CompletionQueue* cq,
        experimental::ClientRpcInfo* rpc_info)
       : call_hook_(call_hook),
         cq_(cq),
@@ -55,7 +57,7 @@ class Call final {
         max_receive_message_size_(-1),
         client_rpc_info_(rpc_info) {}
 
-  Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq,
+  Call(grpc_call* call, CallHook* call_hook, ::grpc_impl::CompletionQueue* cq,
        int max_receive_message_size, experimental::ServerRpcInfo* rpc_info)
       : call_hook_(call_hook),
         cq_(cq),
@@ -68,7 +70,7 @@ class Call final {
   }
 
   grpc_call* call() const { return call_; }
-  CompletionQueue* cq() const { return cq_; }
+  ::grpc_impl::CompletionQueue* cq() const { return cq_; }
 
   int max_receive_message_size() const { return max_receive_message_size_; }
 
@@ -82,7 +84,7 @@ class Call final {
 
  private:
   CallHook* call_hook_;
-  CompletionQueue* cq_;
+  ::grpc_impl::CompletionQueue* cq_;
   grpc_call* call_;
   int max_receive_message_size_;
   experimental::ClientRpcInfo* client_rpc_info_ = nullptr;

include/grpcpp/impl/codegen/call_op_set.h

@@ -48,7 +48,6 @@
 
 namespace grpc {
 
-class CompletionQueue;
 extern CoreCodegenInterface* g_core_codegen_interface;
 
 namespace internal {

include/grpcpp/impl/codegen/channel_interface.h

@@ -24,10 +24,13 @@
 #include <grpcpp/impl/codegen/status.h>
 #include <grpcpp/impl/codegen/time.h>
 
+namespace grpc_impl {
+class CompletionQueue;
+}
+
 namespace grpc {
 class ChannelInterface;
 class ClientContext;
-class CompletionQueue;
 
 template <class R>
 class ClientReader;
@@ -74,7 +77,7 @@ class ChannelInterface {
   /// deadline expires. \a GetState needs to called to get the current state.
   template <typename T>
   void NotifyOnStateChange(grpc_connectivity_state last_observed, T deadline,
-                           CompletionQueue* cq, void* tag) {
+                           ::grpc_impl::CompletionQueue* cq, void* tag) {
     TimePoint<T> deadline_tp(deadline);
     NotifyOnStateChangeImpl(last_observed, deadline_tp.raw_time(), cq, tag);
   }
@@ -127,13 +130,14 @@ class ChannelInterface {
   friend class ::grpc::internal::InterceptedChannel;
   virtual internal::Call CreateCall(const internal::RpcMethod& method,
                                     ClientContext* context,
-                                    CompletionQueue* cq) = 0;
+                                    ::grpc_impl::CompletionQueue* cq) = 0;
   virtual void PerformOpsOnCall(internal::CallOpSetInterface* ops,
                                 internal::Call* call) = 0;
   virtual void* RegisterMethod(const char* method) = 0;
   virtual void NotifyOnStateChangeImpl(grpc_connectivity_state last_observed,
                                        gpr_timespec deadline,
-                                       CompletionQueue* cq, void* tag) = 0;
+                                       ::grpc_impl::CompletionQueue* cq,
+                                       void* tag) = 0;
   virtual bool WaitForStateChangeImpl(grpc_connectivity_state last_observed,
                                       gpr_timespec deadline) = 0;
 
@@ -146,7 +150,7 @@ class ChannelInterface {
   // change (even though this is private and non-API)
   virtual internal::Call CreateCallInternal(const internal::RpcMethod& method,
                                             ClientContext* context,
-                                            CompletionQueue* cq,
+                                            ::grpc_impl::CompletionQueue* cq,
                                             size_t interceptor_pos) {
     return internal::Call();
   }
@@ -159,7 +163,7 @@ class ChannelInterface {
   // Returns nullptr (rather than being pure) since this is a post-1.0 method
   // and adding a new pure method to an interface would be a breaking change
   // (even though this is private and non-API)
-  virtual CompletionQueue* CallbackCQ() { return nullptr; }
+  virtual ::grpc_impl::CompletionQueue* CallbackCQ() { return nullptr; }
 };
 }  // namespace grpc
 

include/grpcpp/impl/codegen/client_callback.h

@@ -29,11 +29,13 @@
 #include <grpcpp/impl/codegen/core_codegen_interface.h>
 #include <grpcpp/impl/codegen/status.h>
 
+namespace grpc_impl {
+class Channel;
+}
+
 namespace grpc {
 
-class Channel;
 class ClientContext;
-class CompletionQueue;
 
 namespace internal {
 class RpcMethod;

include/grpcpp/impl/codegen/client_context.h

@@ -60,12 +60,12 @@ struct grpc_call;
 namespace grpc_impl {
 
 class CallCredentials;
+class Channel;
+class CompletionQueue;
 }  // namespace grpc_impl
 namespace grpc {
 
-class Channel;
 class ChannelInterface;
-class CompletionQueue;
 class ClientContext;
 
 namespace internal {
@@ -397,7 +397,7 @@ class ClientContext {
   friend class ::grpc::testing::InteropClientContextInspector;
   friend class ::grpc::internal::CallOpClientRecvStatus;
   friend class ::grpc::internal::CallOpRecvInitialMetadata;
-  friend class Channel;
+  friend class ::grpc_impl::Channel;
   template <class R>
   friend class ::grpc::ClientReader;
   template <class W>
@@ -430,7 +430,8 @@ class ClientContext {
   }
 
   grpc_call* call() const { return call_; }
-  void set_call(grpc_call* call, const std::shared_ptr<Channel>& channel);
+  void set_call(grpc_call* call,
+                const std::shared_ptr<::grpc_impl::Channel>& channel);
 
   experimental::ClientRpcInfo* set_client_rpc_info(
       const char* method, internal::RpcMethod::RpcType type,
@@ -463,7 +464,7 @@ class ClientContext {
   bool wait_for_ready_explicitly_set_;
   bool idempotent_;
   bool cacheable_;
-  std::shared_ptr<Channel> channel_;
+  std::shared_ptr<::grpc_impl::Channel> channel_;
   grpc::internal::Mutex mu_;
   grpc_call* call_;
   bool call_canceled_;

include/grpcpp/impl/codegen/client_interceptor.h

@@ -26,10 +26,14 @@
 #include <grpcpp/impl/codegen/rpc_method.h>
 #include <grpcpp/impl/codegen/string_ref.h>
 
+namespace grpc_impl {
+
+class Channel;
+}
+
 namespace grpc {
 
 class ClientContext;
-class Channel;
 
 namespace internal {
 class InterceptorBatchMethodsImpl;

include/grpcpp/impl/codegen/client_unary_call.h

@@ -27,9 +27,7 @@
 
 namespace grpc {
 
-class Channel;
 class ClientContext;
-class CompletionQueue;
 
 namespace internal {
 class RpcMethod;

include/grpcpp/impl/codegen/completion_queue.h

@@ -16,401 +16,15 @@
  *
  */
 
-/// A completion queue implements a concurrent producer-consumer queue, with
-/// two main API-exposed methods: \a Next and \a AsyncNext. These
-/// methods are the essential component of the gRPC C++ asynchronous API.
-/// There is also a \a Shutdown method to indicate that a given completion queue
-/// will no longer have regular events. This must be called before the
-/// completion queue is destroyed.
-/// All completion queue APIs are thread-safe and may be used concurrently with
-/// any other completion queue API invocation; it is acceptable to have
-/// multiple threads calling \a Next or \a AsyncNext on the same or different
-/// completion queues, or to call these methods concurrently with a \a Shutdown
-/// elsewhere.
-/// \remark{All other API calls on completion queue should be completed before
-/// a completion queue destructor is called.}
 #ifndef GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_H
 #define GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_H
 
-#include <grpc/impl/codegen/atm.h>
-#include <grpcpp/impl/codegen/completion_queue_tag.h>
-#include <grpcpp/impl/codegen/core_codegen_interface.h>
-#include <grpcpp/impl/codegen/grpc_library.h>
-#include <grpcpp/impl/codegen/status.h>
-#include <grpcpp/impl/codegen/time.h>
+#include <grpcpp/impl/codegen/completion_queue_impl.h>
 
-struct grpc_completion_queue;
-
-namespace grpc_impl {
-
-class Server;
-class ServerBuilder;
-}  // namespace grpc_impl
 namespace grpc {
 
-template <class R>
-class ClientReader;
-template <class W>
-class ClientWriter;
-template <class W, class R>
-class ClientReaderWriter;
-template <class R>
-class ServerReader;
-template <class W>
-class ServerWriter;
-namespace internal {
-template <class W, class R>
-class ServerReaderWriterBody;
-}  // namespace internal
-
-class Channel;
-class ChannelInterface;
-class ClientContext;
-class CompletionQueue;
-class ServerContext;
-class ServerInterface;
-
-namespace internal {
-class CompletionQueueTag;
-class RpcMethod;
-template <class ServiceType, class RequestType, class ResponseType>
-class RpcMethodHandler;
-template <class ServiceType, class RequestType, class ResponseType>
-class ClientStreamingHandler;
-template <class ServiceType, class RequestType, class ResponseType>
-class ServerStreamingHandler;
-template <class ServiceType, class RequestType, class ResponseType>
-class BidiStreamingHandler;
-template <class Streamer, bool WriteNeeded>
-class TemplatedBidiStreamingHandler;
-template <StatusCode code>
-class ErrorMethodHandler;
-template <class InputMessage, class OutputMessage>
-class BlockingUnaryCallImpl;
-template <class Op1, class Op2, class Op3, class Op4, class Op5, class Op6>
-class CallOpSet;
-}  // namespace internal
-
-extern CoreCodegenInterface* g_core_codegen_interface;
-
-/// A thin wrapper around \ref grpc_completion_queue (see \ref
-/// src/core/lib/surface/completion_queue.h).
-/// See \ref doc/cpp/perf_notes.md for notes on best practices for high
-/// performance servers.
-class CompletionQueue : private GrpcLibraryCodegen {
- public:
-  /// Default constructor. Implicitly creates a \a grpc_completion_queue
-  /// instance.
-  CompletionQueue()
-      : CompletionQueue(grpc_completion_queue_attributes{
-            GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING,
-            nullptr}) {}
-
-  /// Wrap \a take, taking ownership of the instance.
-  ///
-  /// \param take The completion queue instance to wrap. Ownership is taken.
-  explicit CompletionQueue(grpc_completion_queue* take);
-
-  /// Destructor. Destroys the owned wrapped completion queue / instance.
-  ~CompletionQueue() {
-    g_core_codegen_interface->grpc_completion_queue_destroy(cq_);
-  }
-
-  /// Tri-state return for AsyncNext: SHUTDOWN, GOT_EVENT, TIMEOUT.
-  enum NextStatus {
-    SHUTDOWN,   ///< The completion queue has been shutdown and fully-drained
-    GOT_EVENT,  ///< Got a new event; \a tag will be filled in with its
-                ///< associated value; \a ok indicating its success.
-    TIMEOUT     ///< deadline was reached.
-  };
-
-  /// Read from the queue, blocking until an event is available or the queue is
-  /// shutting down.
-  ///
-  /// \param tag [out] Updated to point to the read event's tag.
-  /// \param ok [out] true if read a successful event, false otherwise.
-  ///
-  /// Note that each tag sent to the completion queue (through RPC operations
-  /// or alarms) will be delivered out of the completion queue by a call to
-  /// Next (or a related method), regardless of whether the operation succeeded
-  /// or not. Success here means that this operation completed in the normal
-  /// valid manner.
-  ///
-  /// Server-side RPC request: \a ok indicates that the RPC has indeed
-  /// been started. If it is false, the server has been Shutdown
-  /// before this particular call got matched to an incoming RPC.
-  ///
-  /// Client-side StartCall/RPC invocation: \a ok indicates that the RPC is
-  /// going to go to the wire. If it is false, it not going to the wire. This
-  /// would happen if the channel is either permanently broken or
-  /// transiently broken but with the fail-fast option. (Note that async unary
-  /// RPCs don't post a CQ tag at this point, nor do client-streaming
-  /// or bidi-streaming RPCs that have the initial metadata corked option set.)
-  ///
-  /// Client-side Write, Client-side WritesDone, Server-side Write,
-  /// Server-side Finish, Server-side SendInitialMetadata (which is
-  /// typically included in Write or Finish when not done explicitly):
-  /// \a ok means that the data/metadata/status/etc is going to go to the
-  /// wire. If it is false, it not going to the wire because the call
-  /// is already dead (i.e., canceled, deadline expired, other side
-  /// dropped the channel, etc).
-  ///
-  /// Client-side Read, Server-side Read, Client-side
-  /// RecvInitialMetadata (which is typically included in Read if not
-  /// done explicitly): \a ok indicates whether there is a valid message
-  /// that got read. If not, you know that there are certainly no more
-  /// messages that can ever be read from this stream. For the client-side
-  /// operations, this only happens because the call is dead. For the
-  /// server-sider operation, though, this could happen because the client
-  /// has done a WritesDone already.
-  ///
-  /// Client-side Finish: \a ok should always be true
-  ///
-  /// Server-side AsyncNotifyWhenDone: \a ok should always be true
-  ///
-  /// Alarm: \a ok is true if it expired, false if it was canceled
-  ///
-  /// \return true if got an event, false if the queue is fully drained and
-  ///         shut down.
-  bool Next(void** tag, bool* ok) {
-    return (AsyncNextInternal(tag, ok,
-                              g_core_codegen_interface->gpr_inf_future(
-                                  GPR_CLOCK_REALTIME)) != SHUTDOWN);
-  }
-
-  /// Read from the queue, blocking up to \a deadline (or the queue's shutdown).
-  /// Both \a tag and \a ok are updated upon success (if an event is available
-  /// within the \a deadline).  A \a tag points to an arbitrary location usually
-  /// employed to uniquely identify an event.
-  ///
-  /// \param tag [out] Upon success, updated to point to the event's tag.
-  /// \param ok [out] Upon success, true if a successful event, false otherwise
-  ///        See documentation for CompletionQueue::Next for explanation of ok
-  /// \param deadline [in] How long to block in wait for an event.
-  ///
-  /// \return The type of event read.
-  template <typename T>
-  NextStatus AsyncNext(void** tag, bool* ok, const T& deadline) {
-    TimePoint<T> deadline_tp(deadline);
-    return AsyncNextInternal(tag, ok, deadline_tp.raw_time());
-  }
-
-  /// EXPERIMENTAL
-  /// First executes \a F, then reads from the queue, blocking up to
-  /// \a deadline (or the queue's shutdown).
-  /// Both \a tag and \a ok are updated upon success (if an event is available
-  /// within the \a deadline).  A \a tag points to an arbitrary location usually
-  /// employed to uniquely identify an event.
-  ///
-  /// \param f [in] Function to execute before calling AsyncNext on this queue.
-  /// \param tag [out] Upon success, updated to point to the event's tag.
-  /// \param ok [out] Upon success, true if read a regular event, false
-  /// otherwise.
-  /// \param deadline [in] How long to block in wait for an event.
-  ///
-  /// \return The type of event read.
-  template <typename T, typename F>
-  NextStatus DoThenAsyncNext(F&& f, void** tag, bool* ok, const T& deadline) {
-    CompletionQueueTLSCache cache = CompletionQueueTLSCache(this);
-    f();
-    if (cache.Flush(tag, ok)) {
-      return GOT_EVENT;
-    } else {
-      return AsyncNext(tag, ok, deadline);
-    }
-  }
-
-  /// Request the shutdown of the queue.
-  ///
-  /// \warning This method must be called at some point if this completion queue
-  /// is accessed with Next or AsyncNext. \a Next will not return false
-  /// until this method has been called and all pending tags have been drained.
-  /// (Likewise for \a AsyncNext returning \a NextStatus::SHUTDOWN .)
-  /// Only once either one of these methods does that (that is, once the queue
-  /// has been \em drained) can an instance of this class be destroyed.
-  /// Also note that applications must ensure that no work is enqueued on this
-  /// completion queue after this method is called.
-  void Shutdown();
-
-  /// Returns a \em raw pointer to the underlying \a grpc_completion_queue
-  /// instance.
-  ///
-  /// \warning Remember that the returned instance is owned. No transfer of
-  /// owership is performed.
-  grpc_completion_queue* cq() { return cq_; }
-
- protected:
-  /// Private constructor of CompletionQueue only visible to friend classes
-  CompletionQueue(const grpc_completion_queue_attributes& attributes) {
-    cq_ = g_core_codegen_interface->grpc_completion_queue_create(
-        g_core_codegen_interface->grpc_completion_queue_factory_lookup(
-            &attributes),
-        &attributes, NULL);
-    InitialAvalanching();  // reserve this for the future shutdown
-  }
-
- private:
-  // Friend synchronous wrappers so that they can access Pluck(), which is
-  // a semi-private API geared towards the synchronous implementation.
-  template <class R>
-  friend class ::grpc::ClientReader;
-  template <class W>
-  friend class ::grpc::ClientWriter;
-  template <class W, class R>
-  friend class ::grpc::ClientReaderWriter;
-  template <class R>
-  friend class ::grpc::ServerReader;
-  template <class W>
-  friend class ::grpc::ServerWriter;
-  template <class W, class R>
-  friend class ::grpc::internal::ServerReaderWriterBody;
-  template <class ServiceType, class RequestType, class ResponseType>
-  friend class ::grpc::internal::RpcMethodHandler;
-  template <class ServiceType, class RequestType, class ResponseType>
-  friend class ::grpc::internal::ClientStreamingHandler;
-  template <class ServiceType, class RequestType, class ResponseType>
-  friend class ::grpc::internal::ServerStreamingHandler;
-  template <class Streamer, bool WriteNeeded>
-  friend class ::grpc::internal::TemplatedBidiStreamingHandler;
-  template <StatusCode code>
-  friend class ::grpc::internal::ErrorMethodHandler;
-  friend class ::grpc_impl::Server;
-  friend class ::grpc::ServerContext;
-  friend class ::grpc::ServerInterface;
-  template <class InputMessage, class OutputMessage>
-  friend class ::grpc::internal::BlockingUnaryCallImpl;
-
-  // Friends that need access to constructor for callback CQ
-  friend class ::grpc::Channel;
-
-  // For access to Register/CompleteAvalanching
-  template <class Op1, class Op2, class Op3, class Op4, class Op5, class Op6>
-  friend class ::grpc::internal::CallOpSet;
-
-  /// EXPERIMENTAL
-  /// Creates a Thread Local cache to store the first event
-  /// On this completion queue queued from this thread.  Once
-  /// initialized, it must be flushed on the same thread.
-  class CompletionQueueTLSCache {
-   public:
-    CompletionQueueTLSCache(CompletionQueue* cq);
-    ~CompletionQueueTLSCache();
-    bool Flush(void** tag, bool* ok);
-
-   private:
-    CompletionQueue* cq_;
-    bool flushed_;
-  };
-
-  NextStatus AsyncNextInternal(void** tag, bool* ok, gpr_timespec deadline);
-
-  /// Wraps \a grpc_completion_queue_pluck.
-  /// \warning Must not be mixed with calls to \a Next.
-  bool Pluck(internal::CompletionQueueTag* tag) {
-    auto deadline =
-        g_core_codegen_interface->gpr_inf_future(GPR_CLOCK_REALTIME);
-    while (true) {
-      auto ev = g_core_codegen_interface->grpc_completion_queue_pluck(
-          cq_, tag, deadline, nullptr);
-      bool ok = ev.success != 0;
-      void* ignored = tag;
-      if (tag->FinalizeResult(&ignored, &ok)) {
-        GPR_CODEGEN_ASSERT(ignored == tag);
-        return ok;
-      }
-    }
-  }
-
-  /// Performs a single polling pluck on \a tag.
-  /// \warning Must not be mixed with calls to \a Next.
-  ///
-  /// TODO: sreek - This calls tag->FinalizeResult() even if the cq_ is already
-  /// shutdown. This is most likely a bug and if it is a bug, then change this
-  /// implementation to simple call the other TryPluck function with a zero
-  /// timeout. i.e:
-  ///      TryPluck(tag, gpr_time_0(GPR_CLOCK_REALTIME))
-  void TryPluck(internal::CompletionQueueTag* tag) {
-    auto deadline = g_core_codegen_interface->gpr_time_0(GPR_CLOCK_REALTIME);
-    auto ev = g_core_codegen_interface->grpc_completion_queue_pluck(
-        cq_, tag, deadline, nullptr);
-    if (ev.type == GRPC_QUEUE_TIMEOUT) return;
-    bool ok = ev.success != 0;
-    void* ignored = tag;
-    // the tag must be swallowed if using TryPluck
-    GPR_CODEGEN_ASSERT(!tag->FinalizeResult(&ignored, &ok));
-  }
-
-  /// Performs a single polling pluck on \a tag. Calls tag->FinalizeResult if
-  /// the pluck() was successful and returned the tag.
-  ///
-  /// This exects tag->FinalizeResult (if called) to return 'false' i.e expects
-  /// that the tag is internal not something that is returned to the user.
-  void TryPluck(internal::CompletionQueueTag* tag, gpr_timespec deadline) {
-    auto ev = g_core_codegen_interface->grpc_completion_queue_pluck(
-        cq_, tag, deadline, nullptr);
-    if (ev.type == GRPC_QUEUE_TIMEOUT || ev.type == GRPC_QUEUE_SHUTDOWN) {
-      return;
-    }
-
-    bool ok = ev.success != 0;
-    void* ignored = tag;
-    GPR_CODEGEN_ASSERT(!tag->FinalizeResult(&ignored, &ok));
-  }
-
-  /// Manage state of avalanching operations : completion queue tags that
-  /// trigger other completion queue operations. The underlying core completion
-  /// queue should not really shutdown until all avalanching operations have
-  /// been finalized. Note that we maintain the requirement that an avalanche
-  /// registration must take place before CQ shutdown (which must be maintained
-  /// elsewhere)
-  void InitialAvalanching() {
-    gpr_atm_rel_store(&avalanches_in_flight_, static_cast<gpr_atm>(1));
-  }
-  void RegisterAvalanching() {
-    gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_,
-                                 static_cast<gpr_atm>(1));
-  }
-  void CompleteAvalanching() {
-    if (gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_,
-                                     static_cast<gpr_atm>(-1)) == 1) {
-      g_core_codegen_interface->grpc_completion_queue_shutdown(cq_);
-    }
-  }
-
-  grpc_completion_queue* cq_;  // owned
-
-  gpr_atm avalanches_in_flight_;
-};
-
-/// A specific type of completion queue used by the processing of notifications
-/// by servers. Instantiated by \a ServerBuilder.
-class ServerCompletionQueue : public CompletionQueue {
- public:
-  bool IsFrequentlyPolled() { return polling_type_ != GRPC_CQ_NON_LISTENING; }
-
- protected:
-  /// Default constructor
-  ServerCompletionQueue() : polling_type_(GRPC_CQ_DEFAULT_POLLING) {}
-
- private:
-  /// \param completion_type indicates whether this is a NEXT or CALLBACK
-  /// completion queue.
-  /// \param polling_type Informs the GRPC library about the type of polling
-  /// allowed on this completion queue. See grpc_cq_polling_type's description
-  /// in grpc_types.h for more details.
-  /// \param shutdown_cb is the shutdown callback used for CALLBACK api queues
-  ServerCompletionQueue(grpc_cq_completion_type completion_type,
-                        grpc_cq_polling_type polling_type,
-                        grpc_experimental_completion_queue_functor* shutdown_cb)
-      : CompletionQueue(grpc_completion_queue_attributes{
-            GRPC_CQ_CURRENT_VERSION, completion_type, polling_type,
-            shutdown_cb}),
-        polling_type_(polling_type) {}
-
-  grpc_cq_polling_type polling_type_;
-  friend class grpc_impl::ServerBuilder;
-  friend class grpc_impl::Server;
-};
+typedef ::grpc_impl::CompletionQueue CompletionQueue;
+typedef ::grpc_impl::ServerCompletionQueue ServerCompletionQueue;
 
 }  // namespace grpc
 

include/grpcpp/impl/codegen/completion_queue_impl.h

@@ -0,0 +1,422 @@
+/*
+ *
+ * Copyright 2015-2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/// A completion queue implements a concurrent producer-consumer queue, with
+/// two main API-exposed methods: \a Next and \a AsyncNext. These
+/// methods are the essential component of the gRPC C++ asynchronous API.
+/// There is also a \a Shutdown method to indicate that a given completion queue
+/// will no longer have regular events. This must be called before the
+/// completion queue is destroyed.
+/// All completion queue APIs are thread-safe and may be used concurrently with
+/// any other completion queue API invocation; it is acceptable to have
+/// multiple threads calling \a Next or \a AsyncNext on the same or different
+/// completion queues, or to call these methods concurrently with a \a Shutdown
+/// elsewhere.
+/// \remark{All other API calls on completion queue should be completed before
+/// a completion queue destructor is called.}
+#ifndef GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_IMPL_H
+#define GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_IMPL_H
+
+#include <grpc/impl/codegen/atm.h>
+#include <grpcpp/impl/codegen/completion_queue_tag.h>
+#include <grpcpp/impl/codegen/core_codegen_interface.h>
+#include <grpcpp/impl/codegen/grpc_library.h>
+#include <grpcpp/impl/codegen/status.h>
+#include <grpcpp/impl/codegen/time.h>
+
+struct grpc_completion_queue;
+
+namespace grpc_impl {
+
+class Channel;
+class Server;
+class ServerBuilder;
+}  // namespace grpc_impl
+namespace grpc {
+
+template <class R>
+class ClientReader;
+template <class W>
+class ClientWriter;
+template <class W, class R>
+class ClientReaderWriter;
+template <class R>
+class ServerReader;
+template <class W>
+class ServerWriter;
+namespace internal {
+template <class W, class R>
+class ServerReaderWriterBody;
+}  // namespace internal
+
+class ChannelInterface;
+class ClientContext;
+class ServerContext;
+class ServerInterface;
+
+namespace internal {
+class CompletionQueueTag;
+class RpcMethod;
+template <class ServiceType, class RequestType, class ResponseType>
+class RpcMethodHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class ClientStreamingHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class ServerStreamingHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class BidiStreamingHandler;
+template <class Streamer, bool WriteNeeded>
+class TemplatedBidiStreamingHandler;
+template <StatusCode code>
+class ErrorMethodHandler;
+template <class InputMessage, class OutputMessage>
+class BlockingUnaryCallImpl;
+template <class Op1, class Op2, class Op3, class Op4, class Op5, class Op6>
+class CallOpSet;
+}  // namespace internal
+
+extern CoreCodegenInterface* g_core_codegen_interface;
+
+}  // namespace grpc
+
+namespace grpc_impl {
+
+/// A thin wrapper around \ref grpc_completion_queue (see \ref
+/// src/core/lib/surface/completion_queue.h).
+/// See \ref doc/cpp/perf_notes.md for notes on best practices for high
+/// performance servers.
+class CompletionQueue : private ::grpc::GrpcLibraryCodegen {
+ public:
+  /// Default constructor. Implicitly creates a \a grpc_completion_queue
+  /// instance.
+  CompletionQueue()
+      : CompletionQueue(grpc_completion_queue_attributes{
+            GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING,
+            nullptr}) {}
+
+  /// Wrap \a take, taking ownership of the instance.
+  ///
+  /// \param take The completion queue instance to wrap. Ownership is taken.
+  explicit CompletionQueue(grpc_completion_queue* take);
+
+  /// Destructor. Destroys the owned wrapped completion queue / instance.
+  ~CompletionQueue() {
+    ::grpc::g_core_codegen_interface->grpc_completion_queue_destroy(cq_);
+  }
+
+  /// Tri-state return for AsyncNext: SHUTDOWN, GOT_EVENT, TIMEOUT.
+  enum NextStatus {
+    SHUTDOWN,   ///< The completion queue has been shutdown and fully-drained
+    GOT_EVENT,  ///< Got a new event; \a tag will be filled in with its
+                ///< associated value; \a ok indicating its success.
+    TIMEOUT     ///< deadline was reached.
+  };
+
+  /// Read from the queue, blocking until an event is available or the queue is
+  /// shutting down.
+  ///
+  /// \param tag [out] Updated to point to the read event's tag.
+  /// \param ok [out] true if read a successful event, false otherwise.
+  ///
+  /// Note that each tag sent to the completion queue (through RPC operations
+  /// or alarms) will be delivered out of the completion queue by a call to
+  /// Next (or a related method), regardless of whether the operation succeeded
+  /// or not. Success here means that this operation completed in the normal
+  /// valid manner.
+  ///
+  /// Server-side RPC request: \a ok indicates that the RPC has indeed
+  /// been started. If it is false, the server has been Shutdown
+  /// before this particular call got matched to an incoming RPC.
+  ///
+  /// Client-side StartCall/RPC invocation: \a ok indicates that the RPC is
+  /// going to go to the wire. If it is false, it not going to the wire. This
+  /// would happen if the channel is either permanently broken or
+  /// transiently broken but with the fail-fast option. (Note that async unary
+  /// RPCs don't post a CQ tag at this point, nor do client-streaming
+  /// or bidi-streaming RPCs that have the initial metadata corked option set.)
+  ///
+  /// Client-side Write, Client-side WritesDone, Server-side Write,
+  /// Server-side Finish, Server-side SendInitialMetadata (which is
+  /// typically included in Write or Finish when not done explicitly):
+  /// \a ok means that the data/metadata/status/etc is going to go to the
+  /// wire. If it is false, it not going to the wire because the call
+  /// is already dead (i.e., canceled, deadline expired, other side
+  /// dropped the channel, etc).
+  ///
+  /// Client-side Read, Server-side Read, Client-side
+  /// RecvInitialMetadata (which is typically included in Read if not
+  /// done explicitly): \a ok indicates whether there is a valid message
+  /// that got read. If not, you know that there are certainly no more
+  /// messages that can ever be read from this stream. For the client-side
+  /// operations, this only happens because the call is dead. For the
+  /// server-sider operation, though, this could happen because the client
+  /// has done a WritesDone already.
+  ///
+  /// Client-side Finish: \a ok should always be true
+  ///
+  /// Server-side AsyncNotifyWhenDone: \a ok should always be true
+  ///
+  /// Alarm: \a ok is true if it expired, false if it was canceled
+  ///
+  /// \return true if got an event, false if the queue is fully drained and
+  ///         shut down.
+  bool Next(void** tag, bool* ok) {
+    return (AsyncNextInternal(tag, ok,
+                              ::grpc::g_core_codegen_interface->gpr_inf_future(
+                                  GPR_CLOCK_REALTIME)) != SHUTDOWN);
+  }
+
+  /// Read from the queue, blocking up to \a deadline (or the queue's shutdown).
+  /// Both \a tag and \a ok are updated upon success (if an event is available
+  /// within the \a deadline).  A \a tag points to an arbitrary location usually
+  /// employed to uniquely identify an event.
+  ///
+  /// \param tag [out] Upon sucess, updated to point to the event's tag.
+  /// \param ok [out] Upon sucess, true if a successful event, false otherwise
+  ///        See documentation for CompletionQueue::Next for explanation of ok
+  /// \param deadline [in] How long to block in wait for an event.
+  ///
+  /// \return The type of event read.
+  template <typename T>
+  NextStatus AsyncNext(void** tag, bool* ok, const T& deadline) {
+    ::grpc::TimePoint<T> deadline_tp(deadline);
+    return AsyncNextInternal(tag, ok, deadline_tp.raw_time());
+  }
+
+  /// EXPERIMENTAL
+  /// First executes \a F, then reads from the queue, blocking up to
+  /// \a deadline (or the queue's shutdown).
+  /// Both \a tag and \a ok are updated upon success (if an event is available
+  /// within the \a deadline).  A \a tag points to an arbitrary location usually
+  /// employed to uniquely identify an event.
+  ///
+  /// \param f [in] Function to execute before calling AsyncNext on this queue.
+  /// \param tag [out] Upon sucess, updated to point to the event's tag.
+  /// \param ok [out] Upon sucess, true if read a regular event, false
+  /// otherwise.
+  /// \param deadline [in] How long to block in wait for an event.
+  ///
+  /// \return The type of event read.
+  template <typename T, typename F>
+  NextStatus DoThenAsyncNext(F&& f, void** tag, bool* ok, const T& deadline) {
+    CompletionQueueTLSCache cache = CompletionQueueTLSCache(this);
+    f();
+    if (cache.Flush(tag, ok)) {
+      return GOT_EVENT;
+    } else {
+      return AsyncNext(tag, ok, deadline);
+    }
+  }
+
+  /// Request the shutdown of the queue.
+  ///
+  /// \warning This method must be called at some point if this completion queue
+  /// is accessed with Next or AsyncNext. \a Next will not return false
+  /// until this method has been called and all pending tags have been drained.
+  /// (Likewise for \a AsyncNext returning \a NextStatus::SHUTDOWN .)
+  /// Only once either one of these methods does that (that is, once the queue
+  /// has been \em drained) can an instance of this class be destroyed.
+  /// Also note that applications must ensure that no work is enqueued on this
+  /// completion queue after this method is called.
+  void Shutdown();
+
+  /// Returns a \em raw pointer to the underlying \a grpc_completion_queue
+  /// instance.
+  ///
+  /// \warning Remember that the returned instance is owned. No transfer of
+  /// owership is performed.
+  grpc_completion_queue* cq() { return cq_; }
+
+ protected:
+  /// Private constructor of CompletionQueue only visible to friend classes
+  CompletionQueue(const grpc_completion_queue_attributes& attributes) {
+    cq_ = ::grpc::g_core_codegen_interface->grpc_completion_queue_create(
+        ::grpc::g_core_codegen_interface->grpc_completion_queue_factory_lookup(
+            &attributes),
+        &attributes, NULL);
+    InitialAvalanching();  // reserve this for the future shutdown
+  }
+
+ private:
+  // Friend synchronous wrappers so that they can access Pluck(), which is
+  // a semi-private API geared towards the synchronous implementation.
+  template <class R>
+  friend class ::grpc::ClientReader;
+  template <class W>
+  friend class ::grpc::ClientWriter;
+  template <class W, class R>
+  friend class ::grpc::ClientReaderWriter;
+  template <class R>
+  friend class ::grpc::ServerReader;
+  template <class W>
+  friend class ::grpc::ServerWriter;
+  template <class W, class R>
+  friend class ::grpc::internal::ServerReaderWriterBody;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class ::grpc::internal::RpcMethodHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class ::grpc::internal::ClientStreamingHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class ::grpc::internal::ServerStreamingHandler;
+  template <class Streamer, bool WriteNeeded>
+  friend class ::grpc::internal::TemplatedBidiStreamingHandler;
+  template <::grpc::StatusCode code>
+  friend class ::grpc::internal::ErrorMethodHandler;
+  friend class ::grpc_impl::Server;
+  friend class ::grpc::ServerContext;
+  friend class ::grpc::ServerInterface;
+  template <class InputMessage, class OutputMessage>
+  friend class ::grpc::internal::BlockingUnaryCallImpl;
+
+  // Friends that need access to constructor for callback CQ
+  friend class ::grpc_impl::Channel;
+
+  // For access to Register/CompleteAvalanching
+  template <class Op1, class Op2, class Op3, class Op4, class Op5, class Op6>
+  friend class ::grpc::internal::CallOpSet;
+
+  /// EXPERIMENTAL
+  /// Creates a Thread Local cache to store the first event
+  /// On this completion queue queued from this thread.  Once
+  /// initialized, it must be flushed on the same thread.
+  class CompletionQueueTLSCache {
+   public:
+    CompletionQueueTLSCache(CompletionQueue* cq);
+    ~CompletionQueueTLSCache();
+    bool Flush(void** tag, bool* ok);
+
+   private:
+    CompletionQueue* cq_;
+    bool flushed_;
+  };
+
+  NextStatus AsyncNextInternal(void** tag, bool* ok, gpr_timespec deadline);
+
+  /// Wraps \a grpc_completion_queue_pluck.
+  /// \warning Must not be mixed with calls to \a Next.
+  bool Pluck(::grpc::internal::CompletionQueueTag* tag) {
+    auto deadline =
+        ::grpc::g_core_codegen_interface->gpr_inf_future(GPR_CLOCK_REALTIME);
+    while (true) {
+      auto ev = ::grpc::g_core_codegen_interface->grpc_completion_queue_pluck(
+          cq_, tag, deadline, nullptr);
+      bool ok = ev.success != 0;
+      void* ignored = tag;
+      if (tag->FinalizeResult(&ignored, &ok)) {
+        GPR_CODEGEN_ASSERT(ignored == tag);
+        return ok;
+      }
+    }
+  }
+
+  /// Performs a single polling pluck on \a tag.
+  /// \warning Must not be mixed with calls to \a Next.
+  ///
+  /// TODO: sreek - This calls tag->FinalizeResult() even if the cq_ is already
+  /// shutdown. This is most likely a bug and if it is a bug, then change this
+  /// implementation to simple call the other TryPluck function with a zero
+  /// timeout. i.e:
+  ///      TryPluck(tag, gpr_time_0(GPR_CLOCK_REALTIME))
+  void TryPluck(::grpc::internal::CompletionQueueTag* tag) {
+    auto deadline =
+        ::grpc::g_core_codegen_interface->gpr_time_0(GPR_CLOCK_REALTIME);
+    auto ev = ::grpc::g_core_codegen_interface->grpc_completion_queue_pluck(
+        cq_, tag, deadline, nullptr);
+    if (ev.type == GRPC_QUEUE_TIMEOUT) return;
+    bool ok = ev.success != 0;
+    void* ignored = tag;
+    // the tag must be swallowed if using TryPluck
+    GPR_CODEGEN_ASSERT(!tag->FinalizeResult(&ignored, &ok));
+  }
+
+  /// Performs a single polling pluck on \a tag. Calls tag->FinalizeResult if
+  /// the pluck() was successful and returned the tag.
+  ///
+  /// This exects tag->FinalizeResult (if called) to return 'false' i.e expects
+  /// that the tag is internal not something that is returned to the user.
+  void TryPluck(::grpc::internal::CompletionQueueTag* tag,
+                gpr_timespec deadline) {
+    auto ev = ::grpc::g_core_codegen_interface->grpc_completion_queue_pluck(
+        cq_, tag, deadline, nullptr);
+    if (ev.type == GRPC_QUEUE_TIMEOUT || ev.type == GRPC_QUEUE_SHUTDOWN) {
+      return;
+    }
+
+    bool ok = ev.success != 0;
+    void* ignored = tag;
+    GPR_CODEGEN_ASSERT(!tag->FinalizeResult(&ignored, &ok));
+  }
+
+  /// Manage state of avalanching operations : completion queue tags that
+  /// trigger other completion queue operations. The underlying core completion
+  /// queue should not really shutdown until all avalanching operations have
+  /// been finalized. Note that we maintain the requirement that an avalanche
+  /// registration must take place before CQ shutdown (which must be maintained
+  /// elsehwere)
+  void InitialAvalanching() {
+    gpr_atm_rel_store(&avalanches_in_flight_, static_cast<gpr_atm>(1));
+  }
+  void RegisterAvalanching() {
+    gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_,
+                                 static_cast<gpr_atm>(1));
+  }
+  void CompleteAvalanching() {
+    if (gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_,
+                                     static_cast<gpr_atm>(-1)) == 1) {
+      ::grpc::g_core_codegen_interface->grpc_completion_queue_shutdown(cq_);
+    }
+  }
+
+  grpc_completion_queue* cq_;  // owned
+
+  gpr_atm avalanches_in_flight_;
+};
+
+/// A specific type of completion queue used by the processing of notifications
+/// by servers. Instantiated by \a ServerBuilder.
+class ServerCompletionQueue : public CompletionQueue {
+ public:
+  bool IsFrequentlyPolled() { return polling_type_ != GRPC_CQ_NON_LISTENING; }
+
+ protected:
+  /// Default constructor
+  ServerCompletionQueue() : polling_type_(GRPC_CQ_DEFAULT_POLLING) {}
+
+ private:
+  /// \param completion_type indicates whether this is a NEXT or CALLBACK
+  /// completion queue.
+  /// \param polling_type Informs the GRPC library about the type of polling
+  /// allowed on this completion queue. See grpc_cq_polling_type's description
+  /// in grpc_types.h for more details.
+  /// \param shutdown_cb is the shutdown callback used for CALLBACK api queues
+  ServerCompletionQueue(grpc_cq_completion_type completion_type,
+                        grpc_cq_polling_type polling_type,
+                        grpc_experimental_completion_queue_functor* shutdown_cb)
+      : CompletionQueue(grpc_completion_queue_attributes{
+            GRPC_CQ_CURRENT_VERSION, completion_type, polling_type,
+            shutdown_cb}),
+        polling_type_(polling_type) {}
+
+  grpc_cq_polling_type polling_type_;
+  friend class ::grpc_impl::ServerBuilder;
+  friend class ::grpc_impl::Server;
+};
+
+}  // namespace grpc_impl
+
+#endif  // GRPCPP_IMPL_CODEGEN_COMPLETION_QUEUE_IMPL_H

include/grpcpp/impl/codegen/intercepted_channel.h

@@ -21,6 +21,10 @@
 
 #include <grpcpp/impl/codegen/channel_interface.h>
 
+namespace grpc_impl {
+class CompletionQueue;
+}
+
 namespace grpc {
 
 namespace internal {
@@ -46,7 +50,7 @@ class InterceptedChannel : public ChannelInterface {
       : channel_(channel), interceptor_pos_(pos) {}
 
   Call CreateCall(const RpcMethod& method, ClientContext* context,
-                  CompletionQueue* cq) override {
+                  ::grpc_impl::CompletionQueue* cq) override {
     return channel_->CreateCallInternal(method, context, cq, interceptor_pos_);
   }
 
@@ -58,7 +62,8 @@ class InterceptedChannel : public ChannelInterface {
   }
 
   void NotifyOnStateChangeImpl(grpc_connectivity_state last_observed,
-                               gpr_timespec deadline, CompletionQueue* cq,
+                               gpr_timespec deadline,
+                               ::grpc_impl::CompletionQueue* cq,
                                void* tag) override {
     return channel_->NotifyOnStateChangeImpl(last_observed, deadline, cq, tag);
   }
@@ -67,7 +72,9 @@ class InterceptedChannel : public ChannelInterface {
     return channel_->WaitForStateChangeImpl(last_observed, deadline);
   }
 
-  CompletionQueue* CallbackCQ() override { return channel_->CallbackCQ(); }
+  ::grpc_impl::CompletionQueue* CallbackCQ() override {
+    return channel_->CallbackCQ();
+  }
 
   ChannelInterface* channel_;
   size_t interceptor_pos_;

include/grpcpp/impl/codegen/server_context.h

@@ -43,12 +43,12 @@ struct census_context;
 
 namespace grpc_impl {
 
+class CompletionQueue;
 class Server;
 }  // namespace grpc_impl
 namespace grpc {
 class ClientContext;
 class GenericServerContext;
-class CompletionQueue;
 class ServerInterface;
 template <class W, class R>
 class ServerAsyncReader;
@@ -90,6 +90,7 @@ class Call;
 class ServerReactor;
 }  // namespace internal
 
+class ServerInterface;
 namespace testing {
 class InteropServerContextInspector;
 class ServerContextTestSpouse;
@@ -354,7 +355,7 @@ class ServerContext {
 
   gpr_timespec deadline_;
   grpc_call* call_;
-  CompletionQueue* cq_;
+  ::grpc_impl::CompletionQueue* cq_;
   bool sent_initial_metadata_;
   mutable std::shared_ptr<const AuthContext> auth_context_;
   mutable internal::MetadataMap client_metadata_;

include/grpcpp/impl/codegen/server_interface.h

@@ -30,14 +30,15 @@
 
 namespace grpc_impl {
 
+class CompletionQueue;
+class ServerCompletionQueue;
+class Channel;
 class ServerCredentials;
-}
+}  // namespace grpc_impl
 namespace grpc {
 
 class AsyncGenericService;
-class Channel;
 class GenericServerContext;
-class ServerCompletionQueue;
 class ServerContext;
 class Service;
 
@@ -161,7 +162,8 @@ class ServerInterface : public internal::CallHook {
   /// caller is required to keep all completion queues live until the server is
   /// destroyed.
   /// \param num_cqs How many completion queues does \a cqs hold.
-  virtual void Start(ServerCompletionQueue** cqs, size_t num_cqs) = 0;
+  virtual void Start(::grpc_impl::ServerCompletionQueue** cqs,
+                     size_t num_cqs) = 0;
 
   virtual void ShutdownInternal(gpr_timespec deadline) = 0;
 
@@ -176,9 +178,9 @@ class ServerInterface : public internal::CallHook {
    public:
     BaseAsyncRequest(ServerInterface* server, ServerContext* context,
                      internal::ServerAsyncStreamingInterface* stream,
-                     CompletionQueue* call_cq,
-                     ServerCompletionQueue* notification_cq, void* tag,
-                     bool delete_on_finalize);
+                     ::grpc_impl::CompletionQueue* call_cq,
+                     ::grpc_impl::ServerCompletionQueue* notification_cq,
+                     void* tag, bool delete_on_finalize);
     virtual ~BaseAsyncRequest();
 
     bool FinalizeResult(void** tag, bool* status) override;
@@ -190,8 +192,8 @@ class ServerInterface : public internal::CallHook {
     ServerInterface* const server_;
     ServerContext* const context_;
     internal::ServerAsyncStreamingInterface* const stream_;
-    CompletionQueue* const call_cq_;
-    ServerCompletionQueue* const notification_cq_;
+    ::grpc_impl::CompletionQueue* const call_cq_;
+    ::grpc_impl::ServerCompletionQueue* const notification_cq_;
     void* const tag_;
     const bool delete_on_finalize_;
     grpc_call* call_;
@@ -205,16 +207,17 @@ class ServerInterface : public internal::CallHook {
    public:
     RegisteredAsyncRequest(ServerInterface* server, ServerContext* context,
                            internal::ServerAsyncStreamingInterface* stream,
-                           CompletionQueue* call_cq,
-                           ServerCompletionQueue* notification_cq, void* tag,
-                           const char* name, internal::RpcMethod::RpcType type);
+                           ::grpc_impl::CompletionQueue* call_cq,
+                           ::grpc_impl::ServerCompletionQueue* notification_cq,
+                           void* tag, const char* name,
+                           internal::RpcMethod::RpcType type);
 
     virtual bool FinalizeResult(void** tag, bool* status) override {
       /* If we are done intercepting, then there is nothing more for us to do */
       if (done_intercepting_) {
         return BaseAsyncRequest::FinalizeResult(tag, status);
       }
-      call_wrapper_ = internal::Call(
+      call_wrapper_ = ::grpc::internal::Call(
           call_, server_, call_cq_, server_->max_receive_message_size(),
           context_->set_server_rpc_info(name_, type_,
                                         *server_->interceptor_creators()));
@@ -223,7 +226,7 @@ class ServerInterface : public internal::CallHook {
 
    protected:
     void IssueRequest(void* registered_method, grpc_byte_buffer** payload,
-                      ServerCompletionQueue* notification_cq);
+                      ::grpc_impl::ServerCompletionQueue* notification_cq);
     const char* name_;
     const internal::RpcMethod::RpcType type_;
   };
@@ -233,8 +236,9 @@ class ServerInterface : public internal::CallHook {
     NoPayloadAsyncRequest(internal::RpcServiceMethod* registered_method,
                           ServerInterface* server, ServerContext* context,
                           internal::ServerAsyncStreamingInterface* stream,
-                          CompletionQueue* call_cq,
-                          ServerCompletionQueue* notification_cq, void* tag)
+                          ::grpc_impl::CompletionQueue* call_cq,
+                          ::grpc_impl::ServerCompletionQueue* notification_cq,
+                          void* tag)
         : RegisteredAsyncRequest(
               server, context, stream, call_cq, notification_cq, tag,
               registered_method->name(), registered_method->method_type()) {
@@ -250,9 +254,9 @@ class ServerInterface : public internal::CallHook {
     PayloadAsyncRequest(internal::RpcServiceMethod* registered_method,
                         ServerInterface* server, ServerContext* context,
                         internal::ServerAsyncStreamingInterface* stream,
-                        CompletionQueue* call_cq,
-                        ServerCompletionQueue* notification_cq, void* tag,
-                        Message* request)
+                        ::grpc_impl::CompletionQueue* call_cq,
+                        ::grpc_impl::ServerCompletionQueue* notification_cq,
+                        void* tag, Message* request)
         : RegisteredAsyncRequest(
               server, context, stream, call_cq, notification_cq, tag,
               registered_method->name(), registered_method->method_type()),
@@ -307,9 +311,9 @@ class ServerInterface : public internal::CallHook {
     ServerInterface* const server_;
     ServerContext* const context_;
     internal::ServerAsyncStreamingInterface* const stream_;
-    CompletionQueue* const call_cq_;
+    ::grpc_impl::CompletionQueue* const call_cq_;
 
-    ServerCompletionQueue* const notification_cq_;
+    ::grpc_impl::ServerCompletionQueue* const notification_cq_;
     void* const tag_;
     Message* const request_;
     ByteBuffer payload_;
@@ -319,9 +323,9 @@ class ServerInterface : public internal::CallHook {
    public:
     GenericAsyncRequest(ServerInterface* server, GenericServerContext* context,
                         internal::ServerAsyncStreamingInterface* stream,
-                        CompletionQueue* call_cq,
-                        ServerCompletionQueue* notification_cq, void* tag,
-                        bool delete_on_finalize);
+                        ::grpc_impl::CompletionQueue* call_cq,
+                        ::grpc_impl::ServerCompletionQueue* notification_cq,
+                        void* tag, bool delete_on_finalize);
 
     bool FinalizeResult(void** tag, bool* status) override;
 
@@ -333,9 +337,9 @@ class ServerInterface : public internal::CallHook {
   void RequestAsyncCall(internal::RpcServiceMethod* method,
                         ServerContext* context,
                         internal::ServerAsyncStreamingInterface* stream,
-                        CompletionQueue* call_cq,
-                        ServerCompletionQueue* notification_cq, void* tag,
-                        Message* message) {
+                        ::grpc_impl::CompletionQueue* call_cq,
+                        ::grpc_impl::ServerCompletionQueue* notification_cq,
+                        void* tag, Message* message) {
     GPR_CODEGEN_ASSERT(method);
     new PayloadAsyncRequest<Message>(method, this, context, stream, call_cq,
                                      notification_cq, tag, message);
@@ -344,18 +348,19 @@ class ServerInterface : public internal::CallHook {
   void RequestAsyncCall(internal::RpcServiceMethod* method,
                         ServerContext* context,
                         internal::ServerAsyncStreamingInterface* stream,
-                        CompletionQueue* call_cq,
-                        ServerCompletionQueue* notification_cq, void* tag) {
+                        ::grpc_impl::CompletionQueue* call_cq,
+                        ::grpc_impl::ServerCompletionQueue* notification_cq,
+                        void* tag) {
     GPR_CODEGEN_ASSERT(method);
     new NoPayloadAsyncRequest(method, this, context, stream, call_cq,
                               notification_cq, tag);
   }
 
-  void RequestAsyncGenericCall(GenericServerContext* context,
-                               internal::ServerAsyncStreamingInterface* stream,
-                               CompletionQueue* call_cq,
-                               ServerCompletionQueue* notification_cq,
-                               void* tag) {
+  void RequestAsyncGenericCall(
+      GenericServerContext* context,
+      internal::ServerAsyncStreamingInterface* stream,
+      ::grpc_impl::CompletionQueue* call_cq,
+      ::grpc_impl::ServerCompletionQueue* notification_cq, void* tag) {
     new GenericAsyncRequest(this, context, stream, call_cq, notification_cq,
                             tag, true);
   }
@@ -380,7 +385,7 @@ class ServerInterface : public internal::CallHook {
   // Returns nullptr (rather than being pure) since this is a post-1.0 method
   // and adding a new pure method to an interface would be a breaking change
   // (even though this is private and non-API)
-  virtual CompletionQueue* CallbackCQ() { return nullptr; }
+  virtual ::grpc_impl::CompletionQueue* CallbackCQ() { return nullptr; }
 };
 
 }  // namespace grpc

include/grpcpp/impl/codegen/service_type.h

@@ -29,12 +29,11 @@
 namespace grpc_impl {
 
 class Server;
+class CompletionQueue;
 }  // namespace grpc_impl
 namespace grpc {
 
-class CompletionQueue;
 class ServerInterface;
-class ServerCompletionQueue;
 class ServerContext;
 
 namespace internal {

include/grpcpp/security/credentials_impl.h

@@ -24,6 +24,7 @@
 #include <vector>
 
 #include <grpc/grpc_security_constants.h>
+#include <grpcpp/channel.h>
 #include <grpcpp/impl/codegen/client_interceptor.h>
 #include <grpcpp/impl/codegen/grpc_library.h>
 #include <grpcpp/security/auth_context.h>
@@ -35,7 +36,6 @@ struct grpc_call;
 
 namespace grpc_impl {
 
-class Channel;
 class ChannelCredentials;
 class CallCredentials;
 class SecureCallCredentials;

include/grpcpp/server_builder.h

@@ -21,13 +21,6 @@
 
 #include <grpcpp/server_builder_impl.h>
 
-namespace grpc_impl {
-
-class Server;
-class ServerCredentials;
-class ResourceQuota;
-}  // namespace grpc_impl
-
 namespace grpc {
 
 typedef ::grpc_impl::ServerBuilder ServerBuilder;

include/grpcpp/server_builder_impl.h

@@ -37,15 +37,17 @@
 struct grpc_resource_quota;
 
 namespace grpc_impl {
+
+class CompletionQueue;
 class ResourceQuota;
+class Server;
+class ServerCompletionQueue;
 class ServerCredentials;
 }  // namespace grpc_impl
 
 namespace grpc {
 
 class AsyncGenericService;
-class CompletionQueue;
-class ServerCompletionQueue;
 class Service;
 namespace testing {
 class ServerBuilderPluginTest;
@@ -154,7 +156,7 @@ class ServerBuilder {
   /// not polling the completion queue frequently) will have a significantly
   /// negative performance impact and hence should not be used in production
   /// use cases.
-  std::unique_ptr<grpc::ServerCompletionQueue> AddCompletionQueue(
+  std::unique_ptr<grpc_impl::ServerCompletionQueue> AddCompletionQueue(
       bool is_frequently_polled = true);
 
   //////////////////////////////////////////////////////////////////////////////
@@ -360,7 +362,7 @@ class ServerBuilder {
   SyncServerSettings sync_server_settings_;
 
   /// List of completion queues added via \a AddCompletionQueue method.
-  std::vector<grpc::ServerCompletionQueue*> cqs_;
+  std::vector<grpc_impl::ServerCompletionQueue*> cqs_;
 
   std::shared_ptr<grpc_impl::ServerCredentials> creds_;
   std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>> plugins_;

include/grpcpp/server_impl.h

@@ -27,6 +27,7 @@
 
 #include <grpc/compression.h>
 #include <grpc/support/atm.h>
+#include <grpcpp/channel.h>
 #include <grpcpp/completion_queue.h>
 #include <grpcpp/health_check_service_interface.h>
 #include <grpcpp/impl/call.h>
@@ -107,7 +108,7 @@ class Server : public grpc::ServerInterface, private grpc::GrpcLibraryCodegen {
   }
 
   /// Establish a channel for in-process communication
-  std::shared_ptr<grpc::Channel> InProcessChannel(
+  std::shared_ptr<::grpc::Channel> InProcessChannel(
       const grpc::ChannelArguments& args);
 
   /// NOTE: class experimental_type is not part of the public API of this class.
@@ -119,7 +120,7 @@ class Server : public grpc::ServerInterface, private grpc::GrpcLibraryCodegen {
 
     /// Establish a channel for in-process communication with client
     /// interceptors
-    std::shared_ptr<grpc::Channel> InProcessChannelWithInterceptors(
+    std::shared_ptr<::grpc::Channel> InProcessChannelWithInterceptors(
         const grpc::ChannelArguments& args,
         std::vector<std::unique_ptr<
             grpc::experimental::ClientInterceptorFactoryInterface>>

package.xml

@@ -13,8 +13,8 @@
  <date>2018-01-19</date>
  <time>16:06:07</time>
  <version>
-  <release>1.21.0dev</release>
-  <api>1.21.0dev</api>
+  <release>1.22.0dev</release>
+  <api>1.22.0dev</api>
  </version>
  <stability>
   <release>beta</release>
@@ -433,6 +433,7 @@
     <file baseinstalldir="/" name="src/core/lib/slice/slice_hash_table.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/slice/slice_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/slice/slice_string_helpers.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/slice/slice_utils.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/slice/slice_weak_hash_table.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/api_trace.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/call.h" role="src" />
@@ -478,6 +479,7 @@
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.h" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/max_age/max_age_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/message_size/message_size_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/http/client_authority_filter.h" role="src" />
@@ -811,6 +813,7 @@
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv_windows.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc" role="src" />
     <file baseinstalldir="/" name="src/core/ext/filters/census/grpc_context.cc" role="src" />

src/android/test/interop/app/src/main/cpp/grpc-interop.cc

@@ -18,8 +18,8 @@
 
 #include <grpcpp/grpcpp.h>
 #include <jni.h>
-#include <src/core/lib/gpr/env.h>
 
+#include "src/core/lib/security/security_connector/ssl_utils.h"
 #include "test/cpp/interop/interop_client.h"
 
 extern "C" JNIEXPORT void JNICALL
@@ -28,7 +28,7 @@ Java_io_grpc_interop_cpp_InteropActivity_configureSslRoots(JNIEnv* env,
                                                            jstring path_raw) {
   const char* path = env->GetStringUTFChars(path_raw, (jboolean*)0);
 
-  gpr_setenv("GRPC_DEFAULT_SSL_ROOTS_FILE_PATH", path);
+  GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, path);
 }
 
 std::shared_ptr<grpc::testing::InteropClient> GetClient(const char* host,
@@ -45,7 +45,7 @@ std::shared_ptr<grpc::testing::InteropClient> GetClient(const char* host,
     credentials = grpc::InsecureChannelCredentials();
   }
 
-  grpc::testing::ChannelCreationFunc channel_creation_func = 
+  grpc::testing::ChannelCreationFunc channel_creation_func =
       std::bind(grpc::CreateChannel, host_port, credentials);
   return std::shared_ptr<grpc::testing::InteropClient>(
       new grpc::testing::InteropClient(channel_creation_func, true, false));

src/compiler/cpp_generator.cc

@@ -154,14 +154,18 @@ grpc::string GetHeaderIncludes(grpc_generator::File* file,
     PrintIncludes(printer.get(), headers, params.use_system_headers,
                   params.grpc_search_path);
     printer->Print(vars, "\n");
+    printer->Print(vars, "namespace grpc_impl {\n");
+    printer->Print(vars, "class Channel;\n");
+    printer->Print(vars, "class CompletionQueue;\n");
+    printer->Print(vars, "class ServerCompletionQueue;\n");
+    printer->Print(vars, "}  // namespace grpc_impl\n\n");
     printer->Print(vars, "namespace grpc {\n");
     printer->Print(vars, "namespace experimental {\n");
     printer->Print(vars, "template <typename RequestT, typename ResponseT>\n");
     printer->Print(vars, "class MessageAllocator;\n");
     printer->Print(vars, "}  // namespace experimental\n");
-    printer->Print(vars, "class CompletionQueue;\n");
-    printer->Print(vars, "class Channel;\n");
-    printer->Print(vars, "class ServerCompletionQueue;\n");
+    printer->Print(vars, "}  // namespace grpc_impl\n\n");
+    printer->Print(vars, "namespace grpc {\n");
     printer->Print(vars, "class ServerContext;\n");
     printer->Print(vars, "}  // namespace grpc\n\n");
 

src/compiler/cpp_plugin.cc

@@ -18,137 +18,7 @@
 
 // Generates cpp gRPC service interface out of Protobuf IDL.
 //
-
-#include <memory>
-#include <sstream>
-
-#include "src/compiler/config.h"
-
-#include "src/compiler/cpp_generator.h"
-#include "src/compiler/generator_helpers.h"
-#include "src/compiler/protobuf_plugin.h"
-
-class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
- public:
-  CppGrpcGenerator() {}
-  virtual ~CppGrpcGenerator() {}
-
-  virtual bool Generate(const grpc::protobuf::FileDescriptor* file,
-                        const grpc::string& parameter,
-                        grpc::protobuf::compiler::GeneratorContext* context,
-                        grpc::string* error) const {
-    if (file->options().cc_generic_services()) {
-      *error =
-          "cpp grpc proto compiler plugin does not work with generic "
-          "services. To generate cpp grpc APIs, please set \""
-          "cc_generic_service = false\".";
-      return false;
-    }
-
-    grpc_cpp_generator::Parameters generator_parameters;
-    generator_parameters.use_system_headers = true;
-    generator_parameters.generate_mock_code = false;
-    generator_parameters.include_import_headers = false;
-
-    ProtoBufFile pbfile(file);
-
-    if (!parameter.empty()) {
-      std::vector<grpc::string> parameters_list =
-          grpc_generator::tokenize(parameter, ",");
-      for (auto parameter_string = parameters_list.begin();
-           parameter_string != parameters_list.end(); parameter_string++) {
-        std::vector<grpc::string> param =
-            grpc_generator::tokenize(*parameter_string, "=");
-        if (param[0] == "services_namespace") {
-          generator_parameters.services_namespace = param[1];
-        } else if (param[0] == "use_system_headers") {
-          if (param[1] == "true") {
-            generator_parameters.use_system_headers = true;
-          } else if (param[1] == "false") {
-            generator_parameters.use_system_headers = false;
-          } else {
-            *error = grpc::string("Invalid parameter: ") + *parameter_string;
-            return false;
-          }
-        } else if (param[0] == "grpc_search_path") {
-          generator_parameters.grpc_search_path = param[1];
-        } else if (param[0] == "generate_mock_code") {
-          if (param[1] == "true") {
-            generator_parameters.generate_mock_code = true;
-          } else if (param[1] != "false") {
-            *error = grpc::string("Invalid parameter: ") + *parameter_string;
-            return false;
-          }
-        } else if (param[0] == "gmock_search_path") {
-          generator_parameters.gmock_search_path = param[1];
-        } else if (param[0] == "additional_header_includes") {
-          generator_parameters.additional_header_includes =
-              grpc_generator::tokenize(param[1], ":");
-        } else if (param[0] == "message_header_extension") {
-          generator_parameters.message_header_extension = param[1];
-        } else if (param[0] == "include_import_headers") {
-          if (param[1] == "true") {
-            generator_parameters.include_import_headers = true;
-          } else if (param[1] != "false") {
-            *error = grpc::string("Invalid parameter: ") + *parameter_string;
-            return false;
-          }
-        } else {
-          *error = grpc::string("Unknown parameter: ") + *parameter_string;
-          return false;
-        }
-      }
-    }
-
-    grpc::string file_name = grpc_generator::StripProto(file->name());
-
-    grpc::string header_code =
-        grpc_cpp_generator::GetHeaderPrologue(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetHeaderIncludes(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetHeaderServices(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetHeaderEpilogue(&pbfile, generator_parameters);
-    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> header_output(
-        context->Open(file_name + ".grpc.pb.h"));
-    grpc::protobuf::io::CodedOutputStream header_coded_out(header_output.get());
-    header_coded_out.WriteRaw(header_code.data(), header_code.size());
-
-    grpc::string source_code =
-        grpc_cpp_generator::GetSourcePrologue(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetSourceIncludes(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetSourceServices(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetSourceEpilogue(&pbfile, generator_parameters);
-    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> source_output(
-        context->Open(file_name + ".grpc.pb.cc"));
-    grpc::protobuf::io::CodedOutputStream source_coded_out(source_output.get());
-    source_coded_out.WriteRaw(source_code.data(), source_code.size());
-
-    if (!generator_parameters.generate_mock_code) {
-      return true;
-    }
-    grpc::string mock_code =
-        grpc_cpp_generator::GetMockPrologue(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetMockIncludes(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetMockServices(&pbfile, generator_parameters) +
-        grpc_cpp_generator::GetMockEpilogue(&pbfile, generator_parameters);
-    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> mock_output(
-        context->Open(file_name + "_mock.grpc.pb.h"));
-    grpc::protobuf::io::CodedOutputStream mock_coded_out(mock_output.get());
-    mock_coded_out.WriteRaw(mock_code.data(), mock_code.size());
-
-    return true;
-  }
-
- private:
-  // Insert the given code into the given file at the given insertion point.
-  void Insert(grpc::protobuf::compiler::GeneratorContext* context,
-              const grpc::string& filename, const grpc::string& insertion_point,
-              const grpc::string& code) const {
-    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> output(
-        context->OpenForInsert(filename, insertion_point));
-    grpc::protobuf::io::CodedOutputStream coded_out(output.get());
-    coded_out.WriteRaw(code.data(), code.size());
-  }
-};
+#include "src/compiler/cpp_plugin.h"
 
 int main(int argc, char* argv[]) {
   CppGrpcGenerator generator;

src/compiler/cpp_plugin.h

@@ -0,0 +1,154 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_COMPILER_CPP_PLUGIN_H
+#define GRPC_INTERNAL_COMPILER_CPP_PLUGIN_H
+
+#include <memory>
+#include <sstream>
+
+#include "src/compiler/config.h"
+
+#include "src/compiler/cpp_generator.h"
+#include "src/compiler/generator_helpers.h"
+#include "src/compiler/protobuf_plugin.h"
+
+// Cpp Generator for Protobug IDL
+class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
+ public:
+  CppGrpcGenerator() {}
+  virtual ~CppGrpcGenerator() {}
+
+  virtual bool Generate(const grpc::protobuf::FileDescriptor* file,
+                        const grpc::string& parameter,
+                        grpc::protobuf::compiler::GeneratorContext* context,
+                        grpc::string* error) const {
+    if (file->options().cc_generic_services()) {
+      *error =
+          "cpp grpc proto compiler plugin does not work with generic "
+          "services. To generate cpp grpc APIs, please set \""
+          "cc_generic_service = false\".";
+      return false;
+    }
+
+    grpc_cpp_generator::Parameters generator_parameters;
+    generator_parameters.use_system_headers = true;
+    generator_parameters.generate_mock_code = false;
+    generator_parameters.include_import_headers = false;
+
+    ProtoBufFile pbfile(file);
+
+    if (!parameter.empty()) {
+      std::vector<grpc::string> parameters_list =
+          grpc_generator::tokenize(parameter, ",");
+      for (auto parameter_string = parameters_list.begin();
+           parameter_string != parameters_list.end(); parameter_string++) {
+        std::vector<grpc::string> param =
+            grpc_generator::tokenize(*parameter_string, "=");
+        if (param[0] == "services_namespace") {
+          generator_parameters.services_namespace = param[1];
+        } else if (param[0] == "use_system_headers") {
+          if (param[1] == "true") {
+            generator_parameters.use_system_headers = true;
+          } else if (param[1] == "false") {
+            generator_parameters.use_system_headers = false;
+          } else {
+            *error = grpc::string("Invalid parameter: ") + *parameter_string;
+            return false;
+          }
+        } else if (param[0] == "grpc_search_path") {
+          generator_parameters.grpc_search_path = param[1];
+        } else if (param[0] == "generate_mock_code") {
+          if (param[1] == "true") {
+            generator_parameters.generate_mock_code = true;
+          } else if (param[1] != "false") {
+            *error = grpc::string("Invalid parameter: ") + *parameter_string;
+            return false;
+          }
+        } else if (param[0] == "gmock_search_path") {
+          generator_parameters.gmock_search_path = param[1];
+        } else if (param[0] == "additional_header_includes") {
+          generator_parameters.additional_header_includes =
+              grpc_generator::tokenize(param[1], ":");
+        } else if (param[0] == "message_header_extension") {
+          generator_parameters.message_header_extension = param[1];
+        } else if (param[0] == "include_import_headers") {
+          if (param[1] == "true") {
+            generator_parameters.include_import_headers = true;
+          } else if (param[1] != "false") {
+            *error = grpc::string("Invalid parameter: ") + *parameter_string;
+            return false;
+          }
+        } else {
+          *error = grpc::string("Unknown parameter: ") + *parameter_string;
+          return false;
+        }
+      }
+    }
+
+    grpc::string file_name = grpc_generator::StripProto(file->name());
+
+    grpc::string header_code =
+        grpc_cpp_generator::GetHeaderPrologue(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetHeaderIncludes(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetHeaderServices(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetHeaderEpilogue(&pbfile, generator_parameters);
+    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> header_output(
+        context->Open(file_name + ".grpc.pb.h"));
+    grpc::protobuf::io::CodedOutputStream header_coded_out(header_output.get());
+    header_coded_out.WriteRaw(header_code.data(), header_code.size());
+
+    grpc::string source_code =
+        grpc_cpp_generator::GetSourcePrologue(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetSourceIncludes(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetSourceServices(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetSourceEpilogue(&pbfile, generator_parameters);
+    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> source_output(
+        context->Open(file_name + ".grpc.pb.cc"));
+    grpc::protobuf::io::CodedOutputStream source_coded_out(source_output.get());
+    source_coded_out.WriteRaw(source_code.data(), source_code.size());
+
+    if (!generator_parameters.generate_mock_code) {
+      return true;
+    }
+    grpc::string mock_code =
+        grpc_cpp_generator::GetMockPrologue(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetMockIncludes(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetMockServices(&pbfile, generator_parameters) +
+        grpc_cpp_generator::GetMockEpilogue(&pbfile, generator_parameters);
+    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> mock_output(
+        context->Open(file_name + "_mock.grpc.pb.h"));
+    grpc::protobuf::io::CodedOutputStream mock_coded_out(mock_output.get());
+    mock_coded_out.WriteRaw(mock_code.data(), mock_code.size());
+
+    return true;
+  }
+
+ private:
+  // Insert the given code into the given file at the given insertion point.
+  void Insert(grpc::protobuf::compiler::GeneratorContext* context,
+              const grpc::string& filename, const grpc::string& insertion_point,
+              const grpc::string& code) const {
+    std::unique_ptr<grpc::protobuf::io::ZeroCopyOutputStream> output(
+        context->OpenForInsert(filename, insertion_point));
+    grpc::protobuf::io::CodedOutputStream coded_out(output.get());
+    coded_out.WriteRaw(code.data(), code.size());
+  }
+};
+
+#endif  // GRPC_INTERNAL_COMPILER_CPP_PLUGIN_H

src/core/ext/filters/client_channel/backup_poller.cc

@@ -56,9 +56,14 @@ static backup_poller* g_poller = nullptr;  // guarded by g_poller_mu
 // treated as const.
 static int g_poll_interval_ms = DEFAULT_POLL_INTERVAL_MS;
 
-GPR_GLOBAL_CONFIG_DEFINE_INT32(grpc_client_channel_backup_poll_interval_ms,
-                               DEFAULT_POLL_INTERVAL_MS,
-                               "Client channel backup poll interval (ms)");
+GPR_GLOBAL_CONFIG_DEFINE_INT32(
+    grpc_client_channel_backup_poll_interval_ms, DEFAULT_POLL_INTERVAL_MS,
+    "Declares the interval in ms between two backup polls on client channels. "
+    "These polls are run in the timer thread so that gRPC can process "
+    "connection failures while there is no active polling thread. "
+    "They help reconnect disconnected client channels (mostly due to "
+    "idleness), so that the next RPC on this channel won't fail. Set to 0 to "
+    "turn off the backup polls.");
 
 static void init_globals() {
   gpr_mu_init(&g_poller_mu);

src/core/ext/filters/client_channel/channel_connectivity.cc

@@ -125,7 +125,7 @@ static void partly_done(state_watcher* w, bool due_to_completion,
   gpr_mu_lock(&w->mu);
 
   if (due_to_completion) {
-    if (grpc_trace_operation_failures.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_operation_failures)) {
       GRPC_LOG_IF_ERROR("watch_completion_error", GRPC_ERROR_REF(error));
     }
     GRPC_ERROR_UNREF(error);

src/core/ext/filters/client_channel/client_channel.cc

@@ -67,7 +67,6 @@
 #include "src/core/lib/transport/status_metadata.h"
 
 using grpc_core::internal::ClientChannelMethodParsedObject;
-using grpc_core::internal::ProcessedResolverResult;
 using grpc_core::internal::ServerRetryThrottleData;
 
 //
@@ -223,8 +222,9 @@ class ChannelData {
   ~ChannelData();
 
   static bool ProcessResolverResultLocked(
-      void* arg, const Resolver::Result& result, const char** lb_policy_name,
-      RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config);
+      void* arg, Resolver::Result* result, const char** lb_policy_name,
+      RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config,
+      grpc_error** service_config_error);
 
   grpc_error* DoPingLocked(grpc_transport_op* op);
 
@@ -232,6 +232,12 @@ class ChannelData {
 
   static void TryToConnectLocked(void* arg, grpc_error* error_ignored);
 
+  void ProcessLbPolicy(
+      const Resolver::Result& resolver_result,
+      const internal::ClientChannelGlobalParsedObject* parsed_service_config,
+      UniquePtr<char>* lb_policy_name,
+      RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config);
+
   //
   // Fields set at construction and never modified.
   //
@@ -241,6 +247,7 @@ class ChannelData {
   grpc_channel_stack* owning_stack_;
   ClientChannelFactory* client_channel_factory_;
   UniquePtr<char> server_name_;
+  RefCountedPtr<ServiceConfig> default_service_config_;
   // Initialized shortly after construction.
   channelz::ClientChannelNode* channelz_node_ = nullptr;
 
@@ -264,7 +271,8 @@ class ChannelData {
   OrphanablePtr<LoadBalancingPolicy> resolving_lb_policy_;
   grpc_connectivity_state_tracker state_tracker_;
   ExternalConnectivityWatcher::WatcherList external_connectivity_watcher_list_;
-  UniquePtr<char> health_check_service_name_;
+  RefCountedPtr<ServiceConfig> saved_service_config_;
+  bool received_first_resolver_result_ = false;
 
   //
   // Fields accessed from both data plane and control plane combiners.
@@ -942,18 +950,10 @@ class ChannelData::ClientChannelControlHelper
   }
 
   Subchannel* CreateSubchannel(const grpc_channel_args& args) override {
-    grpc_arg args_to_add[2];
-    int num_args_to_add = 0;
-    if (chand_->health_check_service_name_ != nullptr) {
-      args_to_add[0] = grpc_channel_arg_string_create(
-          const_cast<char*>("grpc.temp.health_check"),
-          const_cast<char*>(chand_->health_check_service_name_.get()));
-      num_args_to_add++;
-    }
-    args_to_add[num_args_to_add++] = SubchannelPoolInterface::CreateChannelArg(
+    grpc_arg arg = SubchannelPoolInterface::CreateChannelArg(
         chand_->subchannel_pool_.get());
     grpc_channel_args* new_args =
-        grpc_channel_args_copy_and_add(&args, args_to_add, num_args_to_add);
+        grpc_channel_args_copy_and_add(&args, &arg, 1);
     Subchannel* subchannel =
         chand_->client_channel_factory_->CreateSubchannel(new_args);
     grpc_channel_args_destroy(new_args);
@@ -970,7 +970,7 @@ class ChannelData::ClientChannelControlHelper
       UniquePtr<LoadBalancingPolicy::SubchannelPicker> picker) override {
     grpc_error* disconnect_error =
         chand_->disconnect_error_.Load(MemoryOrder::ACQUIRE);
-    if (grpc_client_channel_routing_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
       const char* extra = disconnect_error == GRPC_ERROR_NONE
                               ? ""
                               : " (ignoring -- channel shutting down)";
@@ -1066,6 +1066,19 @@ ChannelData::ChannelData(grpc_channel_element_args* args, grpc_error** error)
         "filter");
     return;
   }
+  // Get default service config
+  const char* service_config_json = grpc_channel_arg_get_string(
+      grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVICE_CONFIG));
+  // TODO(yashkt): Make sure we set the channel in TRANSIENT_FAILURE on an
+  // invalid default service config
+  if (service_config_json != nullptr) {
+    *error = GRPC_ERROR_NONE;
+    default_service_config_ = ServiceConfig::Create(service_config_json, error);
+    if (*error != GRPC_ERROR_NONE) {
+      default_service_config_.reset();
+      return;
+    }
+  }
   grpc_uri* uri = grpc_uri_parse(server_uri, true);
   if (uri != nullptr && uri->path[0] != '\0') {
     server_name_.reset(
@@ -1105,7 +1118,7 @@ ChannelData::ChannelData(grpc_channel_element_args* args, grpc_error** error)
   } else {
     grpc_pollset_set_add_pollset_set(resolving_lb_policy_->interested_parties(),
                                      interested_parties_);
-    if (grpc_client_channel_routing_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
       gpr_log(GPR_INFO, "chand=%p: created resolving_lb_policy=%p", this,
               resolving_lb_policy_.get());
     }
@@ -1128,40 +1141,172 @@ ChannelData::~ChannelData() {
   gpr_mu_destroy(&info_mu_);
 }
 
+void ChannelData::ProcessLbPolicy(
+    const Resolver::Result& resolver_result,
+    const internal::ClientChannelGlobalParsedObject* parsed_service_config,
+    UniquePtr<char>* lb_policy_name,
+    RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config) {
+  // Prefer the LB policy name found in the service config.
+  if (parsed_service_config != nullptr &&
+      parsed_service_config->parsed_lb_config() != nullptr) {
+    lb_policy_name->reset(
+        gpr_strdup(parsed_service_config->parsed_lb_config()->name()));
+    *lb_policy_config = parsed_service_config->parsed_lb_config();
+    return;
+  }
+  const char* local_policy_name = nullptr;
+  if (parsed_service_config != nullptr &&
+      parsed_service_config->parsed_deprecated_lb_policy() != nullptr) {
+    local_policy_name = parsed_service_config->parsed_deprecated_lb_policy();
+  } else {
+    const grpc_arg* channel_arg =
+        grpc_channel_args_find(resolver_result.args, GRPC_ARG_LB_POLICY_NAME);
+    local_policy_name = grpc_channel_arg_get_string(channel_arg);
+  }
+  // Special case: If at least one balancer address is present, we use
+  // the grpclb policy, regardless of what the resolver has returned.
+  bool found_balancer_address = false;
+  for (size_t i = 0; i < resolver_result.addresses.size(); ++i) {
+    const ServerAddress& address = resolver_result.addresses[i];
+    if (address.IsBalancer()) {
+      found_balancer_address = true;
+      break;
+    }
+  }
+  if (found_balancer_address) {
+    if (local_policy_name != nullptr &&
+        strcmp(local_policy_name, "grpclb") != 0) {
+      gpr_log(GPR_INFO,
+              "resolver requested LB policy %s but provided at least one "
+              "balancer address -- forcing use of grpclb LB policy",
+              local_policy_name);
+    }
+    local_policy_name = "grpclb";
+  }
+  // Use pick_first if nothing was specified and we didn't select grpclb
+  // above.
+  lb_policy_name->reset(gpr_strdup(
+      local_policy_name == nullptr ? "pick_first" : local_policy_name));
+}
+
 // Synchronous callback from ResolvingLoadBalancingPolicy to process a
 // resolver result update.
 bool ChannelData::ProcessResolverResultLocked(
-    void* arg, const Resolver::Result& result, const char** lb_policy_name,
-    RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config) {
+    void* arg, Resolver::Result* result, const char** lb_policy_name,
+    RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config,
+    grpc_error** service_config_error) {
   ChannelData* chand = static_cast<ChannelData*>(arg);
-  ProcessedResolverResult resolver_result(result);
-  char* service_config_json = gpr_strdup(resolver_result.service_config_json());
-  if (grpc_client_channel_routing_trace.enabled()) {
-    gpr_log(GPR_INFO, "chand=%p: resolver returned service config: \"%s\"",
-            chand, service_config_json);
-  }
-  chand->health_check_service_name_.reset(
-      gpr_strdup(resolver_result.health_check_service_name()));
-  // Create service config setter to update channel state in the data
-  // plane combiner.  Destroys itself when done.
-  New<ServiceConfigSetter>(chand, resolver_result.retry_throttle_data(),
-                           resolver_result.service_config());
+  RefCountedPtr<ServiceConfig> service_config;
+  // If resolver did not return a service config or returned an invalid service
+  // config, we need a fallback service config.
+  if (result->service_config_error != GRPC_ERROR_NONE) {
+    // If the service config was invalid, then fallback to the saved service
+    // config. If there is no saved config either, use the default service
+    // config.
+    if (chand->saved_service_config_ != nullptr) {
+      service_config = chand->saved_service_config_;
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+        gpr_log(GPR_INFO,
+                "chand=%p: resolver returned invalid service config. "
+                "Continuing to use previous service config.",
+                chand);
+      }
+    } else if (chand->default_service_config_ != nullptr) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+        gpr_log(GPR_INFO,
+                "chand=%p: resolver returned invalid service config. Using "
+                "default service config provided by client API.",
+                chand);
+      }
+      service_config = chand->default_service_config_;
+    }
+  } else if (result->service_config == nullptr) {
+    if (chand->default_service_config_ != nullptr) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+        gpr_log(GPR_INFO,
+                "chand=%p: resolver returned no service config. Using default "
+                "service config provided by client API.",
+                chand);
+      }
+      service_config = chand->default_service_config_;
+    }
+  } else {
+    service_config = result->service_config;
+  }
+  *service_config_error = GRPC_ERROR_REF(result->service_config_error);
+  if (service_config == nullptr &&
+      result->service_config_error != GRPC_ERROR_NONE) {
+    return false;
+  }
+  // Process service config.
+  UniquePtr<char> service_config_json;
+  const internal::ClientChannelGlobalParsedObject* parsed_service_config =
+      nullptr;
+  if (service_config != nullptr) {
+    parsed_service_config =
+        static_cast<const internal::ClientChannelGlobalParsedObject*>(
+            service_config->GetParsedGlobalServiceConfigObject(
+                internal::ClientChannelServiceConfigParser::ParserIndex()));
+  }
+  // TODO(roth): Eliminate this hack as part of hiding health check
+  // service name from LB policy API.  As part of this, change the API
+  // for this function to pass in result as a const reference.
+  if (parsed_service_config != nullptr &&
+      parsed_service_config->health_check_service_name() != nullptr) {
+    grpc_arg new_arg = grpc_channel_arg_string_create(
+        const_cast<char*>("grpc.temp.health_check"),
+        const_cast<char*>(parsed_service_config->health_check_service_name()));
+    grpc_channel_args* new_args =
+        grpc_channel_args_copy_and_add(result->args, &new_arg, 1);
+    grpc_channel_args_destroy(result->args);
+    result->args = new_args;
+  }
+  // Check if the config has changed.
+  const bool service_config_changed =
+      ((service_config == nullptr) !=
+       (chand->saved_service_config_ == nullptr)) ||
+      (service_config != nullptr &&
+       strcmp(service_config->service_config_json(),
+              chand->saved_service_config_->service_config_json()) != 0);
+  if (service_config_changed) {
+    service_config_json.reset(gpr_strdup(
+        service_config != nullptr ? service_config->service_config_json()
+                                  : ""));
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
+      gpr_log(GPR_INFO,
+              "chand=%p: resolver returned updated service config: \"%s\"",
+              chand, service_config_json.get());
+    }
+    chand->saved_service_config_ = std::move(service_config);
+  }
+  // We want to set the service config at least once. This should not really be
+  // needed, but we are doing it as a defensive approach. This can be removed,
+  // if we feel it is unnecessary.
+  if (service_config_changed || !chand->received_first_resolver_result_) {
+    chand->received_first_resolver_result_ = true;
+    Optional<internal::ClientChannelGlobalParsedObject::RetryThrottling>
+        retry_throttle_data;
+    if (parsed_service_config != nullptr) {
+      retry_throttle_data = parsed_service_config->retry_throttling();
+    }
+    // Create service config setter to update channel state in the data
+    // plane combiner.  Destroys itself when done.
+    New<ServiceConfigSetter>(chand, retry_throttle_data,
+                             chand->saved_service_config_);
+  }
+  UniquePtr<char> processed_lb_policy_name;
+  chand->ProcessLbPolicy(*result, parsed_service_config,
+                         &processed_lb_policy_name, lb_policy_config);
   // Swap out the data used by GetChannelInfo().
-  bool service_config_changed;
   {
     MutexLock lock(&chand->info_mu_);
-    chand->info_lb_policy_name_ = resolver_result.lb_policy_name();
-    service_config_changed =
-        ((service_config_json == nullptr) !=
-         (chand->info_service_config_json_ == nullptr)) ||
-        (service_config_json != nullptr &&
-         strcmp(service_config_json, chand->info_service_config_json_.get()) !=
-             0);
-    chand->info_service_config_json_.reset(service_config_json);
+    chand->info_lb_policy_name_ = std::move(processed_lb_policy_name);
+    if (service_config_json != nullptr) {
+      chand->info_service_config_json_ = std::move(service_config_json);
+    }
   }
   // Return results.
   *lb_policy_name = chand->info_lb_policy_name_.get();
-  *lb_policy_config = resolver_result.lb_policy_config();
   return service_config_changed;
 }
 
@@ -1407,7 +1552,7 @@ void CallData::StartTransportStreamOpBatch(
   }
   // If we've previously been cancelled, immediately fail any new batches.
   if (GPR_UNLIKELY(calld->cancel_error_ != GRPC_ERROR_NONE)) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: failing batch with error: %s",
               chand, calld, grpc_error_string(calld->cancel_error_));
     }
@@ -1426,7 +1571,7 @@ void CallData::StartTransportStreamOpBatch(
     GRPC_ERROR_UNREF(calld->cancel_error_);
     calld->cancel_error_ =
         GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error);
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: recording cancel_error=%s", chand,
               calld, grpc_error_string(calld->cancel_error_));
     }
@@ -1454,7 +1599,7 @@ void CallData::StartTransportStreamOpBatch(
   // the channel combiner, which is more efficient (especially for
   // streaming calls).
   if (calld->subchannel_call_ != nullptr) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: starting batch on subchannel_call=%p", chand,
               calld, calld->subchannel_call_.get());
@@ -1466,7 +1611,7 @@ void CallData::StartTransportStreamOpBatch(
   // For batches containing a send_initial_metadata op, enter the channel
   // combiner to start a pick.
   if (GPR_LIKELY(batch->send_initial_metadata)) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: entering client_channel combiner",
               chand, calld);
     }
@@ -1477,7 +1622,7 @@ void CallData::StartTransportStreamOpBatch(
         GRPC_ERROR_NONE);
   } else {
     // For all other batches, release the call combiner.
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: saved batch, yielding call combiner", chand,
               calld);
@@ -1535,7 +1680,7 @@ void CallData::MaybeCacheSendOpsForBatch(PendingBatch* pending) {
 }
 
 void CallData::FreeCachedSendInitialMetadata(ChannelData* chand) {
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: destroying calld->send_initial_metadata", chand,
             this);
@@ -1544,7 +1689,7 @@ void CallData::FreeCachedSendInitialMetadata(ChannelData* chand) {
 }
 
 void CallData::FreeCachedSendMessage(ChannelData* chand, size_t idx) {
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: destroying calld->send_messages[%" PRIuPTR "]",
             chand, this, idx);
@@ -1553,7 +1698,7 @@ void CallData::FreeCachedSendMessage(ChannelData* chand, size_t idx) {
 }
 
 void CallData::FreeCachedSendTrailingMetadata(ChannelData* chand) {
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: destroying calld->send_trailing_metadata",
             chand, this);
@@ -1630,7 +1775,7 @@ void CallData::PendingBatchesAdd(grpc_call_element* elem,
                                  grpc_transport_stream_op_batch* batch) {
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   const size_t idx = GetBatchIndex(batch);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: adding pending batch at index %" PRIuPTR, chand,
             this, idx);
@@ -1659,7 +1804,7 @@ void CallData::PendingBatchesAdd(grpc_call_element* elem,
     }
     if (GPR_UNLIKELY(bytes_buffered_for_retry_ >
                      chand->per_rpc_retry_buffer_size())) {
-      if (grpc_client_channel_call_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
         gpr_log(GPR_INFO,
                 "chand=%p calld=%p: exceeded retry buffer size, committing",
                 chand, this);
@@ -1672,7 +1817,7 @@ void CallData::PendingBatchesAdd(grpc_call_element* elem,
       // If we are not going to retry and have not yet started, pretend
       // retries are disabled so that we don't bother with retry overhead.
       if (num_attempts_completed_ == 0) {
-        if (grpc_client_channel_call_trace.enabled()) {
+        if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
           gpr_log(GPR_INFO,
                   "chand=%p calld=%p: disabling retries before first attempt",
                   chand, this);
@@ -1713,7 +1858,7 @@ void CallData::MaybeClearPendingBatch(grpc_call_element* elem,
       (!batch->recv_trailing_metadata ||
        batch->payload->recv_trailing_metadata.recv_trailing_metadata_ready ==
            nullptr)) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: clearing pending batch", chand,
               this);
     }
@@ -1736,7 +1881,7 @@ void CallData::PendingBatchesFail(
     grpc_call_element* elem, grpc_error* error,
     YieldCallCombinerPredicate yield_call_combiner_predicate) {
   GPR_ASSERT(error != GRPC_ERROR_NONE);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     size_t num_batches = 0;
     for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) {
       if (pending_batches_[i].batch != nullptr) ++num_batches;
@@ -1790,7 +1935,7 @@ void CallData::PendingBatchesResume(grpc_call_element* elem) {
     return;
   }
   // Retries not enabled; send down batches as-is.
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     size_t num_batches = 0;
     for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) {
       if (pending_batches_[i].batch != nullptr) ++num_batches;
@@ -1831,7 +1976,7 @@ CallData::PendingBatch* CallData::PendingBatchFind(grpc_call_element* elem,
     PendingBatch* pending = &pending_batches_[i];
     grpc_transport_stream_op_batch* batch = pending->batch;
     if (batch != nullptr && predicate(batch)) {
-      if (grpc_client_channel_call_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
         gpr_log(GPR_INFO,
                 "chand=%p calld=%p: %s pending batch at index %" PRIuPTR, chand,
                 this, log_message, i);
@@ -1851,7 +1996,7 @@ void CallData::RetryCommit(grpc_call_element* elem,
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   if (retry_committed_) return;
   retry_committed_ = true;
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: committing retries", chand, this);
   }
   if (retry_state != nullptr) {
@@ -1886,7 +2031,7 @@ void CallData::DoRetry(grpc_call_element* elem,
     }
     next_attempt_time = retry_backoff_->NextAttemptTime();
   }
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: retrying failed call in %" PRId64 " ms", chand,
             this, next_attempt_time - ExecCtx::Get()->Now());
@@ -1916,7 +2061,7 @@ bool CallData::MaybeRetry(grpc_call_element* elem,
     retry_state = static_cast<SubchannelCallRetryState*>(
         batch_data->subchannel_call->GetParentData());
     if (retry_state->retry_dispatched) {
-      if (grpc_client_channel_call_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
         gpr_log(GPR_INFO, "chand=%p calld=%p: retry already dispatched", chand,
                 this);
       }
@@ -1928,14 +2073,14 @@ bool CallData::MaybeRetry(grpc_call_element* elem,
     if (retry_throttle_data_ != nullptr) {
       retry_throttle_data_->RecordSuccess();
     }
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: call succeeded", chand, this);
     }
     return false;
   }
   // Status is not OK.  Check whether the status is retryable.
   if (!retry_policy->retryable_status_codes.Contains(status)) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: status %s not configured as retryable", chand,
               this, grpc_status_code_to_string(status));
@@ -1951,14 +2096,14 @@ bool CallData::MaybeRetry(grpc_call_element* elem,
   // checks, so that we don't fail to record failures due to other factors.
   if (retry_throttle_data_ != nullptr &&
       !retry_throttle_data_->RecordFailure()) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: retries throttled", chand, this);
     }
     return false;
   }
   // Check whether the call is committed.
   if (retry_committed_) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: retries already committed", chand,
               this);
     }
@@ -1967,7 +2112,7 @@ bool CallData::MaybeRetry(grpc_call_element* elem,
   // Check whether we have retries remaining.
   ++num_attempts_completed_;
   if (num_attempts_completed_ >= retry_policy->max_attempts) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO, "chand=%p calld=%p: exceeded %d retry attempts", chand,
               this, retry_policy->max_attempts);
     }
@@ -1975,7 +2120,7 @@ bool CallData::MaybeRetry(grpc_call_element* elem,
   }
   // If the call was cancelled from the surface, don't retry.
   if (cancel_error_ != GRPC_ERROR_NONE) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: call cancelled from surface, not retrying",
               chand, this);
@@ -1988,14 +2133,14 @@ bool CallData::MaybeRetry(grpc_call_element* elem,
     // If the value is "-1" or any other unparseable string, we do not retry.
     uint32_t ms;
     if (!grpc_parse_slice_to_uint32(GRPC_MDVALUE(*server_pushback_md), &ms)) {
-      if (grpc_client_channel_call_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
         gpr_log(GPR_INFO,
                 "chand=%p calld=%p: not retrying due to server push-back",
                 chand, this);
       }
       return false;
     } else {
-      if (grpc_client_channel_call_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
         gpr_log(GPR_INFO, "chand=%p calld=%p: server push-back: retry in %u ms",
                 chand, this, ms);
       }
@@ -2098,7 +2243,7 @@ void CallData::RecvInitialMetadataReady(void* arg, grpc_error* error) {
   grpc_call_element* elem = batch_data->elem;
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   CallData* calld = static_cast<CallData*>(elem->call_data);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: got recv_initial_metadata_ready, error=%s",
             chand, calld, grpc_error_string(error));
@@ -2122,7 +2267,7 @@ void CallData::RecvInitialMetadataReady(void* arg, grpc_error* error) {
   if (GPR_UNLIKELY((retry_state->trailing_metadata_available ||
                     error != GRPC_ERROR_NONE) &&
                    !retry_state->completed_recv_trailing_metadata)) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: deferring recv_initial_metadata_ready "
               "(Trailers-Only)",
@@ -2188,7 +2333,7 @@ void CallData::RecvMessageReady(void* arg, grpc_error* error) {
   grpc_call_element* elem = batch_data->elem;
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   CallData* calld = static_cast<CallData*>(elem->call_data);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: got recv_message_ready, error=%s",
             chand, calld, grpc_error_string(error));
   }
@@ -2210,7 +2355,7 @@ void CallData::RecvMessageReady(void* arg, grpc_error* error) {
   if (GPR_UNLIKELY(
           (retry_state->recv_message == nullptr || error != GRPC_ERROR_NONE) &&
           !retry_state->completed_recv_trailing_metadata)) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: deferring recv_message_ready (nullptr "
               "message and recv_trailing_metadata pending)",
@@ -2348,7 +2493,7 @@ void CallData::AddClosuresToFailUnstartedPendingBatches(
   for (size_t i = 0; i < GPR_ARRAY_SIZE(pending_batches_); ++i) {
     PendingBatch* pending = &pending_batches_[i];
     if (PendingBatchIsUnstarted(pending, retry_state)) {
-      if (grpc_client_channel_call_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
         gpr_log(GPR_INFO,
                 "chand=%p calld=%p: failing unstarted pending batch at index "
                 "%" PRIuPTR,
@@ -2394,7 +2539,7 @@ void CallData::RecvTrailingMetadataReady(void* arg, grpc_error* error) {
   grpc_call_element* elem = batch_data->elem;
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   CallData* calld = static_cast<CallData*>(elem->call_data);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: got recv_trailing_metadata_ready, error=%s",
             chand, calld, grpc_error_string(error));
@@ -2410,7 +2555,7 @@ void CallData::RecvTrailingMetadataReady(void* arg, grpc_error* error) {
       batch_data->batch.payload->recv_trailing_metadata.recv_trailing_metadata;
   calld->GetCallStatus(elem, md_batch, GRPC_ERROR_REF(error), &status,
                        &server_pushback_md);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: call finished, status=%s", chand,
             calld, grpc_status_code_to_string(status));
   }
@@ -2489,7 +2634,7 @@ void CallData::AddClosuresForReplayOrPendingSendOps(
     }
   }
   if (have_pending_send_message_ops || have_pending_send_trailing_metadata_op) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: starting next batch for pending send op(s)",
               chand, this);
@@ -2508,7 +2653,7 @@ void CallData::OnComplete(void* arg, grpc_error* error) {
   grpc_call_element* elem = batch_data->elem;
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   CallData* calld = static_cast<CallData*>(elem->call_data);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     char* batch_str = grpc_transport_stream_op_batch_string(&batch_data->batch);
     gpr_log(GPR_INFO, "chand=%p calld=%p: got on_complete, error=%s, batch=%s",
             chand, calld, grpc_error_string(error), batch_str);
@@ -2584,7 +2729,7 @@ void CallData::AddClosureForSubchannelBatch(
   batch->handler_private.extra_arg = subchannel_call_.get();
   GRPC_CLOSURE_INIT(&batch->handler_private.closure, StartBatchInCallCombiner,
                     batch, grpc_schedule_on_exec_ctx);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     char* batch_str = grpc_transport_stream_op_batch_string(batch);
     gpr_log(GPR_INFO, "chand=%p calld=%p: starting subchannel batch: %s", chand,
             this, batch_str);
@@ -2647,7 +2792,7 @@ void CallData::AddRetriableSendMessageOp(grpc_call_element* elem,
                                          SubchannelCallRetryState* retry_state,
                                          SubchannelCallBatchData* batch_data) {
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: starting calld->send_messages[%" PRIuPTR "]",
             chand, this, retry_state->started_send_message_count);
@@ -2730,7 +2875,7 @@ void CallData::AddRetriableRecvTrailingMetadataOp(
 
 void CallData::StartInternalRecvTrailingMetadata(grpc_call_element* elem) {
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: call failed but recv_trailing_metadata not "
             "started; starting it internally",
@@ -2762,7 +2907,7 @@ CallData::MaybeCreateSubchannelBatchForReplay(
   if (seen_send_initial_metadata_ &&
       !retry_state->started_send_initial_metadata &&
       !pending_send_initial_metadata_) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: replaying previously completed "
               "send_initial_metadata op",
@@ -2778,7 +2923,7 @@ CallData::MaybeCreateSubchannelBatchForReplay(
       retry_state->started_send_message_count ==
           retry_state->completed_send_message_count &&
       !pending_send_message_) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: replaying previously completed "
               "send_message op",
@@ -2798,7 +2943,7 @@ CallData::MaybeCreateSubchannelBatchForReplay(
       retry_state->started_send_message_count == send_messages_.size() &&
       !retry_state->started_send_trailing_metadata &&
       !pending_send_trailing_metadata_) {
-    if (grpc_client_channel_call_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: replaying previously completed "
               "send_trailing_metadata op",
@@ -2883,6 +3028,8 @@ void CallData::AddSubchannelBatchesForPendingBatches(
     // If we're not retrying, just send the batch as-is.
     if (method_params_ == nullptr ||
         method_params_->retry_policy() == nullptr || retry_committed_) {
+      // TODO(roth) : We should probably call
+      // MaybeInjectRecvTrailingMetadataReadyForLoadBalancingPolicy here.
       AddClosureForSubchannelBatch(elem, batch, closures);
       PendingBatchClear(pending);
       continue;
@@ -2941,7 +3088,7 @@ void CallData::StartRetriableSubchannelBatches(void* arg, grpc_error* ignored) {
   grpc_call_element* elem = static_cast<grpc_call_element*>(arg);
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   CallData* calld = static_cast<CallData*>(elem->call_data);
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: constructing retriable batches",
             chand, calld);
   }
@@ -2966,7 +3113,7 @@ void CallData::StartRetriableSubchannelBatches(void* arg, grpc_error* ignored) {
   // Now add pending batches.
   calld->AddSubchannelBatchesForPendingBatches(elem, retry_state, &closures);
   // Start batches on subchannel call.
-  if (grpc_client_channel_call_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_call_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: starting %" PRIuPTR
             " retriable batches on subchannel_call=%p",
@@ -2992,7 +3139,7 @@ void CallData::CreateSubchannelCall(grpc_call_element* elem) {
   grpc_error* error = GRPC_ERROR_NONE;
   subchannel_call_ =
       pick_.pick.connected_subchannel->CreateCall(call_args, &error);
-  if (grpc_client_channel_routing_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: create subchannel_call=%p: error=%s",
             chand, this, subchannel_call_.get(), grpc_error_string(error));
   }
@@ -3012,7 +3159,7 @@ void CallData::PickDone(void* arg, grpc_error* error) {
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
   CallData* calld = static_cast<CallData*>(elem->call_data);
   if (error != GRPC_ERROR_NONE) {
-    if (grpc_client_channel_routing_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: failed to pick subchannel: error=%s", chand,
               calld, grpc_error_string(error));
@@ -3041,7 +3188,7 @@ class CallData::QueuedPickCanceller {
     auto* self = static_cast<QueuedPickCanceller*>(arg);
     auto* chand = static_cast<ChannelData*>(self->elem_->channel_data);
     auto* calld = static_cast<CallData*>(self->elem_->call_data);
-    if (grpc_client_channel_routing_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
       gpr_log(GPR_INFO,
               "chand=%p calld=%p: cancelling queued pick: "
               "error=%s self=%p calld->pick_canceller=%p",
@@ -3065,7 +3212,7 @@ class CallData::QueuedPickCanceller {
 
 void CallData::RemoveCallFromQueuedPicksLocked(grpc_call_element* elem) {
   auto* chand = static_cast<ChannelData*>(elem->channel_data);
-  if (grpc_client_channel_routing_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: removing from queued picks list",
             chand, this);
   }
@@ -3077,7 +3224,7 @@ void CallData::RemoveCallFromQueuedPicksLocked(grpc_call_element* elem) {
 
 void CallData::AddCallToQueuedPicksLocked(grpc_call_element* elem) {
   auto* chand = static_cast<ChannelData*>(elem->channel_data);
-  if (grpc_client_channel_routing_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: adding to queued picks list", chand,
             this);
   }
@@ -3090,7 +3237,7 @@ void CallData::AddCallToQueuedPicksLocked(grpc_call_element* elem) {
 
 void CallData::ApplyServiceConfigToCallLocked(grpc_call_element* elem) {
   ChannelData* chand = static_cast<ChannelData*>(elem->channel_data);
-  if (grpc_client_channel_routing_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO, "chand=%p calld=%p: applying service config to call",
             chand, this);
   }
@@ -3199,7 +3346,7 @@ void CallData::StartPickLocked(void* arg, grpc_error* error) {
   // Attempt pick.
   error = GRPC_ERROR_NONE;
   auto pick_result = chand->picker()->Pick(&calld->pick_.pick, &error);
-  if (grpc_client_channel_routing_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_client_channel_routing_trace)) {
     gpr_log(GPR_INFO,
             "chand=%p calld=%p: LB pick returned %s (connected_subchannel=%p, "
             "error=%s)",

src/core/ext/filters/client_channel/client_channel_channelz.cc

@@ -127,7 +127,9 @@ void SubchannelNode::PopulateConnectivityState(grpc_json* json) {
   if (subchannel_ == nullptr) {
     state = GRPC_CHANNEL_SHUTDOWN;
   } else {
-    state = subchannel_->CheckConnectivity(true /* inhibit_health_checking */);
+    state = subchannel_->CheckConnectivityState(
+        nullptr /* health_check_service_name */,
+        nullptr /* connected_subchannel */);
   }
   json = grpc_json_create_child(nullptr, json, "state", nullptr,
                                 GRPC_JSON_OBJECT, false);

src/core/ext/filters/client_channel/health/health_check_client.cc

@@ -63,7 +63,7 @@ HealthCheckClient::HealthCheckClient(
               .set_jitter(HEALTH_CHECK_RECONNECT_JITTER)
               .set_max_backoff(HEALTH_CHECK_RECONNECT_MAX_BACKOFF_SECONDS *
                                1000)) {
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO, "created HealthCheckClient %p", this);
   }
   GRPC_CLOSURE_INIT(&retry_timer_callback_, OnRetryTimer, this,
@@ -72,7 +72,7 @@ HealthCheckClient::HealthCheckClient(
 }
 
 HealthCheckClient::~HealthCheckClient() {
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO, "destroying HealthCheckClient %p", this);
   }
   GRPC_ERROR_UNREF(error_);
@@ -99,7 +99,7 @@ void HealthCheckClient::SetHealthStatus(grpc_connectivity_state state,
 
 void HealthCheckClient::SetHealthStatusLocked(grpc_connectivity_state state,
                                               grpc_error* error) {
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO, "HealthCheckClient %p: setting state=%d error=%s", this,
             state, grpc_error_string(error));
   }
@@ -115,7 +115,7 @@ void HealthCheckClient::SetHealthStatusLocked(grpc_connectivity_state state,
 }
 
 void HealthCheckClient::Orphan() {
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO, "HealthCheckClient %p: shutting down", this);
   }
   {
@@ -145,7 +145,7 @@ void HealthCheckClient::StartCallLocked() {
   GPR_ASSERT(call_state_ == nullptr);
   SetHealthStatusLocked(GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE);
   call_state_ = MakeOrphanable<CallState>(Ref(), interested_parties_);
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO, "HealthCheckClient %p: created CallState %p", this,
             call_state_.get());
   }
@@ -159,7 +159,7 @@ void HealthCheckClient::StartRetryTimer() {
       GRPC_ERROR_CREATE_FROM_STATIC_STRING(
           "health check call failed; will retry after backoff"));
   grpc_millis next_try = retry_backoff_.NextAttemptTime();
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO, "HealthCheckClient %p: health check call lost...", this);
     grpc_millis timeout = next_try - ExecCtx::Get()->Now();
     if (timeout > 0) {
@@ -184,7 +184,7 @@ void HealthCheckClient::OnRetryTimer(void* arg, grpc_error* error) {
     self->retry_timer_callback_pending_ = false;
     if (!self->shutting_down_ && error == GRPC_ERROR_NONE &&
         self->call_state_ == nullptr) {
-      if (grpc_health_check_client_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
         gpr_log(GPR_INFO, "HealthCheckClient %p: restarting health check call",
                 self);
       }
@@ -277,15 +277,14 @@ bool DecodeResponse(grpc_slice_buffer* slice_buffer, grpc_error** error) {
 HealthCheckClient::CallState::CallState(
     RefCountedPtr<HealthCheckClient> health_check_client,
     grpc_pollset_set* interested_parties)
-    : InternallyRefCounted<CallState>(&grpc_health_check_client_trace),
-      health_check_client_(std::move(health_check_client)),
+    : health_check_client_(std::move(health_check_client)),
       pollent_(grpc_polling_entity_create_from_pollset_set(interested_parties)),
       arena_(Arena::Create(health_check_client_->connected_subchannel_
                                ->GetInitialCallSizeEstimate(0))),
       payload_(context_) {}
 
 HealthCheckClient::CallState::~CallState() {
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO, "HealthCheckClient %p: destroying CallState %p",
             health_check_client_.get(), this);
   }
@@ -322,7 +321,8 @@ void HealthCheckClient::CallState::StartCall() {
       0,  // parent_data_size
   };
   grpc_error* error = GRPC_ERROR_NONE;
-  call_ = health_check_client_->connected_subchannel_->CreateCall(args, &error);
+  call_ = health_check_client_->connected_subchannel_->CreateCall(args, &error)
+              .release();
   if (error != GRPC_ERROR_NONE) {
     gpr_log(GPR_ERROR,
             "HealthCheckClient %p CallState %p: error creating health "
@@ -331,18 +331,22 @@ void HealthCheckClient::CallState::StartCall() {
     GRPC_ERROR_UNREF(error);
     // Schedule instead of running directly, since we must not be
     // holding health_check_client_->mu_ when CallEnded() is called.
-    Ref(DEBUG_LOCATION, "call_end_closure").release();
+    call_->Ref(DEBUG_LOCATION, "call_end_closure").release();
     GRPC_CLOSURE_SCHED(
         GRPC_CLOSURE_INIT(&batch_.handler_private.closure, CallEndedRetry, this,
                           grpc_schedule_on_exec_ctx),
         GRPC_ERROR_NONE);
     return;
   }
+  // Register after-destruction callback.
+  GRPC_CLOSURE_INIT(&after_call_stack_destruction_, AfterCallStackDestruction,
+                    this, grpc_schedule_on_exec_ctx);
+  call_->SetAfterCallStackDestroy(&after_call_stack_destruction_);
   // Initialize payload and batch.
   payload_.context = context_;
   batch_.payload = &payload_;
   // on_complete callback takes ref, handled manually.
-  Ref(DEBUG_LOCATION, "on_complete").release();
+  call_->Ref(DEBUG_LOCATION, "on_complete").release();
   batch_.on_complete = GRPC_CLOSURE_INIT(&on_complete_, OnComplete, this,
                                          grpc_schedule_on_exec_ctx);
   // Add send_initial_metadata op.
@@ -375,7 +379,7 @@ void HealthCheckClient::CallState::StartCall() {
   payload_.recv_initial_metadata.trailing_metadata_available = nullptr;
   payload_.recv_initial_metadata.peer_string = nullptr;
   // recv_initial_metadata_ready callback takes ref, handled manually.
-  Ref(DEBUG_LOCATION, "recv_initial_metadata_ready").release();
+  call_->Ref(DEBUG_LOCATION, "recv_initial_metadata_ready").release();
   payload_.recv_initial_metadata.recv_initial_metadata_ready =
       GRPC_CLOSURE_INIT(&recv_initial_metadata_ready_, RecvInitialMetadataReady,
                         this, grpc_schedule_on_exec_ctx);
@@ -383,7 +387,7 @@ void HealthCheckClient::CallState::StartCall() {
   // Add recv_message op.
   payload_.recv_message.recv_message = &recv_message_;
   // recv_message callback takes ref, handled manually.
-  Ref(DEBUG_LOCATION, "recv_message_ready").release();
+  call_->Ref(DEBUG_LOCATION, "recv_message_ready").release();
   payload_.recv_message.recv_message_ready = GRPC_CLOSURE_INIT(
       &recv_message_ready_, RecvMessageReady, this, grpc_schedule_on_exec_ctx);
   batch_.recv_message = true;
@@ -419,7 +423,7 @@ void HealthCheckClient::CallState::StartBatchInCallCombiner(void* arg,
 
 void HealthCheckClient::CallState::StartBatch(
     grpc_transport_stream_op_batch* batch) {
-  batch->handler_private.extra_arg = call_.get();
+  batch->handler_private.extra_arg = call_;
   GRPC_CLOSURE_INIT(&batch->handler_private.closure, StartBatchInCallCombiner,
                     batch, grpc_schedule_on_exec_ctx);
   GRPC_CALL_COMBINER_START(&call_combiner_, &batch->handler_private.closure,
@@ -430,7 +434,7 @@ void HealthCheckClient::CallState::AfterCallStackDestruction(
     void* arg, grpc_error* error) {
   HealthCheckClient::CallState* self =
       static_cast<HealthCheckClient::CallState*>(arg);
-  self->Unref(DEBUG_LOCATION, "cancel");
+  Delete(self);
 }
 
 void HealthCheckClient::CallState::OnCancelComplete(void* arg,
@@ -438,10 +442,7 @@ void HealthCheckClient::CallState::OnCancelComplete(void* arg,
   HealthCheckClient::CallState* self =
       static_cast<HealthCheckClient::CallState*>(arg);
   GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "health_cancel");
-  GRPC_CLOSURE_INIT(&self->after_call_stack_destruction_,
-                    AfterCallStackDestruction, self, grpc_schedule_on_exec_ctx);
-  self->call_->SetAfterCallStackDestroy(&self->after_call_stack_destruction_);
-  self->call_.reset();
+  self->call_->Unref(DEBUG_LOCATION, "cancel");
 }
 
 void HealthCheckClient::CallState::StartCancel(void* arg, grpc_error* error) {
@@ -458,7 +459,7 @@ void HealthCheckClient::CallState::Cancel() {
   bool expected = false;
   if (cancelled_.CompareExchangeStrong(&expected, true, MemoryOrder::ACQ_REL,
                                        MemoryOrder::ACQUIRE)) {
-    Ref(DEBUG_LOCATION, "cancel").release();
+    call_->Ref(DEBUG_LOCATION, "cancel").release();
     GRPC_CALL_COMBINER_START(
         &call_combiner_,
         GRPC_CLOSURE_CREATE(StartCancel, this, grpc_schedule_on_exec_ctx),
@@ -472,7 +473,7 @@ void HealthCheckClient::CallState::OnComplete(void* arg, grpc_error* error) {
   GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "on_complete");
   grpc_metadata_batch_destroy(&self->send_initial_metadata_);
   grpc_metadata_batch_destroy(&self->send_trailing_metadata_);
-  self->Unref(DEBUG_LOCATION, "on_complete");
+  self->call_->Unref(DEBUG_LOCATION, "on_complete");
 }
 
 void HealthCheckClient::CallState::RecvInitialMetadataReady(void* arg,
@@ -481,7 +482,7 @@ void HealthCheckClient::CallState::RecvInitialMetadataReady(void* arg,
       static_cast<HealthCheckClient::CallState*>(arg);
   GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "recv_initial_metadata_ready");
   grpc_metadata_batch_destroy(&self->recv_initial_metadata_);
-  self->Unref(DEBUG_LOCATION, "recv_initial_metadata_ready");
+  self->call_->Unref(DEBUG_LOCATION, "recv_initial_metadata_ready");
 }
 
 void HealthCheckClient::CallState::DoneReadingRecvMessage(grpc_error* error) {
@@ -490,7 +491,7 @@ void HealthCheckClient::CallState::DoneReadingRecvMessage(grpc_error* error) {
     GRPC_ERROR_UNREF(error);
     Cancel();
     grpc_slice_buffer_destroy_internal(&recv_message_buffer_);
-    Unref(DEBUG_LOCATION, "recv_message_ready");
+    call_->Unref(DEBUG_LOCATION, "recv_message_ready");
     return;
   }
   const bool healthy = DecodeResponse(&recv_message_buffer_, &error);
@@ -563,7 +564,7 @@ void HealthCheckClient::CallState::RecvMessageReady(void* arg,
       static_cast<HealthCheckClient::CallState*>(arg);
   GRPC_CALL_COMBINER_STOP(&self->call_combiner_, "recv_message_ready");
   if (self->recv_message_ == nullptr) {
-    self->Unref(DEBUG_LOCATION, "recv_message_ready");
+    self->call_->Unref(DEBUG_LOCATION, "recv_message_ready");
     return;
   }
   grpc_slice_buffer_init(&self->recv_message_buffer_);
@@ -589,7 +590,7 @@ void HealthCheckClient::CallState::RecvTrailingMetadataReady(
     status = grpc_get_status_code_from_metadata(
         self->recv_trailing_metadata_.idx.named.grpc_status->md);
   }
-  if (grpc_health_check_client_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_health_check_client_trace)) {
     gpr_log(GPR_INFO,
             "HealthCheckClient %p CallState %p: health watch failed with "
             "status %d",
@@ -621,7 +622,7 @@ void HealthCheckClient::CallState::CallEndedRetry(void* arg,
   HealthCheckClient::CallState* self =
       static_cast<HealthCheckClient::CallState*>(arg);
   self->CallEnded(true /* retry */);
-  self->Unref(DEBUG_LOCATION, "call_end_closure");
+  self->call_->Unref(DEBUG_LOCATION, "call_end_closure");
 }
 
 void HealthCheckClient::CallState::CallEnded(bool retry) {
@@ -644,7 +645,9 @@ void HealthCheckClient::CallState::CallEnded(bool retry) {
       }
     }
   }
-  Unref(DEBUG_LOCATION, "call_ended");
+  // When the last ref to the call stack goes away, the CallState object
+  // will be automatically destroyed.
+  call_->Unref(DEBUG_LOCATION, "call_ended");
 }
 
 }  // namespace grpc_core

src/core/ext/filters/client_channel/health/health_check_client.h

@@ -61,7 +61,7 @@ class HealthCheckClient : public InternallyRefCounted<HealthCheckClient> {
 
  private:
   // Contains a call to the backend and all the data related to the call.
-  class CallState : public InternallyRefCounted<CallState> {
+  class CallState : public Orphanable {
    public:
     CallState(RefCountedPtr<HealthCheckClient> health_check_client,
               grpc_pollset_set* interested_parties_);
@@ -101,8 +101,10 @@ class HealthCheckClient : public InternallyRefCounted<HealthCheckClient> {
     grpc_core::CallCombiner call_combiner_;
     grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
 
-    // The streaming call to the backend. Always non-NULL.
-    RefCountedPtr<SubchannelCall> call_;
+    // The streaming call to the backend. Always non-null.
+    // Refs are tracked manually; when the last ref is released, the
+    // CallState object will be automatically destroyed.
+    SubchannelCall* call_;
 
     grpc_transport_stream_op_batch_payload payload_;
     grpc_transport_stream_op_batch batch_;

src/core/ext/filters/client_channel/http_connect_handshaker.cc

@@ -31,7 +31,6 @@
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/handshaker_registry.h"
-#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/http/format_request.h"

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -640,7 +640,7 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
   // If this request is from the pending child policy, ignore it until
   // it reports READY, at which point we swap it into place.
   if (CalledByPendingChild()) {
-    if (grpc_lb_glb_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO,
               "[grpclb %p helper %p] pending child policy %p reports state=%s",
               parent_.get(), this, parent_->pending_child_policy_.get(),
@@ -682,7 +682,7 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
   if (parent_->serverlist_ == nullptr ||
       (!parent_->serverlist_->ContainsAllDropEntries() &&
        state != GRPC_CHANNEL_READY)) {
-    if (grpc_lb_glb_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO,
               "[grpclb %p helper %p] state=%s passing child picker %p as-is",
               parent_.get(), this, grpc_connectivity_state_name(state),
@@ -692,7 +692,7 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
     return;
   }
   // Cases 2 and 3a: wrap picker from the child in our own picker.
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO, "[grpclb %p helper %p] state=%s wrapping child picker %p",
             parent_.get(), this, grpc_connectivity_state_name(state),
             picker.get());
@@ -715,7 +715,7 @@ void GrpcLb::Helper::RequestReresolution() {
           ? parent_->pending_child_policy_.get()
           : parent_->child_policy_.get();
   if (child_ != latest_child_policy) return;
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO,
             "[grpclb %p] Re-resolution requested from %schild policy (%p).",
             parent_.get(), CalledByPendingChild() ? "pending " : "", child_);
@@ -802,7 +802,7 @@ void GrpcLb::BalancerCallState::Orphan() {
 
 void GrpcLb::BalancerCallState::StartQuery() {
   GPR_ASSERT(lb_call_ != nullptr);
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO, "[grpclb %p] lb_calld=%p: Starting LB call %p",
             grpclb_policy_.get(), this, lb_call_);
   }
@@ -1009,7 +1009,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
       lb_calld->client_stats_report_interval_ = GPR_MAX(
           GPR_MS_PER_SEC, grpc_grpclb_duration_to_millis(
                               &initial_response->client_stats_report_interval));
-      if (grpc_lb_glb_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
         gpr_log(GPR_INFO,
                 "[grpclb %p] lb_calld=%p: Received initial LB response "
                 "message; client load reporting interval = %" PRId64
@@ -1017,7 +1017,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
                 grpclb_policy, lb_calld,
                 lb_calld->client_stats_report_interval_);
       }
-    } else if (grpc_lb_glb_trace.enabled()) {
+    } else if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO,
               "[grpclb %p] lb_calld=%p: Received initial LB response message; "
               "client load reporting NOT enabled",
@@ -1030,7 +1030,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
     // Have seen initial response, look for serverlist.
     GPR_ASSERT(lb_calld->lb_call_ != nullptr);
     auto serverlist_wrapper = MakeRefCounted<Serverlist>(serverlist);
-    if (grpc_lb_glb_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
       UniquePtr<char> serverlist_text = serverlist_wrapper->AsText();
       gpr_log(GPR_INFO,
               "[grpclb %p] lb_calld=%p: Serverlist with %" PRIuPTR
@@ -1051,7 +1051,7 @@ void GrpcLb::BalancerCallState::OnBalancerMessageReceivedLocked(
     // Check if the serverlist differs from the previous one.
     if (grpclb_policy->serverlist_ != nullptr &&
         *grpclb_policy->serverlist_ == *serverlist_wrapper) {
-      if (grpc_lb_glb_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
         gpr_log(GPR_INFO,
                 "[grpclb %p] lb_calld=%p: Incoming server list identical to "
                 "current, ignoring.",
@@ -1129,7 +1129,7 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
   BalancerCallState* lb_calld = static_cast<BalancerCallState*>(arg);
   GrpcLb* grpclb_policy = lb_calld->grpclb_policy();
   GPR_ASSERT(lb_calld->lb_call_ != nullptr);
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     char* status_details =
         grpc_slice_to_c_string(lb_calld->lb_call_status_details_);
     gpr_log(GPR_INFO,
@@ -1291,7 +1291,7 @@ GrpcLb::GrpcLb(Args args)
   grpc_uri* uri = grpc_uri_parse(server_uri, true);
   GPR_ASSERT(uri->path[0] != '\0');
   server_name_ = gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path);
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO,
             "[grpclb %p] Will use '%s' as the server name for LB request.",
             this, server_name_);
@@ -1535,7 +1535,7 @@ void GrpcLb::StartBalancerCallLocked() {
   // Init the LB call data.
   GPR_ASSERT(lb_calld_ == nullptr);
   lb_calld_ = MakeOrphanable<BalancerCallState>(Ref());
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO,
             "[grpclb %p] Query for backends (lb_channel: %p, lb_calld: %p)",
             this, lb_channel_, lb_calld_.get());
@@ -1545,7 +1545,7 @@ void GrpcLb::StartBalancerCallLocked() {
 
 void GrpcLb::StartBalancerCallRetryTimerLocked() {
   grpc_millis next_try = lb_call_backoff_.NextAttemptTime();
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO, "[grpclb %p] Connection to LB server lost...", this);
     grpc_millis timeout = next_try - ExecCtx::Get()->Now();
     if (timeout > 0) {
@@ -1572,7 +1572,7 @@ void GrpcLb::OnBalancerCallRetryTimerLocked(void* arg, grpc_error* error) {
   grpclb_policy->retry_timer_callback_pending_ = false;
   if (!grpclb_policy->shutting_down_ && error == GRPC_ERROR_NONE &&
       grpclb_policy->lb_calld_ == nullptr) {
-    if (grpc_lb_glb_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO, "[grpclb %p] Restarting call to LB server",
               grpclb_policy);
     }
@@ -1656,7 +1656,7 @@ OrphanablePtr<LoadBalancingPolicy> GrpcLb::CreateChildPolicyLocked(
     return nullptr;
   }
   helper->set_child(lb_policy.get());
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO, "[grpclb %p] Created new child policy %s (%p)", this,
             name, lb_policy.get());
   }
@@ -1755,7 +1755,7 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
     // Cases 1, 2b, and 3b: create a new child policy.
     // If child_policy_ is null, we set it (case 1), else we set
     // pending_child_policy_ (cases 2b and 3b).
-    if (grpc_lb_glb_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
       gpr_log(GPR_INFO, "[grpclb %p] Creating new %schild policy %s", this,
               child_policy_ == nullptr ? "" : "pending ", child_policy_name);
     }
@@ -1779,7 +1779,7 @@ void GrpcLb::CreateOrUpdateChildPolicyLocked() {
   }
   GPR_ASSERT(policy_to_update != nullptr);
   // Update the policy.
-  if (grpc_lb_glb_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_glb_trace)) {
     gpr_log(GPR_INFO, "[grpclb %p] Updating %schild policy %p", this,
             policy_to_update == pending_child_policy_.get() ? "pending " : "",
             policy_to_update);

src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc

@@ -68,9 +68,8 @@ class PickFirst : public LoadBalancingPolicy {
     PickFirstSubchannelData(
         SubchannelList<PickFirstSubchannelList, PickFirstSubchannelData>*
             subchannel_list,
-        const ServerAddress& address, Subchannel* subchannel,
-        grpc_combiner* combiner)
-        : SubchannelData(subchannel_list, address, subchannel, combiner) {}
+        const ServerAddress& address, Subchannel* subchannel)
+        : SubchannelData(subchannel_list, address, subchannel) {}
 
     void ProcessConnectivityChangeLocked(
         grpc_connectivity_state connectivity_state) override;
@@ -160,13 +159,13 @@ class PickFirst : public LoadBalancingPolicy {
 };
 
 PickFirst::PickFirst(Args args) : LoadBalancingPolicy(std::move(args)) {
-  if (grpc_lb_pick_first_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
     gpr_log(GPR_INFO, "Pick First %p created.", this);
   }
 }
 
 PickFirst::~PickFirst() {
-  if (grpc_lb_pick_first_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
     gpr_log(GPR_INFO, "Destroying Pick First %p", this);
   }
   GPR_ASSERT(subchannel_list_ == nullptr);
@@ -175,7 +174,7 @@ PickFirst::~PickFirst() {
 
 void PickFirst::ShutdownLocked() {
   AutoChildRefsUpdater guard(this);
-  if (grpc_lb_pick_first_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
     gpr_log(GPR_INFO, "Pick First %p Shutting down", this);
   }
   shutdown_ = true;
@@ -245,7 +244,7 @@ void PickFirst::UpdateChildRefsLocked() {
 
 void PickFirst::UpdateLocked(UpdateArgs args) {
   AutoChildRefsUpdater guard(this);
-  if (grpc_lb_pick_first_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
     gpr_log(GPR_INFO,
             "Pick First %p received update with %" PRIuPTR " addresses", this,
             args.addresses.size());
@@ -312,12 +311,13 @@ void PickFirst::UpdateLocked(UpdateArgs args) {
       // here, since we've already checked the initial connectivity
       // state of all subchannels above.
       subchannel_list_->subchannel(0)->StartConnectivityWatchLocked();
+      subchannel_list_->subchannel(0)->subchannel()->AttemptToConnect();
     }
   } else {
     // We do have a selected subchannel (which means it's READY), so keep
     // using it until one of the subchannels in the new list reports READY.
     if (latest_pending_subchannel_list_ != nullptr) {
-      if (grpc_lb_pick_first_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
         gpr_log(GPR_INFO,
                 "Pick First %p Shutting down latest pending subchannel list "
                 "%p, about to be replaced by newer latest %p",
@@ -334,6 +334,9 @@ void PickFirst::UpdateLocked(UpdateArgs args) {
       // state of all subchannels above.
       latest_pending_subchannel_list_->subchannel(0)
           ->StartConnectivityWatchLocked();
+      latest_pending_subchannel_list_->subchannel(0)
+          ->subchannel()
+          ->AttemptToConnect();
     }
   }
 }
@@ -349,7 +352,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
   GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN);
   // Handle updates for the currently selected subchannel.
   if (p->selected_ == this) {
-    if (grpc_lb_pick_first_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
       gpr_log(GPR_INFO,
               "Pick First %p selected subchannel connectivity changed to %s", p,
               grpc_connectivity_state_name(connectivity_state));
@@ -358,7 +361,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
     // pending update, switch to the pending update.
     if (connectivity_state != GRPC_CHANNEL_READY &&
         p->latest_pending_subchannel_list_ != nullptr) {
-      if (grpc_lb_pick_first_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
         gpr_log(GPR_INFO,
                 "Pick First %p promoting pending subchannel list %p to "
                 "replace %p",
@@ -366,7 +369,8 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
                 p->subchannel_list_.get());
       }
       p->selected_ = nullptr;
-      StopConnectivityWatchLocked();
+      CancelConnectivityWatchLocked(
+          "selected subchannel failed; switching to pending update");
       p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
       // Set our state to that of the pending subchannel list.
       if (p->subchannel_list_->in_transient_failure()) {
@@ -391,7 +395,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
         p->idle_ = true;
         p->channel_control_helper()->RequestReresolution();
         p->selected_ = nullptr;
-        StopConnectivityWatchLocked();
+        CancelConnectivityWatchLocked("selected subchannel failed; going IDLE");
         p->channel_control_helper()->UpdateState(
             GRPC_CHANNEL_IDLE,
             UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
@@ -408,8 +412,6 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
               connectivity_state,
               UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
         }
-        // Renew notification.
-        RenewConnectivityWatchLocked();
       }
     }
     return;
@@ -426,13 +428,11 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
   subchannel_list()->set_in_transient_failure(false);
   switch (connectivity_state) {
     case GRPC_CHANNEL_READY: {
-      // Renew notification.
-      RenewConnectivityWatchLocked();
       ProcessUnselectedReadyLocked();
       break;
     }
     case GRPC_CHANNEL_TRANSIENT_FAILURE: {
-      StopConnectivityWatchLocked();
+      CancelConnectivityWatchLocked("connection attempt failed");
       PickFirstSubchannelData* sd = this;
       size_t next_index =
           (sd->Index() + 1) % subchannel_list()->num_subchannels();
@@ -468,8 +468,6 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
             GRPC_CHANNEL_CONNECTING,
             UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
       }
-      // Renew notification.
-      RenewConnectivityWatchLocked();
       break;
     }
     case GRPC_CHANNEL_SHUTDOWN:
@@ -492,7 +490,7 @@ void PickFirst::PickFirstSubchannelData::ProcessUnselectedReadyLocked() {
              subchannel_list() == p->latest_pending_subchannel_list_.get());
   // Case 2.  Promote p->latest_pending_subchannel_list_ to p->subchannel_list_.
   if (subchannel_list() == p->latest_pending_subchannel_list_.get()) {
-    if (grpc_lb_pick_first_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
       gpr_log(GPR_INFO,
               "Pick First %p promoting pending subchannel list %p to "
               "replace %p",
@@ -506,7 +504,7 @@ void PickFirst::PickFirstSubchannelData::ProcessUnselectedReadyLocked() {
   p->channel_control_helper()->UpdateState(
       GRPC_CHANNEL_READY,
       UniquePtr<SubchannelPicker>(New<Picker>(connected_subchannel()->Ref())));
-  if (grpc_lb_pick_first_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_pick_first_trace)) {
     gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", p, subchannel());
   }
 }
@@ -521,8 +519,11 @@ void PickFirst::PickFirstSubchannelData::
   // If current state is READY, select the subchannel now, since we started
   // watching from this state and will not get a notification of it
   // transitioning into this state.
-  if (p->selected_ != this && current_state == GRPC_CHANNEL_READY) {
-    ProcessUnselectedReadyLocked();
+  // If the current state is not READY, attempt to connect.
+  if (current_state == GRPC_CHANNEL_READY) {
+    if (p->selected_ != this) ProcessUnselectedReadyLocked();
+  } else {
+    subchannel()->AttemptToConnect();
   }
 }
 

src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -83,9 +83,8 @@ class RoundRobin : public LoadBalancingPolicy {
     RoundRobinSubchannelData(
         SubchannelList<RoundRobinSubchannelList, RoundRobinSubchannelData>*
             subchannel_list,
-        const ServerAddress& address, Subchannel* subchannel,
-        grpc_combiner* combiner)
-        : SubchannelData(subchannel_list, address, subchannel, combiner) {}
+        const ServerAddress& address, Subchannel* subchannel)
+        : SubchannelData(subchannel_list, address, subchannel) {}
 
     grpc_connectivity_state connectivity_state() const {
       return last_connectivity_state_;
@@ -212,7 +211,7 @@ RoundRobin::Picker::Picker(RoundRobin* parent,
   // TODO(roth): rand(3) is not thread-safe.  This should be replaced with
   // something better as part of https://github.com/grpc/grpc/issues/17891.
   last_picked_index_ = rand() % subchannels_.size();
-  if (grpc_lb_round_robin_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO,
             "[RR %p picker %p] created picker from subchannel_list=%p "
             "with %" PRIuPTR " READY subchannels; last_picked_index_=%" PRIuPTR,
@@ -224,7 +223,7 @@ RoundRobin::Picker::Picker(RoundRobin* parent,
 RoundRobin::PickResult RoundRobin::Picker::Pick(PickArgs* pick,
                                                 grpc_error** error) {
   last_picked_index_ = (last_picked_index_ + 1) % subchannels_.size();
-  if (grpc_lb_round_robin_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO,
             "[RR %p picker %p] returning index %" PRIuPTR
             ", connected_subchannel=%p",
@@ -240,13 +239,13 @@ RoundRobin::PickResult RoundRobin::Picker::Pick(PickArgs* pick,
 //
 
 RoundRobin::RoundRobin(Args args) : LoadBalancingPolicy(std::move(args)) {
-  if (grpc_lb_round_robin_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO, "[RR %p] Created", this);
   }
 }
 
 RoundRobin::~RoundRobin() {
-  if (grpc_lb_round_robin_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO, "[RR %p] Destroying Round Robin policy", this);
   }
   GPR_ASSERT(subchannel_list_ == nullptr);
@@ -255,7 +254,7 @@ RoundRobin::~RoundRobin() {
 
 void RoundRobin::ShutdownLocked() {
   AutoChildRefsUpdater guard(this);
-  if (grpc_lb_round_robin_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO, "[RR %p] Shutting down", this);
   }
   shutdown_ = true;
@@ -320,6 +319,7 @@ void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
   for (size_t i = 0; i < num_subchannels(); i++) {
     if (subchannel(i)->subchannel() != nullptr) {
       subchannel(i)->StartConnectivityWatchLocked();
+      subchannel(i)->subchannel()->AttemptToConnect();
     }
   }
   // Now set the LB policy's state based on the subchannels' states.
@@ -403,7 +403,7 @@ void RoundRobin::RoundRobinSubchannelList::
       // therefore we would not be receiving a notification for them.
       GPR_ASSERT(p->latest_pending_subchannel_list_.get() == this);
       GPR_ASSERT(!shutting_down());
-      if (grpc_lb_round_robin_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
         const size_t old_num_subchannels =
             p->subchannel_list_ != nullptr
                 ? p->subchannel_list_->num_subchannels()
@@ -424,7 +424,7 @@ void RoundRobin::RoundRobinSubchannelList::
 void RoundRobin::RoundRobinSubchannelData::UpdateConnectivityStateLocked(
     grpc_connectivity_state connectivity_state) {
   RoundRobin* p = static_cast<RoundRobin*>(subchannel_list()->policy());
-  if (grpc_lb_round_robin_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(
         GPR_INFO,
         "[RR %p] connectivity changed for subchannel %p, subchannel_list %p "
@@ -448,17 +448,17 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
   // Otherwise, if the subchannel was already in state TRANSIENT_FAILURE
   // when the subchannel list was created, we'd wind up in a constant
   // loop of re-resolution.
+  // Also attempt to reconnect.
   if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
-    if (grpc_lb_round_robin_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
       gpr_log(GPR_INFO,
               "[RR %p] Subchannel %p has gone into TRANSIENT_FAILURE. "
               "Requesting re-resolution",
               p, subchannel());
     }
     p->channel_control_helper()->RequestReresolution();
+    subchannel()->AttemptToConnect();
   }
-  // Renew connectivity watch.
-  RenewConnectivityWatchLocked();
   // Update state counters.
   UpdateConnectivityStateLocked(connectivity_state);
   // Update overall state and renew notification.
@@ -467,13 +467,13 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
 
 void RoundRobin::UpdateLocked(UpdateArgs args) {
   AutoChildRefsUpdater guard(this);
-  if (grpc_lb_round_robin_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
     gpr_log(GPR_INFO, "[RR %p] received update with %" PRIuPTR " addresses",
             this, args.addresses.size());
   }
   // Replace latest_pending_subchannel_list_.
   if (latest_pending_subchannel_list_ != nullptr) {
-    if (grpc_lb_round_robin_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_round_robin_trace)) {
       gpr_log(GPR_INFO,
               "[RR %p] Shutting down previous pending subchannel list %p", this,
               latest_pending_subchannel_list_.get());

src/core/ext/filters/client_channel/lb_policy/subchannel_list.h

@@ -98,15 +98,13 @@ class SubchannelData {
 
   // Synchronously checks the subchannel's connectivity state.
   // Must not be called while there is a connectivity notification
-  // pending (i.e., between calling StartConnectivityWatchLocked() or
-  // RenewConnectivityWatchLocked() and the resulting invocation of
-  // ProcessConnectivityChangeLocked()).
+  // pending (i.e., between calling StartConnectivityWatchLocked() and
+  // calling CancelConnectivityWatchLocked()).
   grpc_connectivity_state CheckConnectivityStateLocked() {
-    GPR_ASSERT(!connectivity_notification_pending_);
-    pending_connectivity_state_unsafe_ = subchannel()->CheckConnectivity(
-        subchannel_list_->inhibit_health_checking());
-    UpdateConnectedSubchannelLocked();
-    return pending_connectivity_state_unsafe_;
+    GPR_ASSERT(pending_watcher_ == nullptr);
+    connectivity_state_ = subchannel()->CheckConnectivityState(
+        subchannel_list_->health_check_service_name(), &connected_subchannel_);
+    return connectivity_state_;
   }
 
   // Resets the connection backoff.
@@ -115,23 +113,11 @@ class SubchannelData {
   void ResetBackoffLocked();
 
   // Starts watching the connectivity state of the subchannel.
-  // ProcessConnectivityChangeLocked() will be called when the
+  // ProcessConnectivityChangeLocked() will be called whenever the
   // connectivity state changes.
   void StartConnectivityWatchLocked();
 
-  // Renews watching the connectivity state of the subchannel.
-  void RenewConnectivityWatchLocked();
-
-  // Stops watching the connectivity state of the subchannel.
-  void StopConnectivityWatchLocked();
-
   // Cancels watching the connectivity state of the subchannel.
-  // Must be called only while there is a connectivity notification
-  // pending (i.e., between calling StartConnectivityWatchLocked() or
-  // RenewConnectivityWatchLocked() and the resulting invocation of
-  // ProcessConnectivityChangeLocked()).
-  // From within ProcessConnectivityChangeLocked(), use
-  // StopConnectivityWatchLocked() instead.
   void CancelConnectivityWatchLocked(const char* reason);
 
   // Cancels any pending connectivity watch and unrefs the subchannel.
@@ -142,44 +128,80 @@ class SubchannelData {
  protected:
   SubchannelData(
       SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list,
-      const ServerAddress& address, Subchannel* subchannel,
-      grpc_combiner* combiner);
+      const ServerAddress& address, Subchannel* subchannel);
 
   virtual ~SubchannelData();
 
-  // After StartConnectivityWatchLocked() or RenewConnectivityWatchLocked()
-  // is called, this method will be invoked when the subchannel's connectivity
-  // state changes.
-  // Implementations must invoke either RenewConnectivityWatchLocked() or
-  // StopConnectivityWatchLocked() before returning.
+  // After StartConnectivityWatchLocked() is called, this method will be
+  // invoked whenever the subchannel's connectivity state changes.
+  // To stop watching, use CancelConnectivityWatchLocked().
   virtual void ProcessConnectivityChangeLocked(
       grpc_connectivity_state connectivity_state) GRPC_ABSTRACT;
 
-  // Unrefs the subchannel.
-  void UnrefSubchannelLocked(const char* reason);
-
  private:
-  // Updates connected_subchannel_ based on pending_connectivity_state_unsafe_.
-  // Returns true if the connectivity state should be reported.
-  bool UpdateConnectedSubchannelLocked();
+  // Watcher for subchannel connectivity state.
+  class Watcher : public Subchannel::ConnectivityStateWatcher {
+   public:
+    Watcher(
+        SubchannelData<SubchannelListType, SubchannelDataType>* subchannel_data,
+        RefCountedPtr<SubchannelListType> subchannel_list)
+        : subchannel_data_(subchannel_data),
+          subchannel_list_(std::move(subchannel_list)) {}
+
+    ~Watcher() { subchannel_list_.reset(DEBUG_LOCATION, "Watcher dtor"); }
+
+    void OnConnectivityStateChange(
+        grpc_connectivity_state new_state,
+        RefCountedPtr<ConnectedSubchannel> connected_subchannel) override;
+
+    grpc_pollset_set* interested_parties() override {
+      return subchannel_list_->policy()->interested_parties();
+    }
+
+   private:
+    // A fire-and-forget class that bounces into the combiner to process
+    // a connectivity state update.
+    class Updater {
+     public:
+      Updater(
+          SubchannelData<SubchannelListType, SubchannelDataType>*
+              subchannel_data,
+          RefCountedPtr<SubchannelList<SubchannelListType, SubchannelDataType>>
+              subchannel_list,
+          grpc_connectivity_state state,
+          RefCountedPtr<ConnectedSubchannel> connected_subchannel);
+
+      ~Updater() {
+        subchannel_list_.reset(DEBUG_LOCATION, "Watcher::Updater dtor");
+      }
+
+     private:
+      static void OnUpdateLocked(void* arg, grpc_error* error);
 
-  static void OnConnectivityChangedLocked(void* arg, grpc_error* error);
+      SubchannelData<SubchannelListType, SubchannelDataType>* subchannel_data_;
+      RefCountedPtr<SubchannelList<SubchannelListType, SubchannelDataType>>
+          subchannel_list_;
+      const grpc_connectivity_state state_;
+      RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
+      grpc_closure closure_;
+    };
+
+    SubchannelData<SubchannelListType, SubchannelDataType>* subchannel_data_;
+    RefCountedPtr<SubchannelListType> subchannel_list_;
+  };
+
+  // Unrefs the subchannel.
+  void UnrefSubchannelLocked(const char* reason);
 
   // Backpointer to owning subchannel list.  Not owned.
   SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list_;
-
-  // The subchannel and connected subchannel.
+  // The subchannel.
   Subchannel* subchannel_;
+  // Will be non-null when the subchannel's state is being watched.
+  Subchannel::ConnectivityStateWatcher* pending_watcher_ = nullptr;
+  // Data updated by the watcher.
+  grpc_connectivity_state connectivity_state_;
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
-
-  // Notification that connectivity has changed on subchannel.
-  grpc_closure connectivity_changed_closure_;
-  // Is a connectivity notification pending?
-  bool connectivity_notification_pending_ = false;
-  // Connectivity state to be updated by
-  // grpc_subchannel_notify_on_state_change(), not guarded by
-  // the combiner.
-  grpc_connectivity_state pending_connectivity_state_unsafe_;
 };
 
 // A list of subchannels.
@@ -213,7 +235,9 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
   // Accessors.
   LoadBalancingPolicy* policy() const { return policy_; }
   TraceFlag* tracer() const { return tracer_; }
-  bool inhibit_health_checking() const { return inhibit_health_checking_; }
+  const char* health_check_service_name() const {
+    return health_check_service_name_.get();
+  }
 
   // Resets connection backoff of all subchannels.
   // TODO(roth): We will probably need to rethink this as part of moving
@@ -251,7 +275,7 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
 
   TraceFlag* tracer_;
 
-  bool inhibit_health_checking_;
+  UniquePtr<char> health_check_service_name_;
 
   grpc_combiner* combiner_;
 
@@ -268,6 +292,67 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
 // implementation -- no user-servicable parts below
 //
 
+//
+// SubchannelData::Watcher
+//
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType, SubchannelDataType>::Watcher::
+    OnConnectivityStateChange(
+        grpc_connectivity_state new_state,
+        RefCountedPtr<ConnectedSubchannel> connected_subchannel) {
+  // Will delete itself.
+  New<Updater>(subchannel_data_,
+               subchannel_list_->Ref(DEBUG_LOCATION, "Watcher::Updater"),
+               new_state, std::move(connected_subchannel));
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+SubchannelData<SubchannelListType, SubchannelDataType>::Watcher::Updater::
+    Updater(
+        SubchannelData<SubchannelListType, SubchannelDataType>* subchannel_data,
+        RefCountedPtr<SubchannelList<SubchannelListType, SubchannelDataType>>
+            subchannel_list,
+        grpc_connectivity_state state,
+        RefCountedPtr<ConnectedSubchannel> connected_subchannel)
+    : subchannel_data_(subchannel_data),
+      subchannel_list_(std::move(subchannel_list)),
+      state_(state),
+      connected_subchannel_(std::move(connected_subchannel)) {
+  GRPC_CLOSURE_INIT(&closure_, &OnUpdateLocked, this,
+                    grpc_combiner_scheduler(subchannel_list_->combiner_));
+  GRPC_CLOSURE_SCHED(&closure_, GRPC_ERROR_NONE);
+}
+
+template <typename SubchannelListType, typename SubchannelDataType>
+void SubchannelData<SubchannelListType, SubchannelDataType>::Watcher::Updater::
+    OnUpdateLocked(void* arg, grpc_error* error) {
+  Updater* self = static_cast<Updater*>(arg);
+  SubchannelData* sd = self->subchannel_data_;
+  if (GRPC_TRACE_FLAG_ENABLED(*sd->subchannel_list_->tracer())) {
+    gpr_log(GPR_INFO,
+            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
+            " (subchannel %p): connectivity changed: state=%s, "
+            "connected_subchannel=%p, shutting_down=%d, pending_watcher=%p",
+            sd->subchannel_list_->tracer()->name(),
+            sd->subchannel_list_->policy(), sd->subchannel_list_, sd->Index(),
+            sd->subchannel_list_->num_subchannels(), sd->subchannel_,
+            grpc_connectivity_state_name(self->state_),
+            self->connected_subchannel_.get(),
+            sd->subchannel_list_->shutting_down(), sd->pending_watcher_);
+  }
+  if (!sd->subchannel_list_->shutting_down() &&
+      sd->pending_watcher_ != nullptr) {
+    sd->connectivity_state_ = self->state_;
+    // Get or release ref to connected subchannel.
+    sd->connected_subchannel_ = std::move(self->connected_subchannel_);
+    // Call the subclass's ProcessConnectivityChangeLocked() method.
+    sd->ProcessConnectivityChangeLocked(sd->connectivity_state_);
+  }
+  // Clean up.
+  Delete(self);
+}
+
 //
 // SubchannelData
 //
@@ -275,30 +360,23 @@ class SubchannelList : public InternallyRefCounted<SubchannelListType> {
 template <typename SubchannelListType, typename SubchannelDataType>
 SubchannelData<SubchannelListType, SubchannelDataType>::SubchannelData(
     SubchannelList<SubchannelListType, SubchannelDataType>* subchannel_list,
-    const ServerAddress& address, Subchannel* subchannel,
-    grpc_combiner* combiner)
+    const ServerAddress& address, Subchannel* subchannel)
     : subchannel_list_(subchannel_list),
       subchannel_(subchannel),
       // We assume that the current state is IDLE.  If not, we'll get a
       // callback telling us that.
-      pending_connectivity_state_unsafe_(GRPC_CHANNEL_IDLE) {
-  GRPC_CLOSURE_INIT(
-      &connectivity_changed_closure_,
-      (&SubchannelData<SubchannelListType,
-                       SubchannelDataType>::OnConnectivityChangedLocked),
-      this, grpc_combiner_scheduler(combiner));
-}
+      connectivity_state_(GRPC_CHANNEL_IDLE) {}
 
 template <typename SubchannelListType, typename SubchannelDataType>
 SubchannelData<SubchannelListType, SubchannelDataType>::~SubchannelData() {
-  UnrefSubchannelLocked("subchannel_data_destroy");
+  GPR_ASSERT(subchannel_ == nullptr);
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>
 void SubchannelData<SubchannelListType, SubchannelDataType>::
     UnrefSubchannelLocked(const char* reason) {
   if (subchannel_ != nullptr) {
-    if (subchannel_list_->tracer()->enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
       gpr_log(GPR_INFO,
               "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
               " (subchannel %p): unreffing subchannel",
@@ -323,65 +401,28 @@ void SubchannelData<SubchannelListType,
 template <typename SubchannelListType, typename SubchannelDataType>
 void SubchannelData<SubchannelListType,
                     SubchannelDataType>::StartConnectivityWatchLocked() {
-  if (subchannel_list_->tracer()->enabled()) {
-    gpr_log(GPR_INFO,
-            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): starting watch: requesting connectivity change "
-            "notification (from %s)",
-            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
-            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-            subchannel_,
-            grpc_connectivity_state_name(pending_connectivity_state_unsafe_));
-  }
-  GPR_ASSERT(!connectivity_notification_pending_);
-  connectivity_notification_pending_ = true;
-  subchannel_list()->Ref(DEBUG_LOCATION, "connectivity_watch").release();
-  subchannel_->NotifyOnStateChange(
-      subchannel_list_->policy()->interested_parties(),
-      &pending_connectivity_state_unsafe_, &connectivity_changed_closure_,
-      subchannel_list_->inhibit_health_checking());
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-void SubchannelData<SubchannelListType,
-                    SubchannelDataType>::RenewConnectivityWatchLocked() {
-  if (subchannel_list_->tracer()->enabled()) {
-    gpr_log(GPR_INFO,
-            "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): renewing watch: requesting connectivity change "
-            "notification (from %s)",
-            subchannel_list_->tracer()->name(), subchannel_list_->policy(),
-            subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-            subchannel_,
-            grpc_connectivity_state_name(pending_connectivity_state_unsafe_));
-  }
-  GPR_ASSERT(connectivity_notification_pending_);
-  subchannel_->NotifyOnStateChange(
-      subchannel_list_->policy()->interested_parties(),
-      &pending_connectivity_state_unsafe_, &connectivity_changed_closure_,
-      subchannel_list_->inhibit_health_checking());
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-void SubchannelData<SubchannelListType,
-                    SubchannelDataType>::StopConnectivityWatchLocked() {
-  if (subchannel_list_->tracer()->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
     gpr_log(GPR_INFO,
             "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-            " (subchannel %p): stopping connectivity watch",
+            " (subchannel %p): starting watch (from %s)",
             subchannel_list_->tracer()->name(), subchannel_list_->policy(),
             subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-            subchannel_);
+            subchannel_, grpc_connectivity_state_name(connectivity_state_));
   }
-  GPR_ASSERT(connectivity_notification_pending_);
-  connectivity_notification_pending_ = false;
-  subchannel_list()->Unref(DEBUG_LOCATION, "connectivity_watch");
+  GPR_ASSERT(pending_watcher_ == nullptr);
+  pending_watcher_ =
+      New<Watcher>(this, subchannel_list()->Ref(DEBUG_LOCATION, "Watcher"));
+  subchannel_->WatchConnectivityState(
+      connectivity_state_,
+      UniquePtr<char>(
+          gpr_strdup(subchannel_list_->health_check_service_name())),
+      UniquePtr<Subchannel::ConnectivityStateWatcher>(pending_watcher_));
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>
 void SubchannelData<SubchannelListType, SubchannelDataType>::
     CancelConnectivityWatchLocked(const char* reason) {
-  if (subchannel_list_->tracer()->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*subchannel_list_->tracer())) {
     gpr_log(GPR_INFO,
             "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
             " (subchannel %p): canceling connectivity watch (%s)",
@@ -389,91 +430,17 @@ void SubchannelData<SubchannelListType, SubchannelDataType>::
             subchannel_list_, Index(), subchannel_list_->num_subchannels(),
             subchannel_, reason);
   }
-  GPR_ASSERT(connectivity_notification_pending_);
-  subchannel_->NotifyOnStateChange(nullptr, nullptr,
-                                   &connectivity_changed_closure_,
-                                   subchannel_list_->inhibit_health_checking());
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-bool SubchannelData<SubchannelListType,
-                    SubchannelDataType>::UpdateConnectedSubchannelLocked() {
-  // If the subchannel is READY, take a ref to the connected subchannel.
-  if (pending_connectivity_state_unsafe_ == GRPC_CHANNEL_READY) {
-    connected_subchannel_ = subchannel_->connected_subchannel();
-    // If the subchannel became disconnected between the time that READY
-    // was reported and the time we got here (e.g., between when a
-    // notification callback is scheduled and when it was actually run in
-    // the combiner), then the connected subchannel may have disappeared out
-    // from under us.  In that case, we don't actually want to consider the
-    // subchannel to be in state READY.  Instead, we use IDLE as the
-    // basis for any future connectivity watch; this is the one state that
-    // the subchannel will never transition back into, so this ensures
-    // that we will get a notification for the next state, even if that state
-    // is READY again (e.g., if the subchannel has transitioned back to
-    // READY before the next watch gets requested).
-    if (connected_subchannel_ == nullptr) {
-      if (subchannel_list_->tracer()->enabled()) {
-        gpr_log(GPR_INFO,
-                "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-                " (subchannel %p): state is READY but connected subchannel is "
-                "null; moving to state IDLE",
-                subchannel_list_->tracer()->name(), subchannel_list_->policy(),
-                subchannel_list_, Index(), subchannel_list_->num_subchannels(),
-                subchannel_);
-      }
-      pending_connectivity_state_unsafe_ = GRPC_CHANNEL_IDLE;
-      return false;
-    }
-  } else {
-    // For any state other than READY, unref the connected subchannel.
-    connected_subchannel_.reset();
-  }
-  return true;
-}
-
-template <typename SubchannelListType, typename SubchannelDataType>
-void SubchannelData<SubchannelListType, SubchannelDataType>::
-    OnConnectivityChangedLocked(void* arg, grpc_error* error) {
-  SubchannelData* sd = static_cast<SubchannelData*>(arg);
-  if (sd->subchannel_list_->tracer()->enabled()) {
-    gpr_log(
-        GPR_INFO,
-        "[%s %p] subchannel list %p index %" PRIuPTR " of %" PRIuPTR
-        " (subchannel %p): connectivity changed: state=%s, error=%s, "
-        "shutting_down=%d",
-        sd->subchannel_list_->tracer()->name(), sd->subchannel_list_->policy(),
-        sd->subchannel_list_, sd->Index(),
-        sd->subchannel_list_->num_subchannels(), sd->subchannel_,
-        grpc_connectivity_state_name(sd->pending_connectivity_state_unsafe_),
-        grpc_error_string(error), sd->subchannel_list_->shutting_down());
-  }
-  // If shutting down, unref subchannel and stop watching.
-  if (sd->subchannel_list_->shutting_down() || error == GRPC_ERROR_CANCELLED) {
-    sd->UnrefSubchannelLocked("connectivity_shutdown");
-    sd->StopConnectivityWatchLocked();
-    return;
-  }
-  // Get or release ref to connected subchannel.
-  if (!sd->UpdateConnectedSubchannelLocked()) {
-    // We don't want to report this connectivity state, so renew the watch.
-    sd->RenewConnectivityWatchLocked();
-    return;
+  if (pending_watcher_ != nullptr) {
+    subchannel_->CancelConnectivityStateWatch(
+        subchannel_list_->health_check_service_name(), pending_watcher_);
+    pending_watcher_ = nullptr;
   }
-  // Call the subclass's ProcessConnectivityChangeLocked() method.
-  sd->ProcessConnectivityChangeLocked(sd->pending_connectivity_state_unsafe_);
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>
 void SubchannelData<SubchannelListType, SubchannelDataType>::ShutdownLocked() {
-  // If there's a pending notification for this subchannel, cancel it;
-  // the callback is responsible for unreffing the subchannel.
-  // Otherwise, unref the subchannel directly.
-  if (connectivity_notification_pending_) {
-    CancelConnectivityWatchLocked("shutdown");
-  } else if (subchannel_ != nullptr) {
-    UnrefSubchannelLocked("shutdown");
-  }
+  if (pending_watcher_ != nullptr) CancelConnectivityWatchLocked("shutdown");
+  UnrefSubchannelLocked("shutdown");
 }
 
 //
@@ -490,23 +457,40 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
       policy_(policy),
       tracer_(tracer),
       combiner_(GRPC_COMBINER_REF(combiner, "subchannel_list")) {
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO,
             "[%s %p] Creating subchannel list %p for %" PRIuPTR " subchannels",
             tracer_->name(), policy, this, addresses.size());
   }
   subchannels_.reserve(addresses.size());
+  // Find health check service name.
+  const bool inhibit_health_checking = grpc_channel_arg_get_bool(
+      grpc_channel_args_find(&args, GRPC_ARG_INHIBIT_HEALTH_CHECKING), false);
+  if (!inhibit_health_checking) {
+    const char* health_check_service_name = grpc_channel_arg_get_string(
+        grpc_channel_args_find(&args, "grpc.temp.health_check"));
+    if (health_check_service_name != nullptr) {
+      health_check_service_name_.reset(gpr_strdup(health_check_service_name));
+    }
+  }
   // We need to remove the LB addresses in order to be able to compare the
   // subchannel keys of subchannels from a different batch of addresses.
-  // We also remove the inhibit-health-checking arg, since we are
+  // We also remove the health-checking-related args, since we are
   // handling that here.
-  inhibit_health_checking_ = grpc_channel_arg_get_bool(
-      grpc_channel_args_find(&args, GRPC_ARG_INHIBIT_HEALTH_CHECKING), false);
-  static const char* keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS,
-                                         GRPC_ARG_INHIBIT_HEALTH_CHECKING};
+  // We remove the service config, since it will be passed into the
+  // subchannel via call context.
+  static const char* keys_to_remove[] = {
+      GRPC_ARG_SUBCHANNEL_ADDRESS, "grpc.temp.health_check",
+      GRPC_ARG_INHIBIT_HEALTH_CHECKING, GRPC_ARG_SERVICE_CONFIG};
   // Create a subchannel for each address.
   for (size_t i = 0; i < addresses.size(); i++) {
-    GPR_ASSERT(!addresses[i].IsBalancer());
+    // TODO(roth): we should ideally hide this from the LB policy code. In
+    // principle, if we're dealing with this special case in the client_channel
+    // code for selecting grpclb, then we should also strip out these addresses
+    // there if we're not using grpclb.
+    if (addresses[i].IsBalancer()) {
+      continue;
+    }
     InlinedVector<grpc_arg, 3> args_to_add;
     const size_t subchannel_address_arg_index = args_to_add.size();
     args_to_add.emplace_back(
@@ -524,7 +508,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
     grpc_channel_args_destroy(new_args);
     if (subchannel == nullptr) {
       // Subchannel could not be created.
-      if (tracer_->enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
         char* address_uri = grpc_sockaddr_to_uri(&addresses[i].address());
         gpr_log(GPR_INFO,
                 "[%s %p] could not create subchannel for address uri %s, "
@@ -534,7 +518,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
       }
       continue;
     }
-    if (tracer_->enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
       char* address_uri = grpc_sockaddr_to_uri(&addresses[i].address());
       gpr_log(GPR_INFO,
               "[%s %p] subchannel list %p index %" PRIuPTR
@@ -543,13 +527,13 @@ SubchannelList<SubchannelListType, SubchannelDataType>::SubchannelList(
               address_uri);
       gpr_free(address_uri);
     }
-    subchannels_.emplace_back(this, addresses[i], subchannel, combiner);
+    subchannels_.emplace_back(this, addresses[i], subchannel);
   }
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>
 SubchannelList<SubchannelListType, SubchannelDataType>::~SubchannelList() {
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO, "[%s %p] Destroying subchannel_list %p", tracer_->name(),
             policy_, this);
   }
@@ -558,7 +542,7 @@ SubchannelList<SubchannelListType, SubchannelDataType>::~SubchannelList() {
 
 template <typename SubchannelListType, typename SubchannelDataType>
 void SubchannelList<SubchannelListType, SubchannelDataType>::ShutdownLocked() {
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO, "[%s %p] Shutting down subchannel_list %p",
             tracer_->name(), policy_, this);
   }

src/core/ext/filters/client_channel/lb_policy/xds/xds.cc

@@ -608,7 +608,7 @@ void XdsLb::FallbackHelper::UpdateState(grpc_connectivity_state state,
   // If this request is from the pending fallback policy, ignore it until
   // it reports READY, at which point we swap it into place.
   if (CalledByPendingFallback()) {
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       gpr_log(
           GPR_INFO,
           "[xdslb %p helper %p] pending fallback policy %p reports state=%s",
@@ -635,7 +635,7 @@ void XdsLb::FallbackHelper::RequestReresolution() {
           ? parent_->pending_fallback_policy_.get()
           : parent_->fallback_policy_.get();
   if (child_ != latest_fallback_policy) return;
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO,
             "[xdslb %p] Re-resolution requested from the fallback policy (%p).",
             parent_.get(), child_);
@@ -755,7 +755,7 @@ void XdsLb::BalancerChannelState::Orphan() {
 
 void XdsLb::BalancerChannelState::StartCallRetryTimerLocked() {
   grpc_millis next_try = lb_call_backoff_.NextAttemptTime();
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO,
             "[xdslb %p] Failed to connect to LB server (lb_chand: %p)...",
             xdslb_policy_.get(), this);
@@ -781,7 +781,7 @@ void XdsLb::BalancerChannelState::OnCallRetryTimerLocked(void* arg,
   lb_chand->retry_timer_callback_pending_ = false;
   if (!lb_chand->shutting_down_ && error == GRPC_ERROR_NONE &&
       lb_chand->lb_calld_ == nullptr) {
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       gpr_log(GPR_INFO,
               "[xdslb %p] Restarting call to LB server (lb_chand: %p)",
               lb_chand->xdslb_policy_.get(), lb_chand);
@@ -796,7 +796,7 @@ void XdsLb::BalancerChannelState::StartCallLocked() {
   GPR_ASSERT(channel_ != nullptr);
   GPR_ASSERT(lb_calld_ == nullptr);
   lb_calld_ = MakeOrphanable<BalancerCallState>(Ref());
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO,
             "[xdslb %p] Query for backends (lb_chand: %p, lb_calld: %p)",
             xdslb_policy_.get(), this, lb_calld_.get());
@@ -932,7 +932,7 @@ void XdsLb::BalancerChannelState::BalancerCallState::Orphan() {
 
 void XdsLb::BalancerChannelState::BalancerCallState::StartQuery() {
   GPR_ASSERT(lb_call_ != nullptr);
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO, "[xdslb %p] Starting LB call (lb_calld: %p, lb_call: %p)",
             xdslb_policy(), this, lb_call_);
   }
@@ -1121,7 +1121,7 @@ void XdsLb::BalancerChannelState::BalancerCallState::
             GPR_MAX(GPR_MS_PER_SEC, interval);
       }
     }
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       if (lb_calld->client_stats_report_interval_ != 0) {
         gpr_log(GPR_INFO,
                 "[xdslb %p] Received initial LB response message; "
@@ -1140,7 +1140,7 @@ void XdsLb::BalancerChannelState::BalancerCallState::
                   response_slice)) != nullptr) {
     // Have seen initial response, look for serverlist.
     GPR_ASSERT(lb_calld->lb_call_ != nullptr);
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       gpr_log(GPR_INFO,
               "[xdslb %p] Serverlist with %" PRIuPTR " servers received",
               xdslb_policy, serverlist->num_servers);
@@ -1159,7 +1159,7 @@ void XdsLb::BalancerChannelState::BalancerCallState::
     // such channels don't have any current call but we have checked this call
     // is a current call.
     if (!lb_calld->lb_chand_->IsCurrentChannel()) {
-      if (grpc_lb_xds_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
         gpr_log(GPR_INFO,
                 "[xdslb %p] Promoting pending LB channel %p to replace "
                 "current LB channel %p",
@@ -1180,7 +1180,7 @@ void XdsLb::BalancerChannelState::BalancerCallState::
     if (!xdslb_policy->locality_serverlist_.empty() &&
         xds_grpclb_serverlist_equals(
             xdslb_policy->locality_serverlist_[0]->serverlist, serverlist)) {
-      if (grpc_lb_xds_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
         gpr_log(GPR_INFO,
                 "[xdslb %p] Incoming server list identical to current, "
                 "ignoring.",
@@ -1252,7 +1252,7 @@ void XdsLb::BalancerChannelState::BalancerCallState::
   XdsLb* xdslb_policy = lb_calld->xdslb_policy();
   BalancerChannelState* lb_chand = lb_calld->lb_chand_.get();
   GPR_ASSERT(lb_calld->lb_call_ != nullptr);
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     char* status_details =
         grpc_slice_to_c_string(lb_calld->lb_call_status_details_);
     gpr_log(GPR_INFO,
@@ -1271,7 +1271,7 @@ void XdsLb::BalancerChannelState::BalancerCallState::
     if (lb_chand != xdslb_policy->LatestLbChannel()) {
       // This channel must be the current one and there is a pending one. Swap
       // in the pending one and we are done.
-      if (grpc_lb_xds_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
         gpr_log(GPR_INFO,
                 "[xdslb %p] Promoting pending LB channel %p to replace "
                 "current LB channel %p",
@@ -1370,7 +1370,7 @@ XdsLb::XdsLb(Args args)
   grpc_uri* uri = grpc_uri_parse(server_uri, true);
   GPR_ASSERT(uri->path[0] != '\0');
   server_name_ = gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path);
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO,
             "[xdslb %p] Will use '%s' as the server name for LB request.", this,
             server_name_);
@@ -1569,7 +1569,7 @@ void XdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) {
   // this callback actually runs, don't fall back.
   if (xdslb_policy->fallback_at_startup_checks_pending_ &&
       !xdslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) {
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       gpr_log(GPR_INFO,
               "[xdslb %p] Child policy not ready after fallback timeout; "
               "entering fallback mode",
@@ -1657,7 +1657,7 @@ void XdsLb::UpdateFallbackPolicyLocked() {
     // Cases 1, 2b, and 3b: create a new child policy.
     // If child_policy_ is null, we set it (case 1), else we set
     // pending_child_policy_ (cases 2b and 3b).
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       gpr_log(GPR_INFO, "[xdslb %p] Creating new %sfallback policy %s", this,
               fallback_policy_ == nullptr ? "" : "pending ",
               fallback_policy_name);
@@ -1681,7 +1681,7 @@ void XdsLb::UpdateFallbackPolicyLocked() {
   }
   GPR_ASSERT(policy_to_update != nullptr);
   // Update the policy.
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(
         GPR_INFO, "[xdslb %p] Updating %sfallback policy %p", this,
         policy_to_update == pending_fallback_policy_.get() ? "pending " : "",
@@ -1707,7 +1707,7 @@ OrphanablePtr<LoadBalancingPolicy> XdsLb::CreateFallbackPolicyLocked(
     return nullptr;
   }
   helper->set_child(lb_policy.get());
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO, "[xdslb %p] Created new fallback policy %s (%p)", this,
             name, lb_policy.get());
   }
@@ -1829,7 +1829,7 @@ XdsLb::LocalityMap::LocalityEntry::CreateChildPolicyLocked(
     return nullptr;
   }
   helper->set_child(lb_policy.get());
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO, "[xdslb %p] Created new child policy %s (%p)", this, name,
             lb_policy.get());
   }
@@ -1920,7 +1920,7 @@ void XdsLb::LocalityMap::LocalityEntry::UpdateLocked(
     // Cases 1, 2b, and 3b: create a new child policy.
     // If child_policy_ is null, we set it (case 1), else we set
     // pending_child_policy_ (cases 2b and 3b).
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       gpr_log(GPR_INFO, "[xdslb %p] Creating new %schild policy %s", this,
               child_policy_ == nullptr ? "" : "pending ", child_policy_name);
     }
@@ -1943,7 +1943,7 @@ void XdsLb::LocalityMap::LocalityEntry::UpdateLocked(
   }
   GPR_ASSERT(policy_to_update != nullptr);
   // Update the policy.
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO, "[xdslb %p] Updating %schild policy %p", this,
             policy_to_update == pending_child_policy_.get() ? "pending " : "",
             policy_to_update);
@@ -2029,7 +2029,7 @@ void XdsLb::LocalityMap::LocalityEntry::Helper::UpdateState(
   // If this request is from the pending child policy, ignore it until
   // it reports READY, at which point we swap it into place.
   if (CalledByPendingChild()) {
-    if (grpc_lb_xds_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
       gpr_log(GPR_INFO,
               "[xdslb %p helper %p] pending child policy %p reports state=%s",
               entry_->parent_.get(), this, entry_->pending_child_policy_.get(),
@@ -2129,7 +2129,7 @@ void XdsLb::LocalityMap::LocalityEntry::Helper::RequestReresolution() {
   if (entry_->pending_child_policy_ != nullptr && !CalledByPendingChild()) {
     return;
   }
-  if (grpc_lb_xds_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) {
     gpr_log(GPR_INFO,
             "[xdslb %p] Re-resolution requested from the internal RR policy "
             "(%p).",

src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc

@@ -32,12 +32,12 @@
 #include "src/core/ext/filters/client_channel/http_connect_handshaker.h"
 #include "src/core/ext/filters/client_channel/lb_policy_registry.h"
 #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h"
+#include "src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/ext/filters/client_channel/server_address.h"
 #include "src/core/ext/filters/client_channel/service_config.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/gpr/host_port.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
@@ -227,65 +227,94 @@ bool ValueInJsonArray(grpc_json* array, const char* value) {
   return false;
 }
 
-char* ChooseServiceConfig(char* service_config_choice_json) {
+char* ChooseServiceConfig(char* service_config_choice_json,
+                          grpc_error** error) {
   grpc_json* choices_json = grpc_json_parse_string(service_config_choice_json);
-  if (choices_json == nullptr || choices_json->type != GRPC_JSON_ARRAY) {
-    gpr_log(GPR_ERROR, "cannot parse service config JSON string");
+  if (choices_json == nullptr) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "Service Config JSON Parsing, error: could not parse");
+    return nullptr;
+  }
+  if (choices_json->type != GRPC_JSON_ARRAY) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "Service Config Choices, error: should be of type array");
     return nullptr;
   }
   char* service_config = nullptr;
+  InlinedVector<grpc_error*, 4> error_list;
+  bool found_choice = false;  // have we found a choice?
   for (grpc_json* choice = choices_json->child; choice != nullptr;
        choice = choice->next) {
     if (choice->type != GRPC_JSON_OBJECT) {
-      gpr_log(GPR_ERROR, "cannot parse service config JSON string");
-      break;
+      error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "Service Config Choice, error: should be of type object"));
+      continue;
     }
     grpc_json* service_config_json = nullptr;
+    bool selected = true;  // has this choice been rejected?
     for (grpc_json* field = choice->child; field != nullptr;
          field = field->next) {
       // Check client language, if specified.
       if (strcmp(field->key, "clientLanguage") == 0) {
-        if (field->type != GRPC_JSON_ARRAY || !ValueInJsonArray(field, "c++")) {
-          service_config_json = nullptr;
-          break;
+        if (field->type != GRPC_JSON_ARRAY) {
+          error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+              "field:clientLanguage error:should be of type array"));
+        } else if (!ValueInJsonArray(field, "c++")) {
+          selected = false;
         }
       }
       // Check client hostname, if specified.
       if (strcmp(field->key, "clientHostname") == 0) {
+        if (field->type != GRPC_JSON_ARRAY) {
+          error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+              "field:clientHostname error:should be of type array"));
+          continue;
+        }
         char* hostname = grpc_gethostname();
-        if (hostname == nullptr || field->type != GRPC_JSON_ARRAY ||
-            !ValueInJsonArray(field, hostname)) {
-          service_config_json = nullptr;
-          break;
+        if (hostname == nullptr || !ValueInJsonArray(field, hostname)) {
+          selected = false;
         }
       }
       // Check percentage, if specified.
       if (strcmp(field->key, "percentage") == 0) {
         if (field->type != GRPC_JSON_NUMBER) {
-          service_config_json = nullptr;
-          break;
+          error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+              "field:percentage error:should be of type number"));
+          continue;
         }
         int random_pct = rand() % 100;
         int percentage;
-        if (sscanf(field->value, "%d", &percentage) != 1 ||
-            random_pct > percentage || percentage == 0) {
-          service_config_json = nullptr;
-          break;
+        if (sscanf(field->value, "%d", &percentage) != 1) {
+          error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+              "field:percentage error:should be of type integer"));
+          continue;
+        }
+        if (random_pct > percentage || percentage == 0) {
+          selected = false;
         }
       }
       // Save service config.
       if (strcmp(field->key, "serviceConfig") == 0) {
         if (field->type == GRPC_JSON_OBJECT) {
           service_config_json = field;
+        } else {
+          error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+              "field:serviceConfig error:should be of type object"));
         }
       }
     }
-    if (service_config_json != nullptr) {
+    if (!found_choice && selected && service_config_json != nullptr) {
       service_config = grpc_json_dump_to_string(service_config_json, 0);
-      break;
+      found_choice = true;
     }
   }
   grpc_json_destroy(choices_json);
+  if (!error_list.empty()) {
+    gpr_free(service_config);
+    service_config = nullptr;
+    *error = GRPC_ERROR_CREATE_FROM_VECTOR("Service Config Choices Parser",
+                                           &error_list);
+  }
   return service_config;
 }
 
@@ -303,17 +332,15 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
     Result result;
     result.addresses = std::move(*r->addresses_);
     if (r->service_config_json_ != nullptr) {
-      char* service_config_string =
-          ChooseServiceConfig(r->service_config_json_);
+      char* service_config_string = ChooseServiceConfig(
+          r->service_config_json_, &result.service_config_error);
       gpr_free(r->service_config_json_);
-      if (service_config_string != nullptr) {
+      if (result.service_config_error == GRPC_ERROR_NONE &&
+          service_config_string != nullptr) {
         GRPC_CARES_TRACE_LOG("resolver:%p selected service config choice: %s",
                              r, service_config_string);
-        grpc_error* service_config_error = GRPC_ERROR_NONE;
-        result.service_config =
-            ServiceConfig::Create(service_config_string, &service_config_error);
-        // Error is currently unused.
-        GRPC_ERROR_UNREF(service_config_error);
+        result.service_config = ServiceConfig::Create(
+            service_config_string, &result.service_config_error);
       }
       gpr_free(service_config_string);
     }
@@ -447,8 +474,9 @@ static bool should_use_ares(const char* resolver_env) {
 #endif /* GRPC_UV */
 
 void grpc_resolver_dns_ares_init() {
-  char* resolver_env = gpr_getenv("GRPC_DNS_RESOLVER");
-  if (should_use_ares(resolver_env)) {
+  grpc_core::UniquePtr<char> resolver =
+      GPR_GLOBAL_CONFIG_GET(grpc_dns_resolver);
+  if (should_use_ares(resolver.get())) {
     gpr_log(GPR_DEBUG, "Using ares dns resolver");
     address_sorting_init();
     grpc_error* error = grpc_ares_init();
@@ -464,16 +492,15 @@ void grpc_resolver_dns_ares_init() {
         grpc_core::UniquePtr<grpc_core::ResolverFactory>(
             grpc_core::New<grpc_core::AresDnsResolverFactory>()));
   }
-  gpr_free(resolver_env);
 }
 
 void grpc_resolver_dns_ares_shutdown() {
-  char* resolver_env = gpr_getenv("GRPC_DNS_RESOLVER");
-  if (should_use_ares(resolver_env)) {
+  grpc_core::UniquePtr<char> resolver =
+      GPR_GLOBAL_CONFIG_GET(grpc_dns_resolver);
+  if (should_use_ares(resolver.get())) {
     address_sorting_shutdown();
     grpc_ares_cleanup();
   }
-  gpr_free(resolver_env);
 }
 
 #else /* GRPC_ARES == 1 */

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc

@@ -84,6 +84,10 @@ struct grpc_ares_ev_driver {
   grpc_timer query_timeout;
   /** cancels queries on a timeout */
   grpc_closure on_timeout_locked;
+  /** alarm to poll ares_process on in case fd events don't happen */
+  grpc_timer ares_backup_poll_alarm;
+  /** polls ares_process on a periodic timer */
+  grpc_closure on_ares_backup_poll_alarm_locked;
 };
 
 static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver);
@@ -130,6 +134,13 @@ static void fd_node_shutdown_locked(fd_node* fdn, const char* reason) {
 
 static void on_timeout_locked(void* arg, grpc_error* error);
 
+static void on_ares_backup_poll_alarm_locked(void* arg, grpc_error* error);
+
+static void noop_inject_channel_config(ares_channel channel) {}
+
+void (*grpc_ares_test_only_inject_config)(ares_channel channel) =
+    noop_inject_channel_config;
+
 grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
                                               grpc_pollset_set* pollset_set,
                                               int query_timeout_ms,
@@ -140,6 +151,7 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
   memset(&opts, 0, sizeof(opts));
   opts.flags |= ARES_FLAG_STAYOPEN;
   int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS);
+  grpc_ares_test_only_inject_config((*ev_driver)->channel);
   GRPC_CARES_TRACE_LOG("request:%p grpc_ares_ev_driver_create_locked", request);
   if (status != ARES_SUCCESS) {
     char* err_msg;
@@ -163,6 +175,9 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver,
       ->polled_fd_factory->ConfigureAresChannelLocked((*ev_driver)->channel);
   GRPC_CLOSURE_INIT(&(*ev_driver)->on_timeout_locked, on_timeout_locked,
                     *ev_driver, grpc_combiner_scheduler(combiner));
+  GRPC_CLOSURE_INIT(&(*ev_driver)->on_ares_backup_poll_alarm_locked,
+                    on_ares_backup_poll_alarm_locked, *ev_driver,
+                    grpc_combiner_scheduler(combiner));
   (*ev_driver)->query_timeout_ms = query_timeout_ms;
   return GRPC_ERROR_NONE;
 }
@@ -174,6 +189,7 @@ void grpc_ares_ev_driver_on_queries_complete_locked(
   // fds; if it's not working, there are no fds to shut down.
   ev_driver->shutting_down = true;
   grpc_timer_cancel(&ev_driver->query_timeout);
+  grpc_timer_cancel(&ev_driver->ares_backup_poll_alarm);
   grpc_ares_ev_driver_unref(ev_driver);
 }
 
@@ -204,6 +220,21 @@ static fd_node* pop_fd_node_locked(fd_node** head, ares_socket_t as) {
   return nullptr;
 }
 
+static grpc_millis calculate_next_ares_backup_poll_alarm_ms(
+    grpc_ares_ev_driver* driver) {
+  // An alternative here could be to use ares_timeout to try to be more
+  // accurate, but that would require using "struct timeval"'s, which just makes
+  // things a bit more complicated. So just poll every second, as suggested
+  // by the c-ares code comments.
+  grpc_millis ms_until_next_ares_backup_poll_alarm = 1000;
+  GRPC_CARES_TRACE_LOG(
+      "request:%p ev_driver=%p. next ares process poll time in "
+      "%" PRId64 " ms",
+      driver->request, driver, ms_until_next_ares_backup_poll_alarm);
+  return ms_until_next_ares_backup_poll_alarm +
+         grpc_core::ExecCtx::Get()->Now();
+}
+
 static void on_timeout_locked(void* arg, grpc_error* error) {
   grpc_ares_ev_driver* driver = static_cast<grpc_ares_ev_driver*>(arg);
   GRPC_CARES_TRACE_LOG(
@@ -216,6 +247,47 @@ static void on_timeout_locked(void* arg, grpc_error* error) {
   grpc_ares_ev_driver_unref(driver);
 }
 
+/* In case of non-responsive DNS servers, dropped packets, etc., c-ares has
+ * intelligent timeout and retry logic, which we can take advantage of by
+ * polling ares_process_fd on time intervals. Overall, the c-ares library is
+ * meant to be called into and given a chance to proceed name resolution:
+ *   a) when fd events happen
+ *   b) when some time has passed without fd events having happened
+ * For the latter, we use this backup poller. Also see
+ * https://github.com/grpc/grpc/pull/17688 description for more details. */
+static void on_ares_backup_poll_alarm_locked(void* arg, grpc_error* error) {
+  grpc_ares_ev_driver* driver = static_cast<grpc_ares_ev_driver*>(arg);
+  GRPC_CARES_TRACE_LOG(
+      "request:%p ev_driver=%p on_ares_backup_poll_alarm_locked. "
+      "driver->shutting_down=%d. "
+      "err=%s",
+      driver->request, driver, driver->shutting_down, grpc_error_string(error));
+  if (!driver->shutting_down && error == GRPC_ERROR_NONE) {
+    fd_node* fdn = driver->fds;
+    while (fdn != nullptr) {
+      if (!fdn->already_shutdown) {
+        GRPC_CARES_TRACE_LOG(
+            "request:%p ev_driver=%p on_ares_backup_poll_alarm_locked; "
+            "ares_process_fd. fd=%s",
+            driver->request, driver, fdn->grpc_polled_fd->GetName());
+        ares_socket_t as = fdn->grpc_polled_fd->GetWrappedAresSocketLocked();
+        ares_process_fd(driver->channel, as, as);
+      }
+      fdn = fdn->next;
+    }
+    if (!driver->shutting_down) {
+      grpc_millis next_ares_backup_poll_alarm =
+          calculate_next_ares_backup_poll_alarm_ms(driver);
+      grpc_ares_ev_driver_ref(driver);
+      grpc_timer_init(&driver->ares_backup_poll_alarm,
+                      next_ares_backup_poll_alarm,
+                      &driver->on_ares_backup_poll_alarm_locked);
+    }
+    grpc_ares_notify_on_event_locked(driver);
+  }
+  grpc_ares_ev_driver_unref(driver);
+}
+
 static void on_readable_locked(void* arg, grpc_error* error) {
   fd_node* fdn = static_cast<fd_node*>(arg);
   GPR_ASSERT(fdn->readable_registered);
@@ -353,6 +425,7 @@ void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) {
   if (!ev_driver->working) {
     ev_driver->working = true;
     grpc_ares_notify_on_event_locked(ev_driver);
+    // Initialize overall DNS resolution timeout alarm
     grpc_millis timeout =
         ev_driver->query_timeout_ms == 0
             ? GRPC_MILLIS_INF_FUTURE
@@ -364,6 +437,13 @@ void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) {
     grpc_ares_ev_driver_ref(ev_driver);
     grpc_timer_init(&ev_driver->query_timeout, timeout,
                     &ev_driver->on_timeout_locked);
+    // Initialize the backup poll alarm
+    grpc_millis next_ares_backup_poll_alarm =
+        calculate_next_ares_backup_poll_alarm_ms(ev_driver);
+    grpc_ares_ev_driver_ref(ev_driver);
+    grpc_timer_init(&ev_driver->ares_backup_poll_alarm,
+                    next_ares_backup_poll_alarm,
+                    &ev_driver->on_ares_backup_poll_alarm_locked);
   }
 }
 

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h

@@ -54,6 +54,9 @@ void grpc_ares_ev_driver_on_queries_complete_locked(
 /* Shutdown all the grpc_fds used by \a ev_driver */
 void grpc_ares_ev_driver_shutdown_locked(grpc_ares_ev_driver* ev_driver);
 
+/* Exposed in this header for C-core tests only */
+extern void (*grpc_ares_test_only_inject_config)(ares_channel channel);
+
 namespace grpc_core {
 
 /* A wrapped fd that integrates with the grpc iomgr of the current platform.

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc

@@ -109,6 +109,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     read_closure_ = read_closure;
     GPR_ASSERT(GRPC_SLICE_LENGTH(read_buf_) == 0);
     grpc_slice_unref_internal(read_buf_);
+    GPR_ASSERT(!read_buf_has_data_);
     read_buf_ = GRPC_SLICE_MALLOC(4192);
     WSABUF buffer;
     buffer.buf = (char*)GRPC_SLICE_START_PTR(read_buf_);
@@ -175,7 +176,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     GRPC_CARES_TRACE_LOG(
         "RecvFrom called on fd:|%s|. Current read buf length:|%d|", GetName(),
         GRPC_SLICE_LENGTH(read_buf_));
-    if (GRPC_SLICE_LENGTH(read_buf_) == 0) {
+    if (!read_buf_has_data_) {
       WSASetLastError(WSAEWOULDBLOCK);
       return -1;
     }
@@ -186,6 +187,9 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     }
     read_buf_ = grpc_slice_sub_no_ref(read_buf_, bytes_read,
                                       GRPC_SLICE_LENGTH(read_buf_));
+    if (GRPC_SLICE_LENGTH(read_buf_) == 0) {
+      read_buf_has_data_ = false;
+    }
     /* c-ares overloads this recv_from virtual socket function to receive
      * data on both UDP and TCP sockets, and from is nullptr for TCP. */
     if (from != nullptr) {
@@ -302,6 +306,11 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     polled_fd->OnIocpReadableInner(error);
   }
 
+  // TODO(apolcyn): improve this error handling to be less conversative.
+  // An e.g. ECONNRESET error here should result in errors when
+  // c-ares reads from this socket later, but it shouldn't necessarily cancel
+  // the entire resolution attempt. Doing so will allow the "inject broken
+  // nameserver list" test to pass on Windows.
   void OnIocpReadableInner(grpc_error* error) {
     if (error == GRPC_ERROR_NONE) {
       if (winsocket_->read_info.wsa_error != 0) {
@@ -323,6 +332,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
     if (error == GRPC_ERROR_NONE) {
       read_buf_ = grpc_slice_sub_no_ref(read_buf_, 0,
                                         winsocket_->read_info.bytes_transfered);
+      read_buf_has_data_ = true;
     } else {
       grpc_slice_unref_internal(read_buf_);
       read_buf_ = grpc_empty_slice();
@@ -370,6 +380,7 @@ class GrpcPolledFdWindows : public GrpcPolledFd {
   char recv_from_source_addr_[200];
   ares_socklen_t recv_from_source_addr_len_;
   grpc_slice read_buf_;
+  bool read_buf_has_data_ = false;
   grpc_slice write_buf_;
   grpc_closure* read_closure_ = nullptr;
   grpc_closure* write_closure_ = nullptr;

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc

@@ -101,7 +101,7 @@ static void log_address_sorting_list(const ServerAddressList& addresses,
 }
 
 void grpc_cares_wrapper_address_sorting_sort(ServerAddressList* addresses) {
-  if (grpc_trace_cares_address_sorting.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_cares_address_sorting)) {
     log_address_sorting_list(*addresses, "input");
   }
   address_sorting_sortable* sortables = (address_sorting_sortable*)gpr_zalloc(
@@ -120,7 +120,7 @@ void grpc_cares_wrapper_address_sorting_sort(ServerAddressList* addresses) {
   }
   gpr_free(sortables);
   *addresses = std::move(sorted);
-  if (grpc_trace_cares_address_sorting.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_cares_address_sorting)) {
     log_address_sorting_list(*addresses, "output");
   }
 }
@@ -154,6 +154,10 @@ void grpc_ares_complete_request_locked(grpc_ares_request* r) {
 static grpc_ares_hostbyname_request* create_hostbyname_request_locked(
     grpc_ares_request* parent_request, char* host, uint16_t port,
     bool is_balancer) {
+  GRPC_CARES_TRACE_LOG(
+      "request:%p create_hostbyname_request_locked host:%s port:%d "
+      "is_balancer:%d",
+      parent_request, host, port, is_balancer);
   grpc_ares_hostbyname_request* hr = static_cast<grpc_ares_hostbyname_request*>(
       gpr_zalloc(sizeof(grpc_ares_hostbyname_request)));
   hr->parent_request = parent_request;
@@ -251,6 +255,8 @@ static void on_srv_query_done_locked(void* arg, int status, int timeouts,
     GRPC_CARES_TRACE_LOG("request:%p on_srv_query_done_locked ARES_SUCCESS", r);
     struct ares_srv_reply* reply;
     const int parse_status = ares_parse_srv_reply(abuf, alen, &reply);
+    GRPC_CARES_TRACE_LOG("request:%p ares_parse_srv_reply: %d", r,
+                         parse_status);
     if (parse_status == ARES_SUCCESS) {
       ares_channel* channel =
           grpc_ares_ev_driver_get_channel_locked(r->ev_driver);

src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h

@@ -26,16 +26,18 @@
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 
-#define GRPC_DNS_ARES_DEFAULT_QUERY_TIMEOUT_MS 10000
+#define GRPC_DNS_ARES_DEFAULT_QUERY_TIMEOUT_MS 120000
 
 extern grpc_core::TraceFlag grpc_trace_cares_address_sorting;
 
 extern grpc_core::TraceFlag grpc_trace_cares_resolver;
 
-#define GRPC_CARES_TRACE_LOG(format, ...)                         \
-  if (grpc_trace_cares_resolver.enabled()) {                      \
-    gpr_log(GPR_DEBUG, "(c-ares resolver) " format, __VA_ARGS__); \
-  }
+#define GRPC_CARES_TRACE_LOG(format, ...)                           \
+  do {                                                              \
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_cares_resolver)) {       \
+      gpr_log(GPR_DEBUG, "(c-ares resolver) " format, __VA_ARGS__); \
+    }                                                               \
+  } while (0)
 
 typedef struct grpc_ares_request grpc_ares_request;
 

src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc

@@ -0,0 +1,28 @@
+//
+// Copyright 2019 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// This is similar to the sockaddr resolver, except that it supports a
+// bunch of query args that are useful for dependency injection in tests.
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h"
+
+GPR_GLOBAL_CONFIG_DEFINE_STRING(
+    grpc_dns_resolver, "",
+    "Declares which DNS resolver to use. The default is ares if gRPC is built "
+    "with c-ares support. Otherwise, the value of this environment variable is "
+    "ignored.")

src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h

@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_DNS_RESOLVER_SELECTION_H
+#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_DNS_RESOLVER_SELECTION_H
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/gprpp/global_config.h"
+
+GPR_GLOBAL_CONFIG_DECLARE_STRING(grpc_dns_resolver);
+
+#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_RESOLVER_DNS_DNS_RESOLVER_SELECTION_H \
+        */

src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc

@@ -26,11 +26,11 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
+#include "src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/ext/filters/client_channel/server_address.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/gpr/env.h"
 #include "src/core/lib/gpr/host_port.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
@@ -274,8 +274,9 @@ class NativeDnsResolverFactory : public ResolverFactory {
 }  // namespace grpc_core
 
 void grpc_resolver_dns_native_init() {
-  char* resolver_env = gpr_getenv("GRPC_DNS_RESOLVER");
-  if (resolver_env != nullptr && gpr_stricmp(resolver_env, "native") == 0) {
+  grpc_core::UniquePtr<char> resolver =
+      GPR_GLOBAL_CONFIG_GET(grpc_dns_resolver);
+  if (gpr_stricmp(resolver.get(), "native") == 0) {
     gpr_log(GPR_DEBUG, "Using native dns resolver");
     grpc_core::ResolverRegistry::Builder::RegisterResolverFactory(
         grpc_core::UniquePtr<grpc_core::ResolverFactory>(
@@ -291,7 +292,6 @@ void grpc_resolver_dns_native_init() {
               grpc_core::New<grpc_core::NativeDnsResolverFactory>()));
     }
   }
-  gpr_free(resolver_env);
 }
 
 void grpc_resolver_dns_native_shutdown() {}

src/core/ext/filters/client_channel/resolver_result_parsing.cc

@@ -58,89 +58,6 @@ void ClientChannelServiceConfigParser::Register() {
           New<ClientChannelServiceConfigParser>()));
 }
 
-ProcessedResolverResult::ProcessedResolverResult(
-    const Resolver::Result& resolver_result)
-    : service_config_(resolver_result.service_config) {
-  // If resolver did not return a service config, use the default
-  // specified via the client API.
-  if (service_config_ == nullptr) {
-    const char* service_config_json = grpc_channel_arg_get_string(
-        grpc_channel_args_find(resolver_result.args, GRPC_ARG_SERVICE_CONFIG));
-    if (service_config_json != nullptr) {
-      grpc_error* error = GRPC_ERROR_NONE;
-      service_config_ = ServiceConfig::Create(service_config_json, &error);
-      // Error is currently unused.
-      GRPC_ERROR_UNREF(error);
-    }
-  }
-  // Process service config.
-  const ClientChannelGlobalParsedObject* parsed_object = nullptr;
-  if (service_config_ != nullptr) {
-    parsed_object = static_cast<const ClientChannelGlobalParsedObject*>(
-        service_config_->GetParsedGlobalServiceConfigObject(
-            ClientChannelServiceConfigParser::ParserIndex()));
-    ProcessServiceConfig(resolver_result, parsed_object);
-  }
-  ProcessLbPolicy(resolver_result, parsed_object);
-}
-
-void ProcessedResolverResult::ProcessServiceConfig(
-    const Resolver::Result& resolver_result,
-    const ClientChannelGlobalParsedObject* parsed_object) {
-  health_check_service_name_ = parsed_object->health_check_service_name();
-  service_config_json_ = service_config_->service_config_json();
-  if (parsed_object != nullptr) {
-    retry_throttle_data_ = parsed_object->retry_throttling();
-  }
-}
-
-void ProcessedResolverResult::ProcessLbPolicy(
-    const Resolver::Result& resolver_result,
-    const ClientChannelGlobalParsedObject* parsed_object) {
-  // Prefer the LB policy name found in the service config.
-  if (parsed_object != nullptr) {
-    if (parsed_object->parsed_lb_config() != nullptr) {
-      lb_policy_name_.reset(
-          gpr_strdup(parsed_object->parsed_lb_config()->name()));
-      lb_policy_config_ = parsed_object->parsed_lb_config();
-    } else {
-      lb_policy_name_.reset(
-          gpr_strdup(parsed_object->parsed_deprecated_lb_policy()));
-    }
-  }
-  // Otherwise, find the LB policy name set by the client API.
-  if (lb_policy_name_ == nullptr) {
-    const grpc_arg* channel_arg =
-        grpc_channel_args_find(resolver_result.args, GRPC_ARG_LB_POLICY_NAME);
-    lb_policy_name_.reset(gpr_strdup(grpc_channel_arg_get_string(channel_arg)));
-  }
-  // Special case: If at least one balancer address is present, we use
-  // the grpclb policy, regardless of what the resolver has returned.
-  bool found_balancer_address = false;
-  for (size_t i = 0; i < resolver_result.addresses.size(); ++i) {
-    const ServerAddress& address = resolver_result.addresses[i];
-    if (address.IsBalancer()) {
-      found_balancer_address = true;
-      break;
-    }
-  }
-  if (found_balancer_address) {
-    if (lb_policy_name_ != nullptr &&
-        strcmp(lb_policy_name_.get(), "grpclb") != 0) {
-      gpr_log(GPR_INFO,
-              "resolver requested LB policy %s but provided at least one "
-              "balancer address -- forcing use of grpclb LB policy",
-              lb_policy_name_.get());
-    }
-    lb_policy_name_.reset(gpr_strdup("grpclb"));
-  }
-  // Use pick_first if nothing was specified and we didn't select grpclb
-  // above.
-  if (lb_policy_name_ == nullptr) {
-    lb_policy_name_.reset(gpr_strdup("pick_first"));
-  }
-}
-
 namespace {
 
 // Parses a JSON field of the form generated for a google.proto.Duration

src/core/ext/filters/client_channel/resolver_result_parsing.h

@@ -118,64 +118,6 @@ class ClientChannelServiceConfigParser : public ServiceConfig::Parser {
   static void Register();
 };
 
-// TODO(yashykt): It would be cleaner to move this logic to the client_channel
-// code. A container of processed fields from the resolver result. Simplifies
-// the usage of resolver result.
-class ProcessedResolverResult {
- public:
-  // Processes the resolver result and populates the relative members
-  // for later consumption.
-  ProcessedResolverResult(const Resolver::Result& resolver_result);
-
-  // Getters. Any managed object's ownership is transferred.
-  const char* service_config_json() { return service_config_json_; }
-
-  RefCountedPtr<ServiceConfig> service_config() { return service_config_; }
-
-  UniquePtr<char> lb_policy_name() { return std::move(lb_policy_name_); }
-  RefCountedPtr<ParsedLoadBalancingConfig> lb_policy_config() {
-    return lb_policy_config_;
-  }
-
-  Optional<ClientChannelGlobalParsedObject::RetryThrottling>
-  retry_throttle_data() {
-    return retry_throttle_data_;
-  }
-
-  const char* health_check_service_name() { return health_check_service_name_; }
-
- private:
-  // Finds the service config; extracts LB config and (maybe) retry throttle
-  // params from it.
-  void ProcessServiceConfig(
-      const Resolver::Result& resolver_result,
-      const ClientChannelGlobalParsedObject* parsed_object);
-
-  // Extracts the LB policy.
-  void ProcessLbPolicy(const Resolver::Result& resolver_result,
-                       const ClientChannelGlobalParsedObject* parsed_object);
-
-  // Parses the service config. Intended to be used by
-  // ServiceConfig::ParseGlobalParams.
-  static void ParseServiceConfig(const grpc_json* field,
-                                 ProcessedResolverResult* parsing_state);
-  // Parses the LB config from service config.
-  void ParseLbConfigFromServiceConfig(const grpc_json* field);
-  // Parses the retry throttle parameters from service config.
-  void ParseRetryThrottleParamsFromServiceConfig(const grpc_json* field);
-
-  // Service config.
-  const char* service_config_json_ = nullptr;
-  RefCountedPtr<ServiceConfig> service_config_;
-  // LB policy.
-  UniquePtr<char> lb_policy_name_;
-  RefCountedPtr<ParsedLoadBalancingConfig> lb_policy_config_;
-  // Retry throttle data.
-  Optional<ClientChannelGlobalParsedObject::RetryThrottling>
-      retry_throttle_data_;
-  const char* health_check_service_name_ = nullptr;
-};
-
 }  // namespace internal
 }  // namespace grpc_core
 

src/core/ext/filters/client_channel/resolving_lb_policy.cc

@@ -77,7 +77,7 @@ class ResolvingLoadBalancingPolicy::ResolverResultHandler
       : parent_(std::move(parent)) {}
 
   ~ResolverResultHandler() {
-    if (parent_->tracer_->enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(*(parent_->tracer_))) {
       gpr_log(GPR_INFO, "resolving_lb=%p: resolver shutdown complete",
               parent_.get());
     }
@@ -125,7 +125,7 @@ class ResolvingLoadBalancingPolicy::ResolvingControlHelper
     // If this request is from the pending child policy, ignore it until
     // it reports READY, at which point we swap it into place.
     if (CalledByPendingChild()) {
-      if (parent_->tracer_->enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(*(parent_->tracer_))) {
         gpr_log(GPR_INFO,
                 "resolving_lb=%p helper=%p: pending child policy %p reports "
                 "state=%s",
@@ -151,7 +151,7 @@ class ResolvingLoadBalancingPolicy::ResolvingControlHelper
     if (parent_->pending_lb_policy_ != nullptr && !CalledByPendingChild()) {
       return;
     }
-    if (parent_->tracer_->enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(*(parent_->tracer_))) {
       gpr_log(GPR_INFO, "resolving_lb=%p: started name re-resolving",
               parent_.get());
     }
@@ -241,7 +241,7 @@ void ResolvingLoadBalancingPolicy::ShutdownLocked() {
     resolver_.reset();
     MutexLock lock(&lb_policy_mu_);
     if (lb_policy_ != nullptr) {
-      if (tracer_->enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
         gpr_log(GPR_INFO, "resolving_lb=%p: shutting down lb_policy=%p", this,
                 lb_policy_.get());
       }
@@ -250,7 +250,7 @@ void ResolvingLoadBalancingPolicy::ShutdownLocked() {
       lb_policy_.reset();
     }
     if (pending_lb_policy_ != nullptr) {
-      if (tracer_->enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
         gpr_log(GPR_INFO, "resolving_lb=%p: shutting down pending lb_policy=%p",
                 this, pending_lb_policy_.get());
       }
@@ -298,7 +298,7 @@ void ResolvingLoadBalancingPolicy::FillChildRefsForChannelz(
 }
 
 void ResolvingLoadBalancingPolicy::StartResolvingLocked() {
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO, "resolving_lb=%p: starting name resolution", this);
   }
   GPR_ASSERT(!started_resolving_);
@@ -314,7 +314,7 @@ void ResolvingLoadBalancingPolicy::OnResolverError(grpc_error* error) {
     GRPC_ERROR_UNREF(error);
     return;
   }
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO, "resolving_lb=%p: resolver transient failure: %s", this,
             grpc_error_string(error));
   }
@@ -398,7 +398,7 @@ void ResolvingLoadBalancingPolicy::CreateOrUpdateLbPolicyLocked(
     // Cases 1, 2b, and 3b: create a new child policy.
     // If lb_policy_ is null, we set it (case 1), else we set
     // pending_lb_policy_ (cases 2b and 3b).
-    if (tracer_->enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
       gpr_log(GPR_INFO, "resolving_lb=%p: Creating new %schild policy %s", this,
               lb_policy_ == nullptr ? "" : "pending ", lb_policy_name);
     }
@@ -419,7 +419,7 @@ void ResolvingLoadBalancingPolicy::CreateOrUpdateLbPolicyLocked(
   }
   GPR_ASSERT(policy_to_update != nullptr);
   // Update the policy.
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO, "resolving_lb=%p: Updating %schild policy %p", this,
             policy_to_update == pending_lb_policy_.get() ? "pending " : "",
             policy_to_update);
@@ -458,7 +458,7 @@ ResolvingLoadBalancingPolicy::CreateLbPolicyLocked(
     return nullptr;
   }
   helper->set_child(lb_policy.get());
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO, "resolving_lb=%p: created new LB policy \"%s\" (%p)",
             this, lb_policy_name, lb_policy.get());
   }
@@ -514,7 +514,7 @@ void ResolvingLoadBalancingPolicy::OnResolverResultChangedLocked(
     Resolver::Result result) {
   // Handle race conditions.
   if (resolver_ == nullptr) return;
-  if (tracer_->enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(*tracer_)) {
     gpr_log(GPR_INFO, "resolving_lb=%p: got resolver result", this);
   }
   // We only want to trace the address resolution in the follow cases:
@@ -532,18 +532,32 @@ void ResolvingLoadBalancingPolicy::OnResolverResultChangedLocked(
   const char* lb_policy_name = nullptr;
   RefCountedPtr<ParsedLoadBalancingConfig> lb_policy_config;
   bool service_config_changed = false;
+  char* service_config_error_string = nullptr;
   if (process_resolver_result_ != nullptr) {
-    service_config_changed =
-        process_resolver_result_(process_resolver_result_user_data_, result,
-                                 &lb_policy_name, &lb_policy_config);
+    grpc_error* service_config_error = GRPC_ERROR_NONE;
+    service_config_changed = process_resolver_result_(
+        process_resolver_result_user_data_, &result, &lb_policy_name,
+        &lb_policy_config, &service_config_error);
+    if (service_config_error != GRPC_ERROR_NONE) {
+      service_config_error_string =
+          gpr_strdup(grpc_error_string(service_config_error));
+      if (lb_policy_name == nullptr) {
+        // Use an empty lb_policy_name as an indicator that we received an
+        // invalid service config and we don't have a fallback service config.
+        OnResolverError(service_config_error);
+      } else {
+        GRPC_ERROR_UNREF(service_config_error);
+      }
+    }
   } else {
     lb_policy_name = child_policy_name_.get();
     lb_policy_config = child_lb_config_;
   }
-  GPR_ASSERT(lb_policy_name != nullptr);
-  // Create or update LB policy, as needed.
-  CreateOrUpdateLbPolicyLocked(lb_policy_name, lb_policy_config,
-                               std::move(result), &trace_strings);
+  if (lb_policy_name != nullptr) {
+    // Create or update LB policy, as needed.
+    CreateOrUpdateLbPolicyLocked(lb_policy_name, lb_policy_config,
+                                 std::move(result), &trace_strings);
+  }
   // Add channel trace event.
   if (channelz_node() != nullptr) {
     if (service_config_changed) {
@@ -551,10 +565,15 @@ void ResolvingLoadBalancingPolicy::OnResolverResultChangedLocked(
       // config in the trace, at the risk of bloating the trace logs.
       trace_strings.push_back(gpr_strdup("Service config changed"));
     }
+    if (service_config_error_string != nullptr) {
+      trace_strings.push_back(service_config_error_string);
+      service_config_error_string = nullptr;
+    }
     MaybeAddTraceMessagesForAddressChangesLocked(resolution_contains_addresses,
                                                  &trace_strings);
     ConcatenateAndAddChannelTraceLocked(&trace_strings);
   }
+  gpr_free(service_config_error_string);
 }
 
 }  // namespace grpc_core

src/core/ext/filters/client_channel/resolving_lb_policy.h

@@ -65,10 +65,13 @@ class ResolvingLoadBalancingPolicy : public LoadBalancingPolicy {
   // Synchronous callback that takes the resolver result and sets
   // lb_policy_name and lb_policy_config to point to the right data.
   // Returns true if the service config has changed since the last result.
+  // If the returned service_config_error is not none and lb_policy_name is
+  // empty, it means that we don't have a valid service config to use, and we
+  // should set the channel to be in TRANSIENT_FAILURE.
   typedef bool (*ProcessResolverResultCallback)(
-      void* user_data, const Resolver::Result& result,
-      const char** lb_policy_name,
-      RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config);
+      void* user_data, Resolver::Result* result, const char** lb_policy_name,
+      RefCountedPtr<ParsedLoadBalancingConfig>* lb_policy_config,
+      grpc_error** service_config_error);
   // If error is set when this returns, then construction failed, and
   // the caller may not use the new object.
   ResolvingLoadBalancingPolicy(

src/core/ext/filters/client_channel/subchannel.cc

@@ -303,8 +303,7 @@ void SubchannelCall::IncrementRefCount(const grpc_core::DebugLocation& location,
 // Subchannel::ConnectedSubchannelStateWatcher
 //
 
-class Subchannel::ConnectedSubchannelStateWatcher
-    : public InternallyRefCounted<ConnectedSubchannelStateWatcher> {
+class Subchannel::ConnectedSubchannelStateWatcher {
  public:
   // Must be instantiated while holding c->mu.
   explicit ConnectedSubchannelStateWatcher(Subchannel* c) : subchannel_(c) {
@@ -312,38 +311,17 @@ class Subchannel::ConnectedSubchannelStateWatcher
     GRPC_SUBCHANNEL_WEAK_REF(subchannel_, "state_watcher");
     GRPC_SUBCHANNEL_WEAK_UNREF(subchannel_, "connecting");
     // Start watching for connectivity state changes.
-    // Callback uses initial ref to this.
     GRPC_CLOSURE_INIT(&on_connectivity_changed_, OnConnectivityChanged, this,
                       grpc_schedule_on_exec_ctx);
     c->connected_subchannel_->NotifyOnStateChange(c->pollset_set_,
                                                   &pending_connectivity_state_,
                                                   &on_connectivity_changed_);
-    // Start health check if needed.
-    grpc_connectivity_state health_state = GRPC_CHANNEL_READY;
-    if (c->health_check_service_name_ != nullptr) {
-      health_check_client_ = MakeOrphanable<HealthCheckClient>(
-          c->health_check_service_name_.get(), c->connected_subchannel_,
-          c->pollset_set_, c->channelz_node_);
-      GRPC_CLOSURE_INIT(&on_health_changed_, OnHealthChanged, this,
-                        grpc_schedule_on_exec_ctx);
-      Ref().release();  // Ref for health callback tracked manually.
-      health_check_client_->NotifyOnHealthChange(&health_state_,
-                                                 &on_health_changed_);
-      health_state = GRPC_CHANNEL_CONNECTING;
-    }
-    // Report initial state.
-    c->SetConnectivityStateLocked(GRPC_CHANNEL_READY, "subchannel_connected");
-    grpc_connectivity_state_set(&c->state_and_health_tracker_, health_state,
-                                "subchannel_connected");
   }
 
   ~ConnectedSubchannelStateWatcher() {
     GRPC_SUBCHANNEL_WEAK_UNREF(subchannel_, "state_watcher");
   }
 
-  // Must be called while holding subchannel_->mu.
-  void Orphan() override { health_check_client_.reset(); }
-
  private:
   static void OnConnectivityChanged(void* arg, grpc_error* error) {
     auto* self = static_cast<ConnectedSubchannelStateWatcher*>(arg);
@@ -363,20 +341,10 @@ class Subchannel::ConnectedSubchannelStateWatcher
                           self->pending_connectivity_state_));
             }
             c->connected_subchannel_.reset();
-            c->connected_subchannel_watcher_.reset();
-            self->last_connectivity_state_ = GRPC_CHANNEL_TRANSIENT_FAILURE;
-            c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                          "reflect_child");
-            grpc_connectivity_state_set(&c->state_and_health_tracker_,
-                                        GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                        "reflect_child");
+            c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE);
             c->backoff_begun_ = false;
             c->backoff_.Reset();
-            c->MaybeStartConnectingLocked();
-          } else {
-            self->last_connectivity_state_ = GRPC_CHANNEL_SHUTDOWN;
           }
-          self->health_check_client_.reset();
           break;
         }
         default: {
@@ -384,96 +352,246 @@ class Subchannel::ConnectedSubchannelStateWatcher
           // a callback for READY, because that was the state we started
           // this watch from.  And a connected subchannel should never go
           // from READY to CONNECTING or IDLE.
-          self->last_connectivity_state_ = self->pending_connectivity_state_;
-          c->SetConnectivityStateLocked(self->pending_connectivity_state_,
-                                        "reflect_child");
-          if (self->pending_connectivity_state_ != GRPC_CHANNEL_READY) {
-            grpc_connectivity_state_set(&c->state_and_health_tracker_,
-                                        self->pending_connectivity_state_,
-                                        "reflect_child");
-          }
+          c->SetConnectivityStateLocked(self->pending_connectivity_state_);
           c->connected_subchannel_->NotifyOnStateChange(
               nullptr, &self->pending_connectivity_state_,
               &self->on_connectivity_changed_);
-          self = nullptr;  // So we don't unref below.
+          return;  // So we don't delete ourself below.
         }
       }
     }
-    // Don't unref until we've released the lock, because this might
+    // Don't delete until we've released the lock, because this might
     // cause the subchannel (which contains the lock) to be destroyed.
-    if (self != nullptr) self->Unref();
+    Delete(self);
+  }
+
+  Subchannel* subchannel_;
+  grpc_closure on_connectivity_changed_;
+  grpc_connectivity_state pending_connectivity_state_ = GRPC_CHANNEL_READY;
+};
+
+//
+// Subchannel::ConnectivityStateWatcherList
+//
+
+void Subchannel::ConnectivityStateWatcherList::AddWatcherLocked(
+    UniquePtr<ConnectivityStateWatcher> watcher) {
+  watcher->next_ = head_;
+  head_ = watcher.release();
+}
+
+void Subchannel::ConnectivityStateWatcherList::RemoveWatcherLocked(
+    ConnectivityStateWatcher* watcher) {
+  for (ConnectivityStateWatcher** w = &head_; *w != nullptr; w = &(*w)->next_) {
+    if (*w == watcher) {
+      *w = watcher->next_;
+      Delete(watcher);
+      return;
+    }
+  }
+  GPR_UNREACHABLE_CODE(return );
+}
+
+void Subchannel::ConnectivityStateWatcherList::NotifyLocked(
+    Subchannel* subchannel, grpc_connectivity_state state) {
+  for (ConnectivityStateWatcher* w = head_; w != nullptr; w = w->next_) {
+    RefCountedPtr<ConnectedSubchannel> connected_subchannel;
+    if (state == GRPC_CHANNEL_READY) {
+      connected_subchannel = subchannel->connected_subchannel_;
+    }
+    // TODO(roth): In principle, it seems wrong to send this notification
+    // to the watcher while holding the subchannel's mutex, since it could
+    // lead to a deadlock if the watcher calls back into the subchannel
+    // before returning back to us.  In practice, this doesn't happen,
+    // because the LB policy code that watches subchannels always bounces
+    // the notification into the client_channel control-plane combiner
+    // before processing it.  But if we ever have any other callers here,
+    // we will probably need to change this.
+    w->OnConnectivityStateChange(state, std::move(connected_subchannel));
+  }
+}
+
+void Subchannel::ConnectivityStateWatcherList::Clear() {
+  while (head_ != nullptr) {
+    ConnectivityStateWatcher* next = head_->next_;
+    Delete(head_);
+    head_ = next;
+  }
+}
+
+//
+// Subchannel::HealthWatcherMap::HealthWatcher
+//
+
+// State needed for tracking the connectivity state with a particular
+// health check service name.
+class Subchannel::HealthWatcherMap::HealthWatcher
+    : public InternallyRefCounted<HealthWatcher> {
+ public:
+  HealthWatcher(Subchannel* c, UniquePtr<char> health_check_service_name,
+                grpc_connectivity_state subchannel_state)
+      : subchannel_(c),
+        health_check_service_name_(std::move(health_check_service_name)),
+        state_(subchannel_state == GRPC_CHANNEL_READY ? GRPC_CHANNEL_CONNECTING
+                                                      : subchannel_state) {
+    GRPC_SUBCHANNEL_WEAK_REF(subchannel_, "health_watcher");
+    GRPC_CLOSURE_INIT(&on_health_changed_, OnHealthChanged, this,
+                      grpc_schedule_on_exec_ctx);
+    // If the subchannel is already connected, start health checking.
+    if (subchannel_state == GRPC_CHANNEL_READY) StartHealthCheckingLocked();
+  }
+
+  ~HealthWatcher() {
+    GRPC_SUBCHANNEL_WEAK_UNREF(subchannel_, "health_watcher");
+  }
+
+  const char* health_check_service_name() const {
+    return health_check_service_name_.get();
+  }
+
+  grpc_connectivity_state state() const { return state_; }
+
+  void AddWatcherLocked(grpc_connectivity_state initial_state,
+                        UniquePtr<ConnectivityStateWatcher> watcher) {
+    if (state_ != initial_state) {
+      RefCountedPtr<ConnectedSubchannel> connected_subchannel;
+      if (state_ == GRPC_CHANNEL_READY) {
+        connected_subchannel = subchannel_->connected_subchannel_;
+      }
+      watcher->OnConnectivityStateChange(state_,
+                                         std::move(connected_subchannel));
+    }
+    watcher_list_.AddWatcherLocked(std::move(watcher));
+  }
+
+  void RemoveWatcherLocked(ConnectivityStateWatcher* watcher) {
+    watcher_list_.RemoveWatcherLocked(watcher);
+  }
+
+  bool HasWatchers() const { return !watcher_list_.empty(); }
+
+  void NotifyLocked(grpc_connectivity_state state) {
+    if (state == GRPC_CHANNEL_READY) {
+      // If we had not already notified for CONNECTING state, do so now.
+      // (We may have missed this earlier, because if the transition
+      // from IDLE to CONNECTING to READY was too quick, the connected
+      // subchannel may not have sent us a notification for CONNECTING.)
+      if (state_ != GRPC_CHANNEL_CONNECTING) {
+        state_ = GRPC_CHANNEL_CONNECTING;
+        watcher_list_.NotifyLocked(subchannel_, state_);
+      }
+      // If we've become connected, start health checking.
+      StartHealthCheckingLocked();
+    } else {
+      state_ = state;
+      watcher_list_.NotifyLocked(subchannel_, state_);
+      // We're not connected, so stop health checking.
+      health_check_client_.reset();
+    }
+  }
+
+  void Orphan() override {
+    watcher_list_.Clear();
+    health_check_client_.reset();
+    Unref();
+  }
+
+ private:
+  void StartHealthCheckingLocked() {
+    GPR_ASSERT(health_check_client_ == nullptr);
+    health_check_client_ = MakeOrphanable<HealthCheckClient>(
+        health_check_service_name_.get(), subchannel_->connected_subchannel_,
+        subchannel_->pollset_set_, subchannel_->channelz_node_);
+    Ref().release();  // Ref for health callback tracked manually.
+    health_check_client_->NotifyOnHealthChange(&state_, &on_health_changed_);
   }
 
   static void OnHealthChanged(void* arg, grpc_error* error) {
-    auto* self = static_cast<ConnectedSubchannelStateWatcher*>(arg);
+    auto* self = static_cast<HealthWatcher*>(arg);
     Subchannel* c = self->subchannel_;
     {
       MutexLock lock(&c->mu_);
-      if (self->health_state_ != GRPC_CHANNEL_SHUTDOWN &&
+      if (self->state_ != GRPC_CHANNEL_SHUTDOWN &&
           self->health_check_client_ != nullptr) {
-        if (self->last_connectivity_state_ == GRPC_CHANNEL_READY) {
-          grpc_connectivity_state_set(&c->state_and_health_tracker_,
-                                      self->health_state_, "health_changed");
-        }
+        self->watcher_list_.NotifyLocked(c, self->state_);
+        // Renew watch.
         self->health_check_client_->NotifyOnHealthChange(
-            &self->health_state_, &self->on_health_changed_);
-        self = nullptr;  // So we don't unref below.
+            &self->state_, &self->on_health_changed_);
+        return;  // So we don't unref below.
       }
     }
     // Don't unref until we've released the lock, because this might
     // cause the subchannel (which contains the lock) to be destroyed.
-    if (self != nullptr) self->Unref();
+    self->Unref();
   }
 
   Subchannel* subchannel_;
-  grpc_closure on_connectivity_changed_;
-  grpc_connectivity_state pending_connectivity_state_ = GRPC_CHANNEL_READY;
-  grpc_connectivity_state last_connectivity_state_ = GRPC_CHANNEL_READY;
+  UniquePtr<char> health_check_service_name_;
   OrphanablePtr<HealthCheckClient> health_check_client_;
   grpc_closure on_health_changed_;
-  grpc_connectivity_state health_state_ = GRPC_CHANNEL_CONNECTING;
+  grpc_connectivity_state state_;
+  ConnectivityStateWatcherList watcher_list_;
 };
 
 //
-// Subchannel::ExternalStateWatcher
+// Subchannel::HealthWatcherMap
 //
 
-struct Subchannel::ExternalStateWatcher {
-  ExternalStateWatcher(Subchannel* subchannel, grpc_pollset_set* pollset_set,
-                       grpc_closure* notify)
-      : subchannel(subchannel), pollset_set(pollset_set), notify(notify) {
-    GRPC_SUBCHANNEL_WEAK_REF(subchannel, "external_state_watcher+init");
-    GRPC_CLOSURE_INIT(&on_state_changed, OnStateChanged, this,
-                      grpc_schedule_on_exec_ctx);
+void Subchannel::HealthWatcherMap::AddWatcherLocked(
+    Subchannel* subchannel, grpc_connectivity_state initial_state,
+    UniquePtr<char> health_check_service_name,
+    UniquePtr<ConnectivityStateWatcher> watcher) {
+  // If the health check service name is not already present in the map,
+  // add it.
+  auto it = map_.find(health_check_service_name.get());
+  HealthWatcher* health_watcher;
+  if (it == map_.end()) {
+    const char* key = health_check_service_name.get();
+    auto w = MakeOrphanable<HealthWatcher>(
+        subchannel, std::move(health_check_service_name), subchannel->state_);
+    health_watcher = w.get();
+    map_[key] = std::move(w);
+  } else {
+    health_watcher = it->second.get();
+  }
+  // Add the watcher to the entry.
+  health_watcher->AddWatcherLocked(initial_state, std::move(watcher));
+}
+
+void Subchannel::HealthWatcherMap::RemoveWatcherLocked(
+    const char* health_check_service_name, ConnectivityStateWatcher* watcher) {
+  auto it = map_.find(health_check_service_name);
+  GPR_ASSERT(it != map_.end());
+  it->second->RemoveWatcherLocked(watcher);
+  // If we just removed the last watcher for this service name, remove
+  // the map entry.
+  if (!it->second->HasWatchers()) map_.erase(it);
+}
+
+void Subchannel::HealthWatcherMap::NotifyLocked(grpc_connectivity_state state) {
+  for (const auto& p : map_) {
+    p.second->NotifyLocked(state);
   }
+}
 
-  static void OnStateChanged(void* arg, grpc_error* error) {
-    ExternalStateWatcher* w = static_cast<ExternalStateWatcher*>(arg);
-    grpc_closure* follow_up = w->notify;
-    if (w->pollset_set != nullptr) {
-      grpc_pollset_set_del_pollset_set(w->subchannel->pollset_set_,
-                                       w->pollset_set);
-    }
-    {
-      MutexLock lock(&w->subchannel->mu_);
-      if (w->subchannel->external_state_watcher_list_ == w) {
-        w->subchannel->external_state_watcher_list_ = w->next;
-      }
-      if (w->next != nullptr) w->next->prev = w->prev;
-      if (w->prev != nullptr) w->prev->next = w->next;
-    }
-    GRPC_SUBCHANNEL_WEAK_UNREF(w->subchannel, "external_state_watcher+done");
-    Delete(w);
-    GRPC_CLOSURE_SCHED(follow_up, GRPC_ERROR_REF(error));
+grpc_connectivity_state
+Subchannel::HealthWatcherMap::CheckConnectivityStateLocked(
+    Subchannel* subchannel, const char* health_check_service_name) {
+  auto it = map_.find(health_check_service_name);
+  if (it == map_.end()) {
+    // If the health check service name is not found in the map, we're
+    // not currently doing a health check for that service name.  If the
+    // subchannel's state without health checking is READY, report
+    // CONNECTING, since that's what we'd be in as soon as we do start a
+    // watch.  Otherwise, report the channel's state without health checking.
+    return subchannel->state_ == GRPC_CHANNEL_READY ? GRPC_CHANNEL_CONNECTING
+                                                    : subchannel->state_;
   }
+  HealthWatcher* health_watcher = it->second.get();
+  return health_watcher->state();
+}
 
-  Subchannel* subchannel;
-  grpc_pollset_set* pollset_set;
-  grpc_closure* notify;
-  grpc_closure on_state_changed;
-  ExternalStateWatcher* next = nullptr;
-  ExternalStateWatcher* prev = nullptr;
-};
+void Subchannel::HealthWatcherMap::ShutdownLocked() { map_.clear(); }
 
 //
 // Subchannel
@@ -560,13 +678,6 @@ Subchannel::Subchannel(SubchannelKey* key, grpc_connector* connector,
   if (new_args != nullptr) grpc_channel_args_destroy(new_args);
   GRPC_CLOSURE_INIT(&on_connecting_finished_, OnConnectingFinished, this,
                     grpc_schedule_on_exec_ctx);
-  grpc_connectivity_state_init(&state_tracker_, GRPC_CHANNEL_IDLE,
-                               "subchannel");
-  grpc_connectivity_state_init(&state_and_health_tracker_, GRPC_CHANNEL_IDLE,
-                               "subchannel");
-  health_check_service_name_ =
-      UniquePtr<char>(gpr_strdup(grpc_channel_arg_get_string(
-          grpc_channel_args_find(args_, "grpc.temp.health_check"))));
   const grpc_arg* arg = grpc_channel_args_find(args_, GRPC_ARG_ENABLE_CHANNELZ);
   const bool channelz_enabled =
       grpc_channel_arg_get_bool(arg, GRPC_ENABLE_CHANNELZ_DEFAULT);
@@ -593,8 +704,6 @@ Subchannel::~Subchannel() {
     channelz_node_->MarkSubchannelDestroyed();
   }
   grpc_channel_args_destroy(args_);
-  grpc_connectivity_state_destroy(&state_tracker_);
-  grpc_connectivity_state_destroy(&state_and_health_tracker_);
   grpc_connector_unref(connector_);
   grpc_pollset_set_destroy(pollset_set_);
   Delete(key_);
@@ -698,55 +807,67 @@ const char* Subchannel::GetTargetAddress() {
   return addr_str;
 }
 
-RefCountedPtr<ConnectedSubchannel> Subchannel::connected_subchannel() {
-  MutexLock lock(&mu_);
-  return connected_subchannel_;
-}
-
 channelz::SubchannelNode* Subchannel::channelz_node() {
   return channelz_node_.get();
 }
 
-grpc_connectivity_state Subchannel::CheckConnectivity(
-    bool inhibit_health_checking) {
-  grpc_connectivity_state_tracker* tracker =
-      inhibit_health_checking ? &state_tracker_ : &state_and_health_tracker_;
-  grpc_connectivity_state state = grpc_connectivity_state_check(tracker);
+grpc_connectivity_state Subchannel::CheckConnectivityState(
+    const char* health_check_service_name,
+    RefCountedPtr<ConnectedSubchannel>* connected_subchannel) {
+  MutexLock lock(&mu_);
+  grpc_connectivity_state state;
+  if (health_check_service_name == nullptr) {
+    state = state_;
+  } else {
+    state = health_watcher_map_.CheckConnectivityStateLocked(
+        this, health_check_service_name);
+  }
+  if (connected_subchannel != nullptr && state == GRPC_CHANNEL_READY) {
+    *connected_subchannel = connected_subchannel_;
+  }
   return state;
 }
 
-void Subchannel::NotifyOnStateChange(grpc_pollset_set* interested_parties,
-                                     grpc_connectivity_state* state,
-                                     grpc_closure* notify,
-                                     bool inhibit_health_checking) {
-  grpc_connectivity_state_tracker* tracker =
-      inhibit_health_checking ? &state_tracker_ : &state_and_health_tracker_;
-  ExternalStateWatcher* w;
-  if (state == nullptr) {
-    MutexLock lock(&mu_);
-    for (w = external_state_watcher_list_; w != nullptr; w = w->next) {
-      if (w->notify == notify) {
-        grpc_connectivity_state_notify_on_state_change(tracker, nullptr,
-                                                       &w->on_state_changed);
-      }
+void Subchannel::WatchConnectivityState(
+    grpc_connectivity_state initial_state,
+    UniquePtr<char> health_check_service_name,
+    UniquePtr<ConnectivityStateWatcher> watcher) {
+  MutexLock lock(&mu_);
+  grpc_pollset_set* interested_parties = watcher->interested_parties();
+  if (interested_parties != nullptr) {
+    grpc_pollset_set_add_pollset_set(pollset_set_, interested_parties);
+  }
+  if (health_check_service_name == nullptr) {
+    if (state_ != initial_state) {
+      watcher->OnConnectivityStateChange(state_, connected_subchannel_);
     }
+    watcher_list_.AddWatcherLocked(std::move(watcher));
   } else {
-    w = New<ExternalStateWatcher>(this, interested_parties, notify);
-    if (interested_parties != nullptr) {
-      grpc_pollset_set_add_pollset_set(pollset_set_, interested_parties);
-    }
-    MutexLock lock(&mu_);
-    if (external_state_watcher_list_ != nullptr) {
-      w->next = external_state_watcher_list_;
-      w->next->prev = w;
-    }
-    external_state_watcher_list_ = w;
-    grpc_connectivity_state_notify_on_state_change(tracker, state,
-                                                   &w->on_state_changed);
-    MaybeStartConnectingLocked();
+    health_watcher_map_.AddWatcherLocked(this, initial_state,
+                                         std::move(health_check_service_name),
+                                         std::move(watcher));
   }
 }
 
+void Subchannel::CancelConnectivityStateWatch(
+    const char* health_check_service_name, ConnectivityStateWatcher* watcher) {
+  MutexLock lock(&mu_);
+  grpc_pollset_set* interested_parties = watcher->interested_parties();
+  if (interested_parties != nullptr) {
+    grpc_pollset_set_del_pollset_set(pollset_set_, interested_parties);
+  }
+  if (health_check_service_name == nullptr) {
+    watcher_list_.RemoveWatcherLocked(watcher);
+  } else {
+    health_watcher_map_.RemoveWatcherLocked(health_check_service_name, watcher);
+  }
+}
+
+void Subchannel::AttemptToConnect() {
+  MutexLock lock(&mu_);
+  MaybeStartConnectingLocked();
+}
+
 void Subchannel::ResetBackoff() {
   MutexLock lock(&mu_);
   backoff_.Reset();
@@ -818,15 +939,19 @@ const char* SubchannelConnectivityStateChangeString(
 
 }  // namespace
 
-void Subchannel::SetConnectivityStateLocked(grpc_connectivity_state state,
-                                            const char* reason) {
+// Note: Must be called with a state that is different from the current state.
+void Subchannel::SetConnectivityStateLocked(grpc_connectivity_state state) {
+  state_ = state;
   if (channelz_node_ != nullptr) {
     channelz_node_->AddTraceEvent(
         channelz::ChannelTrace::Severity::Info,
         grpc_slice_from_static_string(
             SubchannelConnectivityStateChangeString(state)));
   }
-  grpc_connectivity_state_set(&state_tracker_, state, reason);
+  // Notify non-health watchers.
+  watcher_list_.NotifyLocked(this, state);
+  // Notify health watchers.
+  health_watcher_map_.NotifyLocked(state);
 }
 
 void Subchannel::MaybeStartConnectingLocked() {
@@ -842,11 +967,6 @@ void Subchannel::MaybeStartConnectingLocked() {
     // Already connected: don't restart.
     return;
   }
-  if (!grpc_connectivity_state_has_watchers(&state_tracker_) &&
-      !grpc_connectivity_state_has_watchers(&state_and_health_tracker_)) {
-    // Nobody is interested in connecting: so don't just yet.
-    return;
-  }
   connecting_ = true;
   GRPC_SUBCHANNEL_WEAK_REF(this, "connecting");
   if (!backoff_begun_) {
@@ -903,9 +1023,7 @@ void Subchannel::ContinueConnectingLocked() {
   next_attempt_deadline_ = backoff_.NextAttemptTime();
   args.deadline = std::max(next_attempt_deadline_, min_deadline);
   args.channel_args = args_;
-  SetConnectivityStateLocked(GRPC_CHANNEL_CONNECTING, "connecting");
-  grpc_connectivity_state_set(&state_and_health_tracker_,
-                              GRPC_CHANNEL_CONNECTING, "connecting");
+  SetConnectivityStateLocked(GRPC_CHANNEL_CONNECTING);
   grpc_connector_connect(connector_, &args, &connecting_result_,
                          &on_connecting_finished_);
 }
@@ -924,12 +1042,7 @@ void Subchannel::OnConnectingFinished(void* arg, grpc_error* error) {
       GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
     } else {
       gpr_log(GPR_INFO, "Connect failed: %s", grpc_error_string(error));
-      c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                    "connect_failed");
-      grpc_connectivity_state_set(&c->state_and_health_tracker_,
-                                  GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                  "connect_failed");
-      c->MaybeStartConnectingLocked();
+      c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE);
       GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
     }
   }
@@ -982,8 +1095,9 @@ bool Subchannel::PublishTransportLocked() {
   gpr_log(GPR_INFO, "New connected subchannel at %p for subchannel %p",
           connected_subchannel_.get(), this);
   // Instantiate state watcher.  Will clean itself up.
-  connected_subchannel_watcher_ =
-      MakeOrphanable<ConnectedSubchannelStateWatcher>(this);
+  New<ConnectedSubchannelStateWatcher>(this);
+  // Report initial state.
+  SetConnectivityStateLocked(GRPC_CHANNEL_READY);
   return true;
 }
 
@@ -1000,7 +1114,7 @@ void Subchannel::Disconnect() {
   grpc_connector_shutdown(connector_, GRPC_ERROR_CREATE_FROM_STATIC_STRING(
                                           "Subchannel disconnected"));
   connected_subchannel_.reset();
-  connected_subchannel_watcher_.reset();
+  health_watcher_map_.ShutdownLocked();
 }
 
 gpr_atm Subchannel::RefMutate(

src/core/ext/filters/client_channel/subchannel.h

@@ -27,6 +27,7 @@
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/gprpp/arena.h"
+#include "src/core/lib/gprpp/map.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
@@ -77,7 +78,7 @@ class ConnectedSubchannel : public RefCounted<ConnectedSubchannel> {
     grpc_millis deadline;
     Arena* arena;
     grpc_call_context_element* context;
-    grpc_core::CallCombiner* call_combiner;
+    CallCombiner* call_combiner;
     size_t parent_data_size;
   };
 
@@ -175,7 +176,38 @@ class SubchannelCall {
 // A subchannel that knows how to connect to exactly one target address. It
 // provides a target for load balancing.
 class Subchannel {
+ private:
+  class ConnectivityStateWatcherList;  // Forward declaration.
+
  public:
+  class ConnectivityStateWatcher {
+   public:
+    virtual ~ConnectivityStateWatcher() = default;
+
+    // Will be invoked whenever the subchannel's connectivity state
+    // changes.  There will be only one invocation of this method on a
+    // given watcher instance at any given time.
+    //
+    // When the state changes to READY, connected_subchannel will
+    // contain a ref to the connected subchannel.  When it changes from
+    // READY to some other state, the implementation must release its
+    // ref to the connected subchannel.
+    virtual void OnConnectivityStateChange(
+        grpc_connectivity_state new_state,
+        RefCountedPtr<ConnectedSubchannel> connected_subchannel)  // NOLINT
+        GRPC_ABSTRACT;
+
+    virtual grpc_pollset_set* interested_parties() GRPC_ABSTRACT;
+
+    GRPC_ABSTRACT_BASE_CLASS
+
+   private:
+    // For access to next_.
+    friend class Subchannel::ConnectivityStateWatcherList;
+
+    ConnectivityStateWatcher* next_ = nullptr;
+  };
+
   // The ctor and dtor are not intended to use directly.
   Subchannel(SubchannelKey* key, grpc_connector* connector,
              const grpc_channel_args* args);
@@ -201,20 +233,36 @@ class Subchannel {
   // Caller doesn't take ownership.
   const char* GetTargetAddress();
 
-  // Gets the connected subchannel - or nullptr if not connected (which may
-  // happen before it initially connects or during transient failures).
-  RefCountedPtr<ConnectedSubchannel> connected_subchannel();
-
   channelz::SubchannelNode* channelz_node();
 
-  // Polls the current connectivity state of the subchannel.
-  grpc_connectivity_state CheckConnectivity(bool inhibit_health_checking);
-
-  // When the connectivity state of the subchannel changes from \a *state,
-  // invokes \a notify and updates \a *state with the new state.
-  void NotifyOnStateChange(grpc_pollset_set* interested_parties,
-                           grpc_connectivity_state* state, grpc_closure* notify,
-                           bool inhibit_health_checking);
+  // Returns the current connectivity state of the subchannel.
+  // If health_check_service_name is non-null, the returned connectivity
+  // state will be based on the state reported by the backend for that
+  // service name.
+  // If the return value is GRPC_CHANNEL_READY, also sets *connected_subchannel.
+  grpc_connectivity_state CheckConnectivityState(
+      const char* health_check_service_name,
+      RefCountedPtr<ConnectedSubchannel>* connected_subchannel);
+
+  // Starts watching the subchannel's connectivity state.
+  // The first callback to the watcher will be delivered when the
+  // subchannel's connectivity state becomes a value other than
+  // initial_state, which may happen immediately.
+  // Subsequent callbacks will be delivered as the subchannel's state
+  // changes.
+  // The watcher will be destroyed either when the subchannel is
+  // destroyed or when CancelConnectivityStateWatch() is called.
+  void WatchConnectivityState(grpc_connectivity_state initial_state,
+                              UniquePtr<char> health_check_service_name,
+                              UniquePtr<ConnectivityStateWatcher> watcher);
+
+  // Cancels a connectivity state watch.
+  // If the watcher has already been destroyed, this is a no-op.
+  void CancelConnectivityStateWatch(const char* health_check_service_name,
+                                    ConnectivityStateWatcher* watcher);
+
+  // Attempt to connect to the backend.  Has no effect if already connected.
+  void AttemptToConnect();
 
   // Resets the connection backoff of the subchannel.
   // TODO(roth): Move connection backoff out of subchannels and up into LB
@@ -236,12 +284,62 @@ class Subchannel {
                                                  grpc_resolved_address* addr);
 
  private:
-  struct ExternalStateWatcher;
+  // A linked list of ConnectivityStateWatchers that are monitoring the
+  // subchannel's state.
+  class ConnectivityStateWatcherList {
+   public:
+    ~ConnectivityStateWatcherList() { Clear(); }
+
+    void AddWatcherLocked(UniquePtr<ConnectivityStateWatcher> watcher);
+    void RemoveWatcherLocked(ConnectivityStateWatcher* watcher);
+
+    // Notifies all watchers in the list about a change to state.
+    void NotifyLocked(Subchannel* subchannel, grpc_connectivity_state state);
+
+    void Clear();
+
+    bool empty() const { return head_ == nullptr; }
+
+   private:
+    ConnectivityStateWatcher* head_ = nullptr;
+  };
+
+  // A map that tracks ConnectivityStateWatchers using a particular health
+  // check service name.
+  //
+  // There is one entry in the map for each health check service name.
+  // Entries exist only as long as there are watchers using the
+  // corresponding service name.
+  //
+  // A health check client is maintained only while the subchannel is in
+  // state READY.
+  class HealthWatcherMap {
+   public:
+    void AddWatcherLocked(Subchannel* subchannel,
+                          grpc_connectivity_state initial_state,
+                          UniquePtr<char> health_check_service_name,
+                          UniquePtr<ConnectivityStateWatcher> watcher);
+    void RemoveWatcherLocked(const char* health_check_service_name,
+                             ConnectivityStateWatcher* watcher);
+
+    // Notifies the watcher when the subchannel's state changes.
+    void NotifyLocked(grpc_connectivity_state state);
+
+    grpc_connectivity_state CheckConnectivityStateLocked(
+        Subchannel* subchannel, const char* health_check_service_name);
+
+    void ShutdownLocked();
+
+   private:
+    class HealthWatcher;
+
+    Map<const char*, OrphanablePtr<HealthWatcher>, StringLess> map_;
+  };
+
   class ConnectedSubchannelStateWatcher;
 
   // Sets the subchannel's connectivity state to \a state.
-  void SetConnectivityStateLocked(grpc_connectivity_state state,
-                                  const char* reason);
+  void SetConnectivityStateLocked(grpc_connectivity_state state);
 
   // Methods for connection.
   void MaybeStartConnectingLocked();
@@ -279,15 +377,15 @@ class Subchannel {
   grpc_closure on_connecting_finished_;
   // Active connection, or null.
   RefCountedPtr<ConnectedSubchannel> connected_subchannel_;
-  OrphanablePtr<ConnectedSubchannelStateWatcher> connected_subchannel_watcher_;
   bool connecting_ = false;
   bool disconnected_ = false;
 
   // Connectivity state tracking.
-  grpc_connectivity_state_tracker state_tracker_;
-  grpc_connectivity_state_tracker state_and_health_tracker_;
-  UniquePtr<char> health_check_service_name_;
-  ExternalStateWatcher* external_state_watcher_list_ = nullptr;
+  grpc_connectivity_state state_ = GRPC_CHANNEL_IDLE;
+  // The list of watchers without a health check service name.
+  ConnectivityStateWatcherList watcher_list_;
+  // The map of watchers with health check service names.
+  HealthWatcherMap health_watcher_map_;
 
   // Backoff state.
   BackOff backoff_;

src/core/ext/filters/http/message_compress/message_compress_filter.cc

@@ -249,7 +249,7 @@ static void finish_send_message(grpc_call_element* elem) {
   bool did_compress = grpc_msg_compress(calld->message_compression_algorithm,
                                         &calld->slices, &tmp);
   if (did_compress) {
-    if (grpc_compression_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
       const char* algo_name;
       const size_t before_size = calld->slices.length;
       const size_t after_size = tmp.length;
@@ -265,7 +265,7 @@ static void finish_send_message(grpc_call_element* elem) {
     grpc_slice_buffer_swap(&calld->slices, &tmp);
     send_flags |= GRPC_WRITE_INTERNAL_COMPRESS;
   } else {
-    if (grpc_compression_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) {
       const char* algo_name;
       GPR_ASSERT(grpc_message_compression_algorithm_name(
           calld->message_compression_algorithm, &algo_name));

src/core/ext/transport/chttp2/transport/chttp2_plugin.cc

@@ -23,8 +23,11 @@
 #include "src/core/lib/gprpp/global_config.h"
 #include "src/core/lib/transport/metadata.h"
 
-GPR_GLOBAL_CONFIG_DEFINE_BOOL(grpc_experimental_disable_flow_control, false,
-                              "Disable flow control");
+GPR_GLOBAL_CONFIG_DEFINE_BOOL(
+    grpc_experimental_disable_flow_control, false,
+    "If set, flow control will be effectively disabled. Max out all values and "
+    "assume the remote peer does the same. Thus we can ignore any flow control "
+    "bookkeeping, error checking, and decision making");
 
 void grpc_chttp2_plugin_init(void) {
   g_flow_control_enabled =

src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -1249,7 +1249,7 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t,
     return;
   }
   closure->next_data.scratch -= CLOSURE_BARRIER_FIRST_REF_BIT;
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     const char* errstr = grpc_error_string(error);
     gpr_log(
         GPR_INFO,
@@ -1401,7 +1401,7 @@ static void perform_stream_op_locked(void* stream_op,
 
   s->context = op->payload->context;
   s->traced = op->is_traced;
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     char* str = grpc_transport_stream_op_batch_string(op);
     gpr_log(GPR_INFO, "perform_stream_op_locked: %s; on_complete = %p", str,
             op->on_complete);
@@ -1705,7 +1705,7 @@ static void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
     }
   }
 
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     char* str = grpc_transport_stream_op_batch_string(op);
     gpr_log(GPR_INFO, "perform_stream_op[s=%p]: %s", s, str);
     gpr_free(str);
@@ -1872,7 +1872,7 @@ static void perform_transport_op_locked(void* stream_op,
 
 static void perform_transport_op(grpc_transport* gt, grpc_transport_op* op) {
   grpc_chttp2_transport* t = reinterpret_cast<grpc_chttp2_transport*>(gt);
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     char* msg = grpc_transport_op_string(op);
     gpr_log(GPR_INFO, "perform_transport_op[t=%p]: %s", t, msg);
     gpr_free(msg);
@@ -2619,7 +2619,7 @@ static void schedule_bdp_ping_locked(grpc_chttp2_transport* t) {
 
 static void start_bdp_ping_locked(void* tp, grpc_error* error) {
   grpc_chttp2_transport* t = static_cast<grpc_chttp2_transport*>(tp);
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     gpr_log(GPR_INFO, "%s: Start BDP ping err=%s", t->peer_string,
             grpc_error_string(error));
   }
@@ -2635,7 +2635,7 @@ static void start_bdp_ping_locked(void* tp, grpc_error* error) {
 
 static void finish_bdp_ping_locked(void* tp, grpc_error* error) {
   grpc_chttp2_transport* t = static_cast<grpc_chttp2_transport*>(tp);
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     gpr_log(GPR_INFO, "%s: Complete BDP ping err=%s", t->peer_string,
             grpc_error_string(error));
   }
@@ -2767,7 +2767,7 @@ static void start_keepalive_ping_locked(void* arg, grpc_error* error) {
   if (t->channelz_socket != nullptr) {
     t->channelz_socket->RecordKeepaliveSent();
   }
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     gpr_log(GPR_INFO, "%s: Start keepalive ping", t->peer_string);
   }
   GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive watchdog");
@@ -2780,7 +2780,7 @@ static void finish_keepalive_ping_locked(void* arg, grpc_error* error) {
   grpc_chttp2_transport* t = static_cast<grpc_chttp2_transport*>(arg);
   if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_PINGING) {
     if (error == GRPC_ERROR_NONE) {
-      if (grpc_http_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
         gpr_log(GPR_INFO, "%s: Finish keepalive ping", t->peer_string);
       }
       t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_WAITING;
@@ -3090,7 +3090,7 @@ static void benign_reclaimer_locked(void* arg, grpc_error* error) {
       grpc_chttp2_stream_map_size(&t->stream_map) == 0) {
     /* Channel with no active streams: send a goaway to try and make it
      * disconnect cleanly */
-    if (grpc_resource_quota_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
       gpr_log(GPR_INFO, "HTTP2: %s - send goaway to free memory",
               t->peer_string);
     }
@@ -3098,7 +3098,8 @@ static void benign_reclaimer_locked(void* arg, grpc_error* error) {
                 grpc_error_set_int(
                     GRPC_ERROR_CREATE_FROM_STATIC_STRING("Buffers full"),
                     GRPC_ERROR_INT_HTTP2_ERROR, GRPC_HTTP2_ENHANCE_YOUR_CALM));
-  } else if (error == GRPC_ERROR_NONE && grpc_resource_quota_trace.enabled()) {
+  } else if (error == GRPC_ERROR_NONE &&
+             GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
     gpr_log(GPR_INFO,
             "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR
             " streams",
@@ -3119,7 +3120,7 @@ static void destructive_reclaimer_locked(void* arg, grpc_error* error) {
   if (error == GRPC_ERROR_NONE && n > 0) {
     grpc_chttp2_stream* s = static_cast<grpc_chttp2_stream*>(
         grpc_chttp2_stream_map_rand(&t->stream_map));
-    if (grpc_resource_quota_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_resource_quota_trace)) {
       gpr_log(GPR_INFO, "HTTP2: %s - abandon stream id %d", t->peer_string,
               s->id);
     }

src/core/ext/transport/chttp2/transport/flow_control.h

@@ -127,7 +127,7 @@ class FlowControlTrace {
             StreamFlowControl* sfc);
   void Finish();
 
-  const bool enabled_ = grpc_flowctl_trace.enabled();
+  const bool enabled_ = GRPC_TRACE_FLAG_ENABLED(grpc_flowctl_trace);
 
   TransportFlowControl* tfc_;
   StreamFlowControl* sfc_;

src/core/ext/transport/chttp2/transport/frame_settings.cc

@@ -217,19 +217,20 @@ grpc_error* grpc_chttp2_settings_parser_parse(void* p, grpc_chttp2_transport* t,
               parser->incoming_settings[id] != parser->value) {
             t->initial_window_update += static_cast<int64_t>(parser->value) -
                                         parser->incoming_settings[id];
-            if (grpc_http_trace.enabled() || grpc_flowctl_trace.enabled()) {
+            if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) ||
+                GRPC_TRACE_FLAG_ENABLED(grpc_flowctl_trace)) {
               gpr_log(GPR_INFO, "%p[%s] adding %d for initial_window change", t,
                       t->is_client ? "cli" : "svr",
                       static_cast<int>(t->initial_window_update));
             }
           }
           parser->incoming_settings[id] = parser->value;
-          if (grpc_http_trace.enabled()) {
+          if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
             gpr_log(GPR_INFO, "CHTTP2:%s:%s: got setting %s = %d",
                     t->is_client ? "CLI" : "SVR", t->peer_string, sp->name,
                     parser->value);
           }
-        } else if (grpc_http_trace.enabled()) {
+        } else if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
           gpr_log(GPR_ERROR, "CHTTP2: Ignoring unknown setting %d (value %d)",
                   parser->id, parser->value);
         }

src/core/ext/transport/chttp2/transport/hpack_encoder.cc

@@ -461,7 +461,7 @@ static void hpack_enc(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem,
         "Reserved header (colon-prefixed) happening after regular ones.");
   }
 
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     char* k = grpc_slice_to_c_string(GRPC_MDKEY(elem));
     char* v = nullptr;
     if (grpc_is_binary_header(GRPC_MDKEY(elem))) {
@@ -660,7 +660,7 @@ void grpc_chttp2_hpack_compressor_set_max_table_size(
     }
   }
   c->advertise_table_size_change = 1;
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     gpr_log(GPR_INFO, "set max table size from encoder to %d", max_table_size);
   }
 }

src/core/ext/transport/chttp2/transport/hpack_parser.cc

@@ -624,7 +624,7 @@ static const uint8_t inverse_base64[256] = {
 /* emission helpers */
 static grpc_error* on_hdr(grpc_chttp2_hpack_parser* p, grpc_mdelem md,
                           int add_to_table) {
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     char* k = grpc_slice_to_c_string(GRPC_MDKEY(md));
     char* v = nullptr;
     if (grpc_is_binary_header(GRPC_MDKEY(md))) {
@@ -994,7 +994,7 @@ static grpc_error* parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser* p,
 /* finish parsing a max table size change */
 static grpc_error* finish_max_tbl_size(grpc_chttp2_hpack_parser* p,
                                        const uint8_t* cur, const uint8_t* end) {
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", p->index);
   }
   grpc_error* err =

src/core/ext/transport/chttp2/transport/hpack_table.cc

@@ -247,7 +247,7 @@ void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl* tbl,
   if (tbl->max_bytes == max_bytes) {
     return;
   }
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     gpr_log(GPR_INFO, "Update hpack parser max size to %d", max_bytes);
   }
   while (tbl->mem_used > max_bytes) {
@@ -270,7 +270,7 @@ grpc_error* grpc_chttp2_hptbl_set_current_table_size(grpc_chttp2_hptbl* tbl,
     gpr_free(msg);
     return err;
   }
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     gpr_log(GPR_INFO, "Update hpack parser table size to %d", bytes);
   }
   while (tbl->mem_used > bytes) {

src/core/ext/transport/chttp2/transport/internal.h

@@ -238,7 +238,7 @@ class Chttp2IncomingByteStream : public ByteStream {
   // switch to std::shared_ptr<>.
   void Ref() { refs_.Ref(); }
   void Unref() {
-    if (refs_.Unref()) {
+    if (GPR_UNLIKELY(refs_.Unref())) {
       grpc_core::Delete(this);
     }
   }
@@ -771,11 +771,12 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t,
 // extern grpc_core::TraceFlag grpc_http_trace;
 // extern grpc_core::TraceFlag grpc_flowctl_trace;
 
-#define GRPC_CHTTP2_IF_TRACING(stmt) \
-  if (!(grpc_http_trace.enabled()))  \
-    ;                                \
-  else                               \
-    stmt
+#define GRPC_CHTTP2_IF_TRACING(stmt)                \
+  do {                                              \
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { \
+      (stmt);                                       \
+    }                                               \
+  } while (0)
 
 void grpc_chttp2_fake_status(grpc_chttp2_transport* t,
                              grpc_chttp2_stream* stream, grpc_error* error);

src/core/ext/transport/chttp2/transport/parsing.cc

@@ -28,6 +28,7 @@
 
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
+#include "src/core/lib/slice/slice_utils.h"
 #include "src/core/lib/transport/http2_errors.h"
 #include "src/core/lib/transport/static_metadata.h"
 #include "src/core/lib/transport/status_conversion.h"
@@ -304,7 +305,7 @@ static grpc_error* init_frame_parser(grpc_chttp2_transport* t) {
     case GRPC_CHTTP2_FRAME_GOAWAY:
       return init_goaway_parser(t);
     default:
-      if (grpc_http_trace.enabled()) {
+      if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
         gpr_log(GPR_ERROR, "Unknown frame type %02x", t->incoming_frame_type);
       }
       return init_skip_frame_parser(t, 0);
@@ -400,7 +401,7 @@ static void on_initial_header(void* tp, grpc_mdelem md) {
   grpc_chttp2_stream* s = t->incoming_stream;
   GPR_ASSERT(s != nullptr);
 
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     char* key = grpc_slice_to_c_string(GRPC_MDKEY(md));
     char* value =
         grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX | GPR_DUMP_ASCII);
@@ -410,53 +411,42 @@ static void on_initial_header(void* tp, grpc_mdelem md) {
     gpr_free(value);
   }
 
-  if (GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC) {
-    // We don't use grpc_mdelem_eq here to avoid executing additional
-    // instructions. The reasoning is if the payload is not equal, we already
-    // know that the metadata elements are not equal because the md is
-    // confirmed to be static. If we had used grpc_mdelem_eq here, then if the
-    // payloads are not equal, grpc_mdelem_eq executes more instructions to
-    // determine if they're equal or not.
-    if (md.payload == GRPC_MDELEM_GRPC_STATUS_1.payload ||
-        md.payload == GRPC_MDELEM_GRPC_STATUS_2.payload) {
-      s->seen_error = true;
-    }
-  } else {
-    if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) &&
-        !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) {
-      /* TODO(ctiller): check for a status like " 0" */
-      s->seen_error = true;
-    }
-
-    if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_TIMEOUT)) {
-      grpc_millis* cached_timeout = static_cast<grpc_millis*>(
-          grpc_mdelem_get_user_data(md, free_timeout));
-      grpc_millis timeout;
-      if (cached_timeout != nullptr) {
-        timeout = *cached_timeout;
-      } else {
-        if (GPR_UNLIKELY(
-                !grpc_http2_decode_timeout(GRPC_MDVALUE(md), &timeout))) {
-          char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
-          gpr_log(GPR_ERROR, "Ignoring bad timeout value '%s'", val);
-          gpr_free(val);
-          timeout = GRPC_MILLIS_INF_FUTURE;
-        }
-        if (GRPC_MDELEM_IS_INTERNED(md)) {
-          /* store the result */
-          cached_timeout =
-              static_cast<grpc_millis*>(gpr_malloc(sizeof(grpc_millis)));
-          *cached_timeout = timeout;
-          grpc_mdelem_set_user_data(md, free_timeout, cached_timeout);
-        }
+  // If md.payload == GRPC_MDELEM_GRPC_STATUS_1 or GRPC_MDELEM_GRPC_STATUS_2,
+  // then we have seen an error. In fact, if it is a GRPC_STATUS and it's
+  // not equal to GRPC_MDELEM_GRPC_STATUS_0, then we have seen an error.
+  if (grpc_slice_eq_static_interned(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) &&
+      !grpc_mdelem_static_value_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) {
+    /* TODO(ctiller): check for a status like " 0" */
+    s->seen_error = true;
+  } else if (grpc_slice_eq_static_interned(GRPC_MDKEY(md),
+                                           GRPC_MDSTR_GRPC_TIMEOUT)) {
+    grpc_millis* cached_timeout =
+        static_cast<grpc_millis*>(grpc_mdelem_get_user_data(md, free_timeout));
+    grpc_millis timeout;
+    if (cached_timeout != nullptr) {
+      timeout = *cached_timeout;
+    } else {
+      if (GPR_UNLIKELY(
+              !grpc_http2_decode_timeout(GRPC_MDVALUE(md), &timeout))) {
+        char* val = grpc_slice_to_c_string(GRPC_MDVALUE(md));
+        gpr_log(GPR_ERROR, "Ignoring bad timeout value '%s'", val);
+        gpr_free(val);
+        timeout = GRPC_MILLIS_INF_FUTURE;
       }
-      if (timeout != GRPC_MILLIS_INF_FUTURE) {
-        grpc_chttp2_incoming_metadata_buffer_set_deadline(
-            &s->metadata_buffer[0], grpc_core::ExecCtx::Get()->Now() + timeout);
+      if (GRPC_MDELEM_IS_INTERNED(md)) {
+        /* store the result */
+        cached_timeout =
+            static_cast<grpc_millis*>(gpr_malloc(sizeof(grpc_millis)));
+        *cached_timeout = timeout;
+        grpc_mdelem_set_user_data(md, free_timeout, cached_timeout);
       }
-      GRPC_MDELEM_UNREF(md);
-      return;
     }
+    if (timeout != GRPC_MILLIS_INF_FUTURE) {
+      grpc_chttp2_incoming_metadata_buffer_set_deadline(
+          &s->metadata_buffer[0], grpc_core::ExecCtx::Get()->Now() + timeout);
+    }
+    GRPC_MDELEM_UNREF(md);
+    return;
   }
 
   const size_t new_size = s->metadata_buffer[0].size + GRPC_MDELEM_LENGTH(md);
@@ -496,7 +486,7 @@ static void on_trailing_header(void* tp, grpc_mdelem md) {
   grpc_chttp2_stream* s = t->incoming_stream;
   GPR_ASSERT(s != nullptr);
 
-  if (grpc_http_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
     char* key = grpc_slice_to_c_string(GRPC_MDKEY(md));
     char* value =
         grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX | GPR_DUMP_ASCII);
@@ -506,19 +496,11 @@ static void on_trailing_header(void* tp, grpc_mdelem md) {
     gpr_free(value);
   }
 
-  if (GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC) {
-    // We don't use grpc_mdelem_eq here to avoid executing additional
-    // instructions. The reasoning is if the payload is not equal, we already
-    // know that the metadata elements are not equal because the md is
-    // confirmed to be static. If we had used grpc_mdelem_eq here, then if the
-    // payloads are not equal, grpc_mdelem_eq executes more instructions to
-    // determine if they're equal or not.
-    if (md.payload == GRPC_MDELEM_GRPC_STATUS_1.payload ||
-        md.payload == GRPC_MDELEM_GRPC_STATUS_2.payload) {
-      s->seen_error = true;
-    }
-  } else if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) &&
-             !grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) {
+  // If md.payload == GRPC_MDELEM_GRPC_STATUS_1 or GRPC_MDELEM_GRPC_STATUS_2,
+  // then we have seen an error. In fact, if it is a GRPC_STATUS and it's
+  // not equal to GRPC_MDELEM_GRPC_STATUS_0, then we have seen an error.
+  if (grpc_slice_eq_static_interned(GRPC_MDKEY(md), GRPC_MDSTR_GRPC_STATUS) &&
+      !grpc_mdelem_static_value_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) {
     /* TODO(ctiller): check for a status like " 0" */
     s->seen_error = true;
   }
@@ -761,7 +743,7 @@ static grpc_error* parse_frame_slice(grpc_chttp2_transport* t,
   if (GPR_LIKELY(err == GRPC_ERROR_NONE)) {
     return err;
   } else if (grpc_error_get_int(err, GRPC_ERROR_INT_STREAM_ID, &unused)) {
-    if (grpc_http_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
       const char* msg = grpc_error_string(err);
       gpr_log(GPR_ERROR, "%s", msg);
     }

src/core/ext/transport/chttp2/transport/stream_lists.cc

@@ -67,7 +67,7 @@ static bool stream_list_pop(grpc_chttp2_transport* t,
     s->included[id] = 0;
   }
   *stream = s;
-  if (s && grpc_trace_http2_stream_state.enabled()) {
+  if (s && GRPC_TRACE_FLAG_ENABLED(grpc_trace_http2_stream_state)) {
     gpr_log(GPR_INFO, "%p[%d][%s]: pop from %s", t, s->id,
             t->is_client ? "cli" : "svr", stream_list_id_string(id));
   }
@@ -89,7 +89,7 @@ static void stream_list_remove(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
   } else {
     t->lists[id].tail = s->links[id].prev;
   }
-  if (grpc_trace_http2_stream_state.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_http2_stream_state)) {
     gpr_log(GPR_INFO, "%p[%d][%s]: remove from %s", t, s->id,
             t->is_client ? "cli" : "svr", stream_list_id_string(id));
   }
@@ -121,7 +121,7 @@ static void stream_list_add_tail(grpc_chttp2_transport* t,
   }
   t->lists[id].tail = s;
   s->included[id] = 1;
-  if (grpc_trace_http2_stream_state.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_http2_stream_state)) {
     gpr_log(GPR_INFO, "%p[%d][%s]: add to %s", t, s->id,
             t->is_client ? "cli" : "svr", stream_list_id_string(id));
   }

src/core/ext/transport/chttp2/transport/writing.cc

@@ -53,7 +53,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) {
   }
   if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_INFLIGHT])) {
     /* ping already in-flight: wait */
-    if (grpc_http_trace.enabled() || grpc_bdp_estimator_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) ||
+        GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_INFO, "%s: Ping delayed [%p]: already pinging",
               t->is_client ? "CLIENT" : "SERVER", t->peer_string);
     }
@@ -62,7 +63,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) {
   if (t->ping_state.pings_before_data_required == 0 &&
       t->ping_policy.max_pings_without_data != 0) {
     /* need to receive something of substance before sending a ping again */
-    if (grpc_http_trace.enabled() || grpc_bdp_estimator_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) ||
+        GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_INFO, "%s: Ping delayed [%p]: too many recent pings: %d/%d",
               t->is_client ? "CLIENT" : "SERVER", t->peer_string,
               t->ping_state.pings_before_data_required,
@@ -82,7 +84,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) {
 
   if (next_allowed_ping > now) {
     /* not enough elapsed time between successive pings */
-    if (grpc_http_trace.enabled() || grpc_bdp_estimator_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) ||
+        GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
       gpr_log(GPR_INFO,
               "%s: Ping delayed [%p]: not enough time elapsed since last ping. "
               " Last ping %f: Next ping %f: Now %f",
@@ -108,7 +111,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) {
                         grpc_chttp2_ping_create(false, pq->inflight_id));
   GRPC_STATS_INC_HTTP2_PINGS_SENT();
   t->ping_state.last_ping_sent_time = now;
-  if (grpc_http_trace.enabled() || grpc_bdp_estimator_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) ||
+      GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) {
     gpr_log(GPR_INFO, "%s: Ping sent [%s]: %d/%d",
             t->is_client ? "CLIENT" : "SERVER", t->peer_string,
             t->ping_state.pings_before_data_required,
@@ -141,7 +145,7 @@ static bool update_list(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
 
 static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
                          const char* staller) {
-  if (grpc_flowctl_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_flowctl_trace)) {
     gpr_log(
         GPR_DEBUG,
         "%s:%p stream %d moved to stalled list by %s. This is FULLY expected "

src/core/ext/transport/cronet/transport/cronet_transport.cc

@@ -753,15 +753,16 @@ static void convert_metadata_to_cronet_headers(
     } else {
       value = grpc_slice_to_c_string(GRPC_MDVALUE(mdelem));
     }
-    if (grpc_slice_eq(GRPC_MDKEY(mdelem), GRPC_MDSTR_SCHEME) ||
-        grpc_slice_eq(GRPC_MDKEY(mdelem), GRPC_MDSTR_AUTHORITY)) {
+    if (grpc_slice_eq_static_interned(GRPC_MDKEY(mdelem), GRPC_MDSTR_SCHEME) ||
+        grpc_slice_eq_static_interned(GRPC_MDKEY(mdelem),
+                                      GRPC_MDSTR_AUTHORITY)) {
       /* Cronet populates these fields on its own */
       gpr_free(key);
       gpr_free(value);
       continue;
     }
-    if (grpc_slice_eq(GRPC_MDKEY(mdelem), GRPC_MDSTR_METHOD)) {
-      if (grpc_slice_eq(GRPC_MDVALUE(mdelem), GRPC_MDSTR_PUT)) {
+    if (grpc_slice_eq_static_interned(GRPC_MDKEY(mdelem), GRPC_MDSTR_METHOD)) {
+      if (grpc_slice_eq_static_interned(GRPC_MDVALUE(mdelem), GRPC_MDSTR_PUT)) {
         *method = "PUT";
       } else {
         /* POST method in default*/
@@ -771,7 +772,7 @@ static void convert_metadata_to_cronet_headers(
       gpr_free(value);
       continue;
     }
-    if (grpc_slice_eq(GRPC_MDKEY(mdelem), GRPC_MDSTR_PATH)) {
+    if (grpc_slice_eq_static_interned(GRPC_MDKEY(mdelem), GRPC_MDSTR_PATH)) {
       /* Create URL by appending :path value to the hostname */
       gpr_asprintf(pp_url, "https://%s%s", host, value);
       gpr_free(key);
@@ -803,7 +804,8 @@ static void parse_grpc_header(const uint8_t* data, int* length,
 
 static bool header_has_authority(grpc_linked_mdelem* head) {
   while (head != nullptr) {
-    if (grpc_slice_eq(GRPC_MDKEY(head->md), GRPC_MDSTR_AUTHORITY)) {
+    if (grpc_slice_eq_static_interned(GRPC_MDKEY(head->md),
+                                      GRPC_MDSTR_AUTHORITY)) {
       return true;
     }
     head = head->next;

src/core/ext/transport/inproc/inproc_transport.cc

@@ -35,9 +35,11 @@
 #include "src/core/lib/transport/error_utils.h"
 #include "src/core/lib/transport/transport_impl.h"
 
-#define INPROC_LOG(...)                                    \
-  do {                                                     \
-    if (grpc_inproc_trace.enabled()) gpr_log(__VA_ARGS__); \
+#define INPROC_LOG(...)                               \
+  do {                                                \
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_inproc_trace)) { \
+      gpr_log(__VA_ARGS__);                           \
+    }                                                 \
   } while (0)
 
 namespace {
@@ -296,7 +298,7 @@ grpc_error* fill_in_metadata(inproc_stream* s,
                              const grpc_metadata_batch* metadata,
                              uint32_t flags, grpc_metadata_batch* out_md,
                              uint32_t* outflags, bool* markfilled) {
-  if (grpc_inproc_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_inproc_trace)) {
     log_metadata(metadata, s->t->is_client, outflags != nullptr);
   }
 
@@ -907,7 +909,7 @@ void perform_stream_op(grpc_transport* gt, grpc_stream* gs,
   gpr_mu* mu = &s->t->mu->mu;  // save aside in case s gets closed
   gpr_mu_lock(mu);
 
-  if (grpc_inproc_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_inproc_trace)) {
     if (op->send_initial_metadata) {
       log_metadata(op->payload->send_initial_metadata.send_initial_metadata,
                    s->t->is_client, true);

src/core/lib/channel/channel_stack.h

@@ -274,7 +274,11 @@ void grpc_call_log_op(const char* file, int line, gpr_log_severity severity,
 
 extern grpc_core::TraceFlag grpc_trace_channel;
 
-#define GRPC_CALL_LOG_OP(sev, elem, op) \
-  if (grpc_trace_channel.enabled()) grpc_call_log_op(sev, elem, op)
+#define GRPC_CALL_LOG_OP(sev, elem, op)                \
+  do {                                                 \
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_channel)) { \
+      grpc_call_log_op(sev, elem, op);                 \
+    }                                                  \
+  } while (0)
 
 #endif /* GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H */

src/core/lib/channel/handshaker.cc

@@ -95,7 +95,7 @@ void HandshakeManager::ShutdownAllPending(grpc_error* why) {
 }
 
 void HandshakeManager::Add(RefCountedPtr<Handshaker> handshaker) {
-  if (grpc_handshaker_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_handshaker_trace)) {
     gpr_log(
         GPR_INFO,
         "handshake_manager %p: adding handshaker %s [%p] at index %" PRIuPTR,
@@ -126,7 +126,7 @@ void HandshakeManager::Shutdown(grpc_error* why) {
 // on_handshake_done callback.
 // Returns true if we've scheduled the on_handshake_done callback.
 bool HandshakeManager::CallNextHandshakerLocked(grpc_error* error) {
-  if (grpc_handshaker_trace.enabled()) {
+  if (GRPC_TRACE_FLAG_ENABLED(grpc_handshaker_trace)) {
     char* args_str = HandshakerArgsString(&args_);
     gpr_log(GPR_INFO,
             "handshake_manager %p: error=%s shutdown=%d index=%" PRIuPTR
@@ -160,7 +160,7 @@ bool HandshakeManager::CallNextHandshakerLocked(grpc_error* error) {
         args_.read_buffer = nullptr;
       }
     }
-    if (grpc_handshaker_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_handshaker_trace)) {
       gpr_log(GPR_INFO,
               "handshake_manager %p: handshaking complete -- scheduling "
               "on_handshake_done with error=%s",
@@ -173,7 +173,7 @@ bool HandshakeManager::CallNextHandshakerLocked(grpc_error* error) {
     is_shutdown_ = true;
   } else {
     auto handshaker = handshakers_[index_];
-    if (grpc_handshaker_trace.enabled()) {
+    if (GRPC_TRACE_FLAG_ENABLED(grpc_handshaker_trace)) {
       gpr_log(
           GPR_INFO,
           "handshake_manager %p: calling handshaker %s [%p] at index %" PRIuPTR,

src/core/lib/compression/compression.cc

@@ -26,6 +26,7 @@
 #include "src/core/lib/compression/algorithm_metadata.h"
 #include "src/core/lib/compression/compression_internal.h"
 #include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/slice/slice_utils.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/transport/static_metadata.h"
 
@@ -42,16 +43,17 @@ int grpc_compression_algorithm_is_stream(grpc_compression_algorithm algorithm) {
 
 int grpc_compression_algorithm_parse(grpc_slice name,
                                      grpc_compression_algorithm* algorithm) {
-  if (grpc_slice_eq(name, GRPC_MDSTR_IDENTITY)) {
+  if (grpc_slice_eq_static_interned(name, GRPC_MDSTR_IDENTITY)) {
     *algorithm = GRPC_COMPRESS_NONE;
     return 1;
-  } else if (grpc_slice_eq(name, GRPC_MDSTR_DEFLATE)) {
+  } else if (grpc_slice_eq_static_interned(name, GRPC_MDSTR_DEFLATE)) {
     *algorithm = GRPC_COMPRESS_DEFLATE;
     return 1;
-  } else if (grpc_slice_eq(name, GRPC_MDSTR_GZIP)) {
+  } else if (grpc_slice_eq_static_interned(name, GRPC_MDSTR_GZIP)) {
     *algorithm = GRPC_COMPRESS_GZIP;
     return 1;
-  } else if (grpc_slice_eq(name, GRPC_MDSTR_STREAM_SLASH_GZIP)) {
+  } else if (grpc_slice_eq_static_interned(name,
+                                           GRPC_MDSTR_STREAM_SLASH_GZIP)) {
     *algorithm = GRPC_COMPRESS_STREAM_GZIP;
     return 1;
   } else {
@@ -148,10 +150,13 @@ grpc_slice grpc_compression_algorithm_slice(
 
 grpc_compression_algorithm grpc_compression_algorithm_from_slice(
     const grpc_slice& str) {
-  if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_COMPRESS_NONE;
-  if (grpc_slice_eq(str, GRPC_MDSTR_DEFLATE)) return GRPC_COMPRESS_DEFLATE;
-  if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_COMPRESS_GZIP;
-  if (grpc_slice_eq(str, GRPC_MDSTR_STREAM_SLASH_GZIP))
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_IDENTITY))
+    return GRPC_COMPRESS_NONE;
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_DEFLATE))
+    return GRPC_COMPRESS_DEFLATE;
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_GZIP))
+    return GRPC_COMPRESS_GZIP;
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_STREAM_SLASH_GZIP))
     return GRPC_COMPRESS_STREAM_GZIP;
   return GRPC_COMPRESS_ALGORITHMS_COUNT;
 }

src/core/lib/compression/compression_internal.cc

@@ -26,6 +26,7 @@
 #include "src/core/lib/compression/algorithm_metadata.h"
 #include "src/core/lib/compression/compression_internal.h"
 #include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/slice/slice_utils.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/transport/static_metadata.h"
 
@@ -33,18 +34,21 @@
 
 grpc_message_compression_algorithm
 grpc_message_compression_algorithm_from_slice(const grpc_slice& str) {
-  if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY))
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_IDENTITY))
     return GRPC_MESSAGE_COMPRESS_NONE;
-  if (grpc_slice_eq(str, GRPC_MDSTR_DEFLATE))
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_DEFLATE))
     return GRPC_MESSAGE_COMPRESS_DEFLATE;
-  if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_MESSAGE_COMPRESS_GZIP;
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_GZIP))
+    return GRPC_MESSAGE_COMPRESS_GZIP;
   return GRPC_MESSAGE_COMPRESS_ALGORITHMS_COUNT;
 }
 
 grpc_stream_compression_algorithm grpc_stream_compression_algorithm_from_slice(
     const grpc_slice& str) {
-  if (grpc_slice_eq(str, GRPC_MDSTR_IDENTITY)) return GRPC_STREAM_COMPRESS_NONE;
-  if (grpc_slice_eq(str, GRPC_MDSTR_GZIP)) return GRPC_STREAM_COMPRESS_GZIP;
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_IDENTITY))
+    return GRPC_STREAM_COMPRESS_NONE;
+  if (grpc_slice_eq_static_interned(str, GRPC_MDSTR_GZIP))
+    return GRPC_STREAM_COMPRESS_GZIP;
   return GRPC_STREAM_COMPRESS_ALGORITHMS_COUNT;
 }
 
@@ -244,13 +248,13 @@ grpc_message_compression_algorithm grpc_message_compression_algorithm_for_level(
 
 int grpc_message_compression_algorithm_parse(
     grpc_slice value, grpc_message_compression_algorithm* algorithm) {
-  if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) {
+  if (grpc_slice_eq_static_interned(value, GRPC_MDSTR_IDENTITY)) {
     *algorithm = GRPC_MESSAGE_COMPRESS_NONE;
     return 1;
-  } else if (grpc_slice_eq(value, GRPC_MDSTR_DEFLATE)) {
+  } else if (grpc_slice_eq_static_interned(value, GRPC_MDSTR_DEFLATE)) {
     *algorithm = GRPC_MESSAGE_COMPRESS_DEFLATE;
     return 1;
-  } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) {
+  } else if (grpc_slice_eq_static_interned(value, GRPC_MDSTR_GZIP)) {
     *algorithm = GRPC_MESSAGE_COMPRESS_GZIP;
     return 1;
   } else {
@@ -263,10 +267,10 @@ int grpc_message_compression_algorithm_parse(
 
 int grpc_stream_compression_algorithm_parse(
     grpc_slice value, grpc_stream_compression_algorithm* algorithm) {
-  if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) {
+  if (grpc_slice_eq_static_interned(value, GRPC_MDSTR_IDENTITY)) {
     *algorithm = GRPC_STREAM_COMPRESS_NONE;
     return 1;
-  } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) {
+  } else if (grpc_slice_eq_static_interned(value, GRPC_MDSTR_GZIP)) {
     *algorithm = GRPC_STREAM_COMPRESS_GZIP;
     return 1;
   } else {

src/core/lib/compression/stream_compression.cc

@@ -22,6 +22,7 @@
 
 #include "src/core/lib/compression/stream_compression.h"
 #include "src/core/lib/compression/stream_compression_gzip.h"
+#include "src/core/lib/slice/slice_utils.h"
 
 extern const grpc_stream_compression_vtable
     grpc_stream_compression_identity_vtable;
@@ -65,11 +66,11 @@ void grpc_stream_compression_context_destroy(
 int grpc_stream_compression_method_parse(
     grpc_slice value, bool is_compress,
     grpc_stream_compression_method* method) {
-  if (grpc_slice_eq(value, GRPC_MDSTR_IDENTITY)) {
+  if (grpc_slice_eq_static_interned(value, GRPC_MDSTR_IDENTITY)) {
     *method = is_compress ? GRPC_STREAM_COMPRESSION_IDENTITY_COMPRESS
                           : GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS;
     return 1;
-  } else if (grpc_slice_eq(value, GRPC_MDSTR_GZIP)) {
+  } else if (grpc_slice_eq_static_interned(value, GRPC_MDSTR_GZIP)) {
     *method = is_compress ? GRPC_STREAM_COMPRESSION_GZIP_COMPRESS
                           : GRPC_STREAM_COMPRESSION_GZIP_DECOMPRESS;
     return 1;

src/core/lib/debug/trace.cc

@@ -26,7 +26,11 @@
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
-#include "src/core/lib/gpr/env.h"
+
+GPR_GLOBAL_CONFIG_DEFINE_STRING(
+    grpc_trace, "",
+    "A comma separated list of tracers that provide additional insight into "
+    "how gRPC C core is processing requests via debug logs.");
 
 int grpc_tracer_set_enabled(const char* name, int enabled);
 
@@ -133,12 +137,14 @@ static void parse(const char* s) {
   gpr_free(strings);
 }
 
-void grpc_tracer_init(const char* env_var) {
-  char* e = gpr_getenv(env_var);
-  if (e != nullptr) {
-    parse(e);
-    gpr_free(e);
-  }
+void grpc_tracer_init(const char* env_var_name) {
+  (void)env_var_name;  // suppress unused variable error
+  grpc_tracer_init();
+}
+
+void grpc_tracer_init() {
+  grpc_core::UniquePtr<char> value = GPR_GLOBAL_CONFIG_GET(grpc_trace);
+  parse(value.get());
 }
 
 void grpc_tracer_shutdown(void) {}

src/core/lib/debug/trace.h

@@ -24,7 +24,15 @@
 #include <grpc/support/atm.h>
 #include <stdbool.h>
 
+#include "src/core/lib/gprpp/global_config.h"
+
+GPR_GLOBAL_CONFIG_DECLARE_STRING(grpc_trace);
+
+// TODO(veblush): Remove this deprecated function once codes depending on this
+// function are updated in the internal repo.
 void grpc_tracer_init(const char* env_var_name);
+
+void grpc_tracer_init();
 void grpc_tracer_shutdown(void);
 
 #if defined(__has_feature)
@@ -65,6 +73,8 @@ class TraceFlag {
 // wrapped language (wr don't want to force recompilation to get tracing).
 // Internally, however, for performance reasons, we compile them out by
 // default, since internal build systems make recompiling trivial.
+//
+// Prefer GRPC_TRACE_FLAG_ENABLED() macro instead of using enabled() directly.
 #define GRPC_USE_TRACERS  // tracers on by default in OSS
 #if defined(GRPC_USE_TRACERS) || !defined(NDEBUG)
   bool enabled() {
@@ -99,6 +109,8 @@ class TraceFlag {
 #endif
 };
 
+#define GRPC_TRACE_FLAG_ENABLED(f) GPR_UNLIKELY((f).enabled())
+
 #ifndef NDEBUG
 typedef TraceFlag DebugOnlyTraceFlag;
 #else

src/core/lib/gpr/env.h

@@ -34,12 +34,6 @@ char* gpr_getenv(const char* name);
 /* Sets the environment with the specified name to the specified value. */
 void gpr_setenv(const char* name, const char* value);
 
-/* This is a version of gpr_getenv that does not produce any output if it has to
-   use an insecure version of the function. It is ONLY to be used to solve the
-   problem in which we need to check an env variable to configure the verbosity
-   level of logging. So DO NOT USE THIS. */
-const char* gpr_getenv_silent(const char* name, char** dst);
-
 /* Deletes the variable name from the environment. */
 void gpr_unsetenv(const char* name);