Merge branch 'master' into internal_py_proto_library

.gitignore

@@ -135,3 +135,11 @@ bm_*.json
 
 # cmake build files
 /cmake/build
+
+# Visual Studio Code artifacts
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+

.pylintrc

@@ -6,6 +6,8 @@ ignore=
 	src/python/grpcio/grpc/framework/foundation,
 	src/python/grpcio/grpc/framework/interfaces,
 
+extension-pkg-whitelist=grpc._cython.cygrpc
+
 [VARIABLES]
 
 # TODO(https://github.com/PyCQA/pylint/issues/1345): How does the inspection
@@ -17,7 +19,7 @@ dummy-variables-rgx=^ignored_|^unused_
 # NOTE(nathaniel): Not particularly attached to this value; it just seems to
 # be what works for us at the moment (excepting the dead-code-walking Beta
 # API).
-max-args=6
+max-args=7
 
 [MISCELLANEOUS]
 

.vscode/launch.json

@@ -1,30 +0,0 @@
-{
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Mocha Tests",
-            "cwd": "${workspaceRoot}",
-            "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha",
-            "windows": {
-                "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha.cmd"
-            },
-            "runtimeArgs": [
-                "-u",
-                "tdd",
-                "--timeout",
-                "999999",
-                "--colors",
-                "${workspaceRoot}/src/node/test"
-            ],
-            "internalConsoleOptions": "openOnSessionStart"
-        },
-        {
-            "type": "node",
-            "request": "attach",
-            "name": "Attach to Process",
-            "port": 5858
-        }
-    ]
-}

BUILD

@@ -255,7 +255,6 @@ GRPCXX_PUBLIC_HDRS = [
     "include/grpcpp/security/server_credentials.h",
     "include/grpcpp/security/server_credentials_impl.h",
     "include/grpcpp/server.h",
-    "include/grpcpp/server_impl.h",
     "include/grpcpp/server_builder.h",
     "include/grpcpp/server_builder_impl.h",
     "include/grpcpp/server_context.h",
@@ -269,6 +268,7 @@ GRPCXX_PUBLIC_HDRS = [
     "include/grpcpp/support/client_interceptor.h",
     "include/grpcpp/support/config.h",
     "include/grpcpp/support/interceptor.h",
+    "include/grpcpp/support/message_allocator.h",
     "include/grpcpp/support/proto_buffer_reader.h",
     "include/grpcpp/support/proto_buffer_writer.h",
     "include/grpcpp/support/server_callback.h",
@@ -541,7 +541,6 @@ grpc_cc_library(
     name = "gpr_base",
     srcs = [
         "src/core/lib/gpr/alloc.cc",
-        "src/core/lib/gpr/arena.cc",
         "src/core/lib/gpr/atm.cc",
         "src/core/lib/gpr/cpu_iphone.cc",
         "src/core/lib/gpr/cpu_linux.cc",
@@ -574,6 +573,7 @@ grpc_cc_library(
         "src/core/lib/gpr/tmpfile_posix.cc",
         "src/core/lib/gpr/tmpfile_windows.cc",
         "src/core/lib/gpr/wrap_memcpy.cc",
+        "src/core/lib/gprpp/arena.cc",
         "src/core/lib/gprpp/fork.cc",
         "src/core/lib/gprpp/thd_posix.cc",
         "src/core/lib/gprpp/thd_windows.cc",
@@ -598,6 +598,8 @@ grpc_cc_library(
         "src/core/lib/gpr/tmpfile.h",
         "src/core/lib/gpr/useful.h",
         "src/core/lib/gprpp/abstract.h",
+        "src/core/lib/gprpp/arena.h",
+        "src/core/lib/gprpp/atomic.h",
         "src/core/lib/gprpp/fork.h",
         "src/core/lib/gprpp/manual_constructor.h",
         "src/core/lib/gprpp/map.h",
@@ -737,6 +739,7 @@ grpc_cc_library(
         "src/core/lib/channel/handshaker_registry.cc",
         "src/core/lib/channel/status_util.cc",
         "src/core/lib/compression/compression.cc",
+        "src/core/lib/compression/compression_args.cc",
         "src/core/lib/compression/compression_internal.cc",
         "src/core/lib/compression/message_compress.cc",
         "src/core/lib/compression/stream_compression.cc",
@@ -892,6 +895,7 @@ grpc_cc_library(
         "src/core/lib/channel/handshaker_registry.h",
         "src/core/lib/channel/status_util.h",
         "src/core/lib/compression/algorithm_metadata.h",
+        "src/core/lib/compression/compression_args.h",
         "src/core/lib/compression/compression_internal.h",
         "src/core/lib/compression/message_compress.h",
         "src/core/lib/compression/stream_compression.h",
@@ -2137,6 +2141,7 @@ grpc_cc_library(
         "include/grpcpp/impl/codegen/intercepted_channel.h",
         "include/grpcpp/impl/codegen/interceptor.h",
         "include/grpcpp/impl/codegen/interceptor_common.h",
+        "include/grpcpp/impl/codegen/message_allocator.h",
         "include/grpcpp/impl/codegen/metadata_map.h",
         "include/grpcpp/impl/codegen/method_handler_impl.h",
         "include/grpcpp/impl/codegen/rpc_method.h",
@@ -2329,6 +2334,28 @@ grpc_cc_library(
 )
 
 #TODO: Get this into build.yaml once we start using it.
+grpc_cc_library(
+    name = "envoy_lrs_upb",
+    srcs = [
+        "src/core/ext/upb-generated/envoy/api/v2/endpoint/load_report.upb.c",
+        "src/core/ext/upb-generated/envoy/service/load_stats/v2/lrs.upb.c",
+    ],
+    hdrs = [
+        "src/core/ext/upb-generated/envoy/api/v2/endpoint/load_report.upb.h",
+        "src/core/ext/upb-generated/envoy/service/load_stats/v2/lrs.upb.h",
+    ],
+    language = "c++",
+    external_deps = [
+        "upb_lib",
+    ],
+    deps = [
+        ":envoy_core_upb",
+        ":google_api_upb",
+        ":proto_gen_validate_upb",
+    ],
+    tags = ["no_windows"],
+)
+
 grpc_cc_library(
     name = "envoy_ads_upb",
     srcs = [
@@ -2361,6 +2388,7 @@ grpc_cc_library(
         ":google_api_upb",
         ":proto_gen_validate_upb",
     ],
+    tags = ["no_windows"],
 )
 
 grpc_cc_library(
@@ -2468,3 +2496,11 @@ grpc_cc_library(
 )
 
 grpc_generate_one_off_targets()
+
+filegroup(
+    name = "root_certificates",
+    srcs = [
+        "etc/roots.pem",
+    ],
+    visibility = ["//visibility:public"],
+)

BUILD.gn

@@ -131,7 +131,6 @@ config("grpc_config") {
         "include/grpc/support/time.h",
         "src/core/lib/gpr/alloc.cc",
         "src/core/lib/gpr/alloc.h",
-        "src/core/lib/gpr/arena.cc",
         "src/core/lib/gpr/arena.h",
         "src/core/lib/gpr/atm.cc",
         "src/core/lib/gpr/cpu_iphone.cc",
@@ -180,6 +179,8 @@ config("grpc_config") {
         "src/core/lib/gpr/useful.h",
         "src/core/lib/gpr/wrap_memcpy.cc",
         "src/core/lib/gprpp/abstract.h",
+        "src/core/lib/gprpp/arena.cc",
+        "src/core/lib/gprpp/arena.h",
         "src/core/lib/gprpp/atomic.h",
         "src/core/lib/gprpp/fork.cc",
         "src/core/lib/gprpp/fork.h",
@@ -445,6 +446,8 @@ config("grpc_config") {
         "src/core/lib/channel/status_util.h",
         "src/core/lib/compression/algorithm_metadata.h",
         "src/core/lib/compression/compression.cc",
+        "src/core/lib/compression/compression_args.cc",
+        "src/core/lib/compression/compression_args.h",
         "src/core/lib/compression/compression_internal.cc",
         "src/core/lib/compression/compression_internal.h",
         "src/core/lib/compression/message_compress.cc",
@@ -1045,6 +1048,7 @@ config("grpc_config") {
         "include/grpcpp/impl/codegen/intercepted_channel.h",
         "include/grpcpp/impl/codegen/interceptor.h",
         "include/grpcpp/impl/codegen/interceptor_common.h",
+        "include/grpcpp/impl/codegen/message_allocator.h",
         "include/grpcpp/impl/codegen/metadata_map.h",
         "include/grpcpp/impl/codegen/method_handler_impl.h",
         "include/grpcpp/impl/codegen/proto_buffer_reader.h",
@@ -1090,7 +1094,6 @@ config("grpc_config") {
         "include/grpcpp/server_builder.h",
         "include/grpcpp/server_builder_impl.h",
         "include/grpcpp/server_context.h",
-        "include/grpcpp/server_impl.h",
         "include/grpcpp/server_posix.h",
         "include/grpcpp/server_posix_impl.h",
         "include/grpcpp/support/async_stream.h",
@@ -1101,6 +1104,7 @@ config("grpc_config") {
         "include/grpcpp/support/client_interceptor.h",
         "include/grpcpp/support/config.h",
         "include/grpcpp/support/interceptor.h",
+        "include/grpcpp/support/message_allocator.h",
         "include/grpcpp/support/proto_buffer_reader.h",
         "include/grpcpp/support/proto_buffer_writer.h",
         "include/grpcpp/support/server_callback.h",
@@ -1128,6 +1132,7 @@ config("grpc_config") {
         "src/core/lib/channel/handshaker_registry.h",
         "src/core/lib/channel/status_util.h",
         "src/core/lib/compression/algorithm_metadata.h",
+        "src/core/lib/compression/compression_args.h",
         "src/core/lib/compression/compression_internal.h",
         "src/core/lib/compression/message_compress.h",
         "src/core/lib/compression/stream_compression.h",
@@ -1153,6 +1158,7 @@ config("grpc_config") {
         "src/core/lib/gpr/tmpfile.h",
         "src/core/lib/gpr/useful.h",
         "src/core/lib/gprpp/abstract.h",
+        "src/core/lib/gprpp/arena.h",
         "src/core/lib/gprpp/atomic.h",
         "src/core/lib/gprpp/debug_location.h",
         "src/core/lib/gprpp/fork.h",

CMakeLists.txt

@@ -484,6 +484,7 @@ add_dependencies(buildtests_c h2_sockpair+trace_test)
 add_dependencies(buildtests_c h2_sockpair_1byte_test)
 add_dependencies(buildtests_c h2_spiffe_test)
 add_dependencies(buildtests_c h2_ssl_test)
+add_dependencies(buildtests_c h2_ssl_cred_reload_test)
 add_dependencies(buildtests_c h2_ssl_proxy_test)
 if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_c h2_uds_test)
@@ -660,6 +661,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx json_run_localhost)
 endif()
 add_dependencies(buildtests_cxx memory_test)
+add_dependencies(buildtests_cxx message_allocator_end2end_test)
 add_dependencies(buildtests_cxx metrics_client)
 add_dependencies(buildtests_cxx mock_test)
 add_dependencies(buildtests_cxx nonblocking_test)
@@ -699,6 +701,7 @@ add_dependencies(buildtests_cxx server_crash_test_client)
 add_dependencies(buildtests_cxx server_early_return_test)
 add_dependencies(buildtests_cxx server_interceptors_end2end_test)
 add_dependencies(buildtests_cxx server_request_call_test)
+add_dependencies(buildtests_cxx service_config_test)
 add_dependencies(buildtests_cxx shutdown_test)
 add_dependencies(buildtests_cxx slice_hash_table_test)
 add_dependencies(buildtests_cxx slice_weak_hash_table_test)
@@ -836,7 +839,6 @@ endif (gRPC_BUILD_TESTS)
 
 add_library(gpr
   src/core/lib/gpr/alloc.cc
-  src/core/lib/gpr/arena.cc
   src/core/lib/gpr/atm.cc
   src/core/lib/gpr/cpu_iphone.cc
   src/core/lib/gpr/cpu_linux.cc
@@ -869,6 +871,7 @@ add_library(gpr
   src/core/lib/gpr/tmpfile_posix.cc
   src/core/lib/gpr/tmpfile_windows.cc
   src/core/lib/gpr/wrap_memcpy.cc
+  src/core/lib/gprpp/arena.cc
   src/core/lib/gprpp/fork.cc
   src/core/lib/gprpp/thd_posix.cc
   src/core/lib/gprpp/thd_windows.cc
@@ -981,6 +984,7 @@ add_library(grpc
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
+  src/core/lib/compression/compression_args.cc
   src/core/lib/compression/compression_internal.cc
   src/core/lib/compression/message_compress.cc
   src/core/lib/compression/stream_compression.cc
@@ -1407,6 +1411,7 @@ add_library(grpc_cronet
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
+  src/core/lib/compression/compression_args.cc
   src/core/lib/compression/compression_internal.cc
   src/core/lib/compression/message_compress.cc
   src/core/lib/compression/stream_compression.cc
@@ -1818,6 +1823,7 @@ add_library(grpc_test_util
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
+  src/core/lib/compression/compression_args.cc
   src/core/lib/compression/compression_internal.cc
   src/core/lib/compression/message_compress.cc
   src/core/lib/compression/stream_compression.cc
@@ -2142,6 +2148,7 @@ add_library(grpc_test_util_unsecure
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
+  src/core/lib/compression/compression_args.cc
   src/core/lib/compression/compression_internal.cc
   src/core/lib/compression/message_compress.cc
   src/core/lib/compression/stream_compression.cc
@@ -2442,6 +2449,7 @@ add_library(grpc_unsecure
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
+  src/core/lib/compression/compression_args.cc
   src/core/lib/compression/compression_internal.cc
   src/core/lib/compression/message_compress.cc
   src/core/lib/compression/stream_compression.cc
@@ -3037,7 +3045,6 @@ foreach(_hdr
   include/grpcpp/server_builder.h
   include/grpcpp/server_builder_impl.h
   include/grpcpp/server_context.h
-  include/grpcpp/server_impl.h
   include/grpcpp/server_posix.h
   include/grpcpp/server_posix_impl.h
   include/grpcpp/support/async_stream.h
@@ -3048,6 +3055,7 @@ foreach(_hdr
   include/grpcpp/support/client_interceptor.h
   include/grpcpp/support/config.h
   include/grpcpp/support/interceptor.h
+  include/grpcpp/support/message_allocator.h
   include/grpcpp/support/proto_buffer_reader.h
   include/grpcpp/support/proto_buffer_writer.h
   include/grpcpp/support/server_callback.h
@@ -3163,6 +3171,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -3342,6 +3351,7 @@ add_library(grpc++_cronet
   src/core/lib/channel/handshaker_registry.cc
   src/core/lib/channel/status_util.cc
   src/core/lib/compression/compression.cc
+  src/core/lib/compression/compression_args.cc
   src/core/lib/compression/compression_internal.cc
   src/core/lib/compression/message_compress.cc
   src/core/lib/compression/stream_compression.cc
@@ -3641,7 +3651,6 @@ foreach(_hdr
   include/grpcpp/server_builder.h
   include/grpcpp/server_builder_impl.h
   include/grpcpp/server_context.h
-  include/grpcpp/server_impl.h
   include/grpcpp/server_posix.h
   include/grpcpp/server_posix_impl.h
   include/grpcpp/support/async_stream.h
@@ -3652,6 +3661,7 @@ foreach(_hdr
   include/grpcpp/support/client_interceptor.h
   include/grpcpp/support/config.h
   include/grpcpp/support/interceptor.h
+  include/grpcpp/support/message_allocator.h
   include/grpcpp/support/proto_buffer_reader.h
   include/grpcpp/support/proto_buffer_writer.h
   include/grpcpp/support/server_callback.h
@@ -3767,6 +3777,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -4200,6 +4211,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -4397,6 +4409,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -4621,7 +4634,6 @@ foreach(_hdr
   include/grpcpp/server_builder.h
   include/grpcpp/server_builder_impl.h
   include/grpcpp/server_context.h
-  include/grpcpp/server_impl.h
   include/grpcpp/server_posix.h
   include/grpcpp/server_posix_impl.h
   include/grpcpp/support/async_stream.h
@@ -4632,6 +4644,7 @@ foreach(_hdr
   include/grpcpp/support/client_interceptor.h
   include/grpcpp/support/config.h
   include/grpcpp/support/interceptor.h
+  include/grpcpp/support/message_allocator.h
   include/grpcpp/support/proto_buffer_reader.h
   include/grpcpp/support/proto_buffer_writer.h
   include/grpcpp/support/server_callback.h
@@ -4747,6 +4760,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -12676,6 +12690,7 @@ target_include_directories(client_crash_test_server
 target_link_libraries(client_crash_test_server
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_config
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -13415,6 +13430,7 @@ target_include_directories(golden_file_test
 target_link_libraries(golden_file_test
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_config
   grpc++
   grpc
   gpr
@@ -14496,6 +14512,46 @@ target_link_libraries(memory_test
 )
 
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_executable(message_allocator_end2end_test
+  test/cpp/end2end/message_allocator_end2end_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(message_allocator_end2end_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(message_allocator_end2end_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_util
+  grpc_test_util
+  grpc++
+  grpc
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
@@ -15619,6 +15675,7 @@ target_include_directories(server_crash_test_client
 target_link_libraries(server_crash_test_client
   ${_gRPC_PROTOBUF_LIBRARIES}
   ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_config
   grpc++_test_util
   grpc_test_util
   grpc++
@@ -15763,6 +15820,45 @@ target_link_libraries(server_request_call_test
 )
 
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_executable(service_config_test
+  test/core/client_channel/service_config_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(service_config_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(service_config_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_test_util
+  grpc++
+  grpc
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
@@ -17641,6 +17737,41 @@ target_link_libraries(h2_ssl_test
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 
+add_executable(h2_ssl_cred_reload_test
+  test/core/end2end/fixtures/h2_ssl_cred_reload.cc
+)
+
+
+target_include_directories(h2_ssl_cred_reload_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+)
+
+target_link_libraries(h2_ssl_cred_reload_test
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  end2end_tests
+  grpc_test_util
+  grpc
+  gpr
+)
+
+  # avoid dependency on libstdc++
+  if (_gRPC_CORE_NOSTDCXX_FLAGS)
+    set_target_properties(h2_ssl_cred_reload_test PROPERTIES LINKER_LANGUAGE C)
+    target_compile_options(h2_ssl_cred_reload_test PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${_gRPC_CORE_NOSTDCXX_FLAGS}>)
+  endif()
+
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
 add_executable(h2_ssl_proxy_test
   test/core/end2end/fixtures/h2_ssl_proxy.cc
 )

CONTRIBUTING.md

@@ -110,5 +110,16 @@ How to get your contributions merged smoothly and quickly.
 - Exceptions to the rules can be made if there's a compelling reason for doing
   so.
 
-
+## Obtaining Commit Access
+We grant Commit Access to contributors based on the following criteria:
+* Sustained contribution to the gRPC project.
+* Deep understanding of the areas contributed to, and good consideration of various reliability, usability and performance tradeoffs. 
+* Contributions demonstrate that obtaining Commit Access will significantly reduce friction for the contributors or others. 
+
+In addition to submitting PRs, a Contributor with Commit Access can:
+* Review PRs and merge once other checks and criteria pass.
+* Triage bugs and PRs and assign appropriate labels and reviewers. 
+
+### Obtaining Commit Access without Code Contributions 
+The [gRPC organization](https://github.com/grpc) is comprised of multiple repositories and commit access is usually restricted to one or more of these repositories. Some repositories such as the [grpc.github.io](https://github.com/grpc/grpc.github.io/) do not have code, but the same principle of sustained, high quality contributions, with a good understanding of the fundamentals, apply. 
 

Makefile

@@ -478,11 +478,11 @@ LDFLAGS += $(EXTRA_LDFLAGS)
 DEFINES += $(EXTRA_DEFINES)
 LDLIBS += $(EXTRA_LDLIBS)
 
-HOST_CPPFLAGS = $(CPPFLAGS)
-HOST_CFLAGS = $(CFLAGS)
-HOST_CXXFLAGS = $(CXXFLAGS)
-HOST_LDFLAGS = $(LDFLAGS)
-HOST_LDLIBS = $(LDLIBS)
+HOST_CPPFLAGS += $(CPPFLAGS)
+HOST_CFLAGS += $(CFLAGS)
+HOST_CXXFLAGS += $(CXXFLAGS)
+HOST_LDFLAGS += $(LDFLAGS)
+HOST_LDLIBS += $(LDLIBS)
 
 # These are automatically computed variables.
 # There shouldn't be any need to change anything from now on.
@@ -1233,6 +1233,7 @@ interop_server: $(BINDIR)/$(CONFIG)/interop_server
 interop_test: $(BINDIR)/$(CONFIG)/interop_test
 json_run_localhost: $(BINDIR)/$(CONFIG)/json_run_localhost
 memory_test: $(BINDIR)/$(CONFIG)/memory_test
+message_allocator_end2end_test: $(BINDIR)/$(CONFIG)/message_allocator_end2end_test
 metrics_client: $(BINDIR)/$(CONFIG)/metrics_client
 mock_test: $(BINDIR)/$(CONFIG)/mock_test
 nonblocking_test: $(BINDIR)/$(CONFIG)/nonblocking_test
@@ -1262,6 +1263,7 @@ server_crash_test_client: $(BINDIR)/$(CONFIG)/server_crash_test_client
 server_early_return_test: $(BINDIR)/$(CONFIG)/server_early_return_test
 server_interceptors_end2end_test: $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test
 server_request_call_test: $(BINDIR)/$(CONFIG)/server_request_call_test
+service_config_test: $(BINDIR)/$(CONFIG)/service_config_test
 shutdown_test: $(BINDIR)/$(CONFIG)/shutdown_test
 slice_hash_table_test: $(BINDIR)/$(CONFIG)/slice_hash_table_test
 slice_weak_hash_table_test: $(BINDIR)/$(CONFIG)/slice_weak_hash_table_test
@@ -1315,6 +1317,7 @@ h2_sockpair+trace_test: $(BINDIR)/$(CONFIG)/h2_sockpair+trace_test
 h2_sockpair_1byte_test: $(BINDIR)/$(CONFIG)/h2_sockpair_1byte_test
 h2_spiffe_test: $(BINDIR)/$(CONFIG)/h2_spiffe_test
 h2_ssl_test: $(BINDIR)/$(CONFIG)/h2_ssl_test
+h2_ssl_cred_reload_test: $(BINDIR)/$(CONFIG)/h2_ssl_cred_reload_test
 h2_ssl_proxy_test: $(BINDIR)/$(CONFIG)/h2_ssl_proxy_test
 h2_uds_test: $(BINDIR)/$(CONFIG)/h2_uds_test
 inproc_test: $(BINDIR)/$(CONFIG)/inproc_test
@@ -1578,6 +1581,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/h2_sockpair_1byte_test \
   $(BINDIR)/$(CONFIG)/h2_spiffe_test \
   $(BINDIR)/$(CONFIG)/h2_ssl_test \
+  $(BINDIR)/$(CONFIG)/h2_ssl_cred_reload_test \
   $(BINDIR)/$(CONFIG)/h2_ssl_proxy_test \
   $(BINDIR)/$(CONFIG)/h2_uds_test \
   $(BINDIR)/$(CONFIG)/inproc_test \
@@ -1698,6 +1702,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/interop_test \
   $(BINDIR)/$(CONFIG)/json_run_localhost \
   $(BINDIR)/$(CONFIG)/memory_test \
+  $(BINDIR)/$(CONFIG)/message_allocator_end2end_test \
   $(BINDIR)/$(CONFIG)/metrics_client \
   $(BINDIR)/$(CONFIG)/mock_test \
   $(BINDIR)/$(CONFIG)/nonblocking_test \
@@ -1727,6 +1732,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/server_early_return_test \
   $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test \
   $(BINDIR)/$(CONFIG)/server_request_call_test \
+  $(BINDIR)/$(CONFIG)/service_config_test \
   $(BINDIR)/$(CONFIG)/shutdown_test \
   $(BINDIR)/$(CONFIG)/slice_hash_table_test \
   $(BINDIR)/$(CONFIG)/slice_weak_hash_table_test \
@@ -1840,6 +1846,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/interop_test \
   $(BINDIR)/$(CONFIG)/json_run_localhost \
   $(BINDIR)/$(CONFIG)/memory_test \
+  $(BINDIR)/$(CONFIG)/message_allocator_end2end_test \
   $(BINDIR)/$(CONFIG)/metrics_client \
   $(BINDIR)/$(CONFIG)/mock_test \
   $(BINDIR)/$(CONFIG)/nonblocking_test \
@@ -1869,6 +1876,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/server_early_return_test \
   $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test \
   $(BINDIR)/$(CONFIG)/server_request_call_test \
+  $(BINDIR)/$(CONFIG)/service_config_test \
   $(BINDIR)/$(CONFIG)/shutdown_test \
   $(BINDIR)/$(CONFIG)/slice_hash_table_test \
   $(BINDIR)/$(CONFIG)/slice_weak_hash_table_test \
@@ -2338,6 +2346,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/interop_test || ( echo test interop_test failed ; exit 1 )
 	$(E) "[RUN]     Testing memory_test"
 	$(Q) $(BINDIR)/$(CONFIG)/memory_test || ( echo test memory_test failed ; exit 1 )
+	$(E) "[RUN]     Testing message_allocator_end2end_test"
+	$(Q) $(BINDIR)/$(CONFIG)/message_allocator_end2end_test || ( echo test message_allocator_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing mock_test"
 	$(Q) $(BINDIR)/$(CONFIG)/mock_test || ( echo test mock_test failed ; exit 1 )
 	$(E) "[RUN]     Testing nonblocking_test"
@@ -2382,6 +2392,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/server_interceptors_end2end_test || ( echo test server_interceptors_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing server_request_call_test"
 	$(Q) $(BINDIR)/$(CONFIG)/server_request_call_test || ( echo test server_request_call_test failed ; exit 1 )
+	$(E) "[RUN]     Testing service_config_test"
+	$(Q) $(BINDIR)/$(CONFIG)/service_config_test || ( echo test service_config_test failed ; exit 1 )
 	$(E) "[RUN]     Testing shutdown_test"
 	$(Q) $(BINDIR)/$(CONFIG)/shutdown_test || ( echo test shutdown_test failed ; exit 1 )
 	$(E) "[RUN]     Testing slice_hash_table_test"
@@ -3308,7 +3320,6 @@ endif
 
 LIBGPR_SRC = \
     src/core/lib/gpr/alloc.cc \
-    src/core/lib/gpr/arena.cc \
     src/core/lib/gpr/atm.cc \
     src/core/lib/gpr/cpu_iphone.cc \
     src/core/lib/gpr/cpu_linux.cc \
@@ -3341,6 +3352,7 @@ LIBGPR_SRC = \
     src/core/lib/gpr/tmpfile_posix.cc \
     src/core/lib/gpr/tmpfile_windows.cc \
     src/core/lib/gpr/wrap_memcpy.cc \
+    src/core/lib/gprpp/arena.cc \
     src/core/lib/gprpp/fork.cc \
     src/core/lib/gprpp/thd_posix.cc \
     src/core/lib/gprpp/thd_windows.cc \
@@ -3432,6 +3444,7 @@ LIBGRPC_SRC = \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
+    src/core/lib/compression/compression_args.cc \
     src/core/lib/compression/compression_internal.cc \
     src/core/lib/compression/message_compress.cc \
     src/core/lib/compression/stream_compression.cc \
@@ -3852,6 +3865,7 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
+    src/core/lib/compression/compression_args.cc \
     src/core/lib/compression/compression_internal.cc \
     src/core/lib/compression/message_compress.cc \
     src/core/lib/compression/stream_compression.cc \
@@ -4256,6 +4270,7 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
+    src/core/lib/compression/compression_args.cc \
     src/core/lib/compression/compression_internal.cc \
     src/core/lib/compression/message_compress.cc \
     src/core/lib/compression/stream_compression.cc \
@@ -4567,6 +4582,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
+    src/core/lib/compression/compression_args.cc \
     src/core/lib/compression/compression_internal.cc \
     src/core/lib/compression/message_compress.cc \
     src/core/lib/compression/stream_compression.cc \
@@ -4841,6 +4857,7 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
+    src/core/lib/compression/compression_args.cc \
     src/core/lib/compression/compression_internal.cc \
     src/core/lib/compression/message_compress.cc \
     src/core/lib/compression/stream_compression.cc \
@@ -5368,7 +5385,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/server_builder.h \
     include/grpcpp/server_builder_impl.h \
     include/grpcpp/server_context.h \
-    include/grpcpp/server_impl.h \
     include/grpcpp/server_posix.h \
     include/grpcpp/server_posix_impl.h \
     include/grpcpp/support/async_stream.h \
@@ -5379,6 +5395,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/support/client_interceptor.h \
     include/grpcpp/support/config.h \
     include/grpcpp/support/interceptor.h \
+    include/grpcpp/support/message_allocator.h \
     include/grpcpp/support/proto_buffer_reader.h \
     include/grpcpp/support/proto_buffer_writer.h \
     include/grpcpp/support/server_callback.h \
@@ -5494,6 +5511,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -5717,6 +5735,7 @@ LIBGRPC++_CRONET_SRC = \
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
+    src/core/lib/compression/compression_args.cc \
     src/core/lib/compression/compression_internal.cc \
     src/core/lib/compression/message_compress.cc \
     src/core/lib/compression/stream_compression.cc \
@@ -5980,7 +5999,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/server_builder.h \
     include/grpcpp/server_builder_impl.h \
     include/grpcpp/server_context.h \
-    include/grpcpp/server_impl.h \
     include/grpcpp/server_posix.h \
     include/grpcpp/server_posix_impl.h \
     include/grpcpp/support/async_stream.h \
@@ -5991,6 +6009,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/support/client_interceptor.h \
     include/grpcpp/support/config.h \
     include/grpcpp/support/interceptor.h \
+    include/grpcpp/support/message_allocator.h \
     include/grpcpp/support/proto_buffer_reader.h \
     include/grpcpp/support/proto_buffer_writer.h \
     include/grpcpp/support/server_callback.h \
@@ -6106,6 +6125,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -6511,6 +6531,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -6679,6 +6700,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -6909,7 +6931,6 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/server_builder.h \
     include/grpcpp/server_builder_impl.h \
     include/grpcpp/server_context.h \
-    include/grpcpp/server_impl.h \
     include/grpcpp/server_posix.h \
     include/grpcpp/server_posix_impl.h \
     include/grpcpp/support/async_stream.h \
@@ -6920,6 +6941,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/support/client_interceptor.h \
     include/grpcpp/support/config.h \
     include/grpcpp/support/interceptor.h \
+    include/grpcpp/support/message_allocator.h \
     include/grpcpp/support/proto_buffer_reader.h \
     include/grpcpp/support/proto_buffer_writer.h \
     include/grpcpp/support/server_callback.h \
@@ -7035,6 +7057,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -15614,16 +15637,16 @@ $(BINDIR)/$(CONFIG)/client_crash_test_server: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/client_crash_test_server: $(PROTOBUF_DEP) $(CLIENT_CRASH_TEST_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(BINDIR)/$(CONFIG)/client_crash_test_server: $(PROTOBUF_DEP) $(CLIENT_CRASH_TEST_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(CLIENT_CRASH_TEST_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/client_crash_test_server
+	$(Q) $(LDXX) $(LDFLAGS) $(CLIENT_CRASH_TEST_SERVER_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/client_crash_test_server
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/end2end/client_crash_test_server.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/cpp/end2end/client_crash_test_server.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_client_crash_test_server: $(CLIENT_CRASH_TEST_SERVER_OBJS:.o=.dep)
 
@@ -16361,18 +16384,18 @@ $(BINDIR)/$(CONFIG)/golden_file_test: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/golden_file_test: $(PROTOBUF_DEP) $(GOLDEN_FILE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(BINDIR)/$(CONFIG)/golden_file_test: $(PROTOBUF_DEP) $(GOLDEN_FILE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(GOLDEN_FILE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/golden_file_test
+	$(Q) $(LDXX) $(LDFLAGS) $(GOLDEN_FILE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/golden_file_test
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/compiler_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/compiler_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
-$(OBJDIR)/$(CONFIG)/test/cpp/codegen/golden_file_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/cpp/codegen/golden_file_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_golden_file_test: $(GOLDEN_FILE_TEST_OBJS:.o=.dep)
 
@@ -17397,6 +17420,49 @@ endif
 endif
 
 
+MESSAGE_ALLOCATOR_END2END_TEST_SRC = \
+    test/cpp/end2end/message_allocator_end2end_test.cc \
+
+MESSAGE_ALLOCATOR_END2END_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MESSAGE_ALLOCATOR_END2END_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/message_allocator_end2end_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+.
+
+$(BINDIR)/$(CONFIG)/message_allocator_end2end_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/message_allocator_end2end_test: $(PROTOBUF_DEP) $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/message_allocator_end2end_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/cpp/end2end/message_allocator_end2end_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_message_allocator_end2end_test: $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 METRICS_CLIENT_SRC = \
     $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc \
     test/cpp/interop/metrics_client.cc \
@@ -18534,16 +18600,16 @@ $(BINDIR)/$(CONFIG)/server_crash_test_client: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/server_crash_test_client: $(PROTOBUF_DEP) $(SERVER_CRASH_TEST_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(BINDIR)/$(CONFIG)/server_crash_test_client: $(PROTOBUF_DEP) $(SERVER_CRASH_TEST_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(SERVER_CRASH_TEST_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/server_crash_test_client
+	$(Q) $(LDXX) $(LDFLAGS) $(SERVER_CRASH_TEST_CLIENT_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/server_crash_test_client
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/end2end/server_crash_test_client.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/cpp/end2end/server_crash_test_client.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_server_crash_test_client: $(SERVER_CRASH_TEST_CLIENT_OBJS:.o=.dep)
 
@@ -18693,6 +18759,49 @@ endif
 $(OBJDIR)/$(CONFIG)/test/cpp/server/server_request_call_test.o: $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc
 
 
+SERVICE_CONFIG_TEST_SRC = \
+    test/core/client_channel/service_config_test.cc \
+
+SERVICE_CONFIG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SERVICE_CONFIG_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/service_config_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+.
+
+$(BINDIR)/$(CONFIG)/service_config_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/service_config_test: $(PROTOBUF_DEP) $(SERVICE_CONFIG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(SERVICE_CONFIG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/service_config_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/client_channel/service_config_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_service_config_test: $(SERVICE_CONFIG_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(SERVICE_CONFIG_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 SHUTDOWN_TEST_SRC = \
     test/cpp/end2end/shutdown_test.cc \
 
@@ -20621,6 +20730,38 @@ endif
 endif
 
 
+H2_SSL_CRED_RELOAD_TEST_SRC = \
+    test/core/end2end/fixtures/h2_ssl_cred_reload.cc \
+
+H2_SSL_CRED_RELOAD_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(H2_SSL_CRED_RELOAD_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/h2_ssl_cred_reload_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/h2_ssl_cred_reload_test: $(H2_SSL_CRED_RELOAD_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libend2end_tests.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(H2_SSL_CRED_RELOAD_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libend2end_tests.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/h2_ssl_cred_reload_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/end2end/fixtures/h2_ssl_cred_reload.o:  $(LIBDIR)/$(CONFIG)/libend2end_tests.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_h2_ssl_cred_reload_test: $(H2_SSL_CRED_RELOAD_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(H2_SSL_CRED_RELOAD_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 H2_SSL_PROXY_TEST_SRC = \
     test/core/end2end/fixtures/h2_ssl_proxy.cc \
 

WORKSPACE

@@ -9,12 +9,13 @@ grpc_deps()
 grpc_test_only_deps()
 
 register_execution_platforms(
-    "//third_party/toolchains:rbe_ubuntu1604",
-    "//third_party/toolchains:rbe_ubuntu1604_large",
+    "//third_party/toolchains:local",
+    "//third_party/toolchains:local_large",
+    "//third_party/toolchains:rbe_windows",
 )
 
 register_toolchains(
-    "//third_party/toolchains:cc-toolchain-clang-x86_64-default",
+    "//third_party/toolchains/bazel_0.23.2_rbe_windows:cc-toolchain-x64_windows",
 )
 
 git_repository(
@@ -43,3 +44,24 @@ http_archive(
 load("//bazel:grpc_python_deps.bzl", "grpc_python_deps")
 
 grpc_python_deps()
+
+load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
+
+# Create toolchain configuration for remote execution.
+rbe_autoconfig(
+    name = "rbe_default",
+)
+
+load("@bazel_toolchains//rules:environments.bzl", "clang_env")
+load("@bazel_skylib//lib:dicts.bzl", "dicts")
+
+# Create msan toolchain configuration for remote execution.
+rbe_autoconfig(
+    name = "rbe_msan",
+    env = dicts.add(
+        clang_env(),
+        {
+            "BAZEL_LINKOPTS": "-lc++:-lc++abi:-lm",
+        },
+    ),
+)

bazel/grpc_build_system.bzl

@@ -112,10 +112,7 @@ def grpc_cc_library(
         visibility = visibility,
         testonly = testonly,
         linkopts = linkopts,
-        includes = [
-            "include",
-            "src/core/ext/upb-generated",
-        ],
+        includes = ["include"] + if_not_windows(["src/core/ext/upb-generated"]),
         alwayslink = alwayslink,
         data = data,
         tags = tags,

bazel/grpc_deps.bzl

@@ -110,6 +110,8 @@ def grpc_deps():
         http_archive(
             name = "boringssl",
             # on the chromium-stable-with-bazel branch
+            # NOTE: This URL generates a tarball containing dynamic date
+            # information, so the sha256 is not consistent.
             url = "https://boringssl.googlesource.com/boringssl/+archive/afc30d43eef92979b05776ec0963c9cede5fb80f.tar.gz",
         )
 
@@ -117,6 +119,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_madler_zlib",
             build_file = "@com_github_grpc_grpc//third_party:zlib.BUILD",
+            sha256 = "6d4d6640ca3121620995ee255945161821218752b551a1a180f4215f7d124d45",
             strip_prefix = "zlib-cacf7f1d4e3d44d871b605da3b647f07d718623f",
             url = "https://github.com/madler/zlib/archive/cacf7f1d4e3d44d871b605da3b647f07d718623f.tar.gz",
         )
@@ -124,6 +127,7 @@ def grpc_deps():
     if "com_google_protobuf" not in native.existing_rules():
         http_archive(
             name = "com_google_protobuf",
+            sha256 = "cf9e2fb1d2cd30ec9d51ff1749045208bd641f290f64b85046485934b0e03783",
             strip_prefix = "protobuf-582743bf40c5d3639a70f98f183914a2c0cd0680",
             url = "https://github.com/google/protobuf/archive/582743bf40c5d3639a70f98f183914a2c0cd0680.tar.gz",
         )
@@ -132,6 +136,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_nanopb_nanopb",
             build_file = "@com_github_grpc_grpc//third_party:nanopb.BUILD",
+            sha256 = "8bbbb1e78d4ddb0a1919276924ab10d11b631df48b657d960e0c795a25515735",
             strip_prefix = "nanopb-f8ac463766281625ad710900479130c7fcb4d63b",
             url = "https://github.com/nanopb/nanopb/archive/f8ac463766281625ad710900479130c7fcb4d63b.tar.gz",
         )
@@ -140,6 +145,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_google_googletest",
             build_file = "@com_github_grpc_grpc//third_party:gtest.BUILD",
+            sha256 = "175a22300b3450e27e5f2e6f95cc9abca74617cbc21a1e0ed19bdfbd22ea0305",
             strip_prefix = "googletest-ec44c6c1675c25b9827aacd08c02433cccde7780",
             url = "https://github.com/google/googletest/archive/ec44c6c1675c25b9827aacd08c02433cccde7780.tar.gz",
         )
@@ -147,6 +153,7 @@ def grpc_deps():
     if "com_github_gflags_gflags" not in native.existing_rules():
         http_archive(
             name = "com_github_gflags_gflags",
+            sha256 = "63ae70ea3e05780f7547d03503a53de3a7d2d83ad1caaa443a31cb20aea28654",
             strip_prefix = "gflags-28f50e0fed19872e0fd50dd23ce2ee8cd759338e",
             url = "https://github.com/gflags/gflags/archive/28f50e0fed19872e0fd50dd23ce2ee8cd759338e.tar.gz",
         )
@@ -154,6 +161,7 @@ def grpc_deps():
     if "com_github_google_benchmark" not in native.existing_rules():
         http_archive(
             name = "com_github_google_benchmark",
+            sha256 = "c7682e9007ddfd94072647abab3e89ffd9084089460ae47d67060974467b58bf",
             strip_prefix = "benchmark-e776aa0275e293707b6a0901e0e8d8a8a3679508",
             url = "https://github.com/google/benchmark/archive/e776aa0275e293707b6a0901e0e8d8a8a3679508.tar.gz",
         )
@@ -162,6 +170,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_cares_cares",
             build_file = "@com_github_grpc_grpc//third_party:cares/cares.BUILD",
+            sha256 = "e8c2751ddc70fed9dc6f999acd92e232d5846f009ee1674f8aee81f19b2b915a",
             strip_prefix = "c-ares-e982924acee7f7313b4baa4ee5ec000c5e373c30",
             url = "https://github.com/c-ares/c-ares/archive/e982924acee7f7313b4baa4ee5ec000c5e373c30.tar.gz",
         )
@@ -169,19 +178,20 @@ def grpc_deps():
     if "com_google_absl" not in native.existing_rules():
         http_archive(
             name = "com_google_absl",
+            sha256 = "5fe2a3a8f8378e81d4d3db6541f48030e04233ccd2d6c7e9d981ed577b314ae8",
             strip_prefix = "abseil-cpp-308ce31528a7edfa39f5f6d36142278a0ae1bf45",
             url = "https://github.com/abseil/abseil-cpp/archive/308ce31528a7edfa39f5f6d36142278a0ae1bf45.tar.gz",
         )
 
-    if "com_github_bazelbuild_bazeltoolchains" not in native.existing_rules():
+    if "bazel_toolchains" not in native.existing_rules():
         http_archive(
-            name = "com_github_bazelbuild_bazeltoolchains",
-            strip_prefix = "bazel-toolchains-37419a124bdb9af2fec5b99a973d359b6b899b61",
+            name = "bazel_toolchains",
+            sha256 = "67335b3563d9b67dc2550b8f27cc689b64fadac491e69ce78763d9ba894cc5cc",
+            strip_prefix = "bazel-toolchains-cddc376d428ada2927ad359211c3e356bd9c9fbb",
             urls = [
-                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
-                "https://github.com/bazelbuild/bazel-toolchains/archive/37419a124bdb9af2fec5b99a973d359b6b899b61.tar.gz",
+                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz",
+                "https://github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz",
             ],
-            sha256 = "ee854b5de299138c1f4a2edb5573d22b21d975acfc7aa938f36d30b49ef97498",
         )
 
     if "bazel_skylib" not in native.existing_rules():
@@ -195,6 +205,7 @@ def grpc_deps():
     if "io_opencensus_cpp" not in native.existing_rules():
         http_archive(
             name = "io_opencensus_cpp",
+            sha256 = "3436ca23dc1b3345186defd0f46d64244079ba3d3234a0c5d6ef5e8d5d06c8b5",
             strip_prefix = "opencensus-cpp-9b1e354e89bf3d92aedc00af45b418ce870f3d77",
             url = "https://github.com/census-instrumentation/opencensus-cpp/archive/9b1e354e89bf3d92aedc00af45b418ce870f3d77.tar.gz",
         )
@@ -202,6 +213,7 @@ def grpc_deps():
     if "upb" not in native.existing_rules():
         http_archive(
             name = "upb",
+            sha256 = "0e749a8973968397f849a3b42e28ee9c85dc418c2477954c2a6a4495f323241d",
             strip_prefix = "upb-ed9faae0993704b033c594b072d65e1bf19207fa",
             url = "https://github.com/google/upb/archive/ed9faae0993704b033c594b072d65e1bf19207fa.tar.gz",
         )
@@ -223,6 +235,7 @@ def grpc_test_only_deps():
     if "com_github_twisted_twisted" not in native.existing_rules():
         http_archive(
             name = "com_github_twisted_twisted",
+            sha256 = "ca17699d0d62eafc5c28daf2c7d0a18e62ae77b4137300b6c7d7868b39b06139",
             strip_prefix = "twisted-twisted-17.5.0",
             url = "https://github.com/twisted/twisted/archive/twisted-17.5.0.zip",
             build_file = "@com_github_grpc_grpc//third_party:twisted.BUILD",
@@ -231,6 +244,7 @@ def grpc_test_only_deps():
     if "com_github_yaml_pyyaml" not in native.existing_rules():
         http_archive(
             name = "com_github_yaml_pyyaml",
+            sha256 = "6b4314b1b2051ddb9d4fcd1634e1fa9c1bb4012954273c9ff3ef689f6ec6c93e",
             strip_prefix = "pyyaml-3.12",
             url = "https://github.com/yaml/pyyaml/archive/3.12.zip",
             build_file = "@com_github_grpc_grpc//third_party:yaml.BUILD",
@@ -239,6 +253,7 @@ def grpc_test_only_deps():
     if "com_github_twisted_incremental" not in native.existing_rules():
         http_archive(
             name = "com_github_twisted_incremental",
+            sha256 = "f0ca93359ee70243ff7fbf2d904a6291810bd88cb80ed4aca6fa77f318a41a36",
             strip_prefix = "incremental-incremental-17.5.0",
             url = "https://github.com/twisted/incremental/archive/incremental-17.5.0.zip",
             build_file = "@com_github_grpc_grpc//third_party:incremental.BUILD",
@@ -247,6 +262,7 @@ def grpc_test_only_deps():
     if "com_github_zopefoundation_zope_interface" not in native.existing_rules():
         http_archive(
             name = "com_github_zopefoundation_zope_interface",
+            sha256 = "e9579fc6149294339897be3aa9ecd8a29217c0b013fe6f44fcdae00e3204198a",
             strip_prefix = "zope.interface-4.4.3",
             url = "https://github.com/zopefoundation/zope.interface/archive/4.4.3.zip",
             build_file = "@com_github_grpc_grpc//third_party:zope_interface.BUILD",
@@ -255,6 +271,7 @@ def grpc_test_only_deps():
     if "com_github_twisted_constantly" not in native.existing_rules():
         http_archive(
             name = "com_github_twisted_constantly",
+            sha256 = "2702cd322161a579d2c0dbf94af4e57712eedc7bd7bbbdc554a230544f7d346c",
             strip_prefix = "constantly-15.1.0",
             url = "https://github.com/twisted/constantly/archive/15.1.0.zip",
             build_file = "@com_github_grpc_grpc//third_party:constantly.BUILD",

build.yaml

@@ -114,7 +114,6 @@ filegroups:
 - name: gpr_base
   src:
   - src/core/lib/gpr/alloc.cc
-  - src/core/lib/gpr/arena.cc
   - src/core/lib/gpr/atm.cc
   - src/core/lib/gpr/cpu_iphone.cc
   - src/core/lib/gpr/cpu_linux.cc
@@ -147,6 +146,7 @@ filegroups:
   - src/core/lib/gpr/tmpfile_posix.cc
   - src/core/lib/gpr/tmpfile_windows.cc
   - src/core/lib/gpr/wrap_memcpy.cc
+  - src/core/lib/gprpp/arena.cc
   - src/core/lib/gprpp/fork.cc
   - src/core/lib/gprpp/thd_posix.cc
   - src/core/lib/gprpp/thd_windows.cc
@@ -191,6 +191,7 @@ filegroups:
   - src/core/lib/gpr/tmpfile.h
   - src/core/lib/gpr/useful.h
   - src/core/lib/gprpp/abstract.h
+  - src/core/lib/gprpp/arena.h
   - src/core/lib/gprpp/atomic.h
   - src/core/lib/gprpp/fork.h
   - src/core/lib/gprpp/manual_constructor.h
@@ -245,6 +246,7 @@ filegroups:
   - src/core/lib/channel/handshaker_registry.cc
   - src/core/lib/channel/status_util.cc
   - src/core/lib/compression/compression.cc
+  - src/core/lib/compression/compression_args.cc
   - src/core/lib/compression/compression_internal.cc
   - src/core/lib/compression/message_compress.cc
   - src/core/lib/compression/stream_compression.cc
@@ -418,6 +420,7 @@ filegroups:
   - src/core/lib/channel/handshaker_registry.h
   - src/core/lib/channel/status_util.h
   - src/core/lib/compression/algorithm_metadata.h
+  - src/core/lib/compression/compression_args.h
   - src/core/lib/compression/compression_internal.h
   - src/core/lib/compression/message_compress.h
   - src/core/lib/compression/stream_compression.h
@@ -1256,6 +1259,7 @@ filegroups:
   - include/grpcpp/impl/codegen/intercepted_channel.h
   - include/grpcpp/impl/codegen/interceptor.h
   - include/grpcpp/impl/codegen/interceptor_common.h
+  - include/grpcpp/impl/codegen/message_allocator.h
   - include/grpcpp/impl/codegen/metadata_map.h
   - include/grpcpp/impl/codegen/method_handler_impl.h
   - include/grpcpp/impl/codegen/rpc_method.h
@@ -1384,7 +1388,6 @@ filegroups:
   - include/grpcpp/server_builder.h
   - include/grpcpp/server_builder_impl.h
   - include/grpcpp/server_context.h
-  - include/grpcpp/server_impl.h
   - include/grpcpp/server_posix.h
   - include/grpcpp/server_posix_impl.h
   - include/grpcpp/support/async_stream.h
@@ -1395,6 +1398,7 @@ filegroups:
   - include/grpcpp/support/client_interceptor.h
   - include/grpcpp/support/config.h
   - include/grpcpp/support/interceptor.h
+  - include/grpcpp/support/message_allocator.h
   - include/grpcpp/support/proto_buffer_reader.h
   - include/grpcpp/support/proto_buffer_writer.h
   - include/grpcpp/support/server_callback.h
@@ -4519,6 +4523,7 @@ targets:
   src:
   - test/cpp/end2end/client_crash_test_server.cc
   deps:
+  - grpc++_test_config
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -4731,6 +4736,7 @@ targets:
   - src/proto/grpc/testing/compiler_test.proto
   - test/cpp/codegen/golden_file_test.cc
   deps:
+  - grpc++_test_config
   - grpc++
   - grpc
   - gpr
@@ -5068,6 +5074,19 @@ targets:
   uses:
   - grpc++_test
   uses_polling: false
+- name: message_allocator_end2end_test
+  gtest: true
+  cpu_cost: 0.5
+  build: test
+  language: c++
+  src:
+  - test/cpp/end2end/message_allocator_end2end_test.cc
+  deps:
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr
 - name: metrics_client
   build: test
   run: false
@@ -5433,6 +5452,7 @@ targets:
   src:
   - test/cpp/end2end/server_crash_test_client.cc
   deps:
+  - grpc++_test_config
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -5480,6 +5500,19 @@ targets:
   - grpc++_unsecure
   - grpc_unsecure
   - gpr
+- name: service_config_test
+  gtest: true
+  build: test
+  language: c++
+  src:
+  - test/core/client_channel/service_config_test.cc
+  deps:
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr
+  uses:
+  - grpc++_test
 - name: shutdown_test
   gtest: true
   build: test

config.m4

@@ -45,7 +45,6 @@ if test "$PHP_GRPC" != "no"; then
     third_party/address_sorting/address_sorting_posix.c \
     third_party/address_sorting/address_sorting_windows.c \
     src/core/lib/gpr/alloc.cc \
-    src/core/lib/gpr/arena.cc \
     src/core/lib/gpr/atm.cc \
     src/core/lib/gpr/cpu_iphone.cc \
     src/core/lib/gpr/cpu_linux.cc \
@@ -78,6 +77,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/gpr/tmpfile_posix.cc \
     src/core/lib/gpr/tmpfile_windows.cc \
     src/core/lib/gpr/wrap_memcpy.cc \
+    src/core/lib/gprpp/arena.cc \
     src/core/lib/gprpp/fork.cc \
     src/core/lib/gprpp/thd_posix.cc \
     src/core/lib/gprpp/thd_windows.cc \
@@ -97,6 +97,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/channel/handshaker_registry.cc \
     src/core/lib/channel/status_util.cc \
     src/core/lib/compression/compression.cc \
+    src/core/lib/compression/compression_args.cc \
     src/core/lib/compression/compression_internal.cc \
     src/core/lib/compression/message_compress.cc \
     src/core/lib/compression/stream_compression.cc \

config.w32

@@ -20,7 +20,6 @@ if (PHP_GRPC != "no") {
     "third_party\\address_sorting\\address_sorting_posix.c " +
     "third_party\\address_sorting\\address_sorting_windows.c " +
     "src\\core\\lib\\gpr\\alloc.cc " +
-    "src\\core\\lib\\gpr\\arena.cc " +
     "src\\core\\lib\\gpr\\atm.cc " +
     "src\\core\\lib\\gpr\\cpu_iphone.cc " +
     "src\\core\\lib\\gpr\\cpu_linux.cc " +
@@ -53,6 +52,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\gpr\\tmpfile_posix.cc " +
     "src\\core\\lib\\gpr\\tmpfile_windows.cc " +
     "src\\core\\lib\\gpr\\wrap_memcpy.cc " +
+    "src\\core\\lib\\gprpp\\arena.cc " +
     "src\\core\\lib\\gprpp\\fork.cc " +
     "src\\core\\lib\\gprpp\\thd_posix.cc " +
     "src\\core\\lib\\gprpp\\thd_windows.cc " +
@@ -72,6 +72,7 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\channel\\handshaker_registry.cc " +
     "src\\core\\lib\\channel\\status_util.cc " +
     "src\\core\\lib\\compression\\compression.cc " +
+    "src\\core\\lib\\compression\\compression_args.cc " +
     "src\\core\\lib\\compression\\compression_internal.cc " +
     "src\\core\\lib\\compression\\message_compress.cc " +
     "src\\core\\lib\\compression\\stream_compression.cc " +

doc/core/grpc-polling-engines.md

@@ -135,7 +135,7 @@ Code at `src/core/lib/iomgr/ev_epollex_posix.cc`
   - The same FD can be in multiple `Pollable`s (even if one of the `Pollable`s is of type PO_FD)
   - There cannot be two `Pollable`s of type PO_FD for the same fd
 
-- Why do we need `Pollable` of type PO_FD and PO_EMTPY ?
+- Why do we need `Pollable` of type PO_FD and PO_EMPTY ?
   - The main reason is the Sync client API
     - We create one new completion queue per call. If we didn’t have PO_EMPTY and PO_FD type pollables, then every call on a given channel will effectively have to create a `Pollable` and hence an epollset. This is because every completion queue automatically creates a pollset and the channel fd will have to be put in that pollset. This clearly requires an epollset to put that fd. Creating an epollset per call (even if we delete the epollset once the call is completed) would mean a lot of sys calls to create/delete epoll fds. This is clearly not a good idea.
     - With these new types of `Pollable`s, all pollsets (corresponding to the new per-call completion queue) will initially point to PO_EMPTY global epollset. Then once the channel fd is added to the pollset, the pollset will point to the `Pollable` of type PO_FD containing just that fd (i.e it will reuse the existing `Pollable`). This way, the epoll fd creation/deletion churn is avoided.

doc/environment_variables.md

@@ -50,6 +50,7 @@ some configuration as environment variables that can be set.
     resolver and load balancing policy interaction
   - compression - traces compression operations
   - connectivity_state - traces connectivity state changes to channels
+  - cronet - traces state in the cronet transport engine
   - executor - traces grpc's internal thread pool ('the executor')
   - fd_trace - traces fd create(), shutdown() and close() calls for channel fds.
     Also traces epoll fd create()/close() calls in epollex polling engine
@@ -145,13 +146,3 @@ some configuration as environment variables that can be set.
 * grpc_cfstream
   set to 1 to turn on CFStream experiment. With this experiment gRPC uses CFStream API to make TCP
   connections. The option is only available on iOS platform and when macro GRPC_CFSTREAM is defined.
-
-* GRPC_ARENA_INIT_STRATEGY
-  Selects the initialization strategy for blocks allocated in the arena. Valid
-  values are:
-  - no_init (default): Do not initialize the arena block.
-  - zero_init: Initialize the arena blocks with 0.
-  - non_zero_init: Initialize the arena blocks with a non-zero value.
-
-  NOTE: This environment variable is experimental and will be removed. Thus, it
-        should not be relied upon.

doc/python/sphinx/grpc.rst

@@ -172,3 +172,9 @@ Future Interfaces
 .. autoexception:: FutureTimeoutError
 .. autoexception:: FutureCancelledError
 .. autoclass:: Future
+
+
+Compression
+^^^^^^^^^^^
+
+.. autoclass:: Compression

examples/csharp/HelloworldXamarin/iOS/AppDelegate.cs

@@ -53,7 +53,7 @@ namespace HelloworldXamarin.iOS
         public override void DidEnterBackground(UIApplication application)
         {
             // Use this method to release shared resources, save user data, invalidate timers and store the application state.
-            // If your application supports background exection this method is called instead of WillTerminate when the user quits.
+            // If your application supports background execution this method is called instead of WillTerminate when the user quits.
         }
 
         public override void WillEnterForeground(UIApplication application)

examples/objective-c/auth_sample/MakeRPCViewController.m

@@ -46,8 +46,16 @@ static NSString * const kTestHostAddress = @"grpc-test.sandbox.googleapis.com";
 }
 @end
 
+@interface MakeRPCViewController ()<GRPCProtoResponseHandler>
+
+@end
+
 @implementation MakeRPCViewController
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)viewWillAppear:(BOOL)animated {
 
   // Create a service client and a proto request as usual.
@@ -57,28 +65,30 @@ static NSString * const kTestHostAddress = @"grpc-test.sandbox.googleapis.com";
   request.fillUsername = YES;
   request.fillOauthScope = YES;
 
-  // Create a not-yet-started RPC. We want to set the request headers on this object before starting
-  // it.
-  ProtoRPC *call =
-      [client RPCToUnaryCallWithRequest:request handler:^(AUTHResponse *response, NSError *error) {
-        if (response) {
-          // This test server responds with the email and scope of the access token it receives.
-          self.mainLabel.text = [NSString stringWithFormat:@"Used scope: %@ on behalf of user %@",
-                                 response.oauthScope, response.username];
-
-        } else {
-          self.mainLabel.text = error.UIDescription;
-        }
-      }];
-
-  // Set the access token to be used.
-  NSString *accessToken = GIDSignIn.sharedInstance.currentUser.authentication.accessToken;
-  call.requestHeaders[@"Authorization"] = [@"Bearer " stringByAppendingString:accessToken];
-
-  // Start the RPC.
+  // Set the request header with call options
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.oauth2AccessToken = GIDSignIn.sharedInstance.currentUser.authentication.accessToken;
+  GRPCUnaryProtoCall *call = [client unaryCallWithMessage:request
+                                          responseHandler:self
+                                              callOptions:options];
   [call start];
 
   self.mainLabel.text = @"Waiting for RPC to complete...";
 }
 
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  AUTHResponse *response = (AUTHResponse *)message;
+  if (response) {
+    // This test server responds with the email and scope of the access token it receives.
+    self.mainLabel.text = [NSString stringWithFormat:@"Used scope: %@ on behalf of user %@",
+                           response.oauthScope, response.username];
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    self.mainLabel.text = error.UIDescription;
+  }
+}
+
 @end

examples/objective-c/helloworld/main.m

@@ -25,20 +25,41 @@
 
 static NSString * const kHostAddress = @"localhost:50051";
 
+@interface HLWResponseHandler : NSObject<GRPCProtoResponseHandler>
+
+@end
+
+// A response handler object dispatching messages to main queue
+@implementation HLWResponseHandler
+
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  NSLog(@"%@", message);
+}
+
+@end
+
 int main(int argc, char * argv[]) {
   @autoreleasepool {
-    [GRPCCall useInsecureConnectionsForHost:kHostAddress];
-    [GRPCCall setUserAgentPrefix:@"HelloWorld/1.0" forHost:kHostAddress];
-
     HLWGreeter *client = [[HLWGreeter alloc] initWithHost:kHostAddress];
 
     HLWHelloRequest *request = [HLWHelloRequest message];
     request.name = @"Objective-C";
 
-    [client sayHelloWithRequest:request handler:^(HLWHelloReply *response, NSError *error) {
-      NSLog(@"%@", response.message);
-    }];
-    
+    GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+    // this example does not use TLS (secure channel); use insecure channel instead
+    options.transportType = GRPCTransportTypeInsecure;
+    options.userAgentPrefix = @"HelloWorld/1.0";
+
+    GRPCUnaryProtoCall *call = [client sayHelloWithMessage:request
+                                           responseHandler:[[HLWResponseHandler alloc] init]
+                                               callOptions:options];
+
+    [call start];
+
     return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
   }
 }

examples/objective-c/helloworld_macos/main.m

@@ -24,19 +24,40 @@
 
 static NSString * const kHostAddress = @"localhost:50051";
 
+@interface HLWResponseHandler : NSObject<GRPCProtoResponseHandler>
+
+@end
+
+// A response handler object dispatching messages to main queue
+@implementation HLWResponseHandler
+
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  NSLog(@"%@", message);
+}
+
+@end
+
 int main(int argc, const char * argv[]) {
   @autoreleasepool {
-    [GRPCCall useInsecureConnectionsForHost:kHostAddress];
-    [GRPCCall setUserAgentPrefix:@"HelloWorld/1.0" forHost:kHostAddress];
-
     HLWGreeter *client = [[HLWGreeter alloc] initWithHost:kHostAddress];
 
     HLWHelloRequest *request = [HLWHelloRequest message];
     request.name = @"Objective-C";
 
-    [client sayHelloWithRequest:request handler:^(HLWHelloReply *response, NSError *error) {
-      NSLog(@"%@", response.message);
-    }];
+    GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+    // this example does not use TLS (secure channel); use insecure channel instead
+    options.transportType = GRPCTransportTypeInsecure;
+    options.userAgentPrefix = @"HelloWorld/1.0";
+
+    GRPCUnaryProtoCall *call = [client sayHelloWithMessage:request
+                                           responseHandler:[[HLWResponseHandler alloc] init]
+                                               callOptions:options];
+
+    [call start];
   }
 
   return NSApplicationMain(argc, argv);

examples/objective-c/route_guide/ViewControllers.m

@@ -17,10 +17,7 @@
  */
 
 #import <UIKit/UIKit.h>
-#import <GRPCClient/GRPCCall+Tests.h>
 #import <RouteGuide/RouteGuide.pbrpc.h>
-#import <RxLibrary/GRXWriter+Immediate.h>
-#import <RxLibrary/GRXWriter+Transformations.h>
 
 static NSString * const kHostAddress = @"localhost:50051";
 
@@ -65,7 +62,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * Run the getFeature demo. Calls getFeature with a point known to have a feature and a point known
  * not to have a feature.
  */
-@interface GetFeatureViewController : UIViewController
+@interface GetFeatureViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -75,39 +72,56 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
-- (void)execRequest {
-  void (^handler)(RTGFeature *response, NSError *error) = ^(RTGFeature *response, NSError *error) {
-    // TODO(makdharma): Remove boilerplate by consolidating into one log function.
-    if (response.name.length) {
-      NSString *str =[NSString stringWithFormat:@"%@\nFound feature called %@ at %@.", self.outputLabel.text, response.location, response.name];
-      self.outputLabel.text = str;
-      NSLog(@"Found feature called %@ at %@.", response.name, response.location);
-    } else if (response) {
-      NSString *str =[NSString stringWithFormat:@"%@\nFound no features at %@",  self.outputLabel.text,response.location];
-      self.outputLabel.text = str;
-      NSLog(@"Found no features at %@", response.location);
-    } else {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-  };
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGFeature *response = (RTGFeature *)message;
 
+  // TODO(makdharma): Remove boilerplate by consolidating into one log function.
+  if (response.name.length != 0) {
+    NSString *str =[NSString stringWithFormat:@"%@\nFound feature called %@ at %@.", self.outputLabel.text, response.location, response.name];
+    self.outputLabel.text = str;
+    NSLog(@"Found feature called %@ at %@.", response.name, response.location);
+  } else if (response) {
+    NSString *str =[NSString stringWithFormat:@"%@\nFound no features at %@",  self.outputLabel.text,response.location];
+    self.outputLabel.text = str;
+    NSLog(@"Found no features at %@", response.location);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
+}
+
+- (void)execRequest {
   RTGPoint *point = [RTGPoint message];
   point.latitude = 409146138;
   point.longitude = -746188906;
 
-  [_service getFeatureWithRequest:point handler:handler];
-  [_service getFeatureWithRequest:[RTGPoint message] handler:handler];
+  GRPCUnaryProtoCall *call = [_service getFeatureWithMessage:point
+                                             responseHandler:self
+                                                 callOptions:nil];
+  [call start];
+  call = [_service getFeatureWithMessage:[RTGPoint message]
+                         responseHandler:self
+                             callOptions:nil];
+  [call start];
+
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  // This only needs to be done once per host, before creating service objects for that host.
-  [GRPCCall useInsecureConnectionsForHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {
@@ -126,7 +140,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * Run the listFeatures demo. Calls listFeatures with a rectangle containing all of the features in
  * the pre-generated database. Prints each response as it comes in.
  */
-@interface ListFeaturesViewController : UIViewController
+@interface ListFeaturesViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -136,6 +150,10 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)execRequest {
   RTGRectangle *rectangle = [RTGRectangle message];
   rectangle.lo.latitude = 405E6;
@@ -144,24 +162,36 @@ static NSString * const kHostAddress = @"localhost:50051";
   rectangle.hi.longitude = -745E6;
 
   NSLog(@"Looking for features between %@ and %@", rectangle.lo, rectangle.hi);
-  [_service listFeaturesWithRequest:rectangle
-                      eventHandler:^(BOOL done, RTGFeature *response, NSError *error) {
-    if (response) {
-      NSString *str =[NSString stringWithFormat:@"%@\nFound feature at %@ called %@.", self.outputLabel.text, response.location, response.name];
-      self.outputLabel.text = str;
-      NSLog(@"Found feature at %@ called %@.", response.location, response.name);
-    } else if (error) {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-  }];
+  GRPCUnaryProtoCall *call = [_service listFeaturesWithMessage:rectangle
+                                               responseHandler:self
+                                                   callOptions:nil];
+  [call start];
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGFeature *response = (RTGFeature *)message;
+  if (response) {
+    NSString *str =[NSString stringWithFormat:@"%@\nFound feature at %@ called %@.", self.outputLabel.text, response.location, response.name];
+    self.outputLabel.text = str;
+    NSLog(@"Found feature at %@ called %@.", response.location, response.name);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
+
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {
@@ -173,7 +203,6 @@ static NSString * const kHostAddress = @"localhost:50051";
 
 @end
 
-
 #pragma mark Demo: Record Route
 
 /**
@@ -181,7 +210,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * database with a variable delay in between. Prints the statistics when they are sent from the
  * server.
  */
-@interface RecordRouteViewController : UIViewController
+@interface RecordRouteViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -191,47 +220,71 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)execRequest {
   NSString *dataBasePath = [NSBundle.mainBundle pathForResource:@"route_guide_db"
                                                          ofType:@"json"];
   NSData *dataBaseContent = [NSData dataWithContentsOfFile:dataBasePath];
-  NSArray *features = [NSJSONSerialization JSONObjectWithData:dataBaseContent options:0 error:NULL];
+  NSError *error;
+  NSArray *features = [NSJSONSerialization JSONObjectWithData:dataBaseContent options:0 error:&error];
+
+  if (error) {
+    NSLog(@"Error reading database.");
+    NSString *str = @"Error reading database.";
+    self.outputLabel.text = str;
+    return;
+  }
 
-  GRXWriter *locations = [[GRXWriter writerWithContainer:features] map:^id(id feature) {
+  GRPCStreamingProtoCall *call = [_service recordRouteWithResponseHandler:self
+                                                              callOptions:nil];
+  [call start];
+  for (id feature in features) {
     RTGPoint *location = [RTGPoint message];
     location.longitude = [((NSNumber *) feature[@"location"][@"longitude"]) intValue];
     location.latitude = [((NSNumber *) feature[@"location"][@"latitude"]) intValue];
     NSString *str =[NSString stringWithFormat:@"%@\nVisiting point %@", self.outputLabel.text, location];
     self.outputLabel.text = str;
     NSLog(@"Visiting point %@", location);
-    return location;
-  }];
-
-  [_service recordRouteWithRequestsWriter:locations
-                                 handler:^(RTGRouteSummary *response, NSError *error) {
-    if (response) {
-      NSString *str =[NSString stringWithFormat:
-                      @"%@\nFinished trip with %i points\nPassed %i features\n"
-                      "Travelled %i meters\nIt took %i seconds",
-                      self.outputLabel.text, response.pointCount, response.featureCount,
-                      response.distance, response.elapsedTime];
-      self.outputLabel.text = str;
-      NSLog(@"Finished trip with %i points", response.pointCount);
-      NSLog(@"Passed %i features", response.featureCount);
-      NSLog(@"Travelled %i meters", response.distance);
-      NSLog(@"It took %i seconds", response.elapsedTime);
-    } else {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-  }];
+    [call writeMessage:location];
+  }
+  [call finish];
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGRouteSummary *response = (RTGRouteSummary *)message;
+
+  if (response) {
+    NSString *str =[NSString stringWithFormat:
+                    @"%@\nFinished trip with %i points\nPassed %i features\n"
+                    "Travelled %i meters\nIt took %i seconds",
+                    self.outputLabel.text, response.pointCount, response.featureCount,
+                    response.distance, response.elapsedTime];
+    self.outputLabel.text = str;
+    NSLog(@"Finished trip with %i points", response.pointCount);
+    NSLog(@"Passed %i features", response.featureCount);
+    NSLog(@"Travelled %i meters", response.distance);
+    NSLog(@"It took %i seconds", response.elapsedTime);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
+
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {
@@ -250,7 +303,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * Run the routeChat demo. Send some chat messages, and print any chat messages that are sent from
  * the server.
  */
-@interface RouteChatViewController : UIViewController
+@interface RouteChatViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -260,38 +313,52 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)execRequest {
   NSArray *notes = @[[RTGRouteNote noteWithMessage:@"First message" latitude:0 longitude:0],
                      [RTGRouteNote noteWithMessage:@"Second message" latitude:0 longitude:1],
                      [RTGRouteNote noteWithMessage:@"Third message" latitude:1 longitude:0],
                      [RTGRouteNote noteWithMessage:@"Fourth message" latitude:0 longitude:0]];
-  GRXWriter *notesWriter = [[GRXWriter writerWithContainer:notes] map:^id(RTGRouteNote *note) {
-    NSLog(@"Sending message %@ at %@", note.message, note.location);
-    return note;
-  }];
-
-  [_service routeChatWithRequestsWriter:notesWriter
-                          eventHandler:^(BOOL done, RTGRouteNote *note, NSError *error) {
-    if (note) {
-      NSString *str =[NSString stringWithFormat:@"%@\nGot message %@ at %@",
-                      self.outputLabel.text, note.message, note.location];
-      self.outputLabel.text = str;
-      NSLog(@"Got message %@ at %@", note.message, note.location);
-    } else if (error) {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-    if (done) {
-      NSLog(@"Chat ended.");
-    }
-  }];
+
+  GRPCStreamingProtoCall *call = [_service routeChatWithResponseHandler:self
+                                                            callOptions:nil];
+  [call start];
+  for (RTGRouteNote *note in notes) {
+    [call writeMessage:note];
+  }
+  [call finish];
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGRouteNote *note = (RTGRouteNote *)message;
+  if (note) {
+    NSString *str =[NSString stringWithFormat:@"%@\nGot message %@ at %@",
+                    self.outputLabel.text, note.message, note.location];
+    self.outputLabel.text = str;
+    NSLog(@"Got message %@ at %@", note.message, note.location);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (!error) {
+    NSLog(@"Chat ended.");
+  } else {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
+
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {

examples/python/wait_for_ready/BUILD.bazel

@@ -0,0 +1,32 @@
+# Copyright 2019 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@grpc_python_dependencies//:requirements.bzl", "requirement")
+
+py_library(
+    name = "wait_for_ready_example",
+    testonly = 1,
+    srcs = ["wait_for_ready_example.py"],
+    deps = [
+        "//src/python/grpcio/grpc:grpcio",
+        "//examples:py_helloworld",
+    ],
+)
+
+py_test(
+    name = "test/_wait_for_ready_example_test",
+    srcs = ["test/_wait_for_ready_example_test.py"],
+    deps = [":wait_for_ready_example",],
+    size = "small",
+)

examples/python/wait_for_ready/README.md

@@ -0,0 +1,32 @@
+# gRPC Python Example for Wait-for-ready
+
+The default behavior of an RPC is to fail instantly if the server is not ready yet. This example demonstrates how to change that behavior.
+
+
+### Definition of 'wait-for-ready' semantics
+> If an RPC is issued but the channel is in TRANSIENT_FAILURE or SHUTDOWN states, the RPC is unable to be transmitted promptly. By default, gRPC implementations SHOULD fail such RPCs immediately. This is known as "fail fast," but the usage of the term is historical. RPCs SHOULD NOT fail as a result of the channel being in other states (CONNECTING, READY, or IDLE).
+> 
+> gRPC implementations MAY provide a per-RPC option to not fail RPCs as a result of the channel being in TRANSIENT_FAILURE state. Instead, the implementation queues the RPCs until the channel is READY. This is known as "wait for ready." The RPCs SHOULD still fail before READY if there are unrelated reasons, such as the channel is SHUTDOWN or the RPC's deadline is reached.
+> 
+> From https://github.com/grpc/grpc/blob/master/doc/wait-for-ready.md 
+
+
+### Use cases for 'wait-for-ready'
+
+When developers spin up gRPC clients and servers at the same time, it is very like to fail first couple RPC calls due to unavailability of the server. If developers failed to prepare for this situation, the result can be catastrophic. But with 'wait-for-ready' semantics, developers can initialize the client and server in any order, especially useful in testing.
+
+Also, developers may ensure the server is up before starting client. But in some cases like transient network failure may result in a temporary unavailability of the server. With 'wait-for-ready' semantics, those RPC calls will automatically wait until the server is ready to accept incoming requests.
+
+
+### DEMO Snippets
+
+```Python
+# Per RPC level
+stub = ...Stub(...)
+
+stub.important_transaction_1(..., wait_for_ready=True)
+stub.unimportant_transaction_2(...)
+stub.important_transaction_3(..., wait_for_ready=True)
+stub.unimportant_transaction_4(...)
+# The unimportant transactions can be status report, or health check, etc.
+```

examples/python/wait_for_ready/test/_wait_for_ready_example_test.py

@@ -0,0 +1,31 @@
+# Copyright 2019 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests of the wait-for-ready example."""
+
+import unittest
+import logging
+
+from examples.python.wait_for_ready import wait_for_ready_example
+
+
+class WaitForReadyExampleTest(unittest.TestCase):
+
+    def test_wait_for_ready_example(self):
+        wait_for_ready_example.main()
+        # No unhandled exception raised, no deadlock, test passed!
+
+
+if __name__ == '__main__':
+    logging.basicConfig()
+    unittest.main(verbosity=2)

examples/python/wait_for_ready/wait_for_ready_example.py

@@ -0,0 +1,114 @@
+# Copyright 2019 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""The Python example of utilizing wait-for-ready flag."""
+
+from __future__ import print_function
+import logging
+from concurrent import futures
+from contextlib import contextmanager
+import socket
+import threading
+
+import grpc
+
+from examples.protos import helloworld_pb2
+from examples.protos import helloworld_pb2_grpc
+
+_LOGGER = logging.getLogger(__name__)
+_LOGGER.setLevel(logging.INFO)
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+
+@contextmanager
+def get_free_loopback_tcp_port():
+    tcp_socket = socket.socket(socket.AF_INET6)
+    tcp_socket.bind(('', 0))
+    address_tuple = tcp_socket.getsockname()
+    yield "[::1]:%s" % (address_tuple[1])
+    tcp_socket.close()
+
+
+class Greeter(helloworld_pb2_grpc.GreeterServicer):
+
+    def SayHello(self, request, unused_context):
+        return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
+
+
+def create_server(server_address):
+    server = grpc.server(futures.ThreadPoolExecutor())
+    helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
+    bound_port = server.add_insecure_port(server_address)
+    assert bound_port == int(server_address.split(':')[-1])
+    return server
+
+
+def process(stub, wait_for_ready=None):
+    try:
+        response = stub.SayHello(
+            helloworld_pb2.HelloRequest(name='you'),
+            wait_for_ready=wait_for_ready)
+        message = response.message
+    except grpc.RpcError as rpc_error:
+        assert rpc_error.code() == grpc.StatusCode.UNAVAILABLE
+        assert not wait_for_ready
+        message = rpc_error
+    else:
+        assert wait_for_ready
+    _LOGGER.info("Wait-for-ready %s, client received: %s", "enabled"
+                 if wait_for_ready else "disabled", message)
+
+
+def main():
+    # Pick a random free port
+    with get_free_loopback_tcp_port() as server_address:
+
+        # Register connectivity event to notify main thread
+        transient_failure_event = threading.Event()
+
+        def wait_for_transient_failure(channel_connectivity):
+            if channel_connectivity == grpc.ChannelConnectivity.TRANSIENT_FAILURE:
+                transient_failure_event.set()
+
+        # Create gRPC channel
+        channel = grpc.insecure_channel(server_address)
+        channel.subscribe(wait_for_transient_failure)
+        stub = helloworld_pb2_grpc.GreeterStub(channel)
+
+        # Fire an RPC without wait_for_ready
+        thread_disabled_wait_for_ready = threading.Thread(
+            target=process, args=(stub, False))
+        thread_disabled_wait_for_ready.start()
+        # Fire an RPC with wait_for_ready
+        thread_enabled_wait_for_ready = threading.Thread(
+            target=process, args=(stub, True))
+        thread_enabled_wait_for_ready.start()
+
+    # Wait for the channel entering TRANSIENT FAILURE state.
+    transient_failure_event.wait()
+    server = create_server(server_address)
+    server.start()
+
+    # Expected to fail with StatusCode.UNAVAILABLE.
+    thread_disabled_wait_for_ready.join()
+    # Expected to success.
+    thread_enabled_wait_for_ready.join()
+
+    server.stop(None)
+    channel.close()
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    main()

examples/ruby/without_protobuf/echo_client.rb

@@ -28,7 +28,7 @@ def main
   stub = EchoWithoutProtobuf::Stub.new('localhost:50051', :this_channel_is_insecure)
   user = ARGV.size > 0 ?  ARGV[0] : 'world'
   message = stub.echo("hello #{user}")
-  p "Reponse: #{message}"
+  p "Response: #{message}"
 end
 
 main

gRPC-C++.podspec

@@ -122,7 +122,6 @@ Pod::Spec.new do |s|
                       'include/grpcpp/server_builder.h',
                       'include/grpcpp/server_builder_impl.h',
                       'include/grpcpp/server_context.h',
-                      'include/grpcpp/server_impl.h',
                       'include/grpcpp/server_posix.h',
                       'include/grpcpp/server_posix_impl.h',
                       'include/grpcpp/support/async_stream.h',
@@ -133,6 +132,7 @@ Pod::Spec.new do |s|
                       'include/grpcpp/support/client_interceptor.h',
                       'include/grpcpp/support/config.h',
                       'include/grpcpp/support/interceptor.h',
+                      'include/grpcpp/support/message_allocator.h',
                       'include/grpcpp/support/proto_buffer_reader.h',
                       'include/grpcpp/support/proto_buffer_writer.h',
                       'include/grpcpp/support/server_callback.h',
@@ -167,6 +167,7 @@ Pod::Spec.new do |s|
                       'include/grpcpp/impl/codegen/intercepted_channel.h',
                       'include/grpcpp/impl/codegen/interceptor.h',
                       'include/grpcpp/impl/codegen/interceptor_common.h',
+                      'include/grpcpp/impl/codegen/message_allocator.h',
                       'include/grpcpp/impl/codegen/metadata_map.h',
                       'include/grpcpp/impl/codegen/method_handler_impl.h',
                       'include/grpcpp/impl/codegen/rpc_method.h',
@@ -263,6 +264,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/gpr/tmpfile.h',
                       'src/core/lib/gpr/useful.h',
                       'src/core/lib/gprpp/abstract.h',
+                      'src/core/lib/gprpp/arena.h',
                       'src/core/lib/gprpp/atomic.h',
                       'src/core/lib/gprpp/fork.h',
                       'src/core/lib/gprpp/manual_constructor.h',
@@ -412,6 +414,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/handshaker_registry.h',
                       'src/core/lib/channel/status_util.h',
                       'src/core/lib/compression/algorithm_metadata.h',
+                      'src/core/lib/compression/compression_args.h',
                       'src/core/lib/compression/compression_internal.h',
                       'src/core/lib/compression/message_compress.h',
                       'src/core/lib/compression/stream_compression.h',
@@ -580,6 +583,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/gpr/tmpfile.h',
                               'src/core/lib/gpr/useful.h',
                               'src/core/lib/gprpp/abstract.h',
+                              'src/core/lib/gprpp/arena.h',
                               'src/core/lib/gprpp/atomic.h',
                               'src/core/lib/gprpp/fork.h',
                               'src/core/lib/gprpp/manual_constructor.h',
@@ -604,6 +608,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/channel/handshaker_registry.h',
                               'src/core/lib/channel/status_util.h',
                               'src/core/lib/compression/algorithm_metadata.h',
+                              'src/core/lib/compression/compression_args.h',
                               'src/core/lib/compression/compression_internal.h',
                               'src/core/lib/compression/message_compress.h',
                               'src/core/lib/compression/stream_compression.h',

gRPC-Core.podspec

@@ -205,6 +205,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/gpr/tmpfile.h',
                       'src/core/lib/gpr/useful.h',
                       'src/core/lib/gprpp/abstract.h',
+                      'src/core/lib/gprpp/arena.h',
                       'src/core/lib/gprpp/atomic.h',
                       'src/core/lib/gprpp/fork.h',
                       'src/core/lib/gprpp/manual_constructor.h',
@@ -215,7 +216,6 @@ Pod::Spec.new do |s|
                       'src/core/lib/gprpp/thd.h',
                       'src/core/lib/profiling/timers.h',
                       'src/core/lib/gpr/alloc.cc',
-                      'src/core/lib/gpr/arena.cc',
                       'src/core/lib/gpr/atm.cc',
                       'src/core/lib/gpr/cpu_iphone.cc',
                       'src/core/lib/gpr/cpu_linux.cc',
@@ -248,6 +248,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/gpr/tmpfile_posix.cc',
                       'src/core/lib/gpr/tmpfile_windows.cc',
                       'src/core/lib/gpr/wrap_memcpy.cc',
+                      'src/core/lib/gprpp/arena.cc',
                       'src/core/lib/gprpp/fork.cc',
                       'src/core/lib/gprpp/thd_posix.cc',
                       'src/core/lib/gprpp/thd_windows.cc',
@@ -393,6 +394,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/handshaker_registry.h',
                       'src/core/lib/channel/status_util.h',
                       'src/core/lib/compression/algorithm_metadata.h',
+                      'src/core/lib/compression/compression_args.h',
                       'src/core/lib/compression/compression_internal.h',
                       'src/core/lib/compression/message_compress.h',
                       'src/core/lib/compression/stream_compression.h',
@@ -547,6 +549,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/channel/handshaker_registry.cc',
                       'src/core/lib/channel/status_util.cc',
                       'src/core/lib/compression/compression.cc',
+                      'src/core/lib/compression/compression_args.cc',
                       'src/core/lib/compression/compression_internal.cc',
                       'src/core/lib/compression/message_compress.cc',
                       'src/core/lib/compression/stream_compression.cc',
@@ -884,6 +887,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/gpr/tmpfile.h',
                               'src/core/lib/gpr/useful.h',
                               'src/core/lib/gprpp/abstract.h',
+                              'src/core/lib/gprpp/arena.h',
                               'src/core/lib/gprpp/atomic.h',
                               'src/core/lib/gprpp/fork.h',
                               'src/core/lib/gprpp/manual_constructor.h',
@@ -1033,6 +1037,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/channel/handshaker_registry.h',
                               'src/core/lib/channel/status_util.h',
                               'src/core/lib/compression/algorithm_metadata.h',
+                              'src/core/lib/compression/compression_args.h',
                               'src/core/lib/compression/compression_internal.h',
                               'src/core/lib/compression/message_compress.h',
                               'src/core/lib/compression/stream_compression.h',

grpc.gemspec

@@ -99,6 +99,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gpr/tmpfile.h )
   s.files += %w( src/core/lib/gpr/useful.h )
   s.files += %w( src/core/lib/gprpp/abstract.h )
+  s.files += %w( src/core/lib/gprpp/arena.h )
   s.files += %w( src/core/lib/gprpp/atomic.h )
   s.files += %w( src/core/lib/gprpp/fork.h )
   s.files += %w( src/core/lib/gprpp/manual_constructor.h )
@@ -109,7 +110,6 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gprpp/thd.h )
   s.files += %w( src/core/lib/profiling/timers.h )
   s.files += %w( src/core/lib/gpr/alloc.cc )
-  s.files += %w( src/core/lib/gpr/arena.cc )
   s.files += %w( src/core/lib/gpr/atm.cc )
   s.files += %w( src/core/lib/gpr/cpu_iphone.cc )
   s.files += %w( src/core/lib/gpr/cpu_linux.cc )
@@ -142,6 +142,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/gpr/tmpfile_posix.cc )
   s.files += %w( src/core/lib/gpr/tmpfile_windows.cc )
   s.files += %w( src/core/lib/gpr/wrap_memcpy.cc )
+  s.files += %w( src/core/lib/gprpp/arena.cc )
   s.files += %w( src/core/lib/gprpp/fork.cc )
   s.files += %w( src/core/lib/gprpp/thd_posix.cc )
   s.files += %w( src/core/lib/gprpp/thd_windows.cc )
@@ -327,6 +328,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/channel/handshaker_registry.h )
   s.files += %w( src/core/lib/channel/status_util.h )
   s.files += %w( src/core/lib/compression/algorithm_metadata.h )
+  s.files += %w( src/core/lib/compression/compression_args.h )
   s.files += %w( src/core/lib/compression/compression_internal.h )
   s.files += %w( src/core/lib/compression/message_compress.h )
   s.files += %w( src/core/lib/compression/stream_compression.h )
@@ -481,6 +483,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/channel/handshaker_registry.cc )
   s.files += %w( src/core/lib/channel/status_util.cc )
   s.files += %w( src/core/lib/compression/compression.cc )
+  s.files += %w( src/core/lib/compression/compression_args.cc )
   s.files += %w( src/core/lib/compression/compression_internal.cc )
   s.files += %w( src/core/lib/compression/message_compress.cc )
   s.files += %w( src/core/lib/compression/stream_compression.cc )

grpc.gyp

@@ -218,7 +218,6 @@
       ],
       'sources': [
         'src/core/lib/gpr/alloc.cc',
-        'src/core/lib/gpr/arena.cc',
         'src/core/lib/gpr/atm.cc',
         'src/core/lib/gpr/cpu_iphone.cc',
         'src/core/lib/gpr/cpu_linux.cc',
@@ -251,6 +250,7 @@
         'src/core/lib/gpr/tmpfile_posix.cc',
         'src/core/lib/gpr/tmpfile_windows.cc',
         'src/core/lib/gpr/wrap_memcpy.cc',
+        'src/core/lib/gprpp/arena.cc',
         'src/core/lib/gprpp/fork.cc',
         'src/core/lib/gprpp/thd_posix.cc',
         'src/core/lib/gprpp/thd_windows.cc',
@@ -279,6 +279,7 @@
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
+        'src/core/lib/compression/compression_args.cc',
         'src/core/lib/compression/compression_internal.cc',
         'src/core/lib/compression/message_compress.cc',
         'src/core/lib/compression/stream_compression.cc',
@@ -646,6 +647,7 @@
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
+        'src/core/lib/compression/compression_args.cc',
         'src/core/lib/compression/compression_internal.cc',
         'src/core/lib/compression/message_compress.cc',
         'src/core/lib/compression/stream_compression.cc',
@@ -890,6 +892,7 @@
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
+        'src/core/lib/compression/compression_args.cc',
         'src/core/lib/compression/compression_internal.cc',
         'src/core/lib/compression/message_compress.cc',
         'src/core/lib/compression/stream_compression.cc',
@@ -1110,6 +1113,7 @@
         'src/core/lib/channel/handshaker_registry.cc',
         'src/core/lib/channel/status_util.cc',
         'src/core/lib/compression/compression.cc',
+        'src/core/lib/compression/compression_args.cc',
         'src/core/lib/compression/compression_internal.cc',
         'src/core/lib/compression/message_compress.cc',
         'src/core/lib/compression/stream_compression.cc',

include/grpc/grpc_security.h

@@ -264,7 +264,7 @@ GRPCAPI grpc_call_credentials* grpc_google_refresh_token_credentials_create(
     const char* json_refresh_token, void* reserved);
 
 /** Creates an Oauth2 Access Token credentials with an access token that was
-   aquired by an out of band mechanism. */
+   acquired by an out of band mechanism. */
 GRPCAPI grpc_call_credentials* grpc_access_token_credentials_create(
     const char* access_token, void* reserved);
 

include/grpc/impl/codegen/grpc_types.h

@@ -315,11 +315,11 @@ typedef struct {
 #define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms"
 /* Timeout in milliseconds to wait for the serverlist from the grpclb load
    balancer before using fallback backend addresses from the resolver.
-   If 0, fallback will never be used. Default value is 10000. */
+   If 0, enter fallback mode immediately. Default value is 10000. */
 #define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms"
 /* Timeout in milliseconds to wait for the serverlist from the xDS load
    balancer before using fallback backend addresses from the resolver.
-   If 0, fallback will never be used. Default value is 10000. */
+   If 0, enter fallback mode immediately. Default value is 10000. */
 #define GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS "grpc.xds_fallback_timeout_ms"
 /** If non-zero, grpc server's cronet compression workaround will be enabled */
 #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \

include/grpc/impl/codegen/port_platform.h

@@ -115,6 +115,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #ifdef _LP64
 #define GPR_ARCH_64 1
@@ -144,6 +145,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #define GPR_SUPPORT_CHANNELS_FROM_FD 1
 #elif defined(__linux__)
@@ -170,6 +172,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #ifdef _LP64
 #define GPR_ARCH_64 1
@@ -235,6 +238,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #ifndef GRPC_CFSTREAM
 #define GPR_SUPPORT_CHANNELS_FROM_FD 1
@@ -260,6 +264,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #define GPR_SUPPORT_CHANNELS_FROM_FD 1
 #ifdef _LP64
@@ -283,6 +288,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #define GPR_SUPPORT_CHANNELS_FROM_FD 1
 #ifdef _LP64
@@ -303,6 +309,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #ifdef _LP64
 #define GPR_ARCH_64 1
@@ -325,6 +332,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #ifdef _LP64
 #define GPR_ARCH_64 1
@@ -353,6 +361,7 @@
 #define GPR_POSIX_SUBPROCESS 1
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #ifdef _LP64
 #define GPR_ARCH_64 1
@@ -378,6 +387,7 @@
 #define GPR_POSIX_SYNC 1
 #define GPR_POSIX_STRING 1
 #define GPR_POSIX_TIME 1
+#define GPR_HAS_PTHREAD_H 1
 #define GPR_GETPID_IN_UNISTD_H 1
 #else
 #error "Could not auto-detect platform"

include/grpc/impl/codegen/slice.h

@@ -40,27 +40,6 @@ typedef struct grpc_slice grpc_slice;
    reference ownership semantics (who should call unref?) and mutability
    constraints (is the callee allowed to modify the slice?) */
 
-typedef struct grpc_slice_refcount_vtable {
-  void (*ref)(void*);
-  void (*unref)(void*);
-  int (*eq)(grpc_slice a, grpc_slice b);
-  uint32_t (*hash)(grpc_slice slice);
-} grpc_slice_refcount_vtable;
-
-/** Reference count container for grpc_slice. Contains function pointers to
-   increment and decrement reference counts. Implementations should cleanup
-   when the reference count drops to zero.
-   Typically client code should not touch this, and use grpc_slice_malloc,
-   grpc_slice_new, or grpc_slice_new_with_len instead. */
-typedef struct grpc_slice_refcount {
-  const grpc_slice_refcount_vtable* vtable;
-  /** If a subset of this slice is taken, use this pointer for the refcount.
-     Typically points back to the refcount itself, however iterning
-     implementations can use this to avoid a verification step on each hash
-     or equality check */
-  struct grpc_slice_refcount* sub_refcount;
-} grpc_slice_refcount;
-
 /* Inlined half of grpc_slice is allowed to expand the size of the overall type
    by this many bytes */
 #define GRPC_SLICE_INLINE_EXTRA_SIZE sizeof(void*)
@@ -68,6 +47,7 @@ typedef struct grpc_slice_refcount {
 #define GRPC_SLICE_INLINED_SIZE \
   (sizeof(size_t) + sizeof(uint8_t*) - 1 + GRPC_SLICE_INLINE_EXTRA_SIZE)
 
+struct grpc_slice_refcount;
 /** A grpc_slice s, if initialized, represents the byte range
    s.bytes[0..s.length-1].
 

include/grpc/slice.h

@@ -147,7 +147,7 @@ GPRAPI int grpc_slice_buf_start_eq(grpc_slice a, const void* b, size_t blen);
 GPRAPI int grpc_slice_rchr(grpc_slice s, char c);
 GPRAPI int grpc_slice_chr(grpc_slice s, char c);
 
-/** return the index of the first occurance of \a needle in \a haystack, or -1
+/** return the index of the first occurrence of \a needle in \a haystack, or -1
    if it's not found */
 GPRAPI int grpc_slice_slice(grpc_slice haystack, grpc_slice needle);
 

include/grpcpp/generic/generic_stub_impl.h

@@ -82,6 +82,12 @@ class GenericStub final {
                    const grpc::ByteBuffer* request, grpc::ByteBuffer* response,
                    std::function<void(grpc::Status)> on_completion);
 
+    /// Setup and start a unary call to a named method \a method using
+    /// \a context and specifying the \a request and \a response buffers.
+    void UnaryCall(grpc::ClientContext* context, const grpc::string& method,
+                   const grpc::ByteBuffer* request, grpc::ByteBuffer* response,
+                   grpc::experimental::ClientUnaryReactor* reactor);
+
     /// Setup a call to a named method \a method using \a context and tied to
     /// \a reactor . Like any other bidi streaming RPC, it will not be activated
     /// until StartCall is invoked on its reactor.

include/grpcpp/impl/codegen/async_generic_service.h

@@ -39,7 +39,7 @@ class GenericServerContext final : public ServerContext {
   const grpc::string& host() const { return host_; }
 
  private:
-  friend class grpc_impl::Server;
+  friend class Server;
   friend class ServerInterface;
 
   void Clear() {
@@ -79,8 +79,8 @@ class AsyncGenericService final {
                    ServerCompletionQueue* notification_cq, void* tag);
 
  private:
-  friend class grpc_impl::Server;
-  grpc_impl::Server* server_;
+  friend class Server;
+  Server* server_;
 };
 
 namespace experimental {
@@ -135,14 +135,14 @@ class CallbackGenericService {
   }
 
  private:
-  friend class ::grpc_impl::Server;
+  friend class ::grpc::Server;
 
   internal::CallbackBidiHandler<ByteBuffer, ByteBuffer>* Handler() {
     return new internal::CallbackBidiHandler<ByteBuffer, ByteBuffer>(
         [this] { return CreateReactor(); });
   }
 
-  grpc_impl::Server* server_{nullptr};
+  Server* server_{nullptr};
 };
 }  // namespace experimental
 }  // namespace grpc

include/grpcpp/impl/codegen/async_stream.h

@@ -661,7 +661,7 @@ class ServerAsyncReaderInterface
   /// some failure occurred when trying to do so.
   ///
   /// gRPC doesn't take ownership or a reference to \a msg or \a status, so it
-  /// is safe to to deallocate once Finish returns.
+  /// is safe to deallocate once Finish returns.
   ///
   /// \param[in] tag Tag identifying this request.
   /// \param[in] status To be sent to the client as the result of this call.
@@ -735,7 +735,7 @@ class ServerAsyncReader final : public ServerAsyncReaderInterface<W, R> {
   /// Note: \a msg is not sent if \a status has a non-OK code.
   ///
   /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it
-  /// is safe to to deallocate once Finish returns.
+  /// is safe to deallocate once Finish returns.
   void Finish(const W& msg, const Status& status, void* tag) override {
     finish_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
@@ -830,7 +830,7 @@ class ServerAsyncWriterInterface
   /// in a single step.
   ///
   /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it
-  /// is safe to to deallocate once WriteAndFinish returns.
+  /// is safe to deallocate once WriteAndFinish returns.
   ///
   /// \param[in] msg The message to be written.
   /// \param[in] options The WriteOptions to be used to write this message.
@@ -897,7 +897,7 @@ class ServerAsyncWriter final : public ServerAsyncWriterInterface<W> {
   /// Note: \a status must have an OK code.
   ///
   /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it
-  /// is safe to to deallocate once WriteAndFinish returns.
+  /// is safe to deallocate once WriteAndFinish returns.
   void WriteAndFinish(const W& msg, WriteOptions options, const Status& status,
                       void* tag) override {
     write_ops_.set_output_tag(tag);
@@ -993,7 +993,7 @@ class ServerAsyncReaderWriterInterface
   /// single step.
   ///
   /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it
-  /// is safe to to deallocate once WriteAndFinish returns.
+  /// is safe to deallocate once WriteAndFinish returns.
   ///
   /// \param[in] msg The message to be written.
   /// \param[in] options The WriteOptions to be used to write this message.
@@ -1068,7 +1068,7 @@ class ServerAsyncReaderWriter final
   /// Note: \a status must have an OK code.
   //
   /// gRPC doesn't take ownership or a reference to \a msg and \a status, so it
-  /// is safe to to deallocate once WriteAndFinish returns.
+  /// is safe to deallocate once WriteAndFinish returns.
   void WriteAndFinish(const W& msg, WriteOptions options, const Status& status,
                       void* tag) override {
     write_ops_.set_output_tag(tag);
@@ -1099,7 +1099,7 @@ class ServerAsyncReaderWriter final
   }
 
  private:
-  friend class ::grpc_impl::Server;
+  friend class ::grpc::Server;
 
   void BindCall(::grpc::internal::Call* call) override { call_ = *call; }
 

include/grpcpp/impl/codegen/channel_interface.h

@@ -58,6 +58,7 @@ template <class R>
 class ClientCallbackReaderFactory;
 template <class W>
 class ClientCallbackWriterFactory;
+class ClientCallbackUnaryFactory;
 class InterceptedChannel;
 }  // namespace internal
 
@@ -117,6 +118,7 @@ class ChannelInterface {
   friend class ::grpc::internal::ClientCallbackReaderFactory;
   template <class W>
   friend class ::grpc::internal::ClientCallbackWriterFactory;
+  friend class ::grpc::internal::ClientCallbackUnaryFactory;
   template <class InputMessage, class OutputMessage>
   friend class ::grpc::internal::BlockingUnaryCallImpl;
   template <class InputMessage, class OutputMessage>

include/grpcpp/impl/codegen/client_callback.h

@@ -100,6 +100,7 @@ template <class Response>
 class ClientReadReactor;
 template <class Request>
 class ClientWriteReactor;
+class ClientUnaryReactor;
 
 // NOTE: The streaming objects are not actually implemented in the public API.
 //       These interfaces are provided for mocking only. Typical applications
@@ -157,6 +158,15 @@ class ClientCallbackWriter {
   }
 };
 
+class ClientCallbackUnary {
+ public:
+  virtual ~ClientCallbackUnary() {}
+  virtual void StartCall() = 0;
+
+ protected:
+  void BindReactor(ClientUnaryReactor* reactor);
+};
+
 // The following classes are the reactor interfaces that are to be implemented
 // by the user. They are passed in to the library as an argument to a call on a
 // stub (either a codegen-ed call or a generic call). The streaming RPC is
@@ -346,6 +356,36 @@ class ClientWriteReactor {
   ClientCallbackWriter<Request>* writer_;
 };
 
+/// \a ClientUnaryReactor is a reactor-style interface for a unary RPC.
+/// This is _not_ a common way of invoking a unary RPC. In practice, this
+/// option should be used only if the unary RPC wants to receive initial
+/// metadata without waiting for the response to complete. Most deployments of
+/// RPC systems do not use this option, but it is needed for generality.
+/// All public methods behave as in ClientBidiReactor.
+/// StartCall is included for consistency with the other reactor flavors: even
+/// though there are no StartRead or StartWrite operations to queue before the
+/// call (that is part of the unary call itself) and there is no reactor object
+/// being created as a result of this call, we keep a consistent 2-phase
+/// initiation API among all the reactor flavors.
+class ClientUnaryReactor {
+ public:
+  virtual ~ClientUnaryReactor() {}
+
+  void StartCall() { call_->StartCall(); }
+  virtual void OnDone(const Status& s) {}
+  virtual void OnReadInitialMetadataDone(bool ok) {}
+
+ private:
+  friend class ClientCallbackUnary;
+  void BindCall(ClientCallbackUnary* call) { call_ = call; }
+  ClientCallbackUnary* call_;
+};
+
+// Define function out-of-line from class to avoid forward declaration issue
+inline void ClientCallbackUnary::BindReactor(ClientUnaryReactor* reactor) {
+  reactor->BindCall(this);
+}
+
 }  // namespace experimental
 
 namespace internal {
@@ -512,9 +552,9 @@ class ClientCallbackReaderWriterImpl
     this->BindReactor(reactor);
   }
 
-  ClientContext* context_;
+  ClientContext* const context_;
   Call call_;
-  ::grpc::experimental::ClientBidiReactor<Request, Response>* reactor_;
+  ::grpc::experimental::ClientBidiReactor<Request, Response>* const reactor_;
 
   CallOpSet<CallOpSendInitialMetadata, CallOpRecvInitialMetadata> start_ops_;
   CallbackWithSuccessTag start_tag_;
@@ -651,9 +691,9 @@ class ClientCallbackReaderImpl
     start_ops_.ClientSendClose();
   }
 
-  ClientContext* context_;
+  ClientContext* const context_;
   Call call_;
-  ::grpc::experimental::ClientReadReactor<Response>* reactor_;
+  ::grpc::experimental::ClientReadReactor<Response>* const reactor_;
 
   CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose,
             CallOpRecvInitialMetadata>
@@ -824,9 +864,9 @@ class ClientCallbackWriterImpl
     finish_ops_.AllowNoMessage();
   }
 
-  ClientContext* context_;
+  ClientContext* const context_;
   Call call_;
-  ::grpc::experimental::ClientWriteReactor<Request>* reactor_;
+  ::grpc::experimental::ClientWriteReactor<Request>* const reactor_;
 
   CallOpSet<CallOpSendInitialMetadata, CallOpRecvInitialMetadata> start_ops_;
   CallbackWithSuccessTag start_tag_;
@@ -867,6 +907,109 @@ class ClientCallbackWriterFactory {
   }
 };
 
+class ClientCallbackUnaryImpl final
+    : public ::grpc::experimental::ClientCallbackUnary {
+ public:
+  // always allocated against a call arena, no memory free required
+  static void operator delete(void* ptr, std::size_t size) {
+    assert(size == sizeof(ClientCallbackUnaryImpl));
+  }
+
+  // This operator should never be called as the memory should be freed as part
+  // of the arena destruction. It only exists to provide a matching operator
+  // delete to the operator new so that some compilers will not complain (see
+  // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this
+  // there are no tests catching the compiler warning.
+  static void operator delete(void*, void*) { assert(0); }
+
+  void StartCall() override {
+    // This call initiates two batches, each with a callback
+    // 1. Send initial metadata + write + writes done + recv initial metadata
+    // 2. Read message, recv trailing metadata
+    started_ = true;
+
+    start_tag_.Set(call_.call(),
+                   [this](bool ok) {
+                     reactor_->OnReadInitialMetadataDone(ok);
+                     MaybeFinish();
+                   },
+                   &start_ops_);
+    start_ops_.SendInitialMetadata(&context_->send_initial_metadata_,
+                                   context_->initial_metadata_flags());
+    start_ops_.RecvInitialMetadata(context_);
+    start_ops_.set_core_cq_tag(&start_tag_);
+    call_.PerformOps(&start_ops_);
+
+    finish_tag_.Set(call_.call(), [this](bool ok) { MaybeFinish(); },
+                    &finish_ops_);
+    finish_ops_.ClientRecvStatus(context_, &finish_status_);
+    finish_ops_.set_core_cq_tag(&finish_tag_);
+    call_.PerformOps(&finish_ops_);
+  }
+
+  void MaybeFinish() {
+    if (--callbacks_outstanding_ == 0) {
+      Status s = std::move(finish_status_);
+      auto* reactor = reactor_;
+      auto* call = call_.call();
+      this->~ClientCallbackUnaryImpl();
+      g_core_codegen_interface->grpc_call_unref(call);
+      reactor->OnDone(s);
+    }
+  }
+
+ private:
+  friend class ClientCallbackUnaryFactory;
+
+  template <class Request, class Response>
+  ClientCallbackUnaryImpl(Call call, ClientContext* context, Request* request,
+                          Response* response,
+                          ::grpc::experimental::ClientUnaryReactor* reactor)
+      : context_(context), call_(call), reactor_(reactor) {
+    this->BindReactor(reactor);
+    // TODO(vjpai): don't assert
+    GPR_CODEGEN_ASSERT(start_ops_.SendMessagePtr(request).ok());
+    start_ops_.ClientSendClose();
+    finish_ops_.RecvMessage(response);
+    finish_ops_.AllowNoMessage();
+  }
+
+  ClientContext* const context_;
+  Call call_;
+  ::grpc::experimental::ClientUnaryReactor* const reactor_;
+
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose,
+            CallOpRecvInitialMetadata>
+      start_ops_;
+  CallbackWithSuccessTag start_tag_;
+
+  CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_;
+  CallbackWithSuccessTag finish_tag_;
+  Status finish_status_;
+
+  // This call will have 2 callbacks: start and finish
+  std::atomic_int callbacks_outstanding_{2};
+  bool started_{false};
+};
+
+class ClientCallbackUnaryFactory {
+ public:
+  template <class Request, class Response>
+  static void Create(ChannelInterface* channel,
+                     const ::grpc::internal::RpcMethod& method,
+                     ClientContext* context, const Request* request,
+                     Response* response,
+                     ::grpc::experimental::ClientUnaryReactor* reactor) {
+    Call call = channel->CreateCall(method, context, channel->CallbackCQ());
+
+    g_core_codegen_interface->grpc_call_ref(call.call());
+
+    new (g_core_codegen_interface->grpc_call_arena_alloc(
+        call.call(), sizeof(ClientCallbackUnaryImpl)))
+        ClientCallbackUnaryImpl(call, context, request, response, reactor);
+  }
+};
+
 }  // namespace internal
 }  // namespace grpc
 

include/grpcpp/impl/codegen/client_context.h

@@ -79,6 +79,7 @@ template <class Response>
 class ClientCallbackReaderImpl;
 template <class Request>
 class ClientCallbackWriterImpl;
+class ClientCallbackUnaryImpl;
 }  // namespace internal
 
 template <class R>
@@ -417,6 +418,7 @@ class ClientContext {
   friend class ::grpc::internal::ClientCallbackReaderImpl;
   template <class Request>
   friend class ::grpc::internal::ClientCallbackWriterImpl;
+  friend class ::grpc::internal::ClientCallbackUnaryImpl;
 
   // Used by friend class CallOpClientRecvStatus
   void set_debug_error_string(const grpc::string& debug_error_string) {

include/grpcpp/impl/codegen/completion_queue.h

@@ -43,9 +43,8 @@ struct grpc_completion_queue;
 
 namespace grpc_impl {
 
-class Server;
 class ServerBuilder;
-}  // namespace grpc_impl
+}
 namespace grpc {
 
 template <class R>
@@ -67,6 +66,7 @@ class Channel;
 class ChannelInterface;
 class ClientContext;
 class CompletionQueue;
+class Server;
 class ServerContext;
 class ServerInterface;
 
@@ -183,8 +183,8 @@ class CompletionQueue : private GrpcLibraryCodegen {
   /// within the \a deadline).  A \a tag points to an arbitrary location usually
   /// employed to uniquely identify an event.
   ///
-  /// \param tag [out] Upon sucess, updated to point to the event's tag.
-  /// \param ok [out] Upon sucess, true if a successful event, false otherwise
+  /// \param tag [out] Upon success, updated to point to the event's tag.
+  /// \param ok [out] Upon success, true if a successful event, false otherwise
   ///        See documentation for CompletionQueue::Next for explanation of ok
   /// \param deadline [in] How long to block in wait for an event.
   ///
@@ -203,8 +203,8 @@ class CompletionQueue : private GrpcLibraryCodegen {
   /// employed to uniquely identify an event.
   ///
   /// \param f [in] Function to execute before calling AsyncNext on this queue.
-  /// \param tag [out] Upon sucess, updated to point to the event's tag.
-  /// \param ok [out] Upon sucess, true if read a regular event, false
+  /// \param tag [out] Upon success, updated to point to the event's tag.
+  /// \param ok [out] Upon success, true if read a regular event, false
   /// otherwise.
   /// \param deadline [in] How long to block in wait for an event.
   ///
@@ -274,7 +274,7 @@ class CompletionQueue : private GrpcLibraryCodegen {
   friend class ::grpc::internal::TemplatedBidiStreamingHandler;
   template <StatusCode code>
   friend class ::grpc::internal::ErrorMethodHandler;
-  friend class ::grpc_impl::Server;
+  friend class ::grpc::Server;
   friend class ::grpc::ServerContext;
   friend class ::grpc::ServerInterface;
   template <class InputMessage, class OutputMessage>
@@ -362,7 +362,7 @@ class CompletionQueue : private GrpcLibraryCodegen {
   /// queue should not really shutdown until all avalanching operations have
   /// been finalized. Note that we maintain the requirement that an avalanche
   /// registration must take place before CQ shutdown (which must be maintained
-  /// elsehwere)
+  /// elsewhere)
   void InitialAvalanching() {
     gpr_atm_rel_store(&avalanches_in_flight_, static_cast<gpr_atm>(1));
   }
@@ -408,8 +408,8 @@ class ServerCompletionQueue : public CompletionQueue {
         polling_type_(polling_type) {}
 
   grpc_cq_polling_type polling_type_;
-  friend class grpc_impl::ServerBuilder;
-  friend class grpc_impl::Server;
+  friend class ::grpc_impl::ServerBuilder;
+  friend class Server;
 };
 
 }  // namespace grpc

include/grpcpp/impl/codegen/message_allocator.h

@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCPP_IMPL_CODEGEN_MESSAGE_ALLOCATOR_H
+#define GRPCPP_IMPL_CODEGEN_MESSAGE_ALLOCATOR_H
+
+namespace grpc {
+namespace experimental {
+
+// This is per rpc struct for the allocator. We can potentially put the grpc
+// call arena in here in the future.
+template <typename RequestT, typename ResponseT>
+struct RpcAllocatorInfo {
+  RequestT* request;
+  ResponseT* response;
+  // per rpc allocator internal state. MessageAllocator can set it when
+  // AllocateMessages is called and use it later.
+  void* allocator_state;
+};
+
+// Implementations need to be thread-safe
+template <typename RequestT, typename ResponseT>
+class MessageAllocator {
+ public:
+  virtual ~MessageAllocator() = default;
+  // Allocate both request and response
+  virtual void AllocateMessages(
+      RpcAllocatorInfo<RequestT, ResponseT>* info) = 0;
+  // Optional: deallocate request early, called by
+  // ServerCallbackRpcController::ReleaseRequest
+  virtual void DeallocateRequest(RpcAllocatorInfo<RequestT, ResponseT>* info) {}
+  // Deallocate response and request (if applicable)
+  virtual void DeallocateMessages(
+      RpcAllocatorInfo<RequestT, ResponseT>* info) = 0;
+};
+
+}  // namespace experimental
+}  // namespace grpc
+
+#endif  // GRPCPP_IMPL_CODEGEN_MESSAGE_ALLOCATOR_H

include/grpcpp/impl/codegen/method_handler_impl.h

@@ -86,8 +86,8 @@ class RpcMethodHandler : public MethodHandler {
     param.call->cq()->Pluck(&ops);
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
     auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(
@@ -191,8 +191,8 @@ class ServerStreamingHandler : public MethodHandler {
     param.call->cq()->Pluck(&ops);
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
     auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(
@@ -327,8 +327,8 @@ class ErrorMethodHandler : public MethodHandler {
     param.call->cq()->Pluck(&ops);
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     // We have to destroy any request payload
     if (req != nullptr) {
       g_core_codegen_interface->grpc_byte_buffer_destroy(req);

include/grpcpp/impl/codegen/rpc_service_method.h

@@ -46,21 +46,25 @@ class MethodHandler {
     /// \param context : the ServerContext structure for this server call
     /// \param req : the request payload, if appropriate for this RPC
     /// \param req_status : the request status after any interceptors have run
+    /// \param handler_data: internal data for the handler.
     /// \param requester : used only by the callback API. It is a function
     ///        called by the RPC Controller to request another RPC (and also
     ///        to set up the state required to make that request possible)
     HandlerParameter(Call* c, ServerContext* context, void* req,
-                     Status req_status, std::function<void()> requester)
+                     Status req_status, void* handler_data,
+                     std::function<void()> requester)
         : call(c),
           server_context(context),
           request(req),
           status(req_status),
+          internal_data(handler_data),
           call_requester(std::move(requester)) {}
     ~HandlerParameter() {}
     Call* call;
     ServerContext* server_context;
     void* request;
     Status status;
+    void* internal_data;
     std::function<void()> call_requester;
   };
   virtual void RunHandler(const HandlerParameter& param) = 0;
@@ -71,7 +75,7 @@ class MethodHandler {
      pointer after calling RunHandler. Ownership of the deserialized request is
      retained by the handler. Returns nullptr if deserialization failed. */
   virtual void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                            Status* status) {
+                            Status* status, void** handler_data) {
     GPR_CODEGEN_ASSERT(req == nullptr);
     return nullptr;
   }

include/grpcpp/impl/codegen/server_callback.h

@@ -28,6 +28,7 @@
 #include <grpcpp/impl/codegen/callback_common.h>
 #include <grpcpp/impl/codegen/config.h>
 #include <grpcpp/impl/codegen/core_codegen_interface.h>
+#include <grpcpp/impl/codegen/message_allocator.h>
 #include <grpcpp/impl/codegen/server_context.h>
 #include <grpcpp/impl/codegen/server_interface.h>
 #include <grpcpp/impl/codegen/status.h>
@@ -37,11 +38,43 @@ namespace grpc {
 // Declare base class of all reactors as internal
 namespace internal {
 
+// Forward declarations
+template <class Request, class Response>
+class CallbackClientStreamingHandler;
+template <class Request, class Response>
+class CallbackServerStreamingHandler;
+template <class Request, class Response>
+class CallbackBidiHandler;
+
 class ServerReactor {
  public:
   virtual ~ServerReactor() = default;
   virtual void OnDone() = 0;
   virtual void OnCancel() = 0;
+
+ private:
+  friend class ::grpc::ServerContext;
+  template <class Request, class Response>
+  friend class CallbackClientStreamingHandler;
+  template <class Request, class Response>
+  friend class CallbackServerStreamingHandler;
+  template <class Request, class Response>
+  friend class CallbackBidiHandler;
+
+  // The ServerReactor is responsible for tracking when it is safe to call
+  // OnCancel. This function should not be called until after OnStarted is done
+  // and the RPC has completed with a cancellation. This is tracked by counting
+  // how many of these conditions have been met and calling OnCancel when none
+  // remain unmet.
+
+  void MaybeCallOnCancel() {
+    if (on_cancel_conditions_remaining_.fetch_sub(
+            1, std::memory_order_acq_rel) == 1) {
+      OnCancel();
+    }
+  }
+
+  std::atomic_int on_cancel_conditions_remaining_{2};
 };
 
 }  // namespace internal
@@ -103,6 +136,14 @@ class ServerCallbackRpcController {
   /// to be called before the callback completes.
   virtual void SetCancelCallback(std::function<void()> callback) = 0;
   virtual void ClearCancelCallback() = 0;
+
+  // NOTE: This is an API for advanced users who need custom allocators.
+  // Optionally deallocate request early to reduce the size of working set.
+  // A custom MessageAllocator needs to be registered to make use of this.
+  virtual void FreeRequest() = 0;
+  // NOTE: This is an API for advanced users who need custom allocators.
+  // Get and maybe mutate the allocator state associated with the current RPC.
+  virtual void* GetAllocatorState() = 0;
 };
 
 // NOTE: The actual streaming object classes are provided
@@ -253,7 +294,9 @@ class ServerBidiReactor : public internal::ServerReactor {
   void Finish(Status s) { stream_->Finish(std::move(s)); }
 
   /// Notify the application that a streaming RPC has started and that it is now
-  /// ok to call any operation initation method.
+  /// ok to call any operation initiation method. An RPC is considered started
+  /// after the server has received all initial metadata from the client, which
+  /// is a result of the client calling StartCall().
   ///
   /// \param[in] context The context object now associated with this RPC
   virtual void OnStarted(ServerContext* context) {}
@@ -330,7 +373,7 @@ class ServerReadReactor : public internal::ServerReactor {
   ServerCallbackReader<Request>* reader_;
 };
 
-/// \a ServerReadReactor is the interface for a server-streaming RPC.
+/// \a ServerWriteReactor is the interface for a server-streaming RPC.
 template <class Request, class Response>
 class ServerWriteReactor : public internal::ServerReactor {
  public:
@@ -413,17 +456,24 @@ class CallbackUnaryHandler : public MethodHandler {
                          experimental::ServerCallbackRpcController*)>
           func)
       : func_(func) {}
+
+  void SetMessageAllocator(
+      experimental::MessageAllocator<RequestType, ResponseType>* allocator) {
+    allocator_ = allocator;
+  }
+
   void RunHandler(const HandlerParameter& param) final {
     // Arena allocate a controller structure (that includes request/response)
     g_core_codegen_interface->grpc_call_ref(param.call->call());
+    auto* allocator_info =
+        static_cast<experimental::RpcAllocatorInfo<RequestType, ResponseType>*>(
+            param.internal_data);
     auto* controller = new (g_core_codegen_interface->grpc_call_arena_alloc(
         param.call->call(), sizeof(ServerCallbackRpcControllerImpl)))
-        ServerCallbackRpcControllerImpl(
-            param.server_context, param.call,
-            static_cast<RequestType*>(param.request),
-            std::move(param.call_requester));
+        ServerCallbackRpcControllerImpl(param.server_context, param.call,
+                                        allocator_info, allocator_,
+                                        std::move(param.call_requester));
     Status status = param.status;
-
     if (status.ok()) {
       // Call the actual function handler and expect the user to call finish
       CatchingCallback(func_, param.server_context, controller->request(),
@@ -434,18 +484,41 @@ class CallbackUnaryHandler : public MethodHandler {
     }
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
-    auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(
-        call, sizeof(RequestType))) RequestType();
+    RequestType* request = nullptr;
+    experimental::RpcAllocatorInfo<RequestType, ResponseType>* allocator_info =
+        new (g_core_codegen_interface->grpc_call_arena_alloc(
+            call, sizeof(*allocator_info)))
+            experimental::RpcAllocatorInfo<RequestType, ResponseType>();
+    if (allocator_ != nullptr) {
+      allocator_->AllocateMessages(allocator_info);
+    } else {
+      allocator_info->request =
+          new (g_core_codegen_interface->grpc_call_arena_alloc(
+              call, sizeof(RequestType))) RequestType();
+      allocator_info->response =
+          new (g_core_codegen_interface->grpc_call_arena_alloc(
+              call, sizeof(ResponseType))) ResponseType();
+    }
+    *handler_data = allocator_info;
+    request = allocator_info->request;
     *status = SerializationTraits<RequestType>::Deserialize(&buf, request);
     buf.Release();
     if (status->ok()) {
       return request;
     }
-    request->~RequestType();
+    // Clean up on deserialization failure.
+    if (allocator_ != nullptr) {
+      allocator_->DeallocateMessages(allocator_info);
+    } else {
+      allocator_info->request->~RequestType();
+      allocator_info->response->~ResponseType();
+      allocator_info->request = nullptr;
+      allocator_info->response = nullptr;
+    }
     return nullptr;
   }
 
@@ -453,6 +526,8 @@ class CallbackUnaryHandler : public MethodHandler {
   std::function<void(ServerContext*, const RequestType*, ResponseType*,
                      experimental::ServerCallbackRpcController*)>
       func_;
+  experimental::MessageAllocator<RequestType, ResponseType>* allocator_ =
+      nullptr;
 
   // The implementation class of ServerCallbackRpcController is a private member
   // of CallbackUnaryHandler since it is never exposed anywhere, and this allows
@@ -473,8 +548,9 @@ class CallbackUnaryHandler : public MethodHandler {
       }
       // The response is dropped if the status is not OK.
       if (s.ok()) {
-        finish_ops_.ServerSendStatus(&ctx_->trailing_metadata_,
-                                     finish_ops_.SendMessagePtr(&resp_));
+        finish_ops_.ServerSendStatus(
+            &ctx_->trailing_metadata_,
+            finish_ops_.SendMessagePtr(allocator_info_->response));
       } else {
         finish_ops_.ServerSendStatus(&ctx_->trailing_metadata_, s);
       }
@@ -512,28 +588,50 @@ class CallbackUnaryHandler : public MethodHandler {
 
     void ClearCancelCallback() override { ctx_->ClearCancelCallback(); }
 
+    void FreeRequest() override {
+      if (allocator_ != nullptr) {
+        allocator_->DeallocateRequest(allocator_info_);
+      }
+    }
+
+    void* GetAllocatorState() override {
+      return allocator_info_->allocator_state;
+    }
+
    private:
     friend class CallbackUnaryHandler<RequestType, ResponseType>;
 
-    ServerCallbackRpcControllerImpl(ServerContext* ctx, Call* call,
-                                    const RequestType* req,
-                                    std::function<void()> call_requester)
+    ServerCallbackRpcControllerImpl(
+        ServerContext* ctx, Call* call,
+        experimental::RpcAllocatorInfo<RequestType, ResponseType>*
+            allocator_info,
+        experimental::MessageAllocator<RequestType, ResponseType>* allocator,
+        std::function<void()> call_requester)
         : ctx_(ctx),
           call_(*call),
-          req_(req),
+          allocator_info_(allocator_info),
+          allocator_(allocator),
           call_requester_(std::move(call_requester)) {
       ctx_->BeginCompletionOp(call, [this](bool) { MaybeDone(); }, nullptr);
     }
 
-    ~ServerCallbackRpcControllerImpl() { req_->~RequestType(); }
-
-    const RequestType* request() { return req_; }
-    ResponseType* response() { return &resp_; }
+    const RequestType* request() { return allocator_info_->request; }
+    ResponseType* response() { return allocator_info_->response; }
 
     void MaybeDone() {
       if (--callbacks_outstanding_ == 0) {
         grpc_call* call = call_.call();
         auto call_requester = std::move(call_requester_);
+        if (allocator_ != nullptr) {
+          allocator_->DeallocateMessages(allocator_info_);
+        } else {
+          if (allocator_info_->request != nullptr) {
+            allocator_info_->request->~RequestType();
+          }
+          if (allocator_info_->response != nullptr) {
+            allocator_info_->response->~ResponseType();
+          }
+        }
         this->~ServerCallbackRpcControllerImpl();  // explicitly call destructor
         g_core_codegen_interface->grpc_call_unref(call);
         call_requester();
@@ -549,8 +647,8 @@ class CallbackUnaryHandler : public MethodHandler {
 
     ServerContext* ctx_;
     Call call_;
-    const RequestType* req_;
-    ResponseType resp_;
+    experimental::RpcAllocatorInfo<RequestType, ResponseType>* allocator_info_;
+    experimental::MessageAllocator<RequestType, ResponseType>* allocator_;
     std::function<void()> call_requester_;
     std::atomic_int callbacks_outstanding_{
         2};  // reserve for Finish and CompletionOp
@@ -588,6 +686,8 @@ class CallbackClientStreamingHandler : public MethodHandler {
 
     reader->BindReactor(reactor);
     reactor->OnStarted(param.server_context, reader->response());
+    // The earliest that OnCancel can be called is after OnStarted is done.
+    reactor->MaybeCallOnCancel();
     reader->MaybeDone();
   }
 
@@ -730,11 +830,13 @@ class CallbackServerStreamingHandler : public MethodHandler {
                                  std::move(param.call_requester), reactor);
     writer->BindReactor(reactor);
     reactor->OnStarted(param.server_context, writer->request());
+    // The earliest that OnCancel can be called is after OnStarted is done.
+    reactor->MaybeCallOnCancel();
     writer->MaybeDone();
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
     auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(
@@ -906,6 +1008,8 @@ class CallbackBidiHandler : public MethodHandler {
 
     stream->BindReactor(reactor);
     reactor->OnStarted(param.server_context);
+    // The earliest that OnCancel can be called is after OnStarted is done.
+    reactor->MaybeCallOnCancel();
     stream->MaybeDone();
   }
 

include/grpcpp/impl/codegen/server_context.h

@@ -41,14 +41,11 @@ struct grpc_metadata;
 struct grpc_call;
 struct census_context;
 
-namespace grpc_impl {
-
-class Server;
-}  // namespace grpc_impl
 namespace grpc {
 class ClientContext;
 class GenericServerContext;
 class CompletionQueue;
+class Server;
 class ServerInterface;
 template <class W, class R>
 class ServerAsyncReader;
@@ -272,7 +269,7 @@ class ServerContext {
   friend class ::grpc::testing::InteropServerContextInspector;
   friend class ::grpc::testing::ServerContextTestSpouse;
   friend class ::grpc::ServerInterface;
-  friend class ::grpc_impl::Server;
+  friend class ::grpc::Server;
   template <class W, class R>
   friend class ::grpc::ServerAsyncReader;
   template <class W>

include/grpcpp/impl/codegen/server_interface.h

@@ -149,7 +149,7 @@ class ServerInterface : public internal::CallHook {
   /// 192.168.1.1:31416, [::1]:27182, etc.).
   /// \params creds The credentials associated with the server.
   ///
-  /// \return bound port number on sucess, 0 on failure.
+  /// \return bound port number on success, 0 on failure.
   ///
   /// \warning It's an error to call this method on an already started server.
   virtual int AddListeningPort(const grpc::string& addr,

include/grpcpp/impl/codegen/service_type.h

@@ -26,13 +26,10 @@
 #include <grpcpp/impl/codegen/server_interface.h>
 #include <grpcpp/impl/codegen/status.h>
 
-namespace grpc_impl {
-
-class Server;
-}  // namespace grpc_impl
 namespace grpc {
 
 class CompletionQueue;
+class Server;
 class ServerInterface;
 class ServerCompletionQueue;
 class ServerContext;
@@ -135,6 +132,11 @@ class Service {
           internal::RpcServiceMethod::ApiType::RAW_CALL_BACK);
     }
 
+    internal::MethodHandler* GetHandler(int index) {
+      size_t idx = static_cast<size_t>(index);
+      return service_->methods_[idx]->handler();
+    }
+
    private:
     Service* service_;
   };
@@ -231,7 +233,7 @@ class Service {
   }
 
  private:
-  friend class grpc_impl::Server;
+  friend class Server;
   friend class ServerInterface;
   ServerInterface* server_;
   std::vector<std::unique_ptr<internal::RpcServiceMethod>> methods_;

include/grpcpp/impl/codegen/sync.h

@@ -19,8 +19,15 @@
 #ifndef GRPCPP_IMPL_CODEGEN_SYNC_H
 #define GRPCPP_IMPL_CODEGEN_SYNC_H
 
-#include <grpc/impl/codegen/log.h>
 #include <grpc/impl/codegen/port_platform.h>
+
+#ifdef GPR_HAS_PTHREAD_H
+#include <pthread.h>
+#endif
+
+#include <mutex>
+
+#include <grpc/impl/codegen/log.h>
 #include <grpc/impl/codegen/sync.h>
 
 #include <grpcpp/impl/codegen/core_codegen_interface.h>
@@ -49,7 +56,13 @@ class Mutex {
   const gpr_mu* get() const { return &mu_; }
 
  private:
-  gpr_mu mu_;
+  union {
+    gpr_mu mu_;
+    std::mutex do_not_use_sth_;
+#ifdef GPR_HAS_PTHREAD_H
+    pthread_mutex_t do_not_use_pth_;
+#endif
+  };
 };
 
 // MutexLock is a std::

include/grpcpp/impl/codegen/sync_stream.h

@@ -180,7 +180,7 @@ class ClientReader final : public ClientReaderInterface<R> {
   ///
   //  Side effect:
   ///   Once complete, the initial metadata read from
-  ///   the server will be accessable through the \a ClientContext used to
+  ///   the server will be accessible through the \a ClientContext used to
   ///   construct this object.
   void WaitForInitialMetadata() override {
     GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
@@ -298,7 +298,7 @@ class ClientWriter : public ClientWriterInterface<W> {
   ///
   //  Side effect:
   ///   Once complete, the initial metadata read from the server will be
-  ///   accessable through the \a ClientContext used to construct this object.
+  ///   accessible through the \a ClientContext used to construct this object.
   void WaitForInitialMetadata() {
     GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 
@@ -449,7 +449,7 @@ class ClientReaderWriter final : public ClientReaderWriterInterface<W, R> {
   /// with or after the \a Finish method.
   ///
   /// Once complete, the initial metadata read from the server will be
-  /// accessable through the \a ClientContext used to construct this object.
+  /// accessible through the \a ClientContext used to construct this object.
   void WaitForInitialMetadata() override {
     GPR_CODEGEN_ASSERT(!context_->initial_metadata_received_);
 

include/grpcpp/impl/server_initializer_impl.h

@@ -26,10 +26,10 @@
 
 namespace grpc {
 
+class Server;
 class Service;
 }  // namespace grpc
 namespace grpc_impl {
-class Server;
 
 class ServerInitializer {
  public:

include/grpcpp/security/credentials.h

@@ -98,10 +98,10 @@ class ChannelCredentials : private GrpcLibraryCodegen {
   // This function should have been a pure virtual function, but it is
   // implemented as a virtual function so that it does not break API.
   virtual std::shared_ptr<Channel> CreateChannelWithInterceptors(
-      const grpc::string& target, const ChannelArguments& args,
+      const grpc::string& /* target */, const ChannelArguments& /* args */,
       std::vector<
           std::unique_ptr<experimental::ClientInterceptorFactoryInterface>>
-          interceptor_creators) {
+      /* interceptor_creators */) {
     return nullptr;
   }
 };

include/grpcpp/security/server_credentials.h

@@ -21,10 +21,6 @@
 
 #include <grpcpp/security/server_credentials_impl.h>
 
-namespace grpc_impl {
-
-class Server;
-}  // namespace grpc_impl
 namespace grpc {
 
 typedef ::grpc_impl::ServerCredentials ServerCredentials;

include/grpcpp/security/server_credentials_impl.h

@@ -30,10 +30,10 @@ struct grpc_server;
 
 namespace grpc {
 
+class Server;
 struct SslServerCredentialsOptions;
 }  // namespace grpc
 namespace grpc_impl {
-class Server;
 
 /// Wrapper around \a grpc_server_credentials, a way to authenticate a server.
 class ServerCredentials {
@@ -46,7 +46,7 @@ class ServerCredentials {
       const std::shared_ptr<grpc::AuthMetadataProcessor>& processor) = 0;
 
  private:
-  friend class ::grpc_impl::Server;
+  friend class ::grpc::Server;
 
   /// Tries to bind \a server to the given \a addr (eg, localhost:1234,
   /// 192.168.1.1:31416, [::1]:27182, etc.)

include/grpcpp/server.h

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2019 gRPC authors.
+ * Copyright 2015 gRPC authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,11 +19,333 @@
 #ifndef GRPCPP_SERVER_H
 #define GRPCPP_SERVER_H
 
-#include <grpcpp/server_impl.h>
+#include <condition_variable>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <vector>
 
+#include <grpc/compression.h>
+#include <grpc/support/atm.h>
+#include <grpcpp/completion_queue.h>
+#include <grpcpp/health_check_service_interface.h>
+#include <grpcpp/impl/call.h>
+#include <grpcpp/impl/codegen/client_interceptor.h>
+#include <grpcpp/impl/codegen/grpc_library.h>
+#include <grpcpp/impl/codegen/server_interface.h>
+#include <grpcpp/impl/rpc_service_method.h>
+#include <grpcpp/security/server_credentials.h>
+#include <grpcpp/support/channel_arguments.h>
+#include <grpcpp/support/config.h>
+#include <grpcpp/support/status.h>
+
+struct grpc_server;
+
+namespace grpc_impl {
+
+class ServerInitializer;
+}
 namespace grpc {
 
-typedef ::grpc_impl::Server Server;
+class AsyncGenericService;
+class ServerContext;
+
+/// Represents a gRPC server.
+///
+/// Use a \a grpc::ServerBuilder to create, configure, and start
+/// \a Server instances.
+class Server : public ServerInterface, private GrpcLibraryCodegen {
+ public:
+  ~Server();
+
+  /// Block until the server shuts down.
+  ///
+  /// \warning The server must be either shutting down or some other thread must
+  /// call \a Shutdown for this function to ever return.
+  void Wait() override;
+
+  /// Global callbacks are a set of hooks that are called when server
+  /// events occur.  \a SetGlobalCallbacks method is used to register
+  /// the hooks with gRPC.  Note that
+  /// the \a GlobalCallbacks instance will be shared among all
+  /// \a Server instances in an application and can be set exactly
+  /// once per application.
+  class GlobalCallbacks {
+   public:
+    virtual ~GlobalCallbacks() {}
+    /// Called before server is created.
+    virtual void UpdateArguments(ChannelArguments* args) {}
+    /// Called before application callback for each synchronous server request
+    virtual void PreSynchronousRequest(ServerContext* context) = 0;
+    /// Called after application callback for each synchronous server request
+    virtual void PostSynchronousRequest(ServerContext* context) = 0;
+    /// Called before server is started.
+    virtual void PreServerStart(Server* server) {}
+    /// Called after a server port is added.
+    virtual void AddPort(Server* server, const grpc::string& addr,
+                         ServerCredentials* creds, int port) {}
+  };
+  /// Set the global callback object. Can only be called once per application.
+  /// Does not take ownership of callbacks, and expects the pointed to object
+  /// to be alive until all server objects in the process have been destroyed.
+  /// The same \a GlobalCallbacks object will be used throughout the
+  /// application and is shared among all \a Server objects.
+  static void SetGlobalCallbacks(GlobalCallbacks* callbacks);
+
+  /// Returns a \em raw pointer to the underlying \a grpc_server instance.
+  /// EXPERIMENTAL:  for internal/test use only
+  grpc_server* c_server();
+
+  /// Returns the health check service.
+  HealthCheckServiceInterface* GetHealthCheckService() const {
+    return health_check_service_.get();
+  }
+
+  /// Establish a channel for in-process communication
+  std::shared_ptr<Channel> InProcessChannel(const ChannelArguments& args);
+
+  /// NOTE: class experimental_type is not part of the public API of this class.
+  /// TODO(yashykt): Integrate into public API when this is no longer
+  /// experimental.
+  class experimental_type {
+   public:
+    explicit experimental_type(Server* server) : server_(server) {}
+
+    /// Establish a channel for in-process communication with client
+    /// interceptors
+    std::shared_ptr<Channel> InProcessChannelWithInterceptors(
+        const ChannelArguments& args,
+        std::vector<
+            std::unique_ptr<experimental::ClientInterceptorFactoryInterface>>
+            interceptor_creators);
+
+   private:
+    Server* server_;
+  };
+
+  /// NOTE: The function experimental() is not stable public API. It is a view
+  /// to the experimental components of this class. It may be changed or removed
+  /// at any time.
+  experimental_type experimental() { return experimental_type(this); }
+
+ protected:
+  /// Register a service. This call does not take ownership of the service.
+  /// The service must exist for the lifetime of the Server instance.
+  bool RegisterService(const grpc::string* host, Service* service) override;
+
+  /// Try binding the server to the given \a addr endpoint
+  /// (port, and optionally including IP address to bind to).
+  ///
+  /// It can be invoked multiple times. Should be used before
+  /// starting the server.
+  ///
+  /// \param addr The address to try to bind to the server (eg, localhost:1234,
+  /// 192.168.1.1:31416, [::1]:27182, etc.).
+  /// \param creds The credentials associated with the server.
+  ///
+  /// \return bound port number on success, 0 on failure.
+  ///
+  /// \warning It is an error to call this method on an already started server.
+  int AddListeningPort(const grpc::string& addr,
+                       ServerCredentials* creds) override;
+
+  /// NOTE: This is *NOT* a public API. The server constructors are supposed to
+  /// be used by \a ServerBuilder class only. The constructor will be made
+  /// 'private' very soon.
+  ///
+  /// Server constructors. To be used by \a ServerBuilder only.
+  ///
+  /// \param max_message_size Maximum message length that the channel can
+  /// receive.
+  ///
+  /// \param args The channel args
+  ///
+  /// \param sync_server_cqs The completion queues to use if the server is a
+  /// synchronous server (or a hybrid server). The server polls for new RPCs on
+  /// these queues
+  ///
+  /// \param min_pollers The minimum number of polling threads per server
+  /// completion queue (in param sync_server_cqs) to use for listening to
+  /// incoming requests (used only in case of sync server)
+  ///
+  /// \param max_pollers The maximum number of polling threads per server
+  /// completion queue (in param sync_server_cqs) to use for listening to
+  /// incoming requests (used only in case of sync server)
+  ///
+  /// \param sync_cq_timeout_msec The timeout to use when calling AsyncNext() on
+  /// server completion queues passed via sync_server_cqs param.
+  Server(int max_message_size, ChannelArguments* args,
+         std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
+             sync_server_cqs,
+         int min_pollers, int max_pollers, int sync_cq_timeout_msec,
+         grpc_resource_quota* server_rq = nullptr,
+         std::vector<
+             std::unique_ptr<experimental::ServerInterceptorFactoryInterface>>
+             interceptor_creators = std::vector<std::unique_ptr<
+                 experimental::ServerInterceptorFactoryInterface>>());
+
+  /// Start the server.
+  ///
+  /// \param cqs Completion queues for handling asynchronous services. The
+  /// caller is required to keep all completion queues live until the server is
+  /// destroyed.
+  /// \param num_cqs How many completion queues does \a cqs hold.
+  void Start(ServerCompletionQueue** cqs, size_t num_cqs) override;
+
+  grpc_server* server() override { return server_; }
+
+ private:
+  std::vector<std::unique_ptr<experimental::ServerInterceptorFactoryInterface>>*
+  interceptor_creators() override {
+    return &interceptor_creators_;
+  }
+
+  friend class AsyncGenericService;
+  friend class grpc_impl::ServerBuilder;
+  friend class grpc_impl::ServerInitializer;
+
+  class SyncRequest;
+  class CallbackRequestBase;
+  template <class ServerContextType>
+  class CallbackRequest;
+  class UnimplementedAsyncRequest;
+  class UnimplementedAsyncResponse;
+
+  /// SyncRequestThreadManager is an implementation of ThreadManager. This class
+  /// is responsible for polling for incoming RPCs and calling the RPC handlers.
+  /// This is only used in case of a Sync server (i.e a server exposing a sync
+  /// interface)
+  class SyncRequestThreadManager;
+
+  /// Register a generic service. This call does not take ownership of the
+  /// service. The service must exist for the lifetime of the Server instance.
+  void RegisterAsyncGenericService(AsyncGenericService* service) override;
+
+  /// NOTE: class experimental_registration_type is not part of the public API
+  /// of this class
+  /// TODO(vjpai): Move these contents to the public API of Server when
+  ///              they are no longer experimental
+  class experimental_registration_type final
+      : public experimental_registration_interface {
+   public:
+    explicit experimental_registration_type(Server* server) : server_(server) {}
+    void RegisterCallbackGenericService(
+        experimental::CallbackGenericService* service) override {
+      server_->RegisterCallbackGenericService(service);
+    }
+
+   private:
+    Server* server_;
+  };
+
+  /// TODO(vjpai): Mark this override when experimental type above is deleted
+  void RegisterCallbackGenericService(
+      experimental::CallbackGenericService* service);
+
+  /// NOTE: The function experimental_registration() is not stable public API.
+  /// It is a view to the experimental components of this class. It may be
+  /// changed or removed at any time.
+  experimental_registration_interface* experimental_registration() override {
+    return &experimental_registration_;
+  }
+
+  void PerformOpsOnCall(internal::CallOpSetInterface* ops,
+                        internal::Call* call) override;
+
+  void ShutdownInternal(gpr_timespec deadline) override;
+
+  int max_receive_message_size() const override {
+    return max_receive_message_size_;
+  }
+
+  CompletionQueue* CallbackCQ() override;
+
+  grpc_impl::ServerInitializer* initializer();
+
+  // A vector of interceptor factory objects.
+  // This should be destroyed after health_check_service_ and this requirement
+  // is satisfied by declaring interceptor_creators_ before
+  // health_check_service_. (C++ mandates that member objects be destroyed in
+  // the reverse order of initialization.)
+  std::vector<std::unique_ptr<experimental::ServerInterceptorFactoryInterface>>
+      interceptor_creators_;
+
+  const int max_receive_message_size_;
+
+  /// The following completion queues are ONLY used in case of Sync API
+  /// i.e. if the server has any services with sync methods. The server uses
+  /// these completion queues to poll for new RPCs
+  std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
+      sync_server_cqs_;
+
+  /// List of \a ThreadManager instances (one for each cq in
+  /// the \a sync_server_cqs)
+  std::vector<std::unique_ptr<SyncRequestThreadManager>> sync_req_mgrs_;
+
+  // Outstanding unmatched callback requests, indexed by method.
+  // NOTE: Using a gpr_atm rather than atomic_int because atomic_int isn't
+  //       copyable or movable and thus will cause compilation errors. We
+  //       actually only want to extend the vector before the threaded use
+  //       starts, but this is still a limitation.
+  std::vector<gpr_atm> callback_unmatched_reqs_count_;
+
+  // List of callback requests to start when server actually starts.
+  std::list<CallbackRequestBase*> callback_reqs_to_start_;
+
+  // For registering experimental callback generic service; remove when that
+  // method longer experimental
+  experimental_registration_type experimental_registration_{this};
+
+  // Server status
+  grpc::internal::Mutex mu_;
+  bool started_;
+  bool shutdown_;
+  bool shutdown_notified_;  // Was notify called on the shutdown_cv_
+
+  grpc::internal::CondVar shutdown_cv_;
+
+  // It is ok (but not required) to nest callback_reqs_mu_ under mu_ .
+  // Incrementing callback_reqs_outstanding_ is ok without a lock but it must be
+  // decremented under the lock in case it is the last request and enables the
+  // server shutdown. The increment is performance-critical since it happens
+  // during periods of increasing load; the decrement happens only when memory
+  // is maxed out, during server shutdown, or (possibly in a future version)
+  // during decreasing load, so it is less performance-critical.
+  grpc::internal::Mutex callback_reqs_mu_;
+  grpc::internal::CondVar callback_reqs_done_cv_;
+  std::atomic_int callback_reqs_outstanding_{0};
+
+  std::shared_ptr<GlobalCallbacks> global_callbacks_;
+
+  std::vector<grpc::string> services_;
+  bool has_async_generic_service_{false};
+  bool has_callback_generic_service_{false};
+
+  // Pointer to the wrapped grpc_server.
+  grpc_server* server_;
+
+  std::unique_ptr<grpc_impl::ServerInitializer> server_initializer_;
+
+  std::unique_ptr<HealthCheckServiceInterface> health_check_service_;
+  bool health_check_service_disabled_;
+
+  // When appropriate, use a default callback generic service to handle
+  // unimplemented methods
+  std::unique_ptr<experimental::CallbackGenericService> unimplemented_service_;
+
+  // A special handler for resource exhausted in sync case
+  std::unique_ptr<internal::MethodHandler> resource_exhausted_handler_;
+
+  // Handler for callback generic service, if any
+  std::unique_ptr<internal::MethodHandler> generic_handler_;
+
+  // callback_cq_ references the callbackable completion queue associated
+  // with this server (if any). It is set on the first call to CallbackCQ().
+  // It is _not owned_ by the server; ownership belongs with its internal
+  // shutdown callback tag (invoked when the CQ is fully shutdown).
+  // It is protected by mu_
+  CompletionQueue* callback_cq_ = nullptr;
+};
 
 }  // namespace grpc
 

include/grpcpp/server_builder.h

@@ -23,7 +23,6 @@
 
 namespace grpc_impl {
 
-class Server;
 class ServerCredentials;
 class ResourceQuota;
 }  // namespace grpc_impl

include/grpcpp/server_builder_impl.h

@@ -31,7 +31,6 @@
 #include <grpcpp/impl/codegen/server_interceptor.h>
 #include <grpcpp/impl/server_builder_option.h>
 #include <grpcpp/impl/server_builder_plugin.h>
-#include <grpcpp/server.h>
 #include <grpcpp/support/config.h>
 
 struct grpc_resource_quota;
@@ -45,6 +44,7 @@ namespace grpc {
 
 class AsyncGenericService;
 class CompletionQueue;
+class Server;
 class ServerCompletionQueue;
 class Service;
 

include/grpcpp/support/message_allocator.h

@@ -0,0 +1,24 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCPP_SUPPORT_MESSAGE_ALLOCATOR_H
+#define GRPCPP_SUPPORT_MESSAGE_ALLOCATOR_H
+
+#include <grpcpp/impl/codegen/message_allocator.h>
+
+#endif  // GRPCPP_SUPPORT_MESSAGE_ALLOCATOR_H

package.xml

@@ -104,6 +104,7 @@
     <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/useful.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/abstract.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/arena.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/atomic.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/fork.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/manual_constructor.h" role="src" />
@@ -114,7 +115,6 @@
     <file baseinstalldir="/" name="src/core/lib/gprpp/thd.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/profiling/timers.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/alloc.cc" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/gpr/arena.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/atm.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/cpu_iphone.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/cpu_linux.cc" role="src" />
@@ -147,6 +147,7 @@
     <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/tmpfile_windows.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gpr/wrap_memcpy.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/gprpp/arena.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/fork.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/thd_posix.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/gprpp/thd_windows.cc" role="src" />
@@ -332,6 +333,7 @@
     <file baseinstalldir="/" name="src/core/lib/channel/handshaker_registry.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/status_util.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/algorithm_metadata.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/compression/compression_args.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/compression_internal.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/message_compress.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/stream_compression.h" role="src" />
@@ -486,6 +488,7 @@
     <file baseinstalldir="/" name="src/core/lib/channel/handshaker_registry.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/channel/status_util.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/compression.cc" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/compression/compression_args.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/compression_internal.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/message_compress.cc" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/stream_compression.cc" role="src" />

src/compiler/cpp_generator.cc

@@ -84,7 +84,7 @@ void PrintIncludes(grpc_generator::Printer* printer,
 }
 
 grpc::string GetHeaderPrologue(grpc_generator::File* file,
-                               const Parameters& /*params*/) {
+                               const Parameters& params) {
   grpc::string output;
   {
     // Scope the output stream so it closes and finalizes output to the string.
@@ -94,7 +94,9 @@ grpc::string GetHeaderPrologue(grpc_generator::File* file,
     vars["filename"] = file->filename();
     vars["filename_identifier"] = FilenameIdentifier(file->filename());
     vars["filename_base"] = file->filename_without_ext();
-    vars["message_header_ext"] = kCppGeneratorMessageHeaderExt;
+    vars["message_header_ext"] = params.message_header_extension.empty()
+                                     ? kCppGeneratorMessageHeaderExt
+                                     : params.message_header_extension;
 
     printer->Print(vars, "// Generated by the gRPC C++ plugin.\n");
     printer->Print(vars,
@@ -115,6 +117,13 @@ grpc::string GetHeaderPrologue(grpc_generator::File* file,
   return output;
 }
 
+// Convert from "a/b/c.proto" to "#include \"a/b/c$message_header_ext$\"\n"
+grpc::string ImportInludeFromProtoName(const grpc::string& proto_name) {
+  return grpc::string("#include \"") +
+         proto_name.substr(0, proto_name.size() - 6) +
+         grpc::string("$message_header_ext$\"\n");
+}
+
 grpc::string GetHeaderIncludes(grpc_generator::File* file,
                                const Parameters& params) {
   grpc::string output;
@@ -146,12 +155,30 @@ grpc::string GetHeaderIncludes(grpc_generator::File* file,
                   params.grpc_search_path);
     printer->Print(vars, "\n");
     printer->Print(vars, "namespace grpc {\n");
+    printer->Print(vars, "namespace experimental {\n");
+    printer->Print(vars, "template <typename RequestT, typename ResponseT>\n");
+    printer->Print(vars, "class MessageAllocator;\n");
+    printer->Print(vars, "}  // namespace experimental\n");
     printer->Print(vars, "class CompletionQueue;\n");
     printer->Print(vars, "class Channel;\n");
     printer->Print(vars, "class ServerCompletionQueue;\n");
     printer->Print(vars, "class ServerContext;\n");
     printer->Print(vars, "}  // namespace grpc\n\n");
 
+    vars["message_header_ext"] = params.message_header_extension.empty()
+                                     ? kCppGeneratorMessageHeaderExt
+                                     : params.message_header_extension;
+
+    if (params.include_import_headers) {
+      const std::vector<grpc::string> import_names = file->GetImportNames();
+      for (const auto& import_name : import_names) {
+        const grpc::string include_name =
+            ImportInludeFromProtoName(import_name);
+        printer->Print(vars, include_name.c_str());
+      }
+      printer->PrintRaw("\n");
+    }
+
     if (!file->package().empty()) {
       std::vector<grpc::string> parts = file->package_parts();
 
@@ -584,6 +611,14 @@ void PrintHeaderClientMethodCallbackInterfaces(
                    "virtual void $Method$(::grpc::ClientContext* context, "
                    "const ::grpc::ByteBuffer* request, $Response$* response, "
                    "std::function<void(::grpc::Status)>) = 0;\n");
+    printer->Print(*vars,
+                   "virtual void $Method$(::grpc::ClientContext* context, "
+                   "const $Request$* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) = 0;\n");
+    printer->Print(*vars,
+                   "virtual void $Method$(::grpc::ClientContext* context, "
+                   "const ::grpc::ByteBuffer* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) = 0;\n");
   } else if (ClientOnlyStreaming(method)) {
     printer->Print(*vars,
                    "virtual void $Method$(::grpc::ClientContext* context, "
@@ -650,6 +685,16 @@ void PrintHeaderClientMethodCallback(grpc_generator::Printer* printer,
                    "void $Method$(::grpc::ClientContext* context, "
                    "const ::grpc::ByteBuffer* request, $Response$* response, "
                    "std::function<void(::grpc::Status)>) override;\n");
+    printer->Print(
+        *vars,
+        "void $Method$(::grpc::ClientContext* context, "
+        "const $Request$* request, $Response$* response, "
+        "::grpc::experimental::ClientUnaryReactor* reactor) override;\n");
+    printer->Print(
+        *vars,
+        "void $Method$(::grpc::ClientContext* context, "
+        "const ::grpc::ByteBuffer* request, $Response$* response, "
+        "::grpc::experimental::ClientUnaryReactor* reactor) override;\n");
   } else if (ClientOnlyStreaming(method)) {
     printer->Print(*vars,
                    "void $Method$(::grpc::ClientContext* context, "
@@ -970,7 +1015,15 @@ void PrintHeaderServerMethodCallback(
         "controller) {\n"
         "               return this->$"
         "Method$(context, request, response, controller);\n"
-        "             }));\n");
+        "             }));\n}\n");
+    printer->Print(*vars,
+                   "void SetMessageAllocatorFor_$Method$(\n"
+                   "    ::grpc::experimental::MessageAllocator< "
+                   "$RealRequest$, $RealResponse$>* allocator) {\n"
+                   "  static_cast<::grpc::internal::CallbackUnaryHandler< "
+                   "$RealRequest$, $RealResponse$>*>(\n"
+                   "      ::grpc::Service::experimental().GetHandler($Idx$))\n"
+                   "          ->SetMessageAllocator(allocator);\n");
   } else if (ClientOnlyStreaming(method)) {
     printer->Print(
         *vars,
@@ -1556,7 +1609,7 @@ grpc::string GetHeaderEpilogue(grpc_generator::File* file,
 }
 
 grpc::string GetSourcePrologue(grpc_generator::File* file,
-                               const Parameters& /*params*/) {
+                               const Parameters& params) {
   grpc::string output;
   {
     // Scope the output stream so it closes and finalizes output to the string.
@@ -1565,7 +1618,9 @@ grpc::string GetSourcePrologue(grpc_generator::File* file,
 
     vars["filename"] = file->filename();
     vars["filename_base"] = file->filename_without_ext();
-    vars["message_header_ext"] = kCppGeneratorMessageHeaderExt;
+    vars["message_header_ext"] = params.message_header_extension.empty()
+                                     ? kCppGeneratorMessageHeaderExt
+                                     : params.message_header_extension;
     vars["service_header_ext"] = kCppGeneratorServiceHeaderExt;
 
     printer->Print(vars, "// Generated by the gRPC C++ plugin.\n");
@@ -1587,7 +1642,6 @@ grpc::string GetSourceIncludes(grpc_generator::File* file,
     // Scope the output stream so it closes and finalizes output to the string.
     auto printer = file->CreatePrinter(&output);
     std::map<grpc::string, grpc::string> vars;
-
     static const char* headers_strs[] = {
         "functional",
         "grpcpp/impl/codegen/async_stream.h",
@@ -1647,7 +1701,7 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer,
                    "const $Request$* request, $Response$* response, "
                    "std::function<void(::grpc::Status)> f) {\n");
     printer->Print(*vars,
-                   "  return ::grpc::internal::CallbackUnaryCall"
+                   "  ::grpc::internal::CallbackUnaryCall"
                    "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
                    "context, request, response, std::move(f));\n}\n\n");
 
@@ -1657,10 +1711,30 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer,
                    "const ::grpc::ByteBuffer* request, $Response$* response, "
                    "std::function<void(::grpc::Status)> f) {\n");
     printer->Print(*vars,
-                   "  return ::grpc::internal::CallbackUnaryCall"
+                   "  ::grpc::internal::CallbackUnaryCall"
                    "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
                    "context, request, response, std::move(f));\n}\n\n");
 
+    printer->Print(*vars,
+                   "void $ns$$Service$::Stub::experimental_async::$Method$("
+                   "::grpc::ClientContext* context, "
+                   "const $Request$* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) {\n");
+    printer->Print(*vars,
+                   "  ::grpc::internal::ClientCallbackUnaryFactory::Create"
+                   "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
+                   "context, request, response, reactor);\n}\n\n");
+
+    printer->Print(*vars,
+                   "void $ns$$Service$::Stub::experimental_async::$Method$("
+                   "::grpc::ClientContext* context, "
+                   "const ::grpc::ByteBuffer* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) {\n");
+    printer->Print(*vars,
+                   "  ::grpc::internal::ClientCallbackUnaryFactory::Create"
+                   "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
+                   "context, request, response, reactor);\n}\n\n");
+
     for (auto async_prefix : async_prefixes) {
       (*vars)["AsyncPrefix"] = async_prefix.prefix;
       (*vars)["AsyncStart"] = async_prefix.start;
@@ -2046,7 +2120,7 @@ grpc::string GetSourceEpilogue(grpc_generator::File* file,
 
 // TODO(mmukhi): Make sure we need parameters or not.
 grpc::string GetMockPrologue(grpc_generator::File* file,
-                             const Parameters& /*params*/) {
+                             const Parameters& params) {
   grpc::string output;
   {
     // Scope the output stream so it closes and finalizes output to the string.
@@ -2055,7 +2129,9 @@ grpc::string GetMockPrologue(grpc_generator::File* file,
 
     vars["filename"] = file->filename();
     vars["filename_base"] = file->filename_without_ext();
-    vars["message_header_ext"] = kCppGeneratorMessageHeaderExt;
+    vars["message_header_ext"] = params.message_header_extension.empty()
+                                     ? kCppGeneratorMessageHeaderExt
+                                     : params.message_header_extension;
     vars["service_header_ext"] = kCppGeneratorServiceHeaderExt;
 
     printer->Print(vars, "// Generated by the gRPC C++ plugin.\n");
@@ -2065,6 +2141,15 @@ grpc::string GetMockPrologue(grpc_generator::File* file,
 
     printer->Print(vars, "#include \"$filename_base$$message_header_ext$\"\n");
     printer->Print(vars, "#include \"$filename_base$$service_header_ext$\"\n");
+    if (params.include_import_headers) {
+      const std::vector<grpc::string> import_names = file->GetImportNames();
+      for (const auto& import_name : import_names) {
+        const grpc::string include_name =
+            ImportInludeFromProtoName(import_name);
+        printer->Print(vars, include_name.c_str());
+      }
+      printer->PrintRaw("\n");
+    }
     printer->Print(vars, file->additional_headers().c_str());
     printer->Print(vars, "\n");
   }

src/compiler/cpp_generator.h

@@ -56,6 +56,10 @@ struct Parameters {
   grpc::string gmock_search_path;
   // *EXPERIMENTAL* Additional include files in grpc.pb.h
   std::vector<grpc::string> additional_header_includes;
+  // By default, use "pb.h"
+  grpc::string message_header_extension;
+  // Whether to include headers corresponding to imports in source file.
+  bool include_import_headers;
 };
 
 // Return the prologue of the generated header file.

src/compiler/cpp_plugin.cc

@@ -48,6 +48,7 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
     grpc_cpp_generator::Parameters generator_parameters;
     generator_parameters.use_system_headers = true;
     generator_parameters.generate_mock_code = false;
+    generator_parameters.include_import_headers = false;
 
     ProtoBufFile pbfile(file);
 
@@ -83,6 +84,15 @@ class CppGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
         } else if (param[0] == "additional_header_includes") {
           generator_parameters.additional_header_includes =
               grpc_generator::tokenize(param[1], ":");
+        } else if (param[0] == "message_header_extension") {
+          generator_parameters.message_header_extension = param[1];
+        } else if (param[0] == "include_import_headers") {
+          if (param[1] == "true") {
+            generator_parameters.include_import_headers = true;
+          } else if (param[1] != "false") {
+            *error = grpc::string("Invalid parameter: ") + *parameter_string;
+            return false;
+          }
         } else {
           *error = grpc::string("Unknown parameter: ") + *parameter_string;
           return false;

src/compiler/objective_c_generator.cc

@@ -50,7 +50,8 @@ void PrintProtoRpcDeclarationAsPragma(
 }
 
 template <typename DescriptorType>
-static void PrintAllComments(const DescriptorType* desc, Printer* printer) {
+static void PrintAllComments(const DescriptorType* desc, Printer* printer,
+                             bool deprecated = false) {
   std::vector<grpc::string> comments;
   grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_LEADING_DETACHED,
                              &comments);
@@ -70,17 +71,20 @@ static void PrintAllComments(const DescriptorType* desc, Printer* printer) {
     }
     printer->Print("\n");
   }
-  printer->Print(" *\n");
-  printer->Print(
-      " * This method belongs to a set of APIs that have been deprecated. Using"
-      " the v2 API is recommended.\n");
+  if (deprecated) {
+    printer->Print(" *\n");
+    printer->Print(
+        " * This method belongs to a set of APIs that have been deprecated. "
+        "Using"
+        " the v2 API is recommended.\n");
+  }
   printer->Print(" */\n");
 }
 
 void PrintMethodSignature(Printer* printer, const MethodDescriptor* method,
                           const map< ::grpc::string, ::grpc::string>& vars) {
   // Print comment
-  PrintAllComments(method, printer);
+  PrintAllComments(method, printer, true);
 
   printer->Print(vars, "- ($return_type$)$method_name$With");
   if (method->client_streaming()) {

src/compiler/protobuf_plugin.h

@@ -55,22 +55,23 @@ class ProtoBufMethod : public grpc_generator::Method {
     return method_->output_type()->file()->name();
   }
 
-  bool get_module_and_message_path_input(grpc::string* str,
-                                         grpc::string generator_file_name,
-                                         bool generate_in_pb2_grpc,
-                                         grpc::string import_prefix) const {
+  // TODO(https://github.com/grpc/grpc/issues/18800): Clean this up.
+  bool get_module_and_message_path_input(
+      grpc::string* str, grpc::string generator_file_name,
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const final {
     return grpc_python_generator::GetModuleAndMessagePath(
         method_->input_type(), str, generator_file_name, generate_in_pb2_grpc,
-        import_prefix);
+        import_prefix, prefixes_to_filter);
   }
 
-  bool get_module_and_message_path_output(grpc::string* str,
-                                          grpc::string generator_file_name,
-                                          bool generate_in_pb2_grpc,
-                                          grpc::string import_prefix) const {
+  bool get_module_and_message_path_output(
+      grpc::string* str, grpc::string generator_file_name,
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const final {
     return grpc_python_generator::GetModuleAndMessagePath(
         method_->output_type(), str, generator_file_name, generate_in_pb2_grpc,
-        import_prefix);
+        import_prefix, prefixes_to_filter);
   }
 
   bool NoStreaming() const {
@@ -189,6 +190,15 @@ class ProtoBufFile : public grpc_generator::File {
     return grpc_python_generator::get_all_comments(file_);
   }
 
+  vector<grpc::string> GetImportNames() const {
+    vector<grpc::string> proto_names;
+    for (int i = 0; i < file_->dependency_count(); ++i) {
+      const auto& dep = *file_->dependency(i);
+      proto_names.push_back(dep.name());
+    }
+    return proto_names;
+  }
+
  private:
   const grpc::protobuf::FileDescriptor* file_;
 };

src/compiler/python_generator.cc

@@ -214,13 +214,15 @@ bool PrivateGenerator::PrintBetaServerFactory(
       grpc::string input_message_module_and_class;
       if (!method->get_module_and_message_path_input(
               &input_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       grpc::string output_message_module_and_class;
       if (!method->get_module_and_message_path_output(
               &output_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       method_implementation_constructors.insert(
@@ -320,13 +322,15 @@ bool PrivateGenerator::PrintBetaStubFactory(
       grpc::string input_message_module_and_class;
       if (!method->get_module_and_message_path_input(
               &input_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       grpc::string output_message_module_and_class;
       if (!method->get_module_and_message_path_output(
               &output_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       method_cardinalities.insert(
@@ -425,13 +429,15 @@ bool PrivateGenerator::PrintStub(
         grpc::string request_module_and_class;
         if (!method->get_module_and_message_path_input(
                 &request_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         grpc::string response_module_and_class;
         if (!method->get_module_and_message_path_output(
                 &response_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         StringMap method_dict;
@@ -516,13 +522,15 @@ bool PrivateGenerator::PrintAddServicerToServer(
         grpc::string request_module_and_class;
         if (!method->get_module_and_message_path_input(
                 &request_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         grpc::string response_module_and_class;
         if (!method->get_module_and_message_path_output(
                 &response_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         StringMap method_dict;
@@ -589,17 +597,21 @@ bool PrivateGenerator::PrintPreamble(grpc_generator::Printer* out) {
 
         grpc::string input_type_file_name = method->get_input_type_name();
         grpc::string input_module_name =
-            ModuleName(input_type_file_name, config.import_prefix);
+            ModuleName(input_type_file_name, config.import_prefix,
+                       config.prefixes_to_filter);
         grpc::string input_module_alias =
-            ModuleAlias(input_type_file_name, config.import_prefix);
+            ModuleAlias(input_type_file_name, config.import_prefix,
+                        config.prefixes_to_filter);
         imports_set.insert(
             std::make_tuple(input_module_name, input_module_alias));
 
         grpc::string output_type_file_name = method->get_output_type_name();
         grpc::string output_module_name =
-            ModuleName(output_type_file_name, config.import_prefix);
+            ModuleName(output_type_file_name, config.import_prefix,
+                       config.prefixes_to_filter);
         grpc::string output_module_alias =
-            ModuleAlias(output_type_file_name, config.import_prefix);
+            ModuleAlias(output_type_file_name, config.import_prefix,
+                        config.prefixes_to_filter);
         imports_set.insert(
             std::make_tuple(output_module_name, output_module_alias));
       }

src/compiler/python_generator.h

@@ -20,6 +20,7 @@
 #define GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H
 
 #include <utility>
+#include <vector>
 
 #include "src/compiler/config.h"
 #include "src/compiler/schema_interface.h"
@@ -35,6 +36,7 @@ struct GeneratorConfiguration {
   grpc::string beta_package_root;
   // TODO(https://github.com/google/protobuf/issues/888): Drop this.
   grpc::string import_prefix;
+  std::vector<grpc::string> prefixes_to_filter;
 };
 
 class PythonGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {

src/compiler/python_generator_helpers.h

@@ -49,23 +49,39 @@ namespace {
 typedef vector<const Descriptor*> DescriptorVector;
 typedef vector<grpc::string> StringVector;
 
+static grpc::string StripModulePrefixes(
+    const grpc::string& raw_module_name,
+    const std::vector<grpc::string>& prefixes_to_filter) {
+  for (const auto& prefix : prefixes_to_filter) {
+    if (raw_module_name.rfind(prefix, 0) == 0) {
+      return raw_module_name.substr(prefix.size(),
+                                    raw_module_name.size() - prefix.size());
+    }
+  }
+  return raw_module_name;
+}
+
 // TODO(https://github.com/google/protobuf/issues/888):
 // Export `ModuleName` from protobuf's
 // `src/google/protobuf/compiler/python/python_generator.cc` file.
 grpc::string ModuleName(const grpc::string& filename,
-                        const grpc::string& import_prefix) {
+                        const grpc::string& import_prefix,
+                        const std::vector<grpc::string>& prefixes_to_filter) {
   grpc::string basename = StripProto(filename);
   basename = StringReplace(basename, "-", "_");
   basename = StringReplace(basename, "/", ".");
-  return import_prefix + basename + "_pb2";
+  return StripModulePrefixes(import_prefix + basename + "_pb2",
+                             prefixes_to_filter);
 }
 
 // TODO(https://github.com/google/protobuf/issues/888):
 // Export `ModuleAlias` from protobuf's
 // `src/google/protobuf/compiler/python/python_generator.cc` file.
 grpc::string ModuleAlias(const grpc::string& filename,
-                         const grpc::string& import_prefix) {
-  grpc::string module_name = ModuleName(filename, import_prefix);
+                         const grpc::string& import_prefix,
+                         const std::vector<grpc::string>& prefixes_to_filter) {
+  grpc::string module_name =
+      ModuleName(filename, import_prefix, prefixes_to_filter);
   // We can't have dots in the module name, so we replace each with _dot_.
   // But that could lead to a collision between a.b and a_dot_b, so we also
   // duplicate each underscore.
@@ -74,10 +90,10 @@ grpc::string ModuleAlias(const grpc::string& filename,
   return module_name;
 }
 
-bool GetModuleAndMessagePath(const Descriptor* type, grpc::string* out,
-                             grpc::string generator_file_name,
-                             bool generate_in_pb2_grpc,
-                             grpc::string& import_prefix) {
+bool GetModuleAndMessagePath(
+    const Descriptor* type, grpc::string* out, grpc::string generator_file_name,
+    bool generate_in_pb2_grpc, grpc::string& import_prefix,
+    const std::vector<grpc::string>& prefixes_to_filter) {
   const Descriptor* path_elem_type = type;
   DescriptorVector message_path;
   do {
@@ -93,7 +109,7 @@ bool GetModuleAndMessagePath(const Descriptor* type, grpc::string* out,
 
   grpc::string module;
   if (generator_file_name != file_name || generate_in_pb2_grpc) {
-    module = ModuleAlias(file_name, import_prefix) + ".";
+    module = ModuleAlias(file_name, import_prefix, prefixes_to_filter) + ".";
   } else {
     module = "";
   }

src/compiler/schema_interface.h

@@ -57,10 +57,12 @@ struct Method : public CommentHolder {
 
   virtual bool get_module_and_message_path_input(
       grpc::string* str, grpc::string generator_file_name,
-      bool generate_in_pb2_grpc, grpc::string import_prefix) const = 0;
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const = 0;
   virtual bool get_module_and_message_path_output(
       grpc::string* str, grpc::string generator_file_name,
-      bool generate_in_pb2_grpc, grpc::string import_prefix) const = 0;
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const = 0;
 
   virtual grpc::string get_input_type_name() const = 0;
   virtual grpc::string get_output_type_name() const = 0;
@@ -101,6 +103,7 @@ struct File : public CommentHolder {
   virtual grpc::string package() const = 0;
   virtual std::vector<grpc::string> package_parts() const = 0;
   virtual grpc::string additional_headers() const = 0;
+  virtual std::vector<grpc::string> GetImportNames() const { return {}; }
 
   virtual int service_count() const = 0;
   virtual std::unique_ptr<const Service> service(int i) const = 0;

src/core/ext/filters/client_channel/client_channel_channelz.cc

@@ -49,8 +49,8 @@ ClientChannelNode::ClientChannelNode(grpc_channel* channel,
     : ChannelNode(channel, channel_tracer_max_nodes, is_top_level_channel) {
   client_channel_ =
       grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel));
-  grpc_client_channel_set_channelz_node(client_channel_, this);
   GPR_ASSERT(client_channel_->filter == &grpc_client_channel_filter);
+  grpc_client_channel_set_channelz_node(client_channel_, this);
 }
 
 void ClientChannelNode::PopulateConnectivityState(grpc_json* json) {
@@ -127,8 +127,7 @@ void SubchannelNode::PopulateConnectivityState(grpc_json* json) {
   if (subchannel_ == nullptr) {
     state = GRPC_CHANNEL_SHUTDOWN;
   } else {
-    state = subchannel_->CheckConnectivity(nullptr,
-                                           true /* inhibit_health_checking */);
+    state = subchannel_->CheckConnectivity(true /* inhibit_health_checking */);
   }
   json = grpc_json_create_child(nullptr, json, "state", nullptr,
                                 GRPC_JSON_OBJECT, false);

src/core/ext/filters/client_channel/client_channel_plugin.cc

@@ -49,6 +49,7 @@ static bool append_filter(grpc_channel_stack_builder* builder, void* arg) {
 }
 
 void grpc_client_channel_init(void) {
+  grpc_core::ServiceConfig::Init();
   grpc_core::LoadBalancingPolicyRegistry::Builder::InitRegistry();
   grpc_core::ResolverRegistry::Builder::InitRegistry();
   grpc_core::internal::ServerRetryThrottleMap::Init();
@@ -68,4 +69,5 @@ void grpc_client_channel_shutdown(void) {
   grpc_core::internal::ServerRetryThrottleMap::Shutdown();
   grpc_core::ResolverRegistry::Builder::ShutdownRegistry();
   grpc_core::LoadBalancingPolicyRegistry::Builder::ShutdownRegistry();
+  grpc_core::ServiceConfig::Shutdown();
 }

src/core/ext/filters/client_channel/health/health_check_client.cc

@@ -37,11 +37,10 @@
 #define HEALTH_CHECK_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define HEALTH_CHECK_RECONNECT_JITTER 0.2
 
-grpc_core::TraceFlag grpc_health_check_client_trace(false,
-                                                    "health_check_client");
-
 namespace grpc_core {
 
+TraceFlag grpc_health_check_client_trace(false, "health_check_client");
+
 //
 // HealthCheckClient
 //
@@ -50,7 +49,7 @@ HealthCheckClient::HealthCheckClient(
     const char* service_name,
     RefCountedPtr<ConnectedSubchannel> connected_subchannel,
     grpc_pollset_set* interested_parties,
-    grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> channelz_node)
+    RefCountedPtr<channelz::SubchannelNode> channelz_node)
     : InternallyRefCounted<HealthCheckClient>(&grpc_health_check_client_trace),
       service_name_(service_name),
       connected_subchannel_(std::move(connected_subchannel)),
@@ -281,11 +280,9 @@ HealthCheckClient::CallState::CallState(
     : InternallyRefCounted<CallState>(&grpc_health_check_client_trace),
       health_check_client_(std::move(health_check_client)),
       pollent_(grpc_polling_entity_create_from_pollset_set(interested_parties)),
-      arena_(gpr_arena_create(health_check_client_->connected_subchannel_
-                                  ->GetInitialCallSizeEstimate(0))),
-      payload_(context_) {
-  grpc_call_combiner_init(&call_combiner_);
-}
+      arena_(Arena::Create(health_check_client_->connected_subchannel_
+                               ->GetInitialCallSizeEstimate(0))),
+      payload_(context_) {}
 
 HealthCheckClient::CallState::~CallState() {
   if (grpc_health_check_client_trace.enabled()) {
@@ -303,14 +300,13 @@ HealthCheckClient::CallState::~CallState() {
   // holding to the call stack. Also flush the closures on exec_ctx so that
   // filters that schedule cancel notification closures on exec_ctx do not
   // need to take a ref of the call stack to guarantee closure liveness.
-  grpc_call_combiner_set_notify_on_cancel(&call_combiner_, nullptr);
-  grpc_core::ExecCtx::Get()->Flush();
-  grpc_call_combiner_destroy(&call_combiner_);
-  gpr_arena_destroy(arena_);
+  call_combiner_.SetNotifyOnCancel(nullptr);
+  ExecCtx::Get()->Flush();
+  arena_->Destroy();
 }
 
 void HealthCheckClient::CallState::Orphan() {
-  grpc_call_combiner_cancel(&call_combiner_, GRPC_ERROR_CANCELLED);
+  call_combiner_.Cancel(GRPC_ERROR_CANCELLED);
   Cancel();
 }
 

src/core/ext/filters/client_channel/health/health_check_client.h

@@ -27,7 +27,7 @@
 #include "src/core/ext/filters/client_channel/client_channel_channelz.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/lib/backoff/backoff.h"
-#include "src/core/lib/gpr/arena.h"
+#include "src/core/lib/gprpp/arena.h"
 #include "src/core/lib/gprpp/atomic.h"
 #include "src/core/lib/gprpp/orphanable.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
@@ -97,8 +97,8 @@ class HealthCheckClient : public InternallyRefCounted<HealthCheckClient> {
     RefCountedPtr<HealthCheckClient> health_check_client_;
     grpc_polling_entity pollent_;
 
-    gpr_arena* arena_;
-    grpc_call_combiner call_combiner_;
+    Arena* arena_;
+    grpc_core::CallCombiner call_combiner_;
     grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
 
     // The streaming call to the backend. Always non-NULL.

src/core/ext/filters/client_channel/http_connect_handshaker.h

@@ -25,7 +25,7 @@
 
 /// Channel arg indicating HTTP CONNECT headers (string).
 /// Multiple headers are separated by newlines.  Key/value pairs are
-/// seperated by colons.
+/// separated by colons.
 #define GRPC_ARG_HTTP_CONNECT_HEADERS "grpc.http_connect_headers"
 
 /// Registers handshaker factory.

src/core/ext/filters/client_channel/lb_policy.h

@@ -167,6 +167,9 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
 
   /// A proxy object used by the LB policy to communicate with the client
   /// channel.
+  // TODO(juanlishen): Consider adding a mid-layer subclass that helps handle
+  // things like swapping in pending policy when it's ready. Currently, we are
+  // duplicating the logic in many subclasses.
   class ChannelControlHelper {
    public:
     ChannelControlHelper() = default;
@@ -185,7 +188,6 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
     /// Sets the connectivity state and returns a new picker to be used
     /// by the client channel.
     virtual void UpdateState(grpc_connectivity_state state,
-                             grpc_error* state_error,
                              UniquePtr<SubchannelPicker>) GRPC_ABSTRACT;
 
     /// Requests that the resolver re-resolve.

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -281,7 +281,7 @@ class GrpcLb : public LoadBalancingPolicy {
     Subchannel* CreateSubchannel(const grpc_channel_args& args) override;
     grpc_channel* CreateChannel(const char* target,
                                 const grpc_channel_args& args) override;
-    void UpdateState(grpc_connectivity_state state, grpc_error* state_error,
+    void UpdateState(grpc_connectivity_state state,
                      UniquePtr<SubchannelPicker> picker) override;
     void RequestReresolution() override;
 
@@ -621,12 +621,8 @@ grpc_channel* GrpcLb::Helper::CreateChannel(const char* target,
 }
 
 void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
-                                 grpc_error* state_error,
                                  UniquePtr<SubchannelPicker> picker) {
-  if (parent_->shutting_down_) {
-    GRPC_ERROR_UNREF(state_error);
-    return;
-  }
+  if (parent_->shutting_down_) return;
   // If this request is from the pending child policy, ignore it until
   // it reports READY, at which point we swap it into place.
   if (CalledByPendingChild()) {
@@ -636,10 +632,7 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
               parent_.get(), this, parent_->pending_child_policy_.get(),
               grpc_connectivity_state_name(state));
     }
-    if (state != GRPC_CHANNEL_READY) {
-      GRPC_ERROR_UNREF(state_error);
-      return;
-    }
+    if (state != GRPC_CHANNEL_READY) return;
     grpc_pollset_set_del_pollset_set(
         parent_->child_policy_->interested_parties(),
         parent_->interested_parties());
@@ -647,7 +640,6 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
     parent_->child_policy_ = std::move(parent_->pending_child_policy_);
   } else if (!CalledByCurrentChild()) {
     // This request is from an outdated child, so ignore it.
-    GRPC_ERROR_UNREF(state_error);
     return;
   }
   // Record whether child policy reports READY.
@@ -682,8 +674,7 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
               parent_.get(), this, grpc_connectivity_state_name(state),
               picker.get());
     }
-    parent_->channel_control_helper()->UpdateState(state, state_error,
-                                                   std::move(picker));
+    parent_->channel_control_helper()->UpdateState(state, std::move(picker));
     return;
   }
   // Cases 2 and 3a: wrap picker from the child in our own picker.
@@ -698,10 +689,9 @@ void GrpcLb::Helper::UpdateState(grpc_connectivity_state state,
     client_stats = parent_->lb_calld_->client_stats()->Ref();
   }
   parent_->channel_control_helper()->UpdateState(
-      state, state_error,
-      UniquePtr<SubchannelPicker>(
-          New<Picker>(parent_.get(), parent_->serverlist_, std::move(picker),
-                      std::move(client_stats))));
+      state, UniquePtr<SubchannelPicker>(
+                 New<Picker>(parent_.get(), parent_->serverlist_,
+                             std::move(picker), std::move(client_stats))));
 }
 
 void GrpcLb::Helper::RequestReresolution() {
@@ -1139,13 +1129,13 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
   // we want to retry connecting. Otherwise, we have deliberately ended this
   // call and no further action is required.
   if (lb_calld == grpclb_policy->lb_calld_.get()) {
-    // If we did not receive a serverlist and the fallback-at-startup checks
-    // are pending, go into fallback mode immediately.  This short-circuits
-    // the timeout for the fallback-at-startup case.
-    if (!lb_calld->seen_serverlist_ &&
-        grpclb_policy->fallback_at_startup_checks_pending_) {
+    // If the fallback-at-startup checks are pending, go into fallback mode
+    // immediately.  This short-circuits the timeout for the fallback-at-startup
+    // case.
+    if (grpclb_policy->fallback_at_startup_checks_pending_) {
+      GPR_ASSERT(!lb_calld->seen_serverlist_);
       gpr_log(GPR_INFO,
-              "[grpclb %p] balancer call finished without receiving "
+              "[grpclb %p] Balancer call finished without receiving "
               "serverlist; entering fallback mode",
               grpclb_policy);
       grpclb_policy->fallback_at_startup_checks_pending_ = false;
@@ -1638,20 +1628,16 @@ void GrpcLb::OnFallbackTimerLocked(void* arg, grpc_error* error) {
 
 grpc_channel_args* GrpcLb::CreateChildPolicyArgsLocked(
     bool is_backend_from_grpclb_load_balancer) {
-  grpc_arg args_to_add[2] = {
-      // A channel arg indicating if the target is a backend inferred from a
-      // grpclb load balancer.
-      grpc_channel_arg_integer_create(
-          const_cast<char*>(
-              GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER),
-          is_backend_from_grpclb_load_balancer),
-  };
-  size_t num_args_to_add = 1;
+  InlinedVector<grpc_arg, 2> args_to_add;
+  args_to_add.emplace_back(grpc_channel_arg_integer_create(
+      const_cast<char*>(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER),
+      is_backend_from_grpclb_load_balancer));
   if (is_backend_from_grpclb_load_balancer) {
-    args_to_add[num_args_to_add++] = grpc_channel_arg_integer_create(
-        const_cast<char*>(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1);
+    args_to_add.emplace_back(grpc_channel_arg_integer_create(
+        const_cast<char*>(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1));
   }
-  return grpc_channel_args_copy_and_add(args_, args_to_add, num_args_to_add);
+  return grpc_channel_args_copy_and_add(args_, args_to_add.data(),
+                                        args_to_add.size());
 }
 
 OrphanablePtr<LoadBalancingPolicy> GrpcLb::CreateChildPolicyLocked(

src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc

@@ -73,7 +73,7 @@ class PickFirst : public LoadBalancingPolicy {
         : SubchannelData(subchannel_list, address, subchannel, combiner) {}
 
     void ProcessConnectivityChangeLocked(
-        grpc_connectivity_state connectivity_state, grpc_error* error) override;
+        grpc_connectivity_state connectivity_state) override;
 
     // Processes the connectivity change to READY for an unselected subchannel.
     void ProcessUnselectedReadyLocked();
@@ -189,10 +189,11 @@ void PickFirst::ExitIdleLocked() {
     idle_ = false;
     if (subchannel_list_ == nullptr ||
         subchannel_list_->num_subchannels() == 0) {
-      grpc_error* error =
-          GRPC_ERROR_CREATE_FROM_STATIC_STRING("No addresses to connect to");
+      grpc_error* error = grpc_error_set_int(
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("No addresses to connect to"),
+          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
       channel_control_helper()->UpdateState(
-          GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+          GRPC_CHANNEL_TRANSIENT_FAILURE,
           UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
     } else {
       subchannel_list_->subchannel(0)
@@ -266,9 +267,11 @@ void PickFirst::UpdateLocked(UpdateArgs args) {
     // haven't gotten a non-empty update by the time the application tries
     // to start a new call.)
     if (!idle_) {
-      grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update");
+      grpc_error* error = grpc_error_set_int(
+          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
+          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
       channel_control_helper()->UpdateState(
-          GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+          GRPC_CHANNEL_TRANSIENT_FAILURE,
           UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
     }
     return;
@@ -282,9 +285,7 @@ void PickFirst::UpdateLocked(UpdateArgs args) {
   // check and instead do it in ExitIdleLocked().
   for (size_t i = 0; i < subchannel_list->num_subchannels(); ++i) {
     PickFirstSubchannelData* sd = subchannel_list->subchannel(i);
-    grpc_error* error = GRPC_ERROR_NONE;
-    grpc_connectivity_state state = sd->CheckConnectivityStateLocked(&error);
-    GRPC_ERROR_UNREF(error);
+    grpc_connectivity_state state = sd->CheckConnectivityStateLocked();
     if (state == GRPC_CHANNEL_READY) {
       subchannel_list_ = std::move(subchannel_list);
       sd->StartConnectivityWatchLocked();
@@ -338,7 +339,7 @@ void PickFirst::UpdateLocked(UpdateArgs args) {
 }
 
 void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
-    grpc_connectivity_state connectivity_state, grpc_error* error) {
+    grpc_connectivity_state connectivity_state) {
   PickFirst* p = static_cast<PickFirst*>(subchannel_list()->policy());
   AutoChildRefsUpdater guard(p);
   // The notification must be for a subchannel in either the current or
@@ -369,17 +370,16 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
       p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_);
       // Set our state to that of the pending subchannel list.
       if (p->subchannel_list_->in_transient_failure()) {
-        grpc_error* new_error =
-            GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                "selected subchannel failed; switching to pending update",
-                &error, 1);
+        grpc_error* error = grpc_error_set_int(
+            GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                "selected subchannel failed; switching to pending update"),
+            GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
         p->channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(new_error),
-            UniquePtr<SubchannelPicker>(
-                New<TransientFailurePicker>(new_error)));
+            GRPC_CHANNEL_TRANSIENT_FAILURE,
+            UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
       } else {
         p->channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
+            GRPC_CHANNEL_CONNECTING,
             UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
       }
     } else {
@@ -393,7 +393,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
         p->selected_ = nullptr;
         StopConnectivityWatchLocked();
         p->channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
+            GRPC_CHANNEL_IDLE,
             UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
       } else {
         // This is unlikely but can happen when a subchannel has been asked
@@ -401,19 +401,17 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
         // some connectivity state notifications.
         if (connectivity_state == GRPC_CHANNEL_READY) {
           p->channel_control_helper()->UpdateState(
-              GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
-              UniquePtr<SubchannelPicker>(
-                  New<Picker>(connected_subchannel()->Ref())));
+              GRPC_CHANNEL_READY, UniquePtr<SubchannelPicker>(New<Picker>(
+                                      connected_subchannel()->Ref())));
         } else {  // CONNECTING
           p->channel_control_helper()->UpdateState(
-              connectivity_state, GRPC_ERROR_REF(error),
+              connectivity_state,
               UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
         }
         // Renew notification.
         RenewConnectivityWatchLocked();
       }
     }
-    GRPC_ERROR_UNREF(error);
     return;
   }
   // If we get here, there are two possible cases:
@@ -450,13 +448,13 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
         subchannel_list()->set_in_transient_failure(true);
         // Only report new state in case 1.
         if (subchannel_list() == p->subchannel_list_.get()) {
-          grpc_error* new_error =
-              GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                  "failed to connect to all addresses", &error, 1);
+          grpc_error* error = grpc_error_set_int(
+              GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                  "failed to connect to all addresses"),
+              GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
           p->channel_control_helper()->UpdateState(
-              GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(new_error),
-              UniquePtr<SubchannelPicker>(
-                  New<TransientFailurePicker>(new_error)));
+              GRPC_CHANNEL_TRANSIENT_FAILURE,
+              UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
         }
       }
       sd->CheckConnectivityStateAndStartWatchingLocked();
@@ -467,7 +465,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
       // Only update connectivity state in case 1.
       if (subchannel_list() == p->subchannel_list_.get()) {
         p->channel_control_helper()->UpdateState(
-            GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
+            GRPC_CHANNEL_CONNECTING,
             UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
       }
       // Renew notification.
@@ -477,7 +475,6 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked(
     case GRPC_CHANNEL_SHUTDOWN:
       GPR_UNREACHABLE_CODE(break);
   }
-  GRPC_ERROR_UNREF(error);
 }
 
 void PickFirst::PickFirstSubchannelData::ProcessUnselectedReadyLocked() {
@@ -507,7 +504,7 @@ void PickFirst::PickFirstSubchannelData::ProcessUnselectedReadyLocked() {
   // Cases 1 and 2.
   p->selected_ = this;
   p->channel_control_helper()->UpdateState(
-      GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
+      GRPC_CHANNEL_READY,
       UniquePtr<SubchannelPicker>(New<Picker>(connected_subchannel()->Ref())));
   if (grpc_lb_pick_first_trace.enabled()) {
     gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", p, subchannel());
@@ -518,9 +515,7 @@ void PickFirst::PickFirstSubchannelData::
     CheckConnectivityStateAndStartWatchingLocked() {
   PickFirst* p = static_cast<PickFirst*>(subchannel_list()->policy());
   // Check current state.
-  grpc_error* error = GRPC_ERROR_NONE;
-  grpc_connectivity_state current_state = CheckConnectivityStateLocked(&error);
-  GRPC_ERROR_UNREF(error);
+  grpc_connectivity_state current_state = CheckConnectivityStateLocked();
   // Start watch.
   StartConnectivityWatchLocked();
   // If current state is READY, select the subchannel now, since we started

src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc

@@ -92,11 +92,11 @@ class RoundRobin : public LoadBalancingPolicy {
     }
 
     void UpdateConnectivityStateLocked(
-        grpc_connectivity_state connectivity_state, grpc_error* error);
+        grpc_connectivity_state connectivity_state);
 
    private:
     void ProcessConnectivityChangeLocked(
-        grpc_connectivity_state connectivity_state, grpc_error* error) override;
+        grpc_connectivity_state connectivity_state) override;
 
     grpc_connectivity_state last_connectivity_state_ = GRPC_CHANNEL_IDLE;
   };
@@ -119,7 +119,6 @@ class RoundRobin : public LoadBalancingPolicy {
     }
 
     ~RoundRobinSubchannelList() {
-      GRPC_ERROR_UNREF(last_transient_failure_error_);
       RoundRobin* p = static_cast<RoundRobin*>(policy());
       p->Unref(DEBUG_LOCATION, "subchannel_list");
     }
@@ -129,11 +128,8 @@ class RoundRobin : public LoadBalancingPolicy {
 
     // Updates the counters of subchannels in each state when a
     // subchannel transitions from old_state to new_state.
-    // transient_failure_error is the error that is reported when
-    // new_state is TRANSIENT_FAILURE.
     void UpdateStateCountersLocked(grpc_connectivity_state old_state,
-                                   grpc_connectivity_state new_state,
-                                   grpc_error* transient_failure_error);
+                                   grpc_connectivity_state new_state);
 
     // If this subchannel list is the RR policy's current subchannel
     // list, updates the RR policy's connectivity state based on the
@@ -148,7 +144,6 @@ class RoundRobin : public LoadBalancingPolicy {
     size_t num_ready_ = 0;
     size_t num_connecting_ = 0;
     size_t num_transient_failure_ = 0;
-    grpc_error* last_transient_failure_error_ = GRPC_ERROR_NONE;
   };
 
   class Picker : public SubchannelPicker {
@@ -315,11 +310,10 @@ void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
   // subchannel already used by some other channel may have a non-IDLE
   // state.
   for (size_t i = 0; i < num_subchannels(); ++i) {
-    grpc_error* error = GRPC_ERROR_NONE;
     grpc_connectivity_state state =
-        subchannel(i)->CheckConnectivityStateLocked(&error);
+        subchannel(i)->CheckConnectivityStateLocked();
     if (state != GRPC_CHANNEL_IDLE) {
-      subchannel(i)->UpdateConnectivityStateLocked(state, error);
+      subchannel(i)->UpdateConnectivityStateLocked(state);
     }
   }
   // Start connectivity watch for each subchannel.
@@ -333,8 +327,7 @@ void RoundRobin::RoundRobinSubchannelList::StartWatchingLocked() {
 }
 
 void RoundRobin::RoundRobinSubchannelList::UpdateStateCountersLocked(
-    grpc_connectivity_state old_state, grpc_connectivity_state new_state,
-    grpc_error* transient_failure_error) {
+    grpc_connectivity_state old_state, grpc_connectivity_state new_state) {
   GPR_ASSERT(old_state != GRPC_CHANNEL_SHUTDOWN);
   GPR_ASSERT(new_state != GRPC_CHANNEL_SHUTDOWN);
   if (old_state == GRPC_CHANNEL_READY) {
@@ -354,8 +347,6 @@ void RoundRobin::RoundRobinSubchannelList::UpdateStateCountersLocked(
   } else if (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) {
     ++num_transient_failure_;
   }
-  GRPC_ERROR_UNREF(last_transient_failure_error_);
-  last_transient_failure_error_ = transient_failure_error;
 }
 
 // Sets the RR policy's connectivity state and generates a new picker based
@@ -382,20 +373,21 @@ void RoundRobin::RoundRobinSubchannelList::
   if (num_ready_ > 0) {
     /* 1) READY */
     p->channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
-        UniquePtr<SubchannelPicker>(New<Picker>(p, this)));
+        GRPC_CHANNEL_READY, UniquePtr<SubchannelPicker>(New<Picker>(p, this)));
   } else if (num_connecting_ > 0) {
     /* 2) CONNECTING */
     p->channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
+        GRPC_CHANNEL_CONNECTING,
         UniquePtr<SubchannelPicker>(New<QueuePicker>(p->Ref())));
   } else if (num_transient_failure_ == num_subchannels()) {
     /* 3) TRANSIENT_FAILURE */
+    grpc_error* error =
+        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                               "connections to all backends failing"),
+                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
     p->channel_control_helper()->UpdateState(
         GRPC_CHANNEL_TRANSIENT_FAILURE,
-        GRPC_ERROR_REF(last_transient_failure_error_),
-        UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(
-            GRPC_ERROR_REF(last_transient_failure_error_))));
+        UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
   }
 }
 
@@ -430,7 +422,7 @@ void RoundRobin::RoundRobinSubchannelList::
 }
 
 void RoundRobin::RoundRobinSubchannelData::UpdateConnectivityStateLocked(
-    grpc_connectivity_state connectivity_state, grpc_error* error) {
+    grpc_connectivity_state connectivity_state) {
   RoundRobin* p = static_cast<RoundRobin*>(subchannel_list()->policy());
   if (grpc_lb_round_robin_trace.enabled()) {
     gpr_log(
@@ -443,12 +435,12 @@ void RoundRobin::RoundRobinSubchannelData::UpdateConnectivityStateLocked(
         grpc_connectivity_state_name(connectivity_state));
   }
   subchannel_list()->UpdateStateCountersLocked(last_connectivity_state_,
-                                               connectivity_state, error);
+                                               connectivity_state);
   last_connectivity_state_ = connectivity_state;
 }
 
 void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
-    grpc_connectivity_state connectivity_state, grpc_error* error) {
+    grpc_connectivity_state connectivity_state) {
   RoundRobin* p = static_cast<RoundRobin*>(subchannel_list()->policy());
   GPR_ASSERT(subchannel() != nullptr);
   // If the new state is TRANSIENT_FAILURE, re-resolve.
@@ -468,7 +460,7 @@ void RoundRobin::RoundRobinSubchannelData::ProcessConnectivityChangeLocked(
   // Renew connectivity watch.
   RenewConnectivityWatchLocked();
   // Update state counters.
-  UpdateConnectivityStateLocked(connectivity_state, error);
+  UpdateConnectivityStateLocked(connectivity_state);
   // Update overall state and renew notification.
   subchannel_list()->UpdateRoundRobinStateFromSubchannelStateCountsLocked();
 }
@@ -492,9 +484,11 @@ void RoundRobin::UpdateLocked(UpdateArgs args) {
   if (latest_pending_subchannel_list_->num_subchannels() == 0) {
     // If the new list is empty, immediately promote the new list to the
     // current list and transition to TRANSIENT_FAILURE.
-    grpc_error* error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update");
+    grpc_error* error =
+        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
+                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
     channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(error),
+        GRPC_CHANNEL_TRANSIENT_FAILURE,
         UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
     subchannel_list_ = std::move(latest_pending_subchannel_list_);
   } else if (subchannel_list_ == nullptr) {

src/core/ext/filters/client_channel/lb_policy/subchannel_list.h

@@ -51,7 +51,7 @@ class MySubchannelData
     : public SubchannelData<MySubchannelList, MySubchannelData> {
  public:
   void ProcessConnectivityChangeLocked(
-      grpc_connectivity_state connectivity_state, grpc_error* error) override {
+      grpc_connectivity_state connectivity_state) override {
     // ...code to handle connectivity changes...
   }
 };
@@ -101,10 +101,10 @@ class SubchannelData {
   // pending (i.e., between calling StartConnectivityWatchLocked() or
   // RenewConnectivityWatchLocked() and the resulting invocation of
   // ProcessConnectivityChangeLocked()).
-  grpc_connectivity_state CheckConnectivityStateLocked(grpc_error** error) {
+  grpc_connectivity_state CheckConnectivityStateLocked() {
     GPR_ASSERT(!connectivity_notification_pending_);
     pending_connectivity_state_unsafe_ = subchannel()->CheckConnectivity(
-        error, subchannel_list_->inhibit_health_checking());
+        subchannel_list_->inhibit_health_checking());
     UpdateConnectedSubchannelLocked();
     return pending_connectivity_state_unsafe_;
   }
@@ -153,8 +153,7 @@ class SubchannelData {
   // Implementations must invoke either RenewConnectivityWatchLocked() or
   // StopConnectivityWatchLocked() before returning.
   virtual void ProcessConnectivityChangeLocked(
-      grpc_connectivity_state connectivity_state,
-      grpc_error* error) GRPC_ABSTRACT;
+      grpc_connectivity_state connectivity_state) GRPC_ABSTRACT;
 
   // Unrefs the subchannel.
   void UnrefSubchannelLocked(const char* reason);
@@ -462,8 +461,7 @@ void SubchannelData<SubchannelListType, SubchannelDataType>::
     return;
   }
   // Call the subclass's ProcessConnectivityChangeLocked() method.
-  sd->ProcessConnectivityChangeLocked(sd->pending_connectivity_state_unsafe_,
-                                      GRPC_ERROR_REF(error));
+  sd->ProcessConnectivityChangeLocked(sd->pending_connectivity_state_unsafe_);
 }
 
 template <typename SubchannelListType, typename SubchannelDataType>

src/core/ext/filters/client_channel/lb_policy/xds/xds.cc

@@ -118,6 +118,7 @@ namespace {
 
 constexpr char kXds[] = "xds_experimental";
 constexpr char kDefaultLocalityName[] = "xds_default_locality";
+constexpr uint32_t kDefaultLocalityWeight = 3;
 
 class XdsLb : public LoadBalancingPolicy {
  public:
@@ -239,6 +240,10 @@ class XdsLb : public LoadBalancingPolicy {
     static void OnCallRetryTimerLocked(void* arg, grpc_error* error);
     void StartCallLocked();
 
+    void StartConnectivityWatchLocked();
+    void CancelConnectivityWatchLocked();
+    static void OnConnectivityChangedLocked(void* arg, grpc_error* error);
+
    private:
     // The owning LB policy.
     RefCountedPtr<XdsLb> xdslb_policy_;
@@ -246,6 +251,8 @@ class XdsLb : public LoadBalancingPolicy {
     // The channel and its status.
     grpc_channel* channel_;
     bool shutting_down_ = false;
+    grpc_connectivity_state connectivity_ = GRPC_CHANNEL_IDLE;
+    grpc_closure on_connectivity_changed_;
 
     // The data associated with the current LB call. It holds a ref to this LB
     // channel. It's instantiated every time we query for backends. It's reset
@@ -259,26 +266,73 @@ class XdsLb : public LoadBalancingPolicy {
     bool retry_timer_callback_pending_ = false;
   };
 
+  // Since pickers are UniquePtrs we use this RefCounted wrapper
+  // to control references to it by the xds picker and the locality
+  // entry
+  class PickerRef : public RefCounted<PickerRef> {
+   public:
+    explicit PickerRef(UniquePtr<SubchannelPicker> picker)
+        : picker_(std::move(picker)) {}
+    PickResult Pick(PickArgs* pick, grpc_error** error) {
+      return picker_->Pick(pick, error);
+    }
+
+   private:
+    UniquePtr<SubchannelPicker> picker_;
+  };
+
+  // The picker will use a stateless weighting algorithm to pick the locality to
+  // use for each request.
   class Picker : public SubchannelPicker {
    public:
-    Picker(UniquePtr<SubchannelPicker> child_picker,
-           RefCountedPtr<XdsLbClientStats> client_stats)
-        : child_picker_(std::move(child_picker)),
-          client_stats_(std::move(client_stats)) {}
+    // Maintains a weighted list of pickers from each locality that is in ready
+    // state. The first element in the pair represents the end of a range
+    // proportional to the locality's weight. The start of the range is the
+    // previous value in the vector and is 0 for the first element.
+    using PickerList =
+        InlinedVector<Pair<uint32_t, RefCountedPtr<PickerRef>>, 1>;
+    Picker(RefCountedPtr<XdsLbClientStats> client_stats, PickerList pickers)
+        : client_stats_(std::move(client_stats)),
+          pickers_(std::move(pickers)) {}
 
     PickResult Pick(PickArgs* pick, grpc_error** error) override;
 
    private:
-    UniquePtr<SubchannelPicker> child_picker_;
+    // Calls the picker of the locality that the key falls within
+    PickResult PickFromLocality(const uint32_t key, PickArgs* pick,
+                                grpc_error** error);
     RefCountedPtr<XdsLbClientStats> client_stats_;
+    PickerList pickers_;
+  };
+
+  class FallbackHelper : public ChannelControlHelper {
+   public:
+    explicit FallbackHelper(RefCountedPtr<XdsLb> parent)
+        : parent_(std::move(parent)) {}
+
+    Subchannel* CreateSubchannel(const grpc_channel_args& args) override;
+    grpc_channel* CreateChannel(const char* target,
+                                const grpc_channel_args& args) override;
+    void UpdateState(grpc_connectivity_state state,
+                     UniquePtr<SubchannelPicker> picker) override;
+    void RequestReresolution() override;
+
+    void set_child(LoadBalancingPolicy* child) { child_ = child; }
+
+   private:
+    bool CalledByPendingFallback() const;
+    bool CalledByCurrentFallback() const;
+
+    RefCountedPtr<XdsLb> parent_;
+    LoadBalancingPolicy* child_ = nullptr;
   };
 
   class LocalityMap {
    public:
     class LocalityEntry : public InternallyRefCounted<LocalityEntry> {
      public:
-      explicit LocalityEntry(RefCountedPtr<XdsLb> parent)
-          : parent_(std::move(parent)) {}
+      LocalityEntry(RefCountedPtr<XdsLb> parent, uint32_t locality_weight)
+          : parent_(std::move(parent)), locality_weight_(locality_weight) {}
       ~LocalityEntry() = default;
 
       void UpdateLocked(xds_grpclb_serverlist* serverlist,
@@ -299,7 +353,7 @@ class XdsLb : public LoadBalancingPolicy {
         Subchannel* CreateSubchannel(const grpc_channel_args& args) override;
         grpc_channel* CreateChannel(const char* target,
                                     const grpc_channel_args& args) override;
-        void UpdateState(grpc_connectivity_state state, grpc_error* state_error,
+        void UpdateState(grpc_connectivity_state state,
                          UniquePtr<SubchannelPicker> picker) override;
         void RequestReresolution() override;
         void set_child(LoadBalancingPolicy* child) { child_ = child; }
@@ -323,6 +377,9 @@ class XdsLb : public LoadBalancingPolicy {
       // pending_child_policy_.
       Mutex child_policy_mu_;
       RefCountedPtr<XdsLb> parent_;
+      RefCountedPtr<PickerRef> picker_ref_;
+      grpc_connectivity_state connectivity_state_;
+      uint32_t locality_weight_;
     };
 
     void UpdateLocked(const LocalityList& locality_list,
@@ -346,7 +403,9 @@ class XdsLb : public LoadBalancingPolicy {
       gpr_free(locality_name);
       xds_grpclb_destroy_serverlist(serverlist);
     }
+
     char* locality_name;
+    uint32_t locality_weight;
     // The deserialized response from the balancer. May be nullptr until one
     // such response has arrived.
     xds_grpclb_serverlist* serverlist;
@@ -371,8 +430,13 @@ class XdsLb : public LoadBalancingPolicy {
                                         : lb_chand_.get();
   }
 
-  // Callback to enter fallback mode.
+  // Methods for dealing with fallback state.
+  void MaybeCancelFallbackAtStartupChecks();
   static void OnFallbackTimerLocked(void* arg, grpc_error* error);
+  void UpdateFallbackPolicyLocked();
+  OrphanablePtr<LoadBalancingPolicy> CreateFallbackPolicyLocked(
+      const char* name, const grpc_channel_args* args);
+  void MaybeExitFallbackMode();
 
   // Who the client is trying to communicate with.
   const char* server_name_ = nullptr;
@@ -397,21 +461,39 @@ class XdsLb : public LoadBalancingPolicy {
   // Timeout in milliseconds for the LB call. 0 means no deadline.
   int lb_call_timeout_ms_ = 0;
 
+  // Whether the checks for fallback at startup are ALL pending. There are
+  // several cases where this can be reset:
+  // 1. The fallback timer fires, we enter fallback mode.
+  // 2. Before the fallback timer fires, the LB channel becomes
+  // TRANSIENT_FAILURE or the LB call fails, we enter fallback mode.
+  // 3. Before the fallback timer fires, we receive a response from the
+  // balancer, we cancel the fallback timer and use the response to update the
+  // locality map.
+  bool fallback_at_startup_checks_pending_ = false;
   // Timeout in milliseconds for before using fallback backend addresses.
   // 0 means not using fallback.
-  RefCountedPtr<Config> fallback_policy_config_;
   int lb_fallback_timeout_ms_ = 0;
   // The backend addresses from the resolver.
-  UniquePtr<ServerAddressList> fallback_backend_addresses_;
+  ServerAddressList fallback_backend_addresses_;
   // Fallback timer.
-  bool fallback_timer_callback_pending_ = false;
   grpc_timer lb_fallback_timer_;
   grpc_closure lb_on_fallback_;
 
+  // The policy to use for the fallback backends.
+  RefCountedPtr<Config> fallback_policy_config_;
+  // Lock held when modifying the value of fallback_policy_ or
+  // pending_fallback_policy_.
+  Mutex fallback_policy_mu_;
+  // Non-null iff we are in fallback mode.
+  OrphanablePtr<LoadBalancingPolicy> fallback_policy_;
+  OrphanablePtr<LoadBalancingPolicy> pending_fallback_policy_;
+
   // The policy to use for the backends.
   RefCountedPtr<Config> child_policy_config_;
   // Map of policies to use in the backend
   LocalityMap locality_map_;
+  // TODO(mhaidry) : Add support for multiple maps of localities
+  // with different priorities
   LocalityList locality_serverlist_;
   // TODO(mhaidry) : Add a pending locality map that may be swapped with the
   // the current one when new localities in the pending map are ready
@@ -424,8 +506,12 @@ class XdsLb : public LoadBalancingPolicy {
 
 XdsLb::PickResult XdsLb::Picker::Pick(PickArgs* pick, grpc_error** error) {
   // TODO(roth): Add support for drop handling.
-  // Forward pick to child policy.
-  PickResult result = child_picker_->Pick(pick, error);
+  // Generate a random number between 0 and the total weight
+  const uint32_t key =
+      (rand() * pickers_[pickers_.size() - 1].first) / RAND_MAX;
+  // Forward pick to whichever locality maps to the range in which the
+  // random number falls in.
+  PickResult result = PickFromLocality(key, pick, error);
   // If pick succeeded, add client stats.
   if (result == PickResult::PICK_COMPLETE &&
       pick->connected_subchannel != nullptr && client_stats_ != nullptr) {
@@ -434,17 +520,113 @@ XdsLb::PickResult XdsLb::Picker::Pick(PickArgs* pick, grpc_error** error) {
   return result;
 }
 
+XdsLb::PickResult XdsLb::Picker::PickFromLocality(const uint32_t key,
+                                                  PickArgs* pick,
+                                                  grpc_error** error) {
+  size_t mid = 0;
+  size_t start_index = 0;
+  size_t end_index = pickers_.size() - 1;
+  size_t index = 0;
+  while (end_index > start_index) {
+    mid = (start_index + end_index) / 2;
+    if (pickers_[mid].first > key) {
+      end_index = mid;
+    } else if (pickers_[mid].first < key) {
+      start_index = mid + 1;
+    } else {
+      index = mid + 1;
+      break;
+    }
+  }
+  if (index == 0) index = start_index;
+  GPR_ASSERT(pickers_[index].first > key);
+  return pickers_[index].second->Pick(pick, error);
+}
+
+//
+// XdsLb::FallbackHelper
+//
+
+bool XdsLb::FallbackHelper::CalledByPendingFallback() const {
+  GPR_ASSERT(child_ != nullptr);
+  return child_ == parent_->pending_fallback_policy_.get();
+}
+
+bool XdsLb::FallbackHelper::CalledByCurrentFallback() const {
+  GPR_ASSERT(child_ != nullptr);
+  return child_ == parent_->fallback_policy_.get();
+}
+
+Subchannel* XdsLb::FallbackHelper::CreateSubchannel(
+    const grpc_channel_args& args) {
+  if (parent_->shutting_down_ ||
+      (!CalledByPendingFallback() && !CalledByCurrentFallback())) {
+    return nullptr;
+  }
+  return parent_->channel_control_helper()->CreateSubchannel(args);
+}
+
+grpc_channel* XdsLb::FallbackHelper::CreateChannel(
+    const char* target, const grpc_channel_args& args) {
+  if (parent_->shutting_down_ ||
+      (!CalledByPendingFallback() && !CalledByCurrentFallback())) {
+    return nullptr;
+  }
+  return parent_->channel_control_helper()->CreateChannel(target, args);
+}
+
+void XdsLb::FallbackHelper::UpdateState(grpc_connectivity_state state,
+                                        UniquePtr<SubchannelPicker> picker) {
+  if (parent_->shutting_down_) return;
+  // If this request is from the pending fallback policy, ignore it until
+  // it reports READY, at which point we swap it into place.
+  if (CalledByPendingFallback()) {
+    if (grpc_lb_xds_trace.enabled()) {
+      gpr_log(
+          GPR_INFO,
+          "[xdslb %p helper %p] pending fallback policy %p reports state=%s",
+          parent_.get(), this, parent_->pending_fallback_policy_.get(),
+          grpc_connectivity_state_name(state));
+    }
+    if (state != GRPC_CHANNEL_READY) return;
+    grpc_pollset_set_del_pollset_set(
+        parent_->fallback_policy_->interested_parties(),
+        parent_->interested_parties());
+    MutexLock lock(&parent_->fallback_policy_mu_);
+    parent_->fallback_policy_ = std::move(parent_->pending_fallback_policy_);
+  } else if (!CalledByCurrentFallback()) {
+    // This request is from an outdated fallback policy, so ignore it.
+    return;
+  }
+  parent_->channel_control_helper()->UpdateState(state, std::move(picker));
+}
+
+void XdsLb::FallbackHelper::RequestReresolution() {
+  if (parent_->shutting_down_) return;
+  const LoadBalancingPolicy* latest_fallback_policy =
+      parent_->pending_fallback_policy_ != nullptr
+          ? parent_->pending_fallback_policy_.get()
+          : parent_->fallback_policy_.get();
+  if (child_ != latest_fallback_policy) return;
+  if (grpc_lb_xds_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[xdslb %p] Re-resolution requested from the fallback policy (%p).",
+            parent_.get(), child_);
+  }
+  GPR_ASSERT(parent_->lb_chand_ != nullptr);
+  parent_->channel_control_helper()->RequestReresolution();
+}
+
 //
 // serverlist parsing code
 //
 
 // Returns the backend addresses extracted from the given addresses.
-UniquePtr<ServerAddressList> ExtractBackendAddresses(
-    const ServerAddressList& addresses) {
-  auto backend_addresses = MakeUnique<ServerAddressList>();
+ServerAddressList ExtractBackendAddresses(const ServerAddressList& addresses) {
+  ServerAddressList backend_addresses;
   for (size_t i = 0; i < addresses.size(); ++i) {
     if (!addresses[i].IsBalancer()) {
-      backend_addresses->emplace_back(addresses[i]);
+      backend_addresses.emplace_back(addresses[i]);
     }
   }
   return backend_addresses;
@@ -524,6 +706,9 @@ XdsLb::BalancerChannelState::BalancerChannelState(
               .set_multiplier(GRPC_XDS_RECONNECT_BACKOFF_MULTIPLIER)
               .set_jitter(GRPC_XDS_RECONNECT_JITTER)
               .set_max_backoff(GRPC_XDS_RECONNECT_MAX_BACKOFF_SECONDS * 1000)) {
+  GRPC_CLOSURE_INIT(&on_connectivity_changed_,
+                    &XdsLb::BalancerChannelState::OnConnectivityChangedLocked,
+                    this, grpc_combiner_scheduler(xdslb_policy_->combiner()));
   channel_ = xdslb_policy_->channel_control_helper()->CreateChannel(
       balancer_name, args);
   GPR_ASSERT(channel_ != nullptr);
@@ -592,6 +777,62 @@ void XdsLb::BalancerChannelState::StartCallLocked() {
   lb_calld_->StartQuery();
 }
 
+void XdsLb::BalancerChannelState::StartConnectivityWatchLocked() {
+  grpc_channel_element* client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel_));
+  GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
+  // Ref held by callback.
+  Ref(DEBUG_LOCATION, "watch_lb_channel_connectivity").release();
+  grpc_client_channel_watch_connectivity_state(
+      client_channel_elem,
+      grpc_polling_entity_create_from_pollset_set(
+          xdslb_policy_->interested_parties()),
+      &connectivity_, &on_connectivity_changed_, nullptr);
+}
+
+void XdsLb::BalancerChannelState::CancelConnectivityWatchLocked() {
+  grpc_channel_element* client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel_));
+  GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
+  grpc_client_channel_watch_connectivity_state(
+      client_channel_elem,
+      grpc_polling_entity_create_from_pollset_set(
+          xdslb_policy_->interested_parties()),
+      nullptr, &on_connectivity_changed_, nullptr);
+}
+
+void XdsLb::BalancerChannelState::OnConnectivityChangedLocked(
+    void* arg, grpc_error* error) {
+  BalancerChannelState* self = static_cast<BalancerChannelState*>(arg);
+  if (!self->shutting_down_ &&
+      self->xdslb_policy_->fallback_at_startup_checks_pending_) {
+    if (self->connectivity_ != GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      // Not in TRANSIENT_FAILURE.  Renew connectivity watch.
+      grpc_channel_element* client_channel_elem =
+          grpc_channel_stack_last_element(
+              grpc_channel_get_channel_stack(self->channel_));
+      GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
+      grpc_client_channel_watch_connectivity_state(
+          client_channel_elem,
+          grpc_polling_entity_create_from_pollset_set(
+              self->xdslb_policy_->interested_parties()),
+          &self->connectivity_, &self->on_connectivity_changed_, nullptr);
+      return;  // Early out so we don't drop the ref below.
+    }
+    // In TRANSIENT_FAILURE.  Cancel the fallback timer and go into
+    // fallback mode immediately.
+    gpr_log(GPR_INFO,
+            "[xdslb %p] Balancer channel in state TRANSIENT_FAILURE; "
+            "entering fallback mode",
+            self);
+    self->xdslb_policy_->fallback_at_startup_checks_pending_ = false;
+    grpc_timer_cancel(&self->xdslb_policy_->lb_fallback_timer_);
+    self->xdslb_policy_->UpdateFallbackPolicyLocked();
+  }
+  // Done watching connectivity state, so drop ref.
+  self->Unref(DEBUG_LOCATION, "watch_lb_channel_connectivity");
+}
+
 //
 // XdsLb::BalancerChannelState::BalancerCallState
 //
@@ -837,6 +1078,14 @@ void XdsLb::BalancerChannelState::BalancerCallState::
       (initial_response = xds_grpclb_initial_response_parse(response_slice)) !=
           nullptr) {
     // Have NOT seen initial response, look for initial response.
+    // TODO(juanlishen): When we convert this to use the xds protocol, the
+    // balancer will send us a fallback timeout such that we should go into
+    // fallback mode if we have lost contact with the balancer after a certain
+    // period of time. We will need to save the timeout value here, and then
+    // when the balancer call ends, we will need to start a timer for the
+    // specified period of time, and if the timer fires, we go into fallback
+    // mode. We will also need to cancel the timer when we receive a serverlist
+    // from the balancer.
     if (initial_response->has_client_stats_report_interval) {
       const grpc_millis interval = xds_grpclb_duration_to_millis(
           &initial_response->client_stats_report_interval);
@@ -878,79 +1127,69 @@ void XdsLb::BalancerChannelState::BalancerCallState::
         gpr_free(ipport);
       }
     }
-    /* update serverlist */
-    // TODO(juanlishen): Don't ingore empty serverlist.
-    if (serverlist->num_servers > 0) {
-      // Pending LB channel receives a serverlist; promote it.
-      // Note that this call can't be on a discarded pending channel, because
-      // such channels don't have any current call but we have checked this call
-      // is a current call.
-      if (!lb_calld->lb_chand_->IsCurrentChannel()) {
-        if (grpc_lb_xds_trace.enabled()) {
-          gpr_log(GPR_INFO,
-                  "[xdslb %p] Promoting pending LB channel %p to replace "
-                  "current LB channel %p",
-                  xdslb_policy, lb_calld->lb_chand_.get(),
-                  lb_calld->xdslb_policy()->lb_chand_.get());
-        }
-        lb_calld->xdslb_policy()->lb_chand_ =
-            std::move(lb_calld->xdslb_policy()->pending_lb_chand_);
-      }
-      // Start sending client load report only after we start using the
-      // serverlist returned from the current LB call.
-      if (lb_calld->client_stats_report_interval_ > 0 &&
-          lb_calld->client_stats_ == nullptr) {
-        lb_calld->client_stats_ = MakeRefCounted<XdsLbClientStats>();
-        // TODO(roth): We currently track this ref manually.  Once the
-        // ClosureRef API is ready, we should pass the RefCountedPtr<> along
-        // with the callback.
-        auto self = lb_calld->Ref(DEBUG_LOCATION, "client_load_report");
-        self.release();
-        lb_calld->ScheduleNextClientLoadReportLocked();
-      }
-      if (!xdslb_policy->locality_serverlist_.empty() &&
-          xds_grpclb_serverlist_equals(
-              xdslb_policy->locality_serverlist_[0]->serverlist, serverlist)) {
-        if (grpc_lb_xds_trace.enabled()) {
-          gpr_log(GPR_INFO,
-                  "[xdslb %p] Incoming server list identical to current, "
-                  "ignoring.",
-                  xdslb_policy);
-        }
-        xds_grpclb_destroy_serverlist(serverlist);
-      } else { /* new serverlist */
-        if (!xdslb_policy->locality_serverlist_.empty()) {
-          /* dispose of the old serverlist */
-          xds_grpclb_destroy_serverlist(
-              xdslb_policy->locality_serverlist_[0]->serverlist);
-        } else {
-          /* or dispose of the fallback */
-          xdslb_policy->fallback_backend_addresses_.reset();
-          if (xdslb_policy->fallback_timer_callback_pending_) {
-            grpc_timer_cancel(&xdslb_policy->lb_fallback_timer_);
-          }
-          /* Initialize locality serverlist, currently the list only handles
-           * one child */
-          xdslb_policy->locality_serverlist_.emplace_back(
-              MakeUnique<LocalityServerlistEntry>());
-          xdslb_policy->locality_serverlist_[0]->locality_name =
-              static_cast<char*>(gpr_strdup(kDefaultLocalityName));
-        }
-        // and update the copy in the XdsLb instance. This
-        // serverlist instance will be destroyed either upon the next
-        // update or when the XdsLb instance is destroyed.
-        xdslb_policy->locality_serverlist_[0]->serverlist = serverlist;
-        xdslb_policy->locality_map_.UpdateLocked(
-            xdslb_policy->locality_serverlist_,
-            xdslb_policy->child_policy_config_.get(), xdslb_policy->args_,
-            xdslb_policy);
+    // Pending LB channel receives a serverlist; promote it.
+    // Note that this call can't be on a discarded pending channel, because
+    // such channels don't have any current call but we have checked this call
+    // is a current call.
+    if (!lb_calld->lb_chand_->IsCurrentChannel()) {
+      if (grpc_lb_xds_trace.enabled()) {
+        gpr_log(GPR_INFO,
+                "[xdslb %p] Promoting pending LB channel %p to replace "
+                "current LB channel %p",
+                xdslb_policy, lb_calld->lb_chand_.get(),
+                lb_calld->xdslb_policy()->lb_chand_.get());
       }
-    } else {
+      lb_calld->xdslb_policy()->lb_chand_ =
+          std::move(lb_calld->xdslb_policy()->pending_lb_chand_);
+    }
+    // Start sending client load report only after we start using the
+    // serverlist returned from the current LB call.
+    if (lb_calld->client_stats_report_interval_ > 0 &&
+        lb_calld->client_stats_ == nullptr) {
+      lb_calld->client_stats_ = MakeRefCounted<XdsLbClientStats>();
+      lb_calld->Ref(DEBUG_LOCATION, "client_load_report").release();
+      lb_calld->ScheduleNextClientLoadReportLocked();
+    }
+    if (!xdslb_policy->locality_serverlist_.empty() &&
+        xds_grpclb_serverlist_equals(
+            xdslb_policy->locality_serverlist_[0]->serverlist, serverlist)) {
       if (grpc_lb_xds_trace.enabled()) {
-        gpr_log(GPR_INFO, "[xdslb %p] Received empty server list, ignoring.",
+        gpr_log(GPR_INFO,
+                "[xdslb %p] Incoming server list identical to current, "
+                "ignoring.",
                 xdslb_policy);
       }
       xds_grpclb_destroy_serverlist(serverlist);
+    } else {  // New serverlist.
+      // If the balancer tells us to drop all the calls, we should exit fallback
+      // mode immediately.
+      // TODO(juanlishen): When we add EDS drop, we should change to check
+      // drop_percentage.
+      if (serverlist->num_servers == 0) xdslb_policy->MaybeExitFallbackMode();
+      if (!xdslb_policy->locality_serverlist_.empty()) {
+        xds_grpclb_destroy_serverlist(
+            xdslb_policy->locality_serverlist_[0]->serverlist);
+      } else {
+        // This is the first serverlist we've received, don't enter fallback
+        // mode.
+        xdslb_policy->MaybeCancelFallbackAtStartupChecks();
+        // Initialize locality serverlist, currently the list only handles
+        // one child.
+        xdslb_policy->locality_serverlist_.emplace_back(
+            MakeUnique<LocalityServerlistEntry>());
+        xdslb_policy->locality_serverlist_[0]->locality_name =
+            static_cast<char*>(gpr_strdup(kDefaultLocalityName));
+        xdslb_policy->locality_serverlist_[0]->locality_weight =
+            kDefaultLocalityWeight;
+      }
+      // Update the serverlist in the XdsLb instance. This serverlist
+      // instance will be destroyed either upon the next update or when the
+      // XdsLb instance is destroyed.
+      xdslb_policy->locality_serverlist_[0]->serverlist = serverlist;
+      xdslb_policy->locality_map_.UpdateLocked(
+          xdslb_policy->locality_serverlist_,
+          xdslb_policy->child_policy_config_.get(), xdslb_policy->args_,
+          xdslb_policy);
     }
   } else {
     // No valid initial response or serverlist found.
@@ -1027,6 +1266,18 @@ void XdsLb::BalancerChannelState::BalancerCallState::
         lb_chand->StartCallRetryTimerLocked();
       }
       xdslb_policy->channel_control_helper()->RequestReresolution();
+      // If the fallback-at-startup checks are pending, go into fallback mode
+      // immediately.  This short-circuits the timeout for the
+      // fallback-at-startup case.
+      if (xdslb_policy->fallback_at_startup_checks_pending_) {
+        gpr_log(GPR_INFO,
+                "[xdslb %p] Balancer call finished; entering fallback mode",
+                xdslb_policy);
+        xdslb_policy->fallback_at_startup_checks_pending_ = false;
+        grpc_timer_cancel(&xdslb_policy->lb_fallback_timer_);
+        lb_chand->CancelConnectivityWatchLocked();
+        xdslb_policy->UpdateFallbackPolicyLocked();
+      }
     }
   }
   lb_calld->Unref(DEBUG_LOCATION, "lb_call_ended");
@@ -1102,7 +1353,7 @@ XdsLb::XdsLb(Args args)
   arg = grpc_channel_args_find(args.args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS);
   lb_call_timeout_ms_ = grpc_channel_arg_get_integer(arg, {0, 0, INT_MAX});
   // Record fallback timeout.
-  arg = grpc_channel_args_find(args.args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS);
+  arg = grpc_channel_args_find(args.args, GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS);
   lb_fallback_timeout_ms_ = grpc_channel_arg_get_integer(
       arg, {GRPC_XDS_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX});
 }
@@ -1115,14 +1366,25 @@ XdsLb::~XdsLb() {
 
 void XdsLb::ShutdownLocked() {
   shutting_down_ = true;
-  if (fallback_timer_callback_pending_) {
+  if (fallback_at_startup_checks_pending_) {
     grpc_timer_cancel(&lb_fallback_timer_);
   }
   locality_map_.ShutdownLocked();
-  // We destroy the LB channel here instead of in our destructor because
-  // destroying the channel triggers a last callback to
-  // OnBalancerChannelConnectivityChangedLocked(), and we need to be
-  // alive when that callback is invoked.
+  if (fallback_policy_ != nullptr) {
+    grpc_pollset_set_del_pollset_set(fallback_policy_->interested_parties(),
+                                     interested_parties());
+  }
+  if (pending_fallback_policy_ != nullptr) {
+    grpc_pollset_set_del_pollset_set(
+        pending_fallback_policy_->interested_parties(), interested_parties());
+  }
+  {
+    MutexLock lock(&fallback_policy_mu_);
+    fallback_policy_.reset();
+    pending_fallback_policy_.reset();
+  }
+  // We reset the LB channels here instead of in our destructor because they
+  // hold refs to XdsLb.
   {
     MutexLock lock(&lb_chand_mu_);
     lb_chand_.reset();
@@ -1142,12 +1404,31 @@ void XdsLb::ResetBackoffLocked() {
     grpc_channel_reset_connect_backoff(pending_lb_chand_->channel());
   }
   locality_map_.ResetBackoffLocked();
+  if (fallback_policy_ != nullptr) {
+    fallback_policy_->ResetBackoffLocked();
+  }
+  if (pending_fallback_policy_ != nullptr) {
+    pending_fallback_policy_->ResetBackoffLocked();
+  }
 }
 
 void XdsLb::FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels,
                                      channelz::ChildRefsList* child_channels) {
-  // Delegate to the child_policy_ to fill the children subchannels.
+  // Delegate to the locality_map_ to fill the children subchannels.
   locality_map_.FillChildRefsForChannelz(child_subchannels, child_channels);
+  {
+    // This must be done holding fallback_policy_mu_, since this method does not
+    // run in the combiner.
+    MutexLock lock(&fallback_policy_mu_);
+    if (fallback_policy_ != nullptr) {
+      fallback_policy_->FillChildRefsForChannelz(child_subchannels,
+                                                 child_channels);
+    }
+    if (pending_fallback_policy_ != nullptr) {
+      pending_fallback_policy_->FillChildRefsForChannelz(child_subchannels,
+                                                         child_channels);
+    }
+  }
   MutexLock lock(&lb_chand_mu_);
   if (lb_chand_ != nullptr) {
     grpc_core::channelz::ChannelNode* channel_node =
@@ -1239,57 +1520,213 @@ void XdsLb::ParseLbConfig(Config* xds_config) {
 void XdsLb::UpdateLocked(UpdateArgs args) {
   const bool is_initial_update = lb_chand_ == nullptr;
   ParseLbConfig(args.config.get());
-  // TODO(juanlishen): Pass fallback policy config update after fallback policy
-  // is added.
   if (balancer_name_ == nullptr) {
     gpr_log(GPR_ERROR, "[xdslb %p] LB config parsing fails.", this);
     return;
   }
   ProcessAddressesAndChannelArgsLocked(args.addresses, *args.args);
-  // Update the existing child policy.
-  // Note: We have disabled fallback mode in the code, so this child policy must
-  // have been created from a serverlist.
-  // TODO(vpowar): Handle the fallback_address changes when we add support for
-  // fallback in xDS.
   locality_map_.UpdateLocked(locality_serverlist_, child_policy_config_.get(),
                              args_, this);
-  // If this is the initial update, start the fallback timer.
+  // Update the existing fallback policy. The fallback policy config and/or the
+  // fallback addresses may be new.
+  if (fallback_policy_ != nullptr) UpdateFallbackPolicyLocked();
+  // If this is the initial update, start the fallback-at-startup checks.
   if (is_initial_update) {
-    if (lb_fallback_timeout_ms_ > 0 && locality_serverlist_.empty() &&
-        !fallback_timer_callback_pending_) {
-      grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_;
-      Ref(DEBUG_LOCATION, "on_fallback_timer").release();  // Held by closure
-      GRPC_CLOSURE_INIT(&lb_on_fallback_, &XdsLb::OnFallbackTimerLocked, this,
-                        grpc_combiner_scheduler(combiner()));
-      fallback_timer_callback_pending_ = true;
-      grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_);
-      // TODO(juanlishen): Monitor the connectivity state of the balancer
-      // channel.  If the channel reports TRANSIENT_FAILURE before the
-      // fallback timeout expires, go into fallback mode early.
-    }
+    grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_;
+    Ref(DEBUG_LOCATION, "on_fallback_timer").release();  // Held by closure
+    GRPC_CLOSURE_INIT(&lb_on_fallback_, &XdsLb::OnFallbackTimerLocked, this,
+                      grpc_combiner_scheduler(combiner()));
+    fallback_at_startup_checks_pending_ = true;
+    grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_);
+    // Start watching the channel's connectivity state.  If the channel
+    // goes into state TRANSIENT_FAILURE, we go into fallback mode even if
+    // the fallback timeout has not elapsed.
+    lb_chand_->StartConnectivityWatchLocked();
   }
 }
 
 //
-// code for balancer channel and call
+// fallback-related methods
 //
 
+void XdsLb::MaybeCancelFallbackAtStartupChecks() {
+  if (!fallback_at_startup_checks_pending_) return;
+  gpr_log(GPR_INFO,
+          "[xdslb %p] Cancelling fallback timer and LB channel connectivity "
+          "watch",
+          this);
+  grpc_timer_cancel(&lb_fallback_timer_);
+  lb_chand_->CancelConnectivityWatchLocked();
+  fallback_at_startup_checks_pending_ = false;
+}
+
 void XdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) {
   XdsLb* xdslb_policy = static_cast<XdsLb*>(arg);
-  xdslb_policy->fallback_timer_callback_pending_ = false;
-  // If we receive a serverlist after the timer fires but before this callback
-  // actually runs, don't fall back.
-  if (xdslb_policy->locality_serverlist_.empty() &&
+  // If some fallback-at-startup check is done after the timer fires but before
+  // this callback actually runs, don't fall back.
+  if (xdslb_policy->fallback_at_startup_checks_pending_ &&
       !xdslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) {
     if (grpc_lb_xds_trace.enabled()) {
       gpr_log(GPR_INFO,
-              "[xdslb %p] Fallback timer fired. Not using fallback backends",
+              "[xdslb %p] Child policy not ready after fallback timeout; "
+              "entering fallback mode",
               xdslb_policy);
     }
+    xdslb_policy->fallback_at_startup_checks_pending_ = false;
+    xdslb_policy->UpdateFallbackPolicyLocked();
+    xdslb_policy->lb_chand_->CancelConnectivityWatchLocked();
   }
   xdslb_policy->Unref(DEBUG_LOCATION, "on_fallback_timer");
 }
 
+void XdsLb::UpdateFallbackPolicyLocked() {
+  if (shutting_down_) return;
+  // Construct update args.
+  UpdateArgs update_args;
+  update_args.addresses = fallback_backend_addresses_;
+  update_args.config = fallback_policy_config_ == nullptr
+                           ? nullptr
+                           : fallback_policy_config_->Ref();
+  update_args.args = grpc_channel_args_copy(args_);
+  // If the child policy name changes, we need to create a new child
+  // policy.  When this happens, we leave child_policy_ as-is and store
+  // the new child policy in pending_child_policy_.  Once the new child
+  // policy transitions into state READY, we swap it into child_policy_,
+  // replacing the original child policy.  So pending_child_policy_ is
+  // non-null only between when we apply an update that changes the child
+  // policy name and when the new child reports state READY.
+  //
+  // Updates can arrive at any point during this transition.  We always
+  // apply updates relative to the most recently created child policy,
+  // even if the most recent one is still in pending_child_policy_.  This
+  // is true both when applying the updates to an existing child policy
+  // and when determining whether we need to create a new policy.
+  //
+  // As a result of this, there are several cases to consider here:
+  //
+  // 1. We have no existing child policy (i.e., we have started up but
+  //    have not yet received a serverlist from the balancer or gone
+  //    into fallback mode; in this case, both child_policy_ and
+  //    pending_child_policy_ are null).  In this case, we create a
+  //    new child policy and store it in child_policy_.
+  //
+  // 2. We have an existing child policy and have no pending child policy
+  //    from a previous update (i.e., either there has not been a
+  //    previous update that changed the policy name, or we have already
+  //    finished swapping in the new policy; in this case, child_policy_
+  //    is non-null but pending_child_policy_ is null).  In this case:
+  //    a. If child_policy_->name() equals child_policy_name, then we
+  //       update the existing child policy.
+  //    b. If child_policy_->name() does not equal child_policy_name,
+  //       we create a new policy.  The policy will be stored in
+  //       pending_child_policy_ and will later be swapped into
+  //       child_policy_ by the helper when the new child transitions
+  //       into state READY.
+  //
+  // 3. We have an existing child policy and have a pending child policy
+  //    from a previous update (i.e., a previous update set
+  //    pending_child_policy_ as per case 2b above and that policy has
+  //    not yet transitioned into state READY and been swapped into
+  //    child_policy_; in this case, both child_policy_ and
+  //    pending_child_policy_ are non-null).  In this case:
+  //    a. If pending_child_policy_->name() equals child_policy_name,
+  //       then we update the existing pending child policy.
+  //    b. If pending_child_policy->name() does not equal
+  //       child_policy_name, then we create a new policy.  The new
+  //       policy is stored in pending_child_policy_ (replacing the one
+  //       that was there before, which will be immediately shut down)
+  //       and will later be swapped into child_policy_ by the helper
+  //       when the new child transitions into state READY.
+  const char* fallback_policy_name = fallback_policy_config_ == nullptr
+                                         ? "round_robin"
+                                         : fallback_policy_config_->name();
+  const bool create_policy =
+      // case 1
+      fallback_policy_ == nullptr ||
+      // case 2b
+      (pending_fallback_policy_ == nullptr &&
+       strcmp(fallback_policy_->name(), fallback_policy_name) != 0) ||
+      // case 3b
+      (pending_fallback_policy_ != nullptr &&
+       strcmp(pending_fallback_policy_->name(), fallback_policy_name) != 0);
+  LoadBalancingPolicy* policy_to_update = nullptr;
+  if (create_policy) {
+    // Cases 1, 2b, and 3b: create a new child policy.
+    // If child_policy_ is null, we set it (case 1), else we set
+    // pending_child_policy_ (cases 2b and 3b).
+    if (grpc_lb_xds_trace.enabled()) {
+      gpr_log(GPR_INFO, "[xdslb %p] Creating new %sfallback policy %s", this,
+              fallback_policy_ == nullptr ? "" : "pending ",
+              fallback_policy_name);
+    }
+    auto new_policy =
+        CreateFallbackPolicyLocked(fallback_policy_name, update_args.args);
+    auto& lb_policy = fallback_policy_ == nullptr ? fallback_policy_
+                                                  : pending_fallback_policy_;
+    {
+      MutexLock lock(&fallback_policy_mu_);
+      lb_policy = std::move(new_policy);
+    }
+    policy_to_update = lb_policy.get();
+  } else {
+    // Cases 2a and 3a: update an existing policy.
+    // If we have a pending child policy, send the update to the pending
+    // policy (case 3a), else send it to the current policy (case 2a).
+    policy_to_update = pending_fallback_policy_ != nullptr
+                           ? pending_fallback_policy_.get()
+                           : fallback_policy_.get();
+  }
+  GPR_ASSERT(policy_to_update != nullptr);
+  // Update the policy.
+  if (grpc_lb_xds_trace.enabled()) {
+    gpr_log(
+        GPR_INFO, "[xdslb %p] Updating %sfallback policy %p", this,
+        policy_to_update == pending_fallback_policy_.get() ? "pending " : "",
+        policy_to_update);
+  }
+  policy_to_update->UpdateLocked(std::move(update_args));
+}
+
+OrphanablePtr<LoadBalancingPolicy> XdsLb::CreateFallbackPolicyLocked(
+    const char* name, const grpc_channel_args* args) {
+  FallbackHelper* helper = New<FallbackHelper>(Ref());
+  LoadBalancingPolicy::Args lb_policy_args;
+  lb_policy_args.combiner = combiner();
+  lb_policy_args.args = args;
+  lb_policy_args.channel_control_helper =
+      UniquePtr<ChannelControlHelper>(helper);
+  OrphanablePtr<LoadBalancingPolicy> lb_policy =
+      LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(
+          name, std::move(lb_policy_args));
+  if (GPR_UNLIKELY(lb_policy == nullptr)) {
+    gpr_log(GPR_ERROR, "[xdslb %p] Failure creating fallback policy %s", this,
+            name);
+    return nullptr;
+  }
+  helper->set_child(lb_policy.get());
+  if (grpc_lb_xds_trace.enabled()) {
+    gpr_log(GPR_INFO, "[xdslb %p] Created new fallback policy %s (%p)", this,
+            name, lb_policy.get());
+  }
+  // Add the xDS's interested_parties pollset_set to that of the newly created
+  // child policy. This will make the child policy progress upon activity on xDS
+  // LB, which in turn is tied to the application's call.
+  grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(),
+                                   interested_parties());
+  return lb_policy;
+}
+
+void XdsLb::MaybeExitFallbackMode() {
+  if (fallback_policy_ == nullptr) return;
+  gpr_log(GPR_INFO, "[xdslb %p] Exiting fallback mode", this);
+  fallback_policy_.reset();
+  pending_fallback_policy_.reset();
+}
+
+//
+// XdsLb::LocalityMap
+//
+
 void XdsLb::LocalityMap::PruneLocalities(const LocalityList& locality_list) {
   for (auto iter = map_.begin(); iter != map_.end();) {
     bool found = false;
@@ -1316,8 +1753,8 @@ void XdsLb::LocalityMap::UpdateLocked(
         gpr_strdup(locality_serverlist[i]->locality_name));
     auto iter = map_.find(locality_name);
     if (iter == map_.end()) {
-      OrphanablePtr<LocalityEntry> new_entry =
-          MakeOrphanable<LocalityEntry>(parent->Ref());
+      OrphanablePtr<LocalityEntry> new_entry = MakeOrphanable<LocalityEntry>(
+          parent->Ref(), locality_serverlist[i]->locality_weight);
       MutexLock lock(&child_refs_mu_);
       iter = map_.emplace(std::move(locality_name), std::move(new_entry)).first;
     }
@@ -1329,27 +1766,29 @@ void XdsLb::LocalityMap::UpdateLocked(
   PruneLocalities(locality_serverlist);
 }
 
-void grpc_core::XdsLb::LocalityMap::ShutdownLocked() {
+void XdsLb::LocalityMap::ShutdownLocked() {
   MutexLock lock(&child_refs_mu_);
   map_.clear();
 }
 
-void grpc_core::XdsLb::LocalityMap::ResetBackoffLocked() {
-  for (auto iter = map_.begin(); iter != map_.end(); iter++) {
-    iter->second->ResetBackoffLocked();
+void XdsLb::LocalityMap::ResetBackoffLocked() {
+  for (auto& p : map_) {
+    p.second->ResetBackoffLocked();
   }
 }
 
-void grpc_core::XdsLb::LocalityMap::FillChildRefsForChannelz(
+void XdsLb::LocalityMap::FillChildRefsForChannelz(
     channelz::ChildRefsList* child_subchannels,
     channelz::ChildRefsList* child_channels) {
   MutexLock lock(&child_refs_mu_);
-  for (auto iter = map_.begin(); iter != map_.end(); iter++) {
-    iter->second->FillChildRefsForChannelz(child_subchannels, child_channels);
+  for (auto& p : map_) {
+    p.second->FillChildRefsForChannelz(child_subchannels, child_channels);
   }
 }
 
-// Locality Entry child policy methods
+//
+// XdsLb::LocalityMap::LocalityEntry
+//
 
 grpc_channel_args*
 XdsLb::LocalityMap::LocalityEntry::CreateChildPolicyArgsLocked(
@@ -1404,18 +1843,12 @@ void XdsLb::LocalityMap::LocalityEntry::UpdateLocked(
     LoadBalancingPolicy::Config* child_policy_config,
     const grpc_channel_args* args_in) {
   if (parent_->shutting_down_) return;
-  // This should never be invoked if we do not have serverlist_, as fallback
-  // mode is disabled for xDS plugin.
-  // TODO(juanlishen): Change this as part of implementing fallback mode.
-  GPR_ASSERT(serverlist != nullptr);
-  GPR_ASSERT(serverlist->num_servers > 0);
   // Construct update args.
   UpdateArgs update_args;
   update_args.addresses = ProcessServerlist(serverlist);
   update_args.config =
       child_policy_config == nullptr ? nullptr : child_policy_config->Ref();
   update_args.args = CreateChildPolicyArgsLocked(args_in);
-
   // If the child policy name changes, we need to create a new child
   // policy.  When this happens, we leave child_policy_ as-is and store
   // the new child policy in pending_child_policy_.  Once the new child
@@ -1556,8 +1989,9 @@ void XdsLb::LocalityMap::LocalityEntry::Orphan() {
 }
 
 //
-// LocalityEntry::Helper implementation
+// XdsLb::LocalityEntry::Helper
 //
+
 bool XdsLb::LocalityMap::LocalityEntry::Helper::CalledByPendingChild() const {
   GPR_ASSERT(child_ != nullptr);
   return child_ == entry_->pending_child_policy_.get();
@@ -1587,12 +2021,8 @@ grpc_channel* XdsLb::LocalityMap::LocalityEntry::Helper::CreateChannel(
 }
 
 void XdsLb::LocalityMap::LocalityEntry::Helper::UpdateState(
-    grpc_connectivity_state state, grpc_error* state_error,
-    UniquePtr<SubchannelPicker> picker) {
-  if (entry_->parent_->shutting_down_) {
-    GRPC_ERROR_UNREF(state_error);
-    return;
-  }
+    grpc_connectivity_state state, UniquePtr<SubchannelPicker> picker) {
+  if (entry_->parent_->shutting_down_) return;
   // If this request is from the pending child policy, ignore it until
   // it reports READY, at which point we swap it into place.
   if (CalledByPendingChild()) {
@@ -1602,10 +2032,7 @@ void XdsLb::LocalityMap::LocalityEntry::Helper::UpdateState(
               entry_->parent_.get(), this, entry_->pending_child_policy_.get(),
               grpc_connectivity_state_name(state));
     }
-    if (state != GRPC_CHANNEL_READY) {
-      GRPC_ERROR_UNREF(state_error);
-      return;
-    }
+    if (state != GRPC_CHANNEL_READY) return;
     grpc_pollset_set_del_pollset_set(
         entry_->child_policy_->interested_parties(),
         entry_->parent_->interested_parties());
@@ -1613,21 +2040,83 @@ void XdsLb::LocalityMap::LocalityEntry::Helper::UpdateState(
     entry_->child_policy_ = std::move(entry_->pending_child_policy_);
   } else if (!CalledByCurrentChild()) {
     // This request is from an outdated child, so ignore it.
-    GRPC_ERROR_UNREF(state_error);
     return;
   }
-  // TODO(juanlishen): When in fallback mode, pass the child picker
-  // through without wrapping it.  (Or maybe use a different helper for
-  // the fallback policy?)
+  // At this point, child_ must be the current child policy.
+  if (state == GRPC_CHANNEL_READY) entry_->parent_->MaybeExitFallbackMode();
+  // If we are in fallback mode, ignore update request from the child policy.
+  if (entry_->parent_->fallback_policy_ != nullptr) return;
   GPR_ASSERT(entry_->parent_->lb_chand_ != nullptr);
   RefCountedPtr<XdsLbClientStats> client_stats =
       entry_->parent_->lb_chand_->lb_calld() == nullptr
           ? nullptr
           : entry_->parent_->lb_chand_->lb_calld()->client_stats();
-  entry_->parent_->channel_control_helper()->UpdateState(
-      state, state_error,
-      UniquePtr<SubchannelPicker>(
-          New<Picker>(std::move(picker), std::move(client_stats))));
+  // Cache the picker and its state in the entry
+  entry_->picker_ref_ = MakeRefCounted<PickerRef>(std::move(picker));
+  entry_->connectivity_state_ = state;
+  // Construct a new xds picker which maintains a map of all locality pickers
+  // that are ready. Each locality is represented by a portion of the range
+  // proportional to its weight, such that the total range is the sum of the
+  // weights of all localities
+  uint32_t end = 0;
+  size_t num_connecting = 0;
+  size_t num_idle = 0;
+  size_t num_transient_failures = 0;
+  auto& locality_map = this->entry_->parent_->locality_map_.map_;
+  Picker::PickerList pickers;
+  for (auto& p : locality_map) {
+    const LocalityEntry* entry = p.second.get();
+    grpc_connectivity_state connectivity_state = entry->connectivity_state_;
+    switch (connectivity_state) {
+      case GRPC_CHANNEL_READY: {
+        end += entry->locality_weight_;
+        pickers.push_back(MakePair(end, entry->picker_ref_));
+        break;
+      }
+      case GRPC_CHANNEL_CONNECTING: {
+        num_connecting++;
+        break;
+      }
+      case GRPC_CHANNEL_IDLE: {
+        num_idle++;
+        break;
+      }
+      case GRPC_CHANNEL_TRANSIENT_FAILURE: {
+        num_transient_failures++;
+        break;
+      }
+      default: {
+        gpr_log(GPR_ERROR, "Invalid locality connectivity state - %d",
+                connectivity_state);
+      }
+    }
+  }
+  // Pass on the constructed xds picker if it has any ready pickers in their map
+  // otherwise pass a QueuePicker if any of the locality pickers are in a
+  // connecting or idle state, finally return a transient failure picker if all
+  // locality pickers are in transient failure
+  if (pickers.size() > 0) {
+    entry_->parent_->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_READY,
+        UniquePtr<LoadBalancingPolicy::SubchannelPicker>(
+            New<Picker>(std::move(client_stats), std::move(pickers))));
+  } else if (num_connecting > 0) {
+    entry_->parent_->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_CONNECTING,
+        UniquePtr<SubchannelPicker>(New<QueuePicker>(this->entry_->parent_)));
+  } else if (num_idle > 0) {
+    entry_->parent_->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_IDLE,
+        UniquePtr<SubchannelPicker>(New<QueuePicker>(this->entry_->parent_)));
+  } else {
+    GPR_ASSERT(num_transient_failures == locality_map.size());
+    grpc_error* error =
+        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                               "connections to all localities failing"),
+                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+    entry_->parent_->channel_control_helper()->UpdateState(
+        state, UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
+  }
 }
 
 void XdsLb::LocalityMap::LocalityEntry::Helper::RequestReresolution() {

src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc

@@ -43,6 +43,7 @@
 #include "src/core/lib/gprpp/manual_constructor.h"
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/gethostname.h"
+#include "src/core/lib/iomgr/iomgr_custom.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/json/json.h"
@@ -308,7 +309,11 @@ void AresDnsResolver::OnResolvedLocked(void* arg, grpc_error* error) {
       if (service_config_string != nullptr) {
         GRPC_CARES_TRACE_LOG("resolver:%p selected service config choice: %s",
                              r, service_config_string);
-        result.service_config = ServiceConfig::Create(service_config_string);
+        grpc_error* service_config_error = GRPC_ERROR_NONE;
+        result.service_config =
+            ServiceConfig::Create(service_config_string, &service_config_error);
+        // Error is currently unused.
+        GRPC_ERROR_UNREF(service_config_error);
       }
       gpr_free(service_config_string);
     }
@@ -426,8 +431,11 @@ static grpc_address_resolver_vtable ares_resolver = {
     grpc_resolve_address_ares, blocking_resolve_address_ares};
 
 static bool should_use_ares(const char* resolver_env) {
-  return resolver_env == nullptr || strlen(resolver_env) == 0 ||
-         gpr_stricmp(resolver_env, "ares") == 0;
+  // TODO(lidiz): Remove the "g_custom_iomgr_enabled" flag once c-ares support
+  // custom IO managers (e.g. gevent).
+  return !g_custom_iomgr_enabled &&
+         (resolver_env == nullptr || strlen(resolver_env) == 0 ||
+          gpr_stricmp(resolver_env, "ares") == 0);
 }
 
 void grpc_resolver_dns_ares_init() {

src/core/ext/filters/client_channel/resolver_result_parsing.cc

@@ -52,7 +52,10 @@ ProcessedResolverResult::ProcessedResolverResult(
     const char* service_config_json = grpc_channel_arg_get_string(
         grpc_channel_args_find(resolver_result->args, GRPC_ARG_SERVICE_CONFIG));
     if (service_config_json != nullptr) {
-      service_config_ = ServiceConfig::Create(service_config_json);
+      grpc_error* error = GRPC_ERROR_NONE;
+      service_config_ = ServiceConfig::Create(service_config_json, &error);
+      // Error is currently unused.
+      GRPC_ERROR_UNREF(error);
     }
   } else {
     // Add the service config JSON to channel args so that it's

src/core/ext/filters/client_channel/resolving_lb_policy.cc

@@ -119,13 +119,9 @@ class ResolvingLoadBalancingPolicy::ResolvingControlHelper
     return parent_->channel_control_helper()->CreateChannel(target, args);
   }
 
-  void UpdateState(grpc_connectivity_state state, grpc_error* state_error,
+  void UpdateState(grpc_connectivity_state state,
                    UniquePtr<SubchannelPicker> picker) override {
-    if (parent_->resolver_ == nullptr) {
-      // shutting down.
-      GRPC_ERROR_UNREF(state_error);
-      return;
-    }
+    if (parent_->resolver_ == nullptr) return;  // Shutting down.
     // If this request is from the pending child policy, ignore it until
     // it reports READY, at which point we swap it into place.
     if (CalledByPendingChild()) {
@@ -136,10 +132,7 @@ class ResolvingLoadBalancingPolicy::ResolvingControlHelper
                 parent_.get(), this, child_,
                 grpc_connectivity_state_name(state));
       }
-      if (state != GRPC_CHANNEL_READY) {
-        GRPC_ERROR_UNREF(state_error);
-        return;
-      }
+      if (state != GRPC_CHANNEL_READY) return;
       grpc_pollset_set_del_pollset_set(
           parent_->lb_policy_->interested_parties(),
           parent_->interested_parties());
@@ -147,11 +140,9 @@ class ResolvingLoadBalancingPolicy::ResolvingControlHelper
       parent_->lb_policy_ = std::move(parent_->pending_lb_policy_);
     } else if (!CalledByCurrentChild()) {
       // This request is from an outdated child, so ignore it.
-      GRPC_ERROR_UNREF(state_error);
       return;
     }
-    parent_->channel_control_helper()->UpdateState(state, state_error,
-                                                   std::move(picker));
+    parent_->channel_control_helper()->UpdateState(state, std::move(picker));
   }
 
   void RequestReresolution() override {
@@ -234,8 +225,7 @@ grpc_error* ResolvingLoadBalancingPolicy::Init(const grpc_channel_args& args) {
   }
   // Return our picker to the channel.
   channel_control_helper()->UpdateState(
-      GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE,
-      UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
+      GRPC_CHANNEL_IDLE, UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
   return GRPC_ERROR_NONE;
 }
 
@@ -313,7 +303,7 @@ void ResolvingLoadBalancingPolicy::StartResolvingLocked() {
   GPR_ASSERT(!started_resolving_);
   started_resolving_ = true;
   channel_control_helper()->UpdateState(
-      GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
+      GRPC_CHANNEL_CONNECTING,
       UniquePtr<SubchannelPicker>(New<QueuePicker>(Ref())));
   resolver_->StartLocked();
 }
@@ -334,7 +324,7 @@ void ResolvingLoadBalancingPolicy::OnResolverError(grpc_error* error) {
     grpc_error* state_error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
         "Resolver transient failure", &error, 1);
     channel_control_helper()->UpdateState(
-        GRPC_CHANNEL_TRANSIENT_FAILURE, GRPC_ERROR_REF(state_error),
+        GRPC_CHANNEL_TRANSIENT_FAILURE,
         UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(state_error)));
   }
   GRPC_ERROR_UNREF(error);

src/core/ext/filters/client_channel/service_config.cc

@@ -33,23 +33,201 @@
 
 namespace grpc_core {
 
-RefCountedPtr<ServiceConfig> ServiceConfig::Create(const char* json) {
+namespace {
+typedef InlinedVector<UniquePtr<ServiceConfigParser>,
+                      ServiceConfig::kNumPreallocatedParsers>
+    ServiceConfigParserList;
+ServiceConfigParserList* registered_parsers;
+
+// Consumes all the errors in the vector and forms a referencing error from
+// them. If the vector is empty, return GRPC_ERROR_NONE.
+template <size_t N>
+grpc_error* CreateErrorFromVector(const char* desc,
+                                  InlinedVector<grpc_error*, N>* error_list) {
+  grpc_error* error = GRPC_ERROR_NONE;
+  if (error_list->size() != 0) {
+    error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+        desc, error_list->data(), error_list->size());
+    // Remove refs to all errors in error_list.
+    for (size_t i = 0; i < error_list->size(); i++) {
+      GRPC_ERROR_UNREF((*error_list)[i]);
+    }
+    error_list->clear();
+  }
+  return error;
+}
+}  // namespace
+
+RefCountedPtr<ServiceConfig> ServiceConfig::Create(const char* json,
+                                                   grpc_error** error) {
   UniquePtr<char> service_config_json(gpr_strdup(json));
   UniquePtr<char> json_string(gpr_strdup(json));
+  GPR_DEBUG_ASSERT(error != nullptr);
   grpc_json* json_tree = grpc_json_parse_string(json_string.get());
   if (json_tree == nullptr) {
-    gpr_log(GPR_INFO, "failed to parse JSON for service config");
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "failed to parse JSON for service config");
     return nullptr;
   }
-  return MakeRefCounted<ServiceConfig>(std::move(service_config_json),
-                                       std::move(json_string), json_tree);
+  return MakeRefCounted<ServiceConfig>(
+      std::move(service_config_json), std::move(json_string), json_tree, error);
 }
 
 ServiceConfig::ServiceConfig(UniquePtr<char> service_config_json,
-                             UniquePtr<char> json_string, grpc_json* json_tree)
+                             UniquePtr<char> json_string, grpc_json* json_tree,
+                             grpc_error** error)
     : service_config_json_(std::move(service_config_json)),
       json_string_(std::move(json_string)),
-      json_tree_(json_tree) {}
+      json_tree_(json_tree) {
+  GPR_DEBUG_ASSERT(error != nullptr);
+  if (json_tree->type != GRPC_JSON_OBJECT || json_tree->key != nullptr) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "Malformed service Config JSON object");
+    return;
+  }
+  grpc_error* error_list[2];
+  int error_count = 0;
+  grpc_error* global_error = ParseGlobalParams(json_tree);
+  grpc_error* local_error = ParsePerMethodParams(json_tree);
+  if (global_error != GRPC_ERROR_NONE) {
+    error_list[error_count++] = global_error;
+  }
+  if (local_error != GRPC_ERROR_NONE) {
+    error_list[error_count++] = local_error;
+  }
+  if (error_count > 0) {
+    *error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
+        "Service config parsing error", error_list, error_count);
+    GRPC_ERROR_UNREF(global_error);
+    GRPC_ERROR_UNREF(local_error);
+  }
+}
+
+grpc_error* ServiceConfig::ParseGlobalParams(const grpc_json* json_tree) {
+  GPR_DEBUG_ASSERT(json_tree_->type == GRPC_JSON_OBJECT);
+  GPR_DEBUG_ASSERT(json_tree_->key == nullptr);
+  InlinedVector<grpc_error*, 4> error_list;
+  for (size_t i = 0; i < registered_parsers->size(); i++) {
+    grpc_error* parser_error = GRPC_ERROR_NONE;
+    auto parsed_obj =
+        (*registered_parsers)[i]->ParseGlobalParams(json_tree, &parser_error);
+    if (parser_error != GRPC_ERROR_NONE) {
+      error_list.push_back(parser_error);
+    }
+    parsed_global_service_config_objects_.push_back(std::move(parsed_obj));
+  }
+  return CreateErrorFromVector("Global Params", &error_list);
+}
+
+grpc_error* ServiceConfig::ParseJsonMethodConfigToServiceConfigObjectsTable(
+    const grpc_json* json,
+    SliceHashTable<const ServiceConfigObjectsVector*>::Entry* entries,
+    size_t* idx) {
+  auto objs_vector = MakeUnique<ServiceConfigObjectsVector>();
+  InlinedVector<grpc_error*, 4> error_list;
+  for (size_t i = 0; i < registered_parsers->size(); i++) {
+    grpc_error* parser_error = GRPC_ERROR_NONE;
+    auto parsed_obj =
+        (*registered_parsers)[i]->ParsePerMethodParams(json, &parser_error);
+    if (parser_error != GRPC_ERROR_NONE) {
+      error_list.push_back(parser_error);
+    }
+    objs_vector->push_back(std::move(parsed_obj));
+  }
+  const auto* vector_ptr = objs_vector.get();
+  service_config_objects_vectors_storage_.push_back(std::move(objs_vector));
+  // Construct list of paths.
+  InlinedVector<UniquePtr<char>, 10> paths;
+  for (grpc_json* child = json->child; child != nullptr; child = child->next) {
+    if (child->key == nullptr) continue;
+    if (strcmp(child->key, "name") == 0) {
+      if (child->type != GRPC_JSON_ARRAY) {
+        error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "field:name error:not of type Array"));
+        goto wrap_error;
+      }
+      for (grpc_json* name = child->child; name != nullptr; name = name->next) {
+        grpc_error* parse_error = GRPC_ERROR_NONE;
+        UniquePtr<char> path = ParseJsonMethodName(name, &parse_error);
+        if (path == nullptr) {
+          error_list.push_back(parse_error);
+        } else {
+          GPR_DEBUG_ASSERT(parse_error == GRPC_ERROR_NONE);
+          paths.push_back(std::move(path));
+        }
+      }
+    }
+  }
+  if (paths.size() == 0) {
+    error_list.push_back(
+        GRPC_ERROR_CREATE_FROM_STATIC_STRING("No names specified"));
+  }
+  // Add entry for each path.
+  for (size_t i = 0; i < paths.size(); ++i) {
+    entries[*idx].key = grpc_slice_from_copied_string(paths[i].get());
+    entries[*idx].value = vector_ptr;
+    ++*idx;
+  }
+wrap_error:
+  return CreateErrorFromVector("methodConfig", &error_list);
+}
+
+grpc_error* ServiceConfig::ParsePerMethodParams(const grpc_json* json_tree) {
+  GPR_DEBUG_ASSERT(json_tree_->type == GRPC_JSON_OBJECT);
+  GPR_DEBUG_ASSERT(json_tree_->key == nullptr);
+  SliceHashTable<const ServiceConfigObjectsVector*>::Entry* entries = nullptr;
+  size_t num_entries = 0;
+  InlinedVector<grpc_error*, 4> error_list;
+  for (grpc_json* field = json_tree->child; field != nullptr;
+       field = field->next) {
+    if (field->key == nullptr) {
+      error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "error:Illegal key value - NULL"));
+      continue;
+    }
+    if (strcmp(field->key, "methodConfig") == 0) {
+      if (entries != nullptr) {
+        GPR_ASSERT(false);
+      }
+      if (field->type != GRPC_JSON_ARRAY) {
+        error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "field:methodConfig error:not of type Array"));
+      }
+      for (grpc_json* method = field->child; method != nullptr;
+           method = method->next) {
+        int count = CountNamesInMethodConfig(method);
+        if (count <= 0) {
+          error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+              "field:methodConfig error:No names found"));
+        }
+        num_entries += static_cast<size_t>(count);
+      }
+      entries = static_cast<
+          SliceHashTable<const ServiceConfigObjectsVector*>::Entry*>(gpr_zalloc(
+          num_entries *
+          sizeof(SliceHashTable<const ServiceConfigObjectsVector*>::Entry)));
+      size_t idx = 0;
+      for (grpc_json* method = field->child; method != nullptr;
+           method = method->next) {
+        grpc_error* error = ParseJsonMethodConfigToServiceConfigObjectsTable(
+            method, entries, &idx);
+        if (error != GRPC_ERROR_NONE) {
+          error_list.push_back(error);
+        }
+      }
+      // idx might not be equal to num_entries due to parsing errors
+      num_entries = idx;
+      break;
+    }
+  }
+  if (entries != nullptr) {
+    parsed_method_service_config_objects_table_ =
+        SliceHashTable<const ServiceConfigObjectsVector*>::Create(
+            num_entries, entries, nullptr);
+    gpr_free(entries);
+  }
+  return CreateErrorFromVector("Method Params", &error_list);
+}
 
 ServiceConfig::~ServiceConfig() { grpc_json_destroy(json_tree_); }
 
@@ -84,28 +262,99 @@ int ServiceConfig::CountNamesInMethodConfig(grpc_json* json) {
   return num_names;
 }
 
-UniquePtr<char> ServiceConfig::ParseJsonMethodName(grpc_json* json) {
-  if (json->type != GRPC_JSON_OBJECT) return nullptr;
+UniquePtr<char> ServiceConfig::ParseJsonMethodName(grpc_json* json,
+                                                   grpc_error** error) {
+  if (json->type != GRPC_JSON_OBJECT) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "field:name error:type is not object");
+    return nullptr;
+  }
   const char* service_name = nullptr;
   const char* method_name = nullptr;
   for (grpc_json* child = json->child; child != nullptr; child = child->next) {
-    if (child->key == nullptr) return nullptr;
-    if (child->type != GRPC_JSON_STRING) return nullptr;
+    if (child->key == nullptr) {
+      *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "field:name error:Child entry with no key");
+      return nullptr;
+    }
+    if (child->type != GRPC_JSON_STRING) {
+      *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+          "field:name error:Child entry not of type string");
+      return nullptr;
+    }
     if (strcmp(child->key, "service") == 0) {
-      if (service_name != nullptr) return nullptr;  // Duplicate.
-      if (child->value == nullptr) return nullptr;
+      if (service_name != nullptr) {
+        *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "field:name error: field:service error:Multiple entries");
+        return nullptr;  // Duplicate.
+      }
+      if (child->value == nullptr) {
+        *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "field:name error: field:service error:empty value");
+        return nullptr;
+      }
       service_name = child->value;
     } else if (strcmp(child->key, "method") == 0) {
-      if (method_name != nullptr) return nullptr;  // Duplicate.
-      if (child->value == nullptr) return nullptr;
+      if (method_name != nullptr) {
+        *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "field:name error: field:method error:multiple entries");
+        return nullptr;  // Duplicate.
+      }
+      if (child->value == nullptr) {
+        *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+            "field:name error: field:method error:empty value");
+        return nullptr;
+      }
       method_name = child->value;
     }
   }
-  if (service_name == nullptr) return nullptr;  // Required field.
+  if (service_name == nullptr) {
+    *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+        "field:name error: field:service error:not found");
+    return nullptr;  // Required field.
+  }
   char* path;
   gpr_asprintf(&path, "/%s/%s", service_name,
                method_name == nullptr ? "*" : method_name);
   return UniquePtr<char>(path);
 }
 
+const ServiceConfig::ServiceConfigObjectsVector* const*
+ServiceConfig::GetMethodServiceConfigObjectsVector(const grpc_slice& path) {
+  const auto* value = parsed_method_service_config_objects_table_->Get(path);
+  // If we didn't find a match for the path, try looking for a wildcard
+  // entry (i.e., change "/service/method" to "/service/*").
+  if (value == nullptr) {
+    char* path_str = grpc_slice_to_c_string(path);
+    const char* sep = strrchr(path_str, '/') + 1;
+    const size_t len = (size_t)(sep - path_str);
+    char* buf = (char*)gpr_malloc(len + 2);  // '*' and NUL
+    memcpy(buf, path_str, len);
+    buf[len] = '*';
+    buf[len + 1] = '\0';
+    grpc_slice wildcard_path = grpc_slice_from_copied_string(buf);
+    gpr_free(buf);
+    value = parsed_method_service_config_objects_table_->Get(wildcard_path);
+    grpc_slice_unref_internal(wildcard_path);
+    gpr_free(path_str);
+    if (value == nullptr) return nullptr;
+  }
+  return value;
+}
+
+size_t ServiceConfig::RegisterParser(UniquePtr<ServiceConfigParser> parser) {
+  registered_parsers->push_back(std::move(parser));
+  return registered_parsers->size() - 1;
+}
+
+void ServiceConfig::Init() {
+  GPR_ASSERT(registered_parsers == nullptr);
+  registered_parsers = New<ServiceConfigParserList>();
+}
+
+void ServiceConfig::Shutdown() {
+  Delete(registered_parsers);
+  registered_parsers = nullptr;
+}
+
 }  // namespace grpc_core

src/core/ext/filters/client_channel/service_config.h

@@ -25,6 +25,7 @@
 #include "src/core/lib/gprpp/inlined_vector.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/json/json.h"
 #include "src/core/lib/slice/slice_hash_table.h"
 
@@ -54,11 +55,46 @@
 
 namespace grpc_core {
 
+/// This is the base class that all service config parsers MUST use to store
+/// parsed service config data.
+class ServiceConfigParsedObject {
+ public:
+  virtual ~ServiceConfigParsedObject() = default;
+
+  GRPC_ABSTRACT_BASE_CLASS;
+};
+
+/// This is the base class that all service config parsers should derive from.
+class ServiceConfigParser {
+ public:
+  virtual ~ServiceConfigParser() = default;
+
+  virtual UniquePtr<ServiceConfigParsedObject> ParseGlobalParams(
+      const grpc_json* json, grpc_error** error) {
+    GPR_DEBUG_ASSERT(error != nullptr);
+    return nullptr;
+  }
+
+  virtual UniquePtr<ServiceConfigParsedObject> ParsePerMethodParams(
+      const grpc_json* json, grpc_error** error) {
+    GPR_DEBUG_ASSERT(error != nullptr);
+    return nullptr;
+  }
+
+  GRPC_ABSTRACT_BASE_CLASS;
+};
+
 class ServiceConfig : public RefCounted<ServiceConfig> {
  public:
+  static constexpr int kNumPreallocatedParsers = 4;
+  typedef InlinedVector<UniquePtr<ServiceConfigParsedObject>,
+                        kNumPreallocatedParsers>
+      ServiceConfigObjectsVector;
+
   /// Creates a new service config from parsing \a json_string.
   /// Returns null on parse error.
-  static RefCountedPtr<ServiceConfig> Create(const char* json);
+  static RefCountedPtr<ServiceConfig> Create(const char* json,
+                                             grpc_error** error);
 
   ~ServiceConfig();
 
@@ -96,6 +132,30 @@ class ServiceConfig : public RefCounted<ServiceConfig> {
   static RefCountedPtr<T> MethodConfigTableLookup(
       const SliceHashTable<RefCountedPtr<T>>& table, const grpc_slice& path);
 
+  /// Retrieves the parsed global service config object at index \a index.
+  ServiceConfigParsedObject* GetParsedGlobalServiceConfigObject(int index) {
+    GPR_DEBUG_ASSERT(
+        index < static_cast<int>(parsed_global_service_config_objects_.size()));
+    return parsed_global_service_config_objects_[index].get();
+  }
+
+  /// Retrieves the vector of method service config objects for a given path \a
+  /// path.
+  const ServiceConfigObjectsVector* const* GetMethodServiceConfigObjectsVector(
+      const grpc_slice& path);
+
+  /// Globally register a service config parser. On successful registration, it
+  /// returns the index at which the parser was registered. On failure, -1 is
+  /// returned. Each new service config update will go through all the
+  /// registered parser. Each parser is responsible for reading the service
+  /// config json and returning a parsed object. This parsed object can later be
+  /// retrieved using the same index that was returned at registration time.
+  static size_t RegisterParser(UniquePtr<ServiceConfigParser> parser);
+
+  static void Init();
+
+  static void Shutdown();
+
  private:
   // So New() can call our private ctor.
   template <typename T, typename... Args>
@@ -103,14 +163,20 @@ class ServiceConfig : public RefCounted<ServiceConfig> {
 
   // Takes ownership of \a json_tree.
   ServiceConfig(UniquePtr<char> service_config_json,
-                UniquePtr<char> json_string, grpc_json* json_tree);
+                UniquePtr<char> json_string, grpc_json* json_tree,
+                grpc_error** error);
+
+  // Helper functions to parse the service config
+  grpc_error* ParseGlobalParams(const grpc_json* json_tree);
+  grpc_error* ParsePerMethodParams(const grpc_json* json_tree);
 
   // Returns the number of names specified in the method config \a json.
   static int CountNamesInMethodConfig(grpc_json* json);
 
   // Returns a path string for the JSON name object specified by \a json.
-  // Returns null on error.
-  static UniquePtr<char> ParseJsonMethodName(grpc_json* json);
+  // Returns null on error, and stores error in \a error.
+  static UniquePtr<char> ParseJsonMethodName(grpc_json* json,
+                                             grpc_error** error);
 
   // Parses the method config from \a json.  Adds an entry to \a entries for
   // each name found, incrementing \a idx for each entry added.
@@ -120,9 +186,26 @@ class ServiceConfig : public RefCounted<ServiceConfig> {
       grpc_json* json, CreateValue<T> create_value,
       typename SliceHashTable<RefCountedPtr<T>>::Entry* entries, size_t* idx);
 
+  grpc_error* ParseJsonMethodConfigToServiceConfigObjectsTable(
+      const grpc_json* json,
+      SliceHashTable<const ServiceConfigObjectsVector*>::Entry* entries,
+      size_t* idx);
+
   UniquePtr<char> service_config_json_;
   UniquePtr<char> json_string_;  // Underlying storage for json_tree.
   grpc_json* json_tree_;
+
+  InlinedVector<UniquePtr<ServiceConfigParsedObject>, kNumPreallocatedParsers>
+      parsed_global_service_config_objects_;
+  // A map from the method name to the service config objects vector. Note that
+  // we are using a raw pointer and not a unique pointer so that we can use the
+  // same vector for multiple names.
+  RefCountedPtr<SliceHashTable<const ServiceConfigObjectsVector*>>
+      parsed_method_service_config_objects_table_;
+  // Storage for all the vectors that are being used in
+  // parsed_method_service_config_objects_table_.
+  InlinedVector<UniquePtr<ServiceConfigObjectsVector>, 32>
+      service_config_objects_vectors_storage_;
 };
 
 //
@@ -157,7 +240,10 @@ bool ServiceConfig::ParseJsonMethodConfig(
     if (strcmp(child->key, "name") == 0) {
       if (child->type != GRPC_JSON_ARRAY) return false;
       for (grpc_json* name = child->child; name != nullptr; name = name->next) {
-        UniquePtr<char> path = ParseJsonMethodName(name);
+        grpc_error* error = GRPC_ERROR_NONE;
+        UniquePtr<char> path = ParseJsonMethodName(name, &error);
+        // We are not reporting the error here.
+        GRPC_ERROR_UNREF(error);
         if (path == nullptr) return false;
         paths.push_back(std::move(path));
       }

src/core/ext/filters/client_channel/subchannel.cc

@@ -121,7 +121,7 @@ RefCountedPtr<SubchannelCall> ConnectedSubchannel::CreateCall(
   const size_t allocation_size =
       GetInitialCallSizeEstimate(args.parent_data_size);
   RefCountedPtr<SubchannelCall> call(
-      new (gpr_arena_alloc(args.arena, allocation_size))
+      new (args.arena->Alloc(allocation_size))
           SubchannelCall(Ref(DEBUG_LOCATION, "subchannel_call"), args));
   grpc_call_stack* callstk = SUBCHANNEL_CALL_TO_CALL_STACK(call.get());
   const grpc_call_element_args call_args = {
@@ -332,10 +332,9 @@ class Subchannel::ConnectedSubchannelStateWatcher
       health_state = GRPC_CHANNEL_CONNECTING;
     }
     // Report initial state.
-    c->SetConnectivityStateLocked(GRPC_CHANNEL_READY, GRPC_ERROR_NONE,
-                                  "subchannel_connected");
+    c->SetConnectivityStateLocked(GRPC_CHANNEL_READY, "subchannel_connected");
     grpc_connectivity_state_set(&c->state_and_health_tracker_, health_state,
-                                GRPC_ERROR_NONE, "subchannel_connected");
+                                "subchannel_connected");
   }
 
   ~ConnectedSubchannelStateWatcher() {
@@ -367,11 +366,10 @@ class Subchannel::ConnectedSubchannelStateWatcher
             c->connected_subchannel_watcher_.reset();
             self->last_connectivity_state_ = GRPC_CHANNEL_TRANSIENT_FAILURE;
             c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                          GRPC_ERROR_REF(error),
                                           "reflect_child");
             grpc_connectivity_state_set(&c->state_and_health_tracker_,
                                         GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                        GRPC_ERROR_REF(error), "reflect_child");
+                                        "reflect_child");
             c->backoff_begun_ = false;
             c->backoff_.Reset();
             c->MaybeStartConnectingLocked();
@@ -388,11 +386,11 @@ class Subchannel::ConnectedSubchannelStateWatcher
           // from READY to CONNECTING or IDLE.
           self->last_connectivity_state_ = self->pending_connectivity_state_;
           c->SetConnectivityStateLocked(self->pending_connectivity_state_,
-                                        GRPC_ERROR_REF(error), "reflect_child");
+                                        "reflect_child");
           if (self->pending_connectivity_state_ != GRPC_CHANNEL_READY) {
             grpc_connectivity_state_set(&c->state_and_health_tracker_,
                                         self->pending_connectivity_state_,
-                                        GRPC_ERROR_REF(error), "reflect_child");
+                                        "reflect_child");
           }
           c->connected_subchannel_->NotifyOnStateChange(
               nullptr, &self->pending_connectivity_state_,
@@ -415,8 +413,7 @@ class Subchannel::ConnectedSubchannelStateWatcher
           self->health_check_client_ != nullptr) {
         if (self->last_connectivity_state_ == GRPC_CHANNEL_READY) {
           grpc_connectivity_state_set(&c->state_and_health_tracker_,
-                                      self->health_state_,
-                                      GRPC_ERROR_REF(error), "health_changed");
+                                      self->health_state_, "health_changed");
         }
         self->health_check_client_->NotifyOnHealthChange(
             &self->health_state_, &self->on_health_changed_);
@@ -590,8 +587,11 @@ Subchannel::Subchannel(SubchannelKey* key, grpc_connector* connector,
   const char* service_config_json = grpc_channel_arg_get_string(
       grpc_channel_args_find(args_, GRPC_ARG_SERVICE_CONFIG));
   if (service_config_json != nullptr) {
+    grpc_error* service_config_error = GRPC_ERROR_NONE;
     RefCountedPtr<ServiceConfig> service_config =
-        ServiceConfig::Create(service_config_json);
+        ServiceConfig::Create(service_config_json, &service_config_error);
+    // service_config_error is currently unused.
+    GRPC_ERROR_UNREF(service_config_error);
     if (service_config != nullptr) {
       HealthCheckParams params;
       service_config->ParseGlobalParams(HealthCheckParams::Parse, &params);
@@ -739,11 +739,10 @@ channelz::SubchannelNode* Subchannel::channelz_node() {
 }
 
 grpc_connectivity_state Subchannel::CheckConnectivity(
-    grpc_error** error, bool inhibit_health_checking) {
-  MutexLock lock(&mu_);
+    bool inhibit_health_checking) {
   grpc_connectivity_state_tracker* tracker =
       inhibit_health_checking ? &state_tracker_ : &state_and_health_tracker_;
-  grpc_connectivity_state state = grpc_connectivity_state_get(tracker, error);
+  grpc_connectivity_state state = grpc_connectivity_state_check(tracker);
   return state;
 }
 
@@ -851,7 +850,6 @@ const char* SubchannelConnectivityStateChangeString(
 }  // namespace
 
 void Subchannel::SetConnectivityStateLocked(grpc_connectivity_state state,
-                                            grpc_error* error,
                                             const char* reason) {
   if (channelz_node_ != nullptr) {
     channelz_node_->AddTraceEvent(
@@ -859,7 +857,7 @@ void Subchannel::SetConnectivityStateLocked(grpc_connectivity_state state,
         grpc_slice_from_static_string(
             SubchannelConnectivityStateChangeString(state)));
   }
-  grpc_connectivity_state_set(&state_tracker_, state, error, reason);
+  grpc_connectivity_state_set(&state_tracker_, state, reason);
 }
 
 void Subchannel::MaybeStartConnectingLocked() {
@@ -936,11 +934,9 @@ void Subchannel::ContinueConnectingLocked() {
   next_attempt_deadline_ = backoff_.NextAttemptTime();
   args.deadline = std::max(next_attempt_deadline_, min_deadline);
   args.channel_args = args_;
-  SetConnectivityStateLocked(GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
-                             "connecting");
+  SetConnectivityStateLocked(GRPC_CHANNEL_CONNECTING, "connecting");
   grpc_connectivity_state_set(&state_and_health_tracker_,
-                              GRPC_CHANNEL_CONNECTING, GRPC_ERROR_NONE,
-                              "connecting");
+                              GRPC_CHANNEL_CONNECTING, "connecting");
   grpc_connector_connect(connector_, &args, &connecting_result_,
                          &on_connecting_finished_);
 }
@@ -958,16 +954,11 @@ void Subchannel::OnConnectingFinished(void* arg, grpc_error* error) {
     } else if (c->disconnected_) {
       GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");
     } else {
-      const char* errmsg = grpc_error_string(error);
-      gpr_log(GPR_INFO, "Connect failed: %s", errmsg);
-      error = grpc_error_set_int(
-          GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Connect Failed",
-                                                           &error, 1),
-          GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+      gpr_log(GPR_INFO, "Connect failed: %s", grpc_error_string(error));
       c->SetConnectivityStateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE,
-                                    GRPC_ERROR_REF(error), "connect_failed");
+                                    "connect_failed");
       grpc_connectivity_state_set(&c->state_and_health_tracker_,
-                                  GRPC_CHANNEL_TRANSIENT_FAILURE, error,
+                                  GRPC_CHANNEL_TRANSIENT_FAILURE,
                                   "connect_failed");
       c->MaybeStartConnectingLocked();
       GRPC_SUBCHANNEL_WEAK_UNREF(c, "connecting");

src/core/ext/filters/client_channel/subchannel.h

@@ -26,7 +26,7 @@
 #include "src/core/ext/filters/client_channel/subchannel_pool_interface.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/gpr/arena.h"
+#include "src/core/lib/gprpp/arena.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"
@@ -75,9 +75,9 @@ class ConnectedSubchannel : public RefCounted<ConnectedSubchannel> {
     grpc_slice path;
     gpr_timespec start_time;
     grpc_millis deadline;
-    gpr_arena* arena;
+    Arena* arena;
     grpc_call_context_element* context;
-    grpc_call_combiner* call_combiner;
+    grpc_core::CallCombiner* call_combiner;
     size_t parent_data_size;
   };
 
@@ -208,8 +208,7 @@ class Subchannel {
   channelz::SubchannelNode* channelz_node();
 
   // Polls the current connectivity state of the subchannel.
-  grpc_connectivity_state CheckConnectivity(grpc_error** error,
-                                            bool inhibit_health_checking);
+  grpc_connectivity_state CheckConnectivity(bool inhibit_health_checking);
 
   // When the connectivity state of the subchannel changes from \a *state,
   // invokes \a notify and updates \a *state with the new state.
@@ -242,7 +241,7 @@ class Subchannel {
 
   // Sets the subchannel's connectivity state to \a state.
   void SetConnectivityStateLocked(grpc_connectivity_state state,
-                                  grpc_error* error, const char* reason);
+                                  const char* reason);
 
   // Methods for connection.
   void MaybeStartConnectingLocked();

src/core/ext/filters/deadline/deadline_filter.cc

@@ -68,8 +68,7 @@ static void timer_callback(void* arg, grpc_error* error) {
     error = grpc_error_set_int(
         GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"),
         GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED);
-    grpc_call_combiner_cancel(deadline_state->call_combiner,
-                              GRPC_ERROR_REF(error));
+    deadline_state->call_combiner->Cancel(GRPC_ERROR_REF(error));
     GRPC_CLOSURE_INIT(&deadline_state->timer_callback,
                       send_cancel_op_in_call_combiner, elem,
                       grpc_schedule_on_exec_ctx);
@@ -124,7 +123,7 @@ static void cancel_timer_if_needed(grpc_deadline_state* deadline_state) {
     deadline_state->timer_state = GRPC_DEADLINE_STATE_FINISHED;
     grpc_timer_cancel(&deadline_state->timer);
   } else {
-    // timer was either in STATE_INITAL (nothing to cancel)
+    // timer was either in STATE_INITIAL (nothing to cancel)
     // OR in STATE_FINISHED (again nothing to cancel)
   }
 }
@@ -183,7 +182,7 @@ static void start_timer_after_init(void* arg, grpc_error* error) {
 
 grpc_deadline_state::grpc_deadline_state(grpc_call_element* elem,
                                          grpc_call_stack* call_stack,
-                                         grpc_call_combiner* call_combiner,
+                                         grpc_core::CallCombiner* call_combiner,
                                          grpc_millis deadline)
     : call_stack(call_stack), call_combiner(call_combiner) {
   // Deadline will always be infinite on servers, so the timer will only be

src/core/ext/filters/deadline/deadline_filter.h

@@ -32,12 +32,13 @@ enum grpc_deadline_timer_state {
 // Must be the first field in the filter's call_data.
 struct grpc_deadline_state {
   grpc_deadline_state(grpc_call_element* elem, grpc_call_stack* call_stack,
-                      grpc_call_combiner* call_combiner, grpc_millis deadline);
+                      grpc_core::CallCombiner* call_combiner,
+                      grpc_millis deadline);
   ~grpc_deadline_state();
 
   // We take a reference to the call stack for the timer callback.
   grpc_call_stack* call_stack;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_deadline_timer_state timer_state = GRPC_DEADLINE_STATE_INITIAL;
   grpc_timer timer;
   grpc_closure timer_callback;

src/core/ext/filters/http/client/http_client_filter.cc

@@ -36,7 +36,7 @@
 #define EXPECTED_CONTENT_TYPE "application/grpc"
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
-/* default maximum size of payload eligable for GET request */
+/* default maximum size of payload eligible for GET request */
 static constexpr size_t kMaxPayloadSizeForGet = 2048;
 
 static void recv_initial_metadata_ready(void* user_data, grpc_error* error);
@@ -62,7 +62,7 @@ struct call_data {
 
   ~call_data() { GRPC_ERROR_UNREF(recv_initial_metadata_error); }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   // State for handling send_initial_metadata ops.
   grpc_linked_mdelem method;
   grpc_linked_mdelem scheme;
@@ -107,7 +107,8 @@ static grpc_error* client_filter_incoming_metadata(grpc_call_element* elem,
      * https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md.
      */
     if (b->idx.named.grpc_status != nullptr ||
-        grpc_mdelem_eq(b->idx.named.status->md, GRPC_MDELEM_STATUS_200)) {
+        grpc_mdelem_static_value_eq(b->idx.named.status->md,
+                                    GRPC_MDELEM_STATUS_200)) {
       grpc_metadata_batch_remove(b, b->idx.named.status);
     } else {
       char* val = grpc_dump_slice(GRPC_MDVALUE(b->idx.named.status->md),
@@ -140,8 +141,9 @@ static grpc_error* client_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.content_type != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.content_type->md,
-                        GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
+    if (!grpc_mdelem_static_value_eq(
+            b->idx.named.content_type->md,
+            GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
       if (grpc_slice_buf_start_eq(GRPC_MDVALUE(b->idx.named.content_type->md),
                                   EXPECTED_CONTENT_TYPE,
                                   EXPECTED_CONTENT_TYPE_LENGTH) &&

src/core/ext/filters/http/client/http_client_filter.h

@@ -25,7 +25,7 @@
 /* Processes metadata on the client side for HTTP2 transports */
 extern const grpc_channel_filter grpc_http_client_filter;
 
-/* Channel arg to determine maximum size of payload eligable for GET request */
+/* Channel arg to determine maximum size of payload eligible for GET request */
 #define GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET "grpc.max_payload_size_for_get"
 
 #endif /* GRPC_CORE_EXT_FILTERS_HTTP_CLIENT_HTTP_CLIENT_FILTER_H */

src/core/ext/filters/http/client_authority_filter.cc

@@ -40,7 +40,7 @@ namespace {
 
 struct call_data {
   grpc_linked_mdelem authority_storage;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
 };
 
 struct channel_data {

src/core/ext/filters/http/message_compress/message_compress_filter.cc

@@ -29,6 +29,7 @@
 #include "src/core/ext/filters/http/message_compress/message_compress_filter.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/compression/algorithm_metadata.h"
+#include "src/core/lib/compression/compression_args.h"
 #include "src/core/lib/compression/compression_internal.h"
 #include "src/core/lib/compression/message_compress.h"
 #include "src/core/lib/gpr/string.h"
@@ -71,7 +72,7 @@ struct call_data {
     GRPC_ERROR_UNREF(cancel_error);
   }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_linked_mdelem compression_algorithm_storage;
   grpc_linked_mdelem stream_compression_algorithm_storage;
   grpc_linked_mdelem accept_encoding_storage;

src/core/ext/filters/http/server/http_server_filter.cc

@@ -61,7 +61,7 @@ struct call_data {
     }
   }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
 
   // Outgoing headers to add to send_initial_metadata.
   grpc_linked_mdelem status;
@@ -131,18 +131,19 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   static const char* error_name = "Failed processing incoming headers";
 
   if (b->idx.named.method != nullptr) {
-    if (grpc_mdelem_eq(b->idx.named.method->md, GRPC_MDELEM_METHOD_POST)) {
+    if (grpc_mdelem_static_value_eq(b->idx.named.method->md,
+                                    GRPC_MDELEM_METHOD_POST)) {
       *calld->recv_initial_metadata_flags &=
           ~(GRPC_INITIAL_METADATA_CACHEABLE_REQUEST |
             GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST);
-    } else if (grpc_mdelem_eq(b->idx.named.method->md,
-                              GRPC_MDELEM_METHOD_PUT)) {
+    } else if (grpc_mdelem_static_value_eq(b->idx.named.method->md,
+                                           GRPC_MDELEM_METHOD_PUT)) {
       *calld->recv_initial_metadata_flags &=
           ~GRPC_INITIAL_METADATA_CACHEABLE_REQUEST;
       *calld->recv_initial_metadata_flags |=
           GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST;
-    } else if (grpc_mdelem_eq(b->idx.named.method->md,
-                              GRPC_MDELEM_METHOD_GET)) {
+    } else if (grpc_mdelem_static_value_eq(b->idx.named.method->md,
+                                           GRPC_MDELEM_METHOD_GET)) {
       *calld->recv_initial_metadata_flags |=
           GRPC_INITIAL_METADATA_CACHEABLE_REQUEST;
       *calld->recv_initial_metadata_flags &=
@@ -163,7 +164,8 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.te != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.te->md, GRPC_MDELEM_TE_TRAILERS)) {
+    if (!grpc_mdelem_static_value_eq(b->idx.named.te->md,
+                                     GRPC_MDELEM_TE_TRAILERS)) {
       hs_add_error(error_name, &error,
                    grpc_attach_md_to_error(
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
@@ -178,9 +180,12 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.scheme != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTP) &&
-        !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTPS) &&
-        !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_GRPC)) {
+    if (!grpc_mdelem_static_value_eq(b->idx.named.scheme->md,
+                                     GRPC_MDELEM_SCHEME_HTTP) &&
+        !grpc_mdelem_static_value_eq(b->idx.named.scheme->md,
+                                     GRPC_MDELEM_SCHEME_HTTPS) &&
+        !grpc_mdelem_static_value_eq(b->idx.named.scheme->md,
+                                     GRPC_MDELEM_SCHEME_GRPC)) {
       hs_add_error(error_name, &error,
                    grpc_attach_md_to_error(
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
@@ -196,8 +201,9 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.content_type != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.content_type->md,
-                        GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
+    if (!grpc_mdelem_static_value_eq(
+            b->idx.named.content_type->md,
+            GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
       if (grpc_slice_buf_start_eq(GRPC_MDVALUE(b->idx.named.content_type->md),
                                   EXPECTED_CONTENT_TYPE,
                                   EXPECTED_CONTENT_TYPE_LENGTH) &&

src/core/ext/filters/message_size/message_size_filter.cc

@@ -137,7 +137,7 @@ struct call_data {
 
   ~call_data() { GRPC_ERROR_UNREF(error); }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   message_size_limits limits;
   // Receive closures are chained: we inject this closure as the
   // recv_message_ready up-call on transport_stream_op, and remember to
@@ -319,8 +319,11 @@ static grpc_error* init_channel_elem(grpc_channel_element* elem,
       grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVICE_CONFIG);
   const char* service_config_str = grpc_channel_arg_get_string(channel_arg);
   if (service_config_str != nullptr) {
+    grpc_error* service_config_error = GRPC_ERROR_NONE;
     grpc_core::RefCountedPtr<grpc_core::ServiceConfig> service_config =
-        grpc_core::ServiceConfig::Create(service_config_str);
+        grpc_core::ServiceConfig::Create(service_config_str,
+                                         &service_config_error);
+    GRPC_ERROR_UNREF(service_config_error);
     if (service_config != nullptr) {
       chand->method_limit_table = service_config->CreateMethodConfigTable(
           grpc_core::MessageSizeLimits::CreateFromJson);

src/core/ext/transport/chttp2/alpn/alpn.h

@@ -23,7 +23,7 @@
 
 #include <string.h>
 
-/* Retuns 1 if the version is supported, 0 otherwise. */
+/* Returns 1 if the version is supported, 0 otherwise. */
 int grpc_chttp2_is_alpn_version_supported(const char* version, size_t size);
 
 /* Returns the number of protocol versions to advertise */

src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -119,7 +119,7 @@ static void maybe_start_some_streams(grpc_chttp2_transport* t);
 
 static void connectivity_state_set(grpc_chttp2_transport* t,
                                    grpc_connectivity_state state,
-                                   grpc_error* error, const char* reason);
+                                   const char* reason);
 
 static void benign_reclaimer_locked(void* t, grpc_error* error);
 static void destructive_reclaimer_locked(void* t, grpc_error* error);
@@ -592,8 +592,7 @@ static void close_transport_locked(grpc_chttp2_transport* t,
     }
     GPR_ASSERT(error != GRPC_ERROR_NONE);
     t->closed_with_error = GRPC_ERROR_REF(error);
-    connectivity_state_set(t, GRPC_CHANNEL_SHUTDOWN, GRPC_ERROR_REF(error),
-                           "close_transport");
+    connectivity_state_set(t, GRPC_CHANNEL_SHUTDOWN, "close_transport");
     if (t->ping_state.is_delayed_ping_timer_set) {
       grpc_timer_cancel(&t->ping_state.delayed_ping_timer);
     }
@@ -656,11 +655,12 @@ grpc_chttp2_stream::Reffer::Reffer(grpc_chttp2_stream* s) {
 grpc_chttp2_stream::grpc_chttp2_stream(grpc_chttp2_transport* t,
                                        grpc_stream_refcount* refcount,
                                        const void* server_data,
-                                       gpr_arena* arena)
+                                       grpc_core::Arena* arena)
     : t(t),
       refcount(refcount),
       reffer(this),
-      metadata_buffer{{arena}, {arena}} {
+      metadata_buffer{grpc_chttp2_incoming_metadata_buffer(arena),
+                      grpc_chttp2_incoming_metadata_buffer(arena)} {
   if (server_data) {
     id = static_cast<uint32_t>((uintptr_t)server_data);
     *t->accepting_stream = this;
@@ -741,7 +741,7 @@ grpc_chttp2_stream::~grpc_chttp2_stream() {
 
 static int init_stream(grpc_transport* gt, grpc_stream* gs,
                        grpc_stream_refcount* refcount, const void* server_data,
-                       gpr_arena* arena) {
+                       grpc_core::Arena* arena) {
   GPR_TIMER_SCOPE("init_stream", 0);
   grpc_chttp2_transport* t = reinterpret_cast<grpc_chttp2_transport*>(gt);
   new (gs) grpc_chttp2_stream(t, refcount, server_data, arena);
@@ -1171,8 +1171,7 @@ void grpc_chttp2_add_incoming_goaway(grpc_chttp2_transport* t,
 
   /* lie: use transient failure from the transport to indicate goaway has been
    * received */
-  connectivity_state_set(t, GRPC_CHANNEL_TRANSIENT_FAILURE,
-                         GRPC_ERROR_REF(t->goaway_error), "got_goaway");
+  connectivity_state_set(t, GRPC_CHANNEL_TRANSIENT_FAILURE, "got_goaway");
 }
 
 static void maybe_start_some_streams(grpc_chttp2_transport* t) {
@@ -1194,10 +1193,8 @@ static void maybe_start_some_streams(grpc_chttp2_transport* t) {
     t->next_stream_id += 2;
 
     if (t->next_stream_id >= MAX_CLIENT_STREAM_ID) {
-      connectivity_state_set(
-          t, GRPC_CHANNEL_TRANSIENT_FAILURE,
-          GRPC_ERROR_CREATE_FROM_STATIC_STRING("Stream IDs exhausted"),
-          "no_more_stream_ids");
+      connectivity_state_set(t, GRPC_CHANNEL_TRANSIENT_FAILURE,
+                             "no_more_stream_ids");
     }
 
     grpc_chttp2_stream_map_add(&t->stream_map, s->id, s);
@@ -1285,8 +1282,8 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t,
 
 static bool contains_non_ok_status(grpc_metadata_batch* batch) {
   if (batch->idx.named.grpc_status != nullptr) {
-    return !grpc_mdelem_eq(batch->idx.named.grpc_status->md,
-                           GRPC_MDELEM_GRPC_STATUS_0);
+    return !grpc_mdelem_static_value_eq(batch->idx.named.grpc_status->md,
+                                        GRPC_MDELEM_GRPC_STATUS_0);
   }
   return false;
 }
@@ -1417,7 +1414,7 @@ static void perform_stream_op_locked(void* stream_op,
   // on_complete will be null if and only if there are no send ops in the batch.
   if (on_complete != nullptr) {
     // This batch has send ops. Use final_data as a barrier until enqueue time;
-    // the inital counter is dropped at the end of this function.
+    // the initial counter is dropped at the end of this function.
     on_complete->next_data.scratch = CLOSURE_BARRIER_FIRST_REF_BIT;
     on_complete->error_data.error = GRPC_ERROR_NONE;
   }
@@ -2804,9 +2801,9 @@ static void keepalive_watchdog_fired_locked(void* arg, grpc_error* error) {
 
 static void connectivity_state_set(grpc_chttp2_transport* t,
                                    grpc_connectivity_state state,
-                                   grpc_error* error, const char* reason) {
+                                   const char* reason) {
   GRPC_CHTTP2_IF_TRACING(gpr_log(GPR_INFO, "set connectivity_state=%d", state));
-  grpc_connectivity_state_set(&t->channel_callback.state_tracker, state, error,
+  grpc_connectivity_state_set(&t->channel_callback.state_tracker, state,
                               reason);
 }
 

src/core/ext/transport/chttp2/transport/hpack_encoder.cc

@@ -56,7 +56,7 @@
 /* don't consider adding anything bigger than this to the hpack table */
 #define MAX_DECODER_SPACE_USAGE 512
 
-static grpc_slice_refcount terminal_slice_refcount = {nullptr, nullptr};
+static grpc_slice_refcount terminal_slice_refcount;
 static const grpc_slice terminal_slice = {
     &terminal_slice_refcount, /* refcount */
     {{0, nullptr}}            /* data.refcounted */