Merge branch 'master' into svc_cfg2

.gitignore

@@ -134,3 +134,11 @@ bm_*.json
 
 # cmake build files
 /cmake/build
+
+# Visual Studio Code artifacts
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+

.vscode/launch.json

@@ -1,30 +0,0 @@
-{
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "type": "node",
-            "request": "launch",
-            "name": "Mocha Tests",
-            "cwd": "${workspaceRoot}",
-            "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha",
-            "windows": {
-                "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha.cmd"
-            },
-            "runtimeArgs": [
-                "-u",
-                "tdd",
-                "--timeout",
-                "999999",
-                "--colors",
-                "${workspaceRoot}/src/node/test"
-            ],
-            "internalConsoleOptions": "openOnSessionStart"
-        },
-        {
-            "type": "node",
-            "request": "attach",
-            "name": "Attach to Process",
-            "port": 5858
-        }
-    ]
-}

BUILD

@@ -268,6 +268,7 @@ GRPCXX_PUBLIC_HDRS = [
     "include/grpcpp/support/client_interceptor.h",
     "include/grpcpp/support/config.h",
     "include/grpcpp/support/interceptor.h",
+    "include/grpcpp/support/message_allocator.h",
     "include/grpcpp/support/proto_buffer_reader.h",
     "include/grpcpp/support/proto_buffer_writer.h",
     "include/grpcpp/support/server_callback.h",
@@ -2140,6 +2141,7 @@ grpc_cc_library(
         "include/grpcpp/impl/codegen/intercepted_channel.h",
         "include/grpcpp/impl/codegen/interceptor.h",
         "include/grpcpp/impl/codegen/interceptor_common.h",
+        "include/grpcpp/impl/codegen/message_allocator.h",
         "include/grpcpp/impl/codegen/metadata_map.h",
         "include/grpcpp/impl/codegen/method_handler_impl.h",
         "include/grpcpp/impl/codegen/rpc_method.h",

BUILD.gn

@@ -1049,6 +1049,7 @@ config("grpc_config") {
         "include/grpcpp/impl/codegen/intercepted_channel.h",
         "include/grpcpp/impl/codegen/interceptor.h",
         "include/grpcpp/impl/codegen/interceptor_common.h",
+        "include/grpcpp/impl/codegen/message_allocator.h",
         "include/grpcpp/impl/codegen/metadata_map.h",
         "include/grpcpp/impl/codegen/method_handler_impl.h",
         "include/grpcpp/impl/codegen/proto_buffer_reader.h",
@@ -1104,6 +1105,7 @@ config("grpc_config") {
         "include/grpcpp/support/client_interceptor.h",
         "include/grpcpp/support/config.h",
         "include/grpcpp/support/interceptor.h",
+        "include/grpcpp/support/message_allocator.h",
         "include/grpcpp/support/proto_buffer_reader.h",
         "include/grpcpp/support/proto_buffer_writer.h",
         "include/grpcpp/support/server_callback.h",

CMakeLists.txt

@@ -661,6 +661,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
 add_dependencies(buildtests_cxx json_run_localhost)
 endif()
 add_dependencies(buildtests_cxx memory_test)
+add_dependencies(buildtests_cxx message_allocator_end2end_test)
 add_dependencies(buildtests_cxx metrics_client)
 add_dependencies(buildtests_cxx mock_test)
 add_dependencies(buildtests_cxx nonblocking_test)
@@ -3059,6 +3060,7 @@ foreach(_hdr
   include/grpcpp/support/client_interceptor.h
   include/grpcpp/support/config.h
   include/grpcpp/support/interceptor.h
+  include/grpcpp/support/message_allocator.h
   include/grpcpp/support/proto_buffer_reader.h
   include/grpcpp/support/proto_buffer_writer.h
   include/grpcpp/support/server_callback.h
@@ -3174,6 +3176,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -3664,6 +3667,7 @@ foreach(_hdr
   include/grpcpp/support/client_interceptor.h
   include/grpcpp/support/config.h
   include/grpcpp/support/interceptor.h
+  include/grpcpp/support/message_allocator.h
   include/grpcpp/support/proto_buffer_reader.h
   include/grpcpp/support/proto_buffer_writer.h
   include/grpcpp/support/server_callback.h
@@ -3779,6 +3783,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -4212,6 +4217,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -4409,6 +4415,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -4643,6 +4650,7 @@ foreach(_hdr
   include/grpcpp/support/client_interceptor.h
   include/grpcpp/support/config.h
   include/grpcpp/support/interceptor.h
+  include/grpcpp/support/message_allocator.h
   include/grpcpp/support/proto_buffer_reader.h
   include/grpcpp/support/proto_buffer_writer.h
   include/grpcpp/support/server_callback.h
@@ -4758,6 +4766,7 @@ foreach(_hdr
   include/grpcpp/impl/codegen/intercepted_channel.h
   include/grpcpp/impl/codegen/interceptor.h
   include/grpcpp/impl/codegen/interceptor_common.h
+  include/grpcpp/impl/codegen/message_allocator.h
   include/grpcpp/impl/codegen/metadata_map.h
   include/grpcpp/impl/codegen/method_handler_impl.h
   include/grpcpp/impl/codegen/rpc_method.h
@@ -14507,6 +14516,46 @@ target_link_libraries(memory_test
 )
 
 
+endif (gRPC_BUILD_TESTS)
+if (gRPC_BUILD_TESTS)
+
+add_executable(message_allocator_end2end_test
+  test/cpp/end2end/message_allocator_end2end_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+
+
+target_include_directories(message_allocator_end2end_test
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
+  PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
+  PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
+  PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
+  PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
+  PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
+  PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+  PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR}
+  PRIVATE third_party/googletest/googletest/include
+  PRIVATE third_party/googletest/googletest
+  PRIVATE third_party/googletest/googlemock/include
+  PRIVATE third_party/googletest/googlemock
+  PRIVATE ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(message_allocator_end2end_test
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc++_test_util
+  grpc_test_util
+  grpc++
+  grpc
+  gpr
+  ${_gRPC_GFLAGS_LIBRARIES}
+)
+
+
 endif (gRPC_BUILD_TESTS)
 if (gRPC_BUILD_TESTS)
 

Makefile

@@ -1233,6 +1233,7 @@ interop_server: $(BINDIR)/$(CONFIG)/interop_server
 interop_test: $(BINDIR)/$(CONFIG)/interop_test
 json_run_localhost: $(BINDIR)/$(CONFIG)/json_run_localhost
 memory_test: $(BINDIR)/$(CONFIG)/memory_test
+message_allocator_end2end_test: $(BINDIR)/$(CONFIG)/message_allocator_end2end_test
 metrics_client: $(BINDIR)/$(CONFIG)/metrics_client
 mock_test: $(BINDIR)/$(CONFIG)/mock_test
 nonblocking_test: $(BINDIR)/$(CONFIG)/nonblocking_test
@@ -1701,6 +1702,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/interop_test \
   $(BINDIR)/$(CONFIG)/json_run_localhost \
   $(BINDIR)/$(CONFIG)/memory_test \
+  $(BINDIR)/$(CONFIG)/message_allocator_end2end_test \
   $(BINDIR)/$(CONFIG)/metrics_client \
   $(BINDIR)/$(CONFIG)/mock_test \
   $(BINDIR)/$(CONFIG)/nonblocking_test \
@@ -1844,6 +1846,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/interop_test \
   $(BINDIR)/$(CONFIG)/json_run_localhost \
   $(BINDIR)/$(CONFIG)/memory_test \
+  $(BINDIR)/$(CONFIG)/message_allocator_end2end_test \
   $(BINDIR)/$(CONFIG)/metrics_client \
   $(BINDIR)/$(CONFIG)/mock_test \
   $(BINDIR)/$(CONFIG)/nonblocking_test \
@@ -2343,6 +2346,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/interop_test || ( echo test interop_test failed ; exit 1 )
 	$(E) "[RUN]     Testing memory_test"
 	$(Q) $(BINDIR)/$(CONFIG)/memory_test || ( echo test memory_test failed ; exit 1 )
+	$(E) "[RUN]     Testing message_allocator_end2end_test"
+	$(Q) $(BINDIR)/$(CONFIG)/message_allocator_end2end_test || ( echo test message_allocator_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing mock_test"
 	$(Q) $(BINDIR)/$(CONFIG)/mock_test || ( echo test mock_test failed ; exit 1 )
 	$(E) "[RUN]     Testing nonblocking_test"
@@ -5395,6 +5400,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/support/client_interceptor.h \
     include/grpcpp/support/config.h \
     include/grpcpp/support/interceptor.h \
+    include/grpcpp/support/message_allocator.h \
     include/grpcpp/support/proto_buffer_reader.h \
     include/grpcpp/support/proto_buffer_writer.h \
     include/grpcpp/support/server_callback.h \
@@ -5510,6 +5516,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -6008,6 +6015,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/support/client_interceptor.h \
     include/grpcpp/support/config.h \
     include/grpcpp/support/interceptor.h \
+    include/grpcpp/support/message_allocator.h \
     include/grpcpp/support/proto_buffer_reader.h \
     include/grpcpp/support/proto_buffer_writer.h \
     include/grpcpp/support/server_callback.h \
@@ -6123,6 +6131,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -6528,6 +6537,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -6696,6 +6706,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -6936,6 +6947,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/support/client_interceptor.h \
     include/grpcpp/support/config.h \
     include/grpcpp/support/interceptor.h \
+    include/grpcpp/support/message_allocator.h \
     include/grpcpp/support/proto_buffer_reader.h \
     include/grpcpp/support/proto_buffer_writer.h \
     include/grpcpp/support/server_callback.h \
@@ -7051,6 +7063,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpcpp/impl/codegen/intercepted_channel.h \
     include/grpcpp/impl/codegen/interceptor.h \
     include/grpcpp/impl/codegen/interceptor_common.h \
+    include/grpcpp/impl/codegen/message_allocator.h \
     include/grpcpp/impl/codegen/metadata_map.h \
     include/grpcpp/impl/codegen/method_handler_impl.h \
     include/grpcpp/impl/codegen/rpc_method.h \
@@ -17413,6 +17426,49 @@ endif
 endif
 
 
+MESSAGE_ALLOCATOR_END2END_TEST_SRC = \
+    test/cpp/end2end/message_allocator_end2end_test.cc \
+
+MESSAGE_ALLOCATOR_END2END_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MESSAGE_ALLOCATOR_END2END_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/message_allocator_end2end_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+.
+
+$(BINDIR)/$(CONFIG)/message_allocator_end2end_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/message_allocator_end2end_test: $(PROTOBUF_DEP) $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/message_allocator_end2end_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/cpp/end2end/message_allocator_end2end_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_message_allocator_end2end_test: $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(MESSAGE_ALLOCATOR_END2END_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 METRICS_CLIENT_SRC = \
     $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc \
     test/cpp/interop/metrics_client.cc \

bazel/grpc_deps.bzl

@@ -110,6 +110,8 @@ def grpc_deps():
         http_archive(
             name = "boringssl",
             # on the chromium-stable-with-bazel branch
+            # NOTE: This URL generates a tarball containing dynamic date
+            # information, so the sha256 is not consistent.
             url = "https://boringssl.googlesource.com/boringssl/+archive/afc30d43eef92979b05776ec0963c9cede5fb80f.tar.gz",
         )
 
@@ -117,6 +119,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_madler_zlib",
             build_file = "@com_github_grpc_grpc//third_party:zlib.BUILD",
+            sha256 = "6d4d6640ca3121620995ee255945161821218752b551a1a180f4215f7d124d45",
             strip_prefix = "zlib-cacf7f1d4e3d44d871b605da3b647f07d718623f",
             url = "https://github.com/madler/zlib/archive/cacf7f1d4e3d44d871b605da3b647f07d718623f.tar.gz",
         )
@@ -124,6 +127,7 @@ def grpc_deps():
     if "com_google_protobuf" not in native.existing_rules():
         http_archive(
             name = "com_google_protobuf",
+            sha256 = "cf9e2fb1d2cd30ec9d51ff1749045208bd641f290f64b85046485934b0e03783",
             strip_prefix = "protobuf-582743bf40c5d3639a70f98f183914a2c0cd0680",
             url = "https://github.com/google/protobuf/archive/582743bf40c5d3639a70f98f183914a2c0cd0680.tar.gz",
         )
@@ -132,6 +136,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_nanopb_nanopb",
             build_file = "@com_github_grpc_grpc//third_party:nanopb.BUILD",
+            sha256 = "8bbbb1e78d4ddb0a1919276924ab10d11b631df48b657d960e0c795a25515735",
             strip_prefix = "nanopb-f8ac463766281625ad710900479130c7fcb4d63b",
             url = "https://github.com/nanopb/nanopb/archive/f8ac463766281625ad710900479130c7fcb4d63b.tar.gz",
         )
@@ -140,6 +145,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_google_googletest",
             build_file = "@com_github_grpc_grpc//third_party:gtest.BUILD",
+            sha256 = "175a22300b3450e27e5f2e6f95cc9abca74617cbc21a1e0ed19bdfbd22ea0305",
             strip_prefix = "googletest-ec44c6c1675c25b9827aacd08c02433cccde7780",
             url = "https://github.com/google/googletest/archive/ec44c6c1675c25b9827aacd08c02433cccde7780.tar.gz",
         )
@@ -147,6 +153,7 @@ def grpc_deps():
     if "com_github_gflags_gflags" not in native.existing_rules():
         http_archive(
             name = "com_github_gflags_gflags",
+            sha256 = "63ae70ea3e05780f7547d03503a53de3a7d2d83ad1caaa443a31cb20aea28654",
             strip_prefix = "gflags-28f50e0fed19872e0fd50dd23ce2ee8cd759338e",
             url = "https://github.com/gflags/gflags/archive/28f50e0fed19872e0fd50dd23ce2ee8cd759338e.tar.gz",
         )
@@ -154,6 +161,7 @@ def grpc_deps():
     if "com_github_google_benchmark" not in native.existing_rules():
         http_archive(
             name = "com_github_google_benchmark",
+            sha256 = "c7682e9007ddfd94072647abab3e89ffd9084089460ae47d67060974467b58bf",
             strip_prefix = "benchmark-e776aa0275e293707b6a0901e0e8d8a8a3679508",
             url = "https://github.com/google/benchmark/archive/e776aa0275e293707b6a0901e0e8d8a8a3679508.tar.gz",
         )
@@ -162,6 +170,7 @@ def grpc_deps():
         http_archive(
             name = "com_github_cares_cares",
             build_file = "@com_github_grpc_grpc//third_party:cares/cares.BUILD",
+            sha256 = "e8c2751ddc70fed9dc6f999acd92e232d5846f009ee1674f8aee81f19b2b915a",
             strip_prefix = "c-ares-e982924acee7f7313b4baa4ee5ec000c5e373c30",
             url = "https://github.com/c-ares/c-ares/archive/e982924acee7f7313b4baa4ee5ec000c5e373c30.tar.gz",
         )
@@ -169,6 +178,7 @@ def grpc_deps():
     if "com_google_absl" not in native.existing_rules():
         http_archive(
             name = "com_google_absl",
+            sha256 = "5fe2a3a8f8378e81d4d3db6541f48030e04233ccd2d6c7e9d981ed577b314ae8",
             strip_prefix = "abseil-cpp-308ce31528a7edfa39f5f6d36142278a0ae1bf45",
             url = "https://github.com/abseil/abseil-cpp/archive/308ce31528a7edfa39f5f6d36142278a0ae1bf45.tar.gz",
         )
@@ -176,12 +186,12 @@ def grpc_deps():
     if "bazel_toolchains" not in native.existing_rules():
         http_archive(
             name = "bazel_toolchains",
+            sha256 = "67335b3563d9b67dc2550b8f27cc689b64fadac491e69ce78763d9ba894cc5cc",
             strip_prefix = "bazel-toolchains-cddc376d428ada2927ad359211c3e356bd9c9fbb",
             urls = [
                 "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz",
                 "https://github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz",
             ],
-            sha256 = "67335b3563d9b67dc2550b8f27cc689b64fadac491e69ce78763d9ba894cc5cc",
         )
 
     if "bazel_skylib" not in native.existing_rules():
@@ -195,6 +205,7 @@ def grpc_deps():
     if "io_opencensus_cpp" not in native.existing_rules():
         http_archive(
             name = "io_opencensus_cpp",
+            sha256 = "3436ca23dc1b3345186defd0f46d64244079ba3d3234a0c5d6ef5e8d5d06c8b5",
             strip_prefix = "opencensus-cpp-9b1e354e89bf3d92aedc00af45b418ce870f3d77",
             url = "https://github.com/census-instrumentation/opencensus-cpp/archive/9b1e354e89bf3d92aedc00af45b418ce870f3d77.tar.gz",
         )
@@ -202,6 +213,7 @@ def grpc_deps():
     if "upb" not in native.existing_rules():
         http_archive(
             name = "upb",
+            sha256 = "0e749a8973968397f849a3b42e28ee9c85dc418c2477954c2a6a4495f323241d",
             strip_prefix = "upb-ed9faae0993704b033c594b072d65e1bf19207fa",
             url = "https://github.com/google/upb/archive/ed9faae0993704b033c594b072d65e1bf19207fa.tar.gz",
         )
@@ -223,6 +235,7 @@ def grpc_test_only_deps():
     if "com_github_twisted_twisted" not in native.existing_rules():
         http_archive(
             name = "com_github_twisted_twisted",
+            sha256 = "ca17699d0d62eafc5c28daf2c7d0a18e62ae77b4137300b6c7d7868b39b06139",
             strip_prefix = "twisted-twisted-17.5.0",
             url = "https://github.com/twisted/twisted/archive/twisted-17.5.0.zip",
             build_file = "@com_github_grpc_grpc//third_party:twisted.BUILD",
@@ -231,6 +244,7 @@ def grpc_test_only_deps():
     if "com_github_yaml_pyyaml" not in native.existing_rules():
         http_archive(
             name = "com_github_yaml_pyyaml",
+            sha256 = "6b4314b1b2051ddb9d4fcd1634e1fa9c1bb4012954273c9ff3ef689f6ec6c93e",
             strip_prefix = "pyyaml-3.12",
             url = "https://github.com/yaml/pyyaml/archive/3.12.zip",
             build_file = "@com_github_grpc_grpc//third_party:yaml.BUILD",
@@ -239,6 +253,7 @@ def grpc_test_only_deps():
     if "com_github_twisted_incremental" not in native.existing_rules():
         http_archive(
             name = "com_github_twisted_incremental",
+            sha256 = "f0ca93359ee70243ff7fbf2d904a6291810bd88cb80ed4aca6fa77f318a41a36",
             strip_prefix = "incremental-incremental-17.5.0",
             url = "https://github.com/twisted/incremental/archive/incremental-17.5.0.zip",
             build_file = "@com_github_grpc_grpc//third_party:incremental.BUILD",
@@ -247,6 +262,7 @@ def grpc_test_only_deps():
     if "com_github_zopefoundation_zope_interface" not in native.existing_rules():
         http_archive(
             name = "com_github_zopefoundation_zope_interface",
+            sha256 = "e9579fc6149294339897be3aa9ecd8a29217c0b013fe6f44fcdae00e3204198a",
             strip_prefix = "zope.interface-4.4.3",
             url = "https://github.com/zopefoundation/zope.interface/archive/4.4.3.zip",
             build_file = "@com_github_grpc_grpc//third_party:zope_interface.BUILD",
@@ -255,6 +271,7 @@ def grpc_test_only_deps():
     if "com_github_twisted_constantly" not in native.existing_rules():
         http_archive(
             name = "com_github_twisted_constantly",
+            sha256 = "2702cd322161a579d2c0dbf94af4e57712eedc7bd7bbbdc554a230544f7d346c",
             strip_prefix = "constantly-15.1.0",
             url = "https://github.com/twisted/constantly/archive/15.1.0.zip",
             build_file = "@com_github_grpc_grpc//third_party:constantly.BUILD",

build.yaml

@@ -1260,6 +1260,7 @@ filegroups:
   - include/grpcpp/impl/codegen/intercepted_channel.h
   - include/grpcpp/impl/codegen/interceptor.h
   - include/grpcpp/impl/codegen/interceptor_common.h
+  - include/grpcpp/impl/codegen/message_allocator.h
   - include/grpcpp/impl/codegen/metadata_map.h
   - include/grpcpp/impl/codegen/method_handler_impl.h
   - include/grpcpp/impl/codegen/rpc_method.h
@@ -1398,6 +1399,7 @@ filegroups:
   - include/grpcpp/support/client_interceptor.h
   - include/grpcpp/support/config.h
   - include/grpcpp/support/interceptor.h
+  - include/grpcpp/support/message_allocator.h
   - include/grpcpp/support/proto_buffer_reader.h
   - include/grpcpp/support/proto_buffer_writer.h
   - include/grpcpp/support/server_callback.h
@@ -5071,6 +5073,19 @@ targets:
   uses:
   - grpc++_test
   uses_polling: false
+- name: message_allocator_end2end_test
+  gtest: true
+  cpu_cost: 0.5
+  build: test
+  language: c++
+  src:
+  - test/cpp/end2end/message_allocator_end2end_test.cc
+  deps:
+  - grpc++_test_util
+  - grpc_test_util
+  - grpc++
+  - grpc
+  - gpr
 - name: metrics_client
   build: test
   run: false

doc/environment_variables.md

@@ -50,6 +50,7 @@ some configuration as environment variables that can be set.
     resolver and load balancing policy interaction
   - compression - traces compression operations
   - connectivity_state - traces connectivity state changes to channels
+  - cronet - traces state in the cronet transport engine
   - executor - traces grpc's internal thread pool ('the executor')
   - fd_trace - traces fd create(), shutdown() and close() calls for channel fds.
     Also traces epoll fd create()/close() calls in epollex polling engine

examples/objective-c/auth_sample/MakeRPCViewController.m

@@ -46,8 +46,16 @@ static NSString * const kTestHostAddress = @"grpc-test.sandbox.googleapis.com";
 }
 @end
 
+@interface MakeRPCViewController ()<GRPCProtoResponseHandler>
+
+@end
+
 @implementation MakeRPCViewController
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)viewWillAppear:(BOOL)animated {
 
   // Create a service client and a proto request as usual.
@@ -57,28 +65,30 @@ static NSString * const kTestHostAddress = @"grpc-test.sandbox.googleapis.com";
   request.fillUsername = YES;
   request.fillOauthScope = YES;
 
-  // Create a not-yet-started RPC. We want to set the request headers on this object before starting
-  // it.
-  ProtoRPC *call =
-      [client RPCToUnaryCallWithRequest:request handler:^(AUTHResponse *response, NSError *error) {
-        if (response) {
-          // This test server responds with the email and scope of the access token it receives.
-          self.mainLabel.text = [NSString stringWithFormat:@"Used scope: %@ on behalf of user %@",
-                                 response.oauthScope, response.username];
-
-        } else {
-          self.mainLabel.text = error.UIDescription;
-        }
-      }];
-
-  // Set the access token to be used.
-  NSString *accessToken = GIDSignIn.sharedInstance.currentUser.authentication.accessToken;
-  call.requestHeaders[@"Authorization"] = [@"Bearer " stringByAppendingString:accessToken];
-
-  // Start the RPC.
+  // Set the request header with call options
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.oauth2AccessToken = GIDSignIn.sharedInstance.currentUser.authentication.accessToken;
+  GRPCUnaryProtoCall *call = [client unaryCallWithMessage:request
+                                          responseHandler:self
+                                              callOptions:options];
   [call start];
 
   self.mainLabel.text = @"Waiting for RPC to complete...";
 }
 
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  AUTHResponse *response = (AUTHResponse *)message;
+  if (response) {
+    // This test server responds with the email and scope of the access token it receives.
+    self.mainLabel.text = [NSString stringWithFormat:@"Used scope: %@ on behalf of user %@",
+                           response.oauthScope, response.username];
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    self.mainLabel.text = error.UIDescription;
+  }
+}
+
 @end

examples/objective-c/helloworld/main.m

@@ -25,20 +25,41 @@
 
 static NSString * const kHostAddress = @"localhost:50051";
 
+@interface HLWResponseHandler : NSObject<GRPCProtoResponseHandler>
+
+@end
+
+// A response handler object dispatching messages to main queue
+@implementation HLWResponseHandler
+
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  NSLog(@"%@", message);
+}
+
+@end
+
 int main(int argc, char * argv[]) {
   @autoreleasepool {
-    [GRPCCall useInsecureConnectionsForHost:kHostAddress];
-    [GRPCCall setUserAgentPrefix:@"HelloWorld/1.0" forHost:kHostAddress];
-
     HLWGreeter *client = [[HLWGreeter alloc] initWithHost:kHostAddress];
 
     HLWHelloRequest *request = [HLWHelloRequest message];
     request.name = @"Objective-C";
 
-    [client sayHelloWithRequest:request handler:^(HLWHelloReply *response, NSError *error) {
-      NSLog(@"%@", response.message);
-    }];
-    
+    GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+    // this example does not use TLS (secure channel); use insecure channel instead
+    options.transportType = GRPCTransportTypeInsecure;
+    options.userAgentPrefix = @"HelloWorld/1.0";
+
+    GRPCUnaryProtoCall *call = [client sayHelloWithMessage:request
+                                           responseHandler:[[HLWResponseHandler alloc] init]
+                                               callOptions:options];
+
+    [call start];
+
     return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
   }
 }

examples/objective-c/helloworld_macos/main.m

@@ -24,19 +24,40 @@
 
 static NSString * const kHostAddress = @"localhost:50051";
 
+@interface HLWResponseHandler : NSObject<GRPCProtoResponseHandler>
+
+@end
+
+// A response handler object dispatching messages to main queue
+@implementation HLWResponseHandler
+
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  NSLog(@"%@", message);
+}
+
+@end
+
 int main(int argc, const char * argv[]) {
   @autoreleasepool {
-    [GRPCCall useInsecureConnectionsForHost:kHostAddress];
-    [GRPCCall setUserAgentPrefix:@"HelloWorld/1.0" forHost:kHostAddress];
-
     HLWGreeter *client = [[HLWGreeter alloc] initWithHost:kHostAddress];
 
     HLWHelloRequest *request = [HLWHelloRequest message];
     request.name = @"Objective-C";
 
-    [client sayHelloWithRequest:request handler:^(HLWHelloReply *response, NSError *error) {
-      NSLog(@"%@", response.message);
-    }];
+    GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+    // this example does not use TLS (secure channel); use insecure channel instead
+    options.transportType = GRPCTransportTypeInsecure;
+    options.userAgentPrefix = @"HelloWorld/1.0";
+
+    GRPCUnaryProtoCall *call = [client sayHelloWithMessage:request
+                                           responseHandler:[[HLWResponseHandler alloc] init]
+                                               callOptions:options];
+
+    [call start];
   }
 
   return NSApplicationMain(argc, argv);

examples/objective-c/route_guide/ViewControllers.m

@@ -17,10 +17,7 @@
  */
 
 #import <UIKit/UIKit.h>
-#import <GRPCClient/GRPCCall+Tests.h>
 #import <RouteGuide/RouteGuide.pbrpc.h>
-#import <RxLibrary/GRXWriter+Immediate.h>
-#import <RxLibrary/GRXWriter+Transformations.h>
 
 static NSString * const kHostAddress = @"localhost:50051";
 
@@ -65,7 +62,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * Run the getFeature demo. Calls getFeature with a point known to have a feature and a point known
  * not to have a feature.
  */
-@interface GetFeatureViewController : UIViewController
+@interface GetFeatureViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -75,39 +72,56 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
-- (void)execRequest {
-  void (^handler)(RTGFeature *response, NSError *error) = ^(RTGFeature *response, NSError *error) {
-    // TODO(makdharma): Remove boilerplate by consolidating into one log function.
-    if (response.name.length) {
-      NSString *str =[NSString stringWithFormat:@"%@\nFound feature called %@ at %@.", self.outputLabel.text, response.location, response.name];
-      self.outputLabel.text = str;
-      NSLog(@"Found feature called %@ at %@.", response.name, response.location);
-    } else if (response) {
-      NSString *str =[NSString stringWithFormat:@"%@\nFound no features at %@",  self.outputLabel.text,response.location];
-      self.outputLabel.text = str;
-      NSLog(@"Found no features at %@", response.location);
-    } else {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-  };
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGFeature *response = (RTGFeature *)message;
 
+  // TODO(makdharma): Remove boilerplate by consolidating into one log function.
+  if (response.name.length != 0) {
+    NSString *str =[NSString stringWithFormat:@"%@\nFound feature called %@ at %@.", self.outputLabel.text, response.location, response.name];
+    self.outputLabel.text = str;
+    NSLog(@"Found feature called %@ at %@.", response.name, response.location);
+  } else if (response) {
+    NSString *str =[NSString stringWithFormat:@"%@\nFound no features at %@",  self.outputLabel.text,response.location];
+    self.outputLabel.text = str;
+    NSLog(@"Found no features at %@", response.location);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
+}
+
+- (void)execRequest {
   RTGPoint *point = [RTGPoint message];
   point.latitude = 409146138;
   point.longitude = -746188906;
 
-  [_service getFeatureWithRequest:point handler:handler];
-  [_service getFeatureWithRequest:[RTGPoint message] handler:handler];
+  GRPCUnaryProtoCall *call = [_service getFeatureWithMessage:point
+                                             responseHandler:self
+                                                 callOptions:nil];
+  [call start];
+  call = [_service getFeatureWithMessage:[RTGPoint message]
+                         responseHandler:self
+                             callOptions:nil];
+  [call start];
+
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  // This only needs to be done once per host, before creating service objects for that host.
-  [GRPCCall useInsecureConnectionsForHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {
@@ -126,7 +140,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * Run the listFeatures demo. Calls listFeatures with a rectangle containing all of the features in
  * the pre-generated database. Prints each response as it comes in.
  */
-@interface ListFeaturesViewController : UIViewController
+@interface ListFeaturesViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -136,6 +150,10 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)execRequest {
   RTGRectangle *rectangle = [RTGRectangle message];
   rectangle.lo.latitude = 405E6;
@@ -144,24 +162,36 @@ static NSString * const kHostAddress = @"localhost:50051";
   rectangle.hi.longitude = -745E6;
 
   NSLog(@"Looking for features between %@ and %@", rectangle.lo, rectangle.hi);
-  [_service listFeaturesWithRequest:rectangle
-                      eventHandler:^(BOOL done, RTGFeature *response, NSError *error) {
-    if (response) {
-      NSString *str =[NSString stringWithFormat:@"%@\nFound feature at %@ called %@.", self.outputLabel.text, response.location, response.name];
-      self.outputLabel.text = str;
-      NSLog(@"Found feature at %@ called %@.", response.location, response.name);
-    } else if (error) {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-  }];
+  GRPCUnaryProtoCall *call = [_service listFeaturesWithMessage:rectangle
+                                               responseHandler:self
+                                                   callOptions:nil];
+  [call start];
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGFeature *response = (RTGFeature *)message;
+  if (response) {
+    NSString *str =[NSString stringWithFormat:@"%@\nFound feature at %@ called %@.", self.outputLabel.text, response.location, response.name];
+    self.outputLabel.text = str;
+    NSLog(@"Found feature at %@ called %@.", response.location, response.name);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
+
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {
@@ -173,7 +203,6 @@ static NSString * const kHostAddress = @"localhost:50051";
 
 @end
 
-
 #pragma mark Demo: Record Route
 
 /**
@@ -181,7 +210,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * database with a variable delay in between. Prints the statistics when they are sent from the
  * server.
  */
-@interface RecordRouteViewController : UIViewController
+@interface RecordRouteViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -191,47 +220,71 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)execRequest {
   NSString *dataBasePath = [NSBundle.mainBundle pathForResource:@"route_guide_db"
                                                          ofType:@"json"];
   NSData *dataBaseContent = [NSData dataWithContentsOfFile:dataBasePath];
-  NSArray *features = [NSJSONSerialization JSONObjectWithData:dataBaseContent options:0 error:NULL];
+  NSError *error;
+  NSArray *features = [NSJSONSerialization JSONObjectWithData:dataBaseContent options:0 error:&error];
+
+  if (error) {
+    NSLog(@"Error reading database.");
+    NSString *str = @"Error reading database.";
+    self.outputLabel.text = str;
+    return;
+  }
 
-  GRXWriter *locations = [[GRXWriter writerWithContainer:features] map:^id(id feature) {
+  GRPCStreamingProtoCall *call = [_service recordRouteWithResponseHandler:self
+                                                              callOptions:nil];
+  [call start];
+  for (id feature in features) {
     RTGPoint *location = [RTGPoint message];
     location.longitude = [((NSNumber *) feature[@"location"][@"longitude"]) intValue];
     location.latitude = [((NSNumber *) feature[@"location"][@"latitude"]) intValue];
     NSString *str =[NSString stringWithFormat:@"%@\nVisiting point %@", self.outputLabel.text, location];
     self.outputLabel.text = str;
     NSLog(@"Visiting point %@", location);
-    return location;
-  }];
-
-  [_service recordRouteWithRequestsWriter:locations
-                                 handler:^(RTGRouteSummary *response, NSError *error) {
-    if (response) {
-      NSString *str =[NSString stringWithFormat:
-                      @"%@\nFinished trip with %i points\nPassed %i features\n"
-                      "Travelled %i meters\nIt took %i seconds",
-                      self.outputLabel.text, response.pointCount, response.featureCount,
-                      response.distance, response.elapsedTime];
-      self.outputLabel.text = str;
-      NSLog(@"Finished trip with %i points", response.pointCount);
-      NSLog(@"Passed %i features", response.featureCount);
-      NSLog(@"Travelled %i meters", response.distance);
-      NSLog(@"It took %i seconds", response.elapsedTime);
-    } else {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-  }];
+    [call writeMessage:location];
+  }
+  [call finish];
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGRouteSummary *response = (RTGRouteSummary *)message;
+
+  if (response) {
+    NSString *str =[NSString stringWithFormat:
+                    @"%@\nFinished trip with %i points\nPassed %i features\n"
+                    "Travelled %i meters\nIt took %i seconds",
+                    self.outputLabel.text, response.pointCount, response.featureCount,
+                    response.distance, response.elapsedTime];
+    self.outputLabel.text = str;
+    NSLog(@"Finished trip with %i points", response.pointCount);
+    NSLog(@"Passed %i features", response.featureCount);
+    NSLog(@"Travelled %i meters", response.distance);
+    NSLog(@"It took %i seconds", response.elapsedTime);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (error) {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
+
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {
@@ -250,7 +303,7 @@ static NSString * const kHostAddress = @"localhost:50051";
  * Run the routeChat demo. Send some chat messages, and print any chat messages that are sent from
  * the server.
  */
-@interface RouteChatViewController : UIViewController
+@interface RouteChatViewController : UIViewController<GRPCProtoResponseHandler>
 
 @property (weak, nonatomic) IBOutlet UILabel *outputLabel;
 
@@ -260,38 +313,52 @@ static NSString * const kHostAddress = @"localhost:50051";
   RTGRouteGuide *_service;
 }
 
+- (dispatch_queue_t)dispatchQueue {
+  return dispatch_get_main_queue();
+}
+
 - (void)execRequest {
   NSArray *notes = @[[RTGRouteNote noteWithMessage:@"First message" latitude:0 longitude:0],
                      [RTGRouteNote noteWithMessage:@"Second message" latitude:0 longitude:1],
                      [RTGRouteNote noteWithMessage:@"Third message" latitude:1 longitude:0],
                      [RTGRouteNote noteWithMessage:@"Fourth message" latitude:0 longitude:0]];
-  GRXWriter *notesWriter = [[GRXWriter writerWithContainer:notes] map:^id(RTGRouteNote *note) {
-    NSLog(@"Sending message %@ at %@", note.message, note.location);
-    return note;
-  }];
-
-  [_service routeChatWithRequestsWriter:notesWriter
-                          eventHandler:^(BOOL done, RTGRouteNote *note, NSError *error) {
-    if (note) {
-      NSString *str =[NSString stringWithFormat:@"%@\nGot message %@ at %@",
-                      self.outputLabel.text, note.message, note.location];
-      self.outputLabel.text = str;
-      NSLog(@"Got message %@ at %@", note.message, note.location);
-    } else if (error) {
-      NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
-      self.outputLabel.text = str;
-      NSLog(@"RPC error: %@", error);
-    }
-    if (done) {
-      NSLog(@"Chat ended.");
-    }
-  }];
+
+  GRPCStreamingProtoCall *call = [_service routeChatWithResponseHandler:self
+                                                            callOptions:nil];
+  [call start];
+  for (RTGRouteNote *note in notes) {
+    [call writeMessage:note];
+  }
+  [call finish];
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  RTGRouteNote *note = (RTGRouteNote *)message;
+  if (note) {
+    NSString *str =[NSString stringWithFormat:@"%@\nGot message %@ at %@",
+                    self.outputLabel.text, note.message, note.location];
+    self.outputLabel.text = str;
+    NSLog(@"Got message %@ at %@", note.message, note.location);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (!error) {
+    NSLog(@"Chat ended.");
+  } else {
+    NSString *str =[NSString stringWithFormat:@"%@\nRPC error: %@", self.outputLabel.text, error];
+    self.outputLabel.text = str;
+    NSLog(@"RPC error: %@", error);
+  }
 }
 
 - (void)viewDidLoad {
   [super viewDidLoad];
 
-  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress];
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = GRPCTransportTypeInsecure;
+
+  _service = [[RTGRouteGuide alloc] initWithHost:kHostAddress callOptions:options];
 }
 
 - (void)viewDidAppear:(BOOL)animated {

gRPC-C++.podspec

@@ -132,6 +132,7 @@ Pod::Spec.new do |s|
                       'include/grpcpp/support/client_interceptor.h',
                       'include/grpcpp/support/config.h',
                       'include/grpcpp/support/interceptor.h',
+                      'include/grpcpp/support/message_allocator.h',
                       'include/grpcpp/support/proto_buffer_reader.h',
                       'include/grpcpp/support/proto_buffer_writer.h',
                       'include/grpcpp/support/server_callback.h',
@@ -166,6 +167,7 @@ Pod::Spec.new do |s|
                       'include/grpcpp/impl/codegen/intercepted_channel.h',
                       'include/grpcpp/impl/codegen/interceptor.h',
                       'include/grpcpp/impl/codegen/interceptor_common.h',
+                      'include/grpcpp/impl/codegen/message_allocator.h',
                       'include/grpcpp/impl/codegen/metadata_map.h',
                       'include/grpcpp/impl/codegen/method_handler_impl.h',
                       'include/grpcpp/impl/codegen/rpc_method.h',

include/grpc/impl/codegen/grpc_types.h

@@ -315,11 +315,11 @@ typedef struct {
 #define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms"
 /* Timeout in milliseconds to wait for the serverlist from the grpclb load
    balancer before using fallback backend addresses from the resolver.
-   If 0, fallback will never be used. Default value is 10000. */
+   If 0, enter fallback mode immediately. Default value is 10000. */
 #define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms"
 /* Timeout in milliseconds to wait for the serverlist from the xDS load
    balancer before using fallback backend addresses from the resolver.
-   If 0, fallback will never be used. Default value is 10000. */
+   If 0, enter fallback mode immediately. Default value is 10000. */
 #define GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS "grpc.xds_fallback_timeout_ms"
 /** If non-zero, grpc server's cronet compression workaround will be enabled */
 #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \

include/grpc/impl/codegen/slice.h

@@ -40,27 +40,6 @@ typedef struct grpc_slice grpc_slice;
    reference ownership semantics (who should call unref?) and mutability
    constraints (is the callee allowed to modify the slice?) */
 
-typedef struct grpc_slice_refcount_vtable {
-  void (*ref)(void*);
-  void (*unref)(void*);
-  int (*eq)(grpc_slice a, grpc_slice b);
-  uint32_t (*hash)(grpc_slice slice);
-} grpc_slice_refcount_vtable;
-
-/** Reference count container for grpc_slice. Contains function pointers to
-   increment and decrement reference counts. Implementations should cleanup
-   when the reference count drops to zero.
-   Typically client code should not touch this, and use grpc_slice_malloc,
-   grpc_slice_new, or grpc_slice_new_with_len instead. */
-typedef struct grpc_slice_refcount {
-  const grpc_slice_refcount_vtable* vtable;
-  /** If a subset of this slice is taken, use this pointer for the refcount.
-     Typically points back to the refcount itself, however iterning
-     implementations can use this to avoid a verification step on each hash
-     or equality check */
-  struct grpc_slice_refcount* sub_refcount;
-} grpc_slice_refcount;
-
 /* Inlined half of grpc_slice is allowed to expand the size of the overall type
    by this many bytes */
 #define GRPC_SLICE_INLINE_EXTRA_SIZE sizeof(void*)
@@ -68,6 +47,7 @@ typedef struct grpc_slice_refcount {
 #define GRPC_SLICE_INLINED_SIZE \
   (sizeof(size_t) + sizeof(uint8_t*) - 1 + GRPC_SLICE_INLINE_EXTRA_SIZE)
 
+struct grpc_slice_refcount;
 /** A grpc_slice s, if initialized, represents the byte range
    s.bytes[0..s.length-1].
 

include/grpcpp/generic/generic_stub_impl.h

@@ -82,6 +82,12 @@ class GenericStub final {
                    const grpc::ByteBuffer* request, grpc::ByteBuffer* response,
                    std::function<void(grpc::Status)> on_completion);
 
+    /// Setup and start a unary call to a named method \a method using
+    /// \a context and specifying the \a request and \a response buffers.
+    void UnaryCall(grpc::ClientContext* context, const grpc::string& method,
+                   const grpc::ByteBuffer* request, grpc::ByteBuffer* response,
+                   grpc::experimental::ClientUnaryReactor* reactor);
+
     /// Setup a call to a named method \a method using \a context and tied to
     /// \a reactor . Like any other bidi streaming RPC, it will not be activated
     /// until StartCall is invoked on its reactor.

include/grpcpp/impl/codegen/channel_interface.h

@@ -58,6 +58,7 @@ template <class R>
 class ClientCallbackReaderFactory;
 template <class W>
 class ClientCallbackWriterFactory;
+class ClientCallbackUnaryFactory;
 class InterceptedChannel;
 }  // namespace internal
 
@@ -117,6 +118,7 @@ class ChannelInterface {
   friend class ::grpc::internal::ClientCallbackReaderFactory;
   template <class W>
   friend class ::grpc::internal::ClientCallbackWriterFactory;
+  friend class ::grpc::internal::ClientCallbackUnaryFactory;
   template <class InputMessage, class OutputMessage>
   friend class ::grpc::internal::BlockingUnaryCallImpl;
   template <class InputMessage, class OutputMessage>

include/grpcpp/impl/codegen/client_callback.h

@@ -100,6 +100,7 @@ template <class Response>
 class ClientReadReactor;
 template <class Request>
 class ClientWriteReactor;
+class ClientUnaryReactor;
 
 // NOTE: The streaming objects are not actually implemented in the public API.
 //       These interfaces are provided for mocking only. Typical applications
@@ -157,6 +158,15 @@ class ClientCallbackWriter {
   }
 };
 
+class ClientCallbackUnary {
+ public:
+  virtual ~ClientCallbackUnary() {}
+  virtual void StartCall() = 0;
+
+ protected:
+  void BindReactor(ClientUnaryReactor* reactor);
+};
+
 // The following classes are the reactor interfaces that are to be implemented
 // by the user. They are passed in to the library as an argument to a call on a
 // stub (either a codegen-ed call or a generic call). The streaming RPC is
@@ -346,6 +356,36 @@ class ClientWriteReactor {
   ClientCallbackWriter<Request>* writer_;
 };
 
+/// \a ClientUnaryReactor is a reactor-style interface for a unary RPC.
+/// This is _not_ a common way of invoking a unary RPC. In practice, this
+/// option should be used only if the unary RPC wants to receive initial
+/// metadata without waiting for the response to complete. Most deployments of
+/// RPC systems do not use this option, but it is needed for generality.
+/// All public methods behave as in ClientBidiReactor.
+/// StartCall is included for consistency with the other reactor flavors: even
+/// though there are no StartRead or StartWrite operations to queue before the
+/// call (that is part of the unary call itself) and there is no reactor object
+/// being created as a result of this call, we keep a consistent 2-phase
+/// initiation API among all the reactor flavors.
+class ClientUnaryReactor {
+ public:
+  virtual ~ClientUnaryReactor() {}
+
+  void StartCall() { call_->StartCall(); }
+  virtual void OnDone(const Status& s) {}
+  virtual void OnReadInitialMetadataDone(bool ok) {}
+
+ private:
+  friend class ClientCallbackUnary;
+  void BindCall(ClientCallbackUnary* call) { call_ = call; }
+  ClientCallbackUnary* call_;
+};
+
+// Define function out-of-line from class to avoid forward declaration issue
+inline void ClientCallbackUnary::BindReactor(ClientUnaryReactor* reactor) {
+  reactor->BindCall(this);
+}
+
 }  // namespace experimental
 
 namespace internal {
@@ -512,9 +552,9 @@ class ClientCallbackReaderWriterImpl
     this->BindReactor(reactor);
   }
 
-  ClientContext* context_;
+  ClientContext* const context_;
   Call call_;
-  ::grpc::experimental::ClientBidiReactor<Request, Response>* reactor_;
+  ::grpc::experimental::ClientBidiReactor<Request, Response>* const reactor_;
 
   CallOpSet<CallOpSendInitialMetadata, CallOpRecvInitialMetadata> start_ops_;
   CallbackWithSuccessTag start_tag_;
@@ -651,9 +691,9 @@ class ClientCallbackReaderImpl
     start_ops_.ClientSendClose();
   }
 
-  ClientContext* context_;
+  ClientContext* const context_;
   Call call_;
-  ::grpc::experimental::ClientReadReactor<Response>* reactor_;
+  ::grpc::experimental::ClientReadReactor<Response>* const reactor_;
 
   CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose,
             CallOpRecvInitialMetadata>
@@ -824,9 +864,9 @@ class ClientCallbackWriterImpl
     finish_ops_.AllowNoMessage();
   }
 
-  ClientContext* context_;
+  ClientContext* const context_;
   Call call_;
-  ::grpc::experimental::ClientWriteReactor<Request>* reactor_;
+  ::grpc::experimental::ClientWriteReactor<Request>* const reactor_;
 
   CallOpSet<CallOpSendInitialMetadata, CallOpRecvInitialMetadata> start_ops_;
   CallbackWithSuccessTag start_tag_;
@@ -867,6 +907,109 @@ class ClientCallbackWriterFactory {
   }
 };
 
+class ClientCallbackUnaryImpl final
+    : public ::grpc::experimental::ClientCallbackUnary {
+ public:
+  // always allocated against a call arena, no memory free required
+  static void operator delete(void* ptr, std::size_t size) {
+    assert(size == sizeof(ClientCallbackUnaryImpl));
+  }
+
+  // This operator should never be called as the memory should be freed as part
+  // of the arena destruction. It only exists to provide a matching operator
+  // delete to the operator new so that some compilers will not complain (see
+  // https://github.com/grpc/grpc/issues/11301) Note at the time of adding this
+  // there are no tests catching the compiler warning.
+  static void operator delete(void*, void*) { assert(0); }
+
+  void StartCall() override {
+    // This call initiates two batches, each with a callback
+    // 1. Send initial metadata + write + writes done + recv initial metadata
+    // 2. Read message, recv trailing metadata
+    started_ = true;
+
+    start_tag_.Set(call_.call(),
+                   [this](bool ok) {
+                     reactor_->OnReadInitialMetadataDone(ok);
+                     MaybeFinish();
+                   },
+                   &start_ops_);
+    start_ops_.SendInitialMetadata(&context_->send_initial_metadata_,
+                                   context_->initial_metadata_flags());
+    start_ops_.RecvInitialMetadata(context_);
+    start_ops_.set_core_cq_tag(&start_tag_);
+    call_.PerformOps(&start_ops_);
+
+    finish_tag_.Set(call_.call(), [this](bool ok) { MaybeFinish(); },
+                    &finish_ops_);
+    finish_ops_.ClientRecvStatus(context_, &finish_status_);
+    finish_ops_.set_core_cq_tag(&finish_tag_);
+    call_.PerformOps(&finish_ops_);
+  }
+
+  void MaybeFinish() {
+    if (--callbacks_outstanding_ == 0) {
+      Status s = std::move(finish_status_);
+      auto* reactor = reactor_;
+      auto* call = call_.call();
+      this->~ClientCallbackUnaryImpl();
+      g_core_codegen_interface->grpc_call_unref(call);
+      reactor->OnDone(s);
+    }
+  }
+
+ private:
+  friend class ClientCallbackUnaryFactory;
+
+  template <class Request, class Response>
+  ClientCallbackUnaryImpl(Call call, ClientContext* context, Request* request,
+                          Response* response,
+                          ::grpc::experimental::ClientUnaryReactor* reactor)
+      : context_(context), call_(call), reactor_(reactor) {
+    this->BindReactor(reactor);
+    // TODO(vjpai): don't assert
+    GPR_CODEGEN_ASSERT(start_ops_.SendMessagePtr(request).ok());
+    start_ops_.ClientSendClose();
+    finish_ops_.RecvMessage(response);
+    finish_ops_.AllowNoMessage();
+  }
+
+  ClientContext* const context_;
+  Call call_;
+  ::grpc::experimental::ClientUnaryReactor* const reactor_;
+
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose,
+            CallOpRecvInitialMetadata>
+      start_ops_;
+  CallbackWithSuccessTag start_tag_;
+
+  CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_;
+  CallbackWithSuccessTag finish_tag_;
+  Status finish_status_;
+
+  // This call will have 2 callbacks: start and finish
+  std::atomic_int callbacks_outstanding_{2};
+  bool started_{false};
+};
+
+class ClientCallbackUnaryFactory {
+ public:
+  template <class Request, class Response>
+  static void Create(ChannelInterface* channel,
+                     const ::grpc::internal::RpcMethod& method,
+                     ClientContext* context, const Request* request,
+                     Response* response,
+                     ::grpc::experimental::ClientUnaryReactor* reactor) {
+    Call call = channel->CreateCall(method, context, channel->CallbackCQ());
+
+    g_core_codegen_interface->grpc_call_ref(call.call());
+
+    new (g_core_codegen_interface->grpc_call_arena_alloc(
+        call.call(), sizeof(ClientCallbackUnaryImpl)))
+        ClientCallbackUnaryImpl(call, context, request, response, reactor);
+  }
+};
+
 }  // namespace internal
 }  // namespace grpc
 

include/grpcpp/impl/codegen/client_context.h

@@ -79,6 +79,7 @@ template <class Response>
 class ClientCallbackReaderImpl;
 template <class Request>
 class ClientCallbackWriterImpl;
+class ClientCallbackUnaryImpl;
 }  // namespace internal
 
 template <class R>
@@ -417,6 +418,7 @@ class ClientContext {
   friend class ::grpc::internal::ClientCallbackReaderImpl;
   template <class Request>
   friend class ::grpc::internal::ClientCallbackWriterImpl;
+  friend class ::grpc::internal::ClientCallbackUnaryImpl;
 
   // Used by friend class CallOpClientRecvStatus
   void set_debug_error_string(const grpc::string& debug_error_string) {

include/grpcpp/impl/codegen/message_allocator.h

@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCPP_IMPL_CODEGEN_MESSAGE_ALLOCATOR_H
+#define GRPCPP_IMPL_CODEGEN_MESSAGE_ALLOCATOR_H
+
+namespace grpc {
+namespace experimental {
+
+// This is per rpc struct for the allocator. We can potentially put the grpc
+// call arena in here in the future.
+template <typename RequestT, typename ResponseT>
+struct RpcAllocatorInfo {
+  RequestT* request;
+  ResponseT* response;
+  // per rpc allocator internal state. MessageAllocator can set it when
+  // AllocateMessages is called and use it later.
+  void* allocator_state;
+};
+
+// Implementations need to be thread-safe
+template <typename RequestT, typename ResponseT>
+class MessageAllocator {
+ public:
+  virtual ~MessageAllocator() = default;
+  // Allocate both request and response
+  virtual void AllocateMessages(
+      RpcAllocatorInfo<RequestT, ResponseT>* info) = 0;
+  // Optional: deallocate request early, called by
+  // ServerCallbackRpcController::ReleaseRequest
+  virtual void DeallocateRequest(RpcAllocatorInfo<RequestT, ResponseT>* info) {}
+  // Deallocate response and request (if applicable)
+  virtual void DeallocateMessages(
+      RpcAllocatorInfo<RequestT, ResponseT>* info) = 0;
+};
+
+}  // namespace experimental
+}  // namespace grpc
+
+#endif  // GRPCPP_IMPL_CODEGEN_MESSAGE_ALLOCATOR_H

include/grpcpp/impl/codegen/method_handler_impl.h

@@ -86,8 +86,8 @@ class RpcMethodHandler : public MethodHandler {
     param.call->cq()->Pluck(&ops);
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
     auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(
@@ -191,8 +191,8 @@ class ServerStreamingHandler : public MethodHandler {
     param.call->cq()->Pluck(&ops);
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
     auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(
@@ -327,8 +327,8 @@ class ErrorMethodHandler : public MethodHandler {
     param.call->cq()->Pluck(&ops);
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     // We have to destroy any request payload
     if (req != nullptr) {
       g_core_codegen_interface->grpc_byte_buffer_destroy(req);

include/grpcpp/impl/codegen/rpc_service_method.h

@@ -46,21 +46,25 @@ class MethodHandler {
     /// \param context : the ServerContext structure for this server call
     /// \param req : the request payload, if appropriate for this RPC
     /// \param req_status : the request status after any interceptors have run
+    /// \param handler_data: internal data for the handler.
     /// \param requester : used only by the callback API. It is a function
     ///        called by the RPC Controller to request another RPC (and also
     ///        to set up the state required to make that request possible)
     HandlerParameter(Call* c, ServerContext* context, void* req,
-                     Status req_status, std::function<void()> requester)
+                     Status req_status, void* handler_data,
+                     std::function<void()> requester)
         : call(c),
           server_context(context),
           request(req),
           status(req_status),
+          internal_data(handler_data),
           call_requester(std::move(requester)) {}
     ~HandlerParameter() {}
     Call* call;
     ServerContext* server_context;
     void* request;
     Status status;
+    void* internal_data;
     std::function<void()> call_requester;
   };
   virtual void RunHandler(const HandlerParameter& param) = 0;
@@ -71,7 +75,7 @@ class MethodHandler {
      pointer after calling RunHandler. Ownership of the deserialized request is
      retained by the handler. Returns nullptr if deserialization failed. */
   virtual void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                            Status* status) {
+                            Status* status, void** handler_data) {
     GPR_CODEGEN_ASSERT(req == nullptr);
     return nullptr;
   }

include/grpcpp/impl/codegen/server_callback.h

@@ -28,6 +28,7 @@
 #include <grpcpp/impl/codegen/callback_common.h>
 #include <grpcpp/impl/codegen/config.h>
 #include <grpcpp/impl/codegen/core_codegen_interface.h>
+#include <grpcpp/impl/codegen/message_allocator.h>
 #include <grpcpp/impl/codegen/server_context.h>
 #include <grpcpp/impl/codegen/server_interface.h>
 #include <grpcpp/impl/codegen/status.h>
@@ -135,6 +136,14 @@ class ServerCallbackRpcController {
   /// to be called before the callback completes.
   virtual void SetCancelCallback(std::function<void()> callback) = 0;
   virtual void ClearCancelCallback() = 0;
+
+  // NOTE: This is an API for advanced users who need custom allocators.
+  // Optionally deallocate request early to reduce the size of working set.
+  // A custom MessageAllocator needs to be registered to make use of this.
+  virtual void FreeRequest() = 0;
+  // NOTE: This is an API for advanced users who need custom allocators.
+  // Get and maybe mutate the allocator state associated with the current RPC.
+  virtual void* GetAllocatorState() = 0;
 };
 
 // NOTE: The actual streaming object classes are provided
@@ -364,7 +373,7 @@ class ServerReadReactor : public internal::ServerReactor {
   ServerCallbackReader<Request>* reader_;
 };
 
-/// \a ServerReadReactor is the interface for a server-streaming RPC.
+/// \a ServerWriteReactor is the interface for a server-streaming RPC.
 template <class Request, class Response>
 class ServerWriteReactor : public internal::ServerReactor {
  public:
@@ -447,17 +456,24 @@ class CallbackUnaryHandler : public MethodHandler {
                          experimental::ServerCallbackRpcController*)>
           func)
       : func_(func) {}
+
+  void SetMessageAllocator(
+      experimental::MessageAllocator<RequestType, ResponseType>* allocator) {
+    allocator_ = allocator;
+  }
+
   void RunHandler(const HandlerParameter& param) final {
     // Arena allocate a controller structure (that includes request/response)
     g_core_codegen_interface->grpc_call_ref(param.call->call());
+    auto* allocator_info =
+        static_cast<experimental::RpcAllocatorInfo<RequestType, ResponseType>*>(
+            param.internal_data);
     auto* controller = new (g_core_codegen_interface->grpc_call_arena_alloc(
         param.call->call(), sizeof(ServerCallbackRpcControllerImpl)))
-        ServerCallbackRpcControllerImpl(
-            param.server_context, param.call,
-            static_cast<RequestType*>(param.request),
-            std::move(param.call_requester));
+        ServerCallbackRpcControllerImpl(param.server_context, param.call,
+                                        allocator_info, allocator_,
+                                        std::move(param.call_requester));
     Status status = param.status;
-
     if (status.ok()) {
       // Call the actual function handler and expect the user to call finish
       CatchingCallback(func_, param.server_context, controller->request(),
@@ -468,18 +484,41 @@ class CallbackUnaryHandler : public MethodHandler {
     }
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
-    auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(
-        call, sizeof(RequestType))) RequestType();
+    RequestType* request = nullptr;
+    experimental::RpcAllocatorInfo<RequestType, ResponseType>* allocator_info =
+        new (g_core_codegen_interface->grpc_call_arena_alloc(
+            call, sizeof(*allocator_info)))
+            experimental::RpcAllocatorInfo<RequestType, ResponseType>();
+    if (allocator_ != nullptr) {
+      allocator_->AllocateMessages(allocator_info);
+    } else {
+      allocator_info->request =
+          new (g_core_codegen_interface->grpc_call_arena_alloc(
+              call, sizeof(RequestType))) RequestType();
+      allocator_info->response =
+          new (g_core_codegen_interface->grpc_call_arena_alloc(
+              call, sizeof(ResponseType))) ResponseType();
+    }
+    *handler_data = allocator_info;
+    request = allocator_info->request;
     *status = SerializationTraits<RequestType>::Deserialize(&buf, request);
     buf.Release();
     if (status->ok()) {
       return request;
     }
-    request->~RequestType();
+    // Clean up on deserialization failure.
+    if (allocator_ != nullptr) {
+      allocator_->DeallocateMessages(allocator_info);
+    } else {
+      allocator_info->request->~RequestType();
+      allocator_info->response->~ResponseType();
+      allocator_info->request = nullptr;
+      allocator_info->response = nullptr;
+    }
     return nullptr;
   }
 
@@ -487,6 +526,8 @@ class CallbackUnaryHandler : public MethodHandler {
   std::function<void(ServerContext*, const RequestType*, ResponseType*,
                      experimental::ServerCallbackRpcController*)>
       func_;
+  experimental::MessageAllocator<RequestType, ResponseType>* allocator_ =
+      nullptr;
 
   // The implementation class of ServerCallbackRpcController is a private member
   // of CallbackUnaryHandler since it is never exposed anywhere, and this allows
@@ -507,8 +548,9 @@ class CallbackUnaryHandler : public MethodHandler {
       }
       // The response is dropped if the status is not OK.
       if (s.ok()) {
-        finish_ops_.ServerSendStatus(&ctx_->trailing_metadata_,
-                                     finish_ops_.SendMessagePtr(&resp_));
+        finish_ops_.ServerSendStatus(
+            &ctx_->trailing_metadata_,
+            finish_ops_.SendMessagePtr(allocator_info_->response));
       } else {
         finish_ops_.ServerSendStatus(&ctx_->trailing_metadata_, s);
       }
@@ -546,28 +588,50 @@ class CallbackUnaryHandler : public MethodHandler {
 
     void ClearCancelCallback() override { ctx_->ClearCancelCallback(); }
 
+    void FreeRequest() override {
+      if (allocator_ != nullptr) {
+        allocator_->DeallocateRequest(allocator_info_);
+      }
+    }
+
+    void* GetAllocatorState() override {
+      return allocator_info_->allocator_state;
+    }
+
    private:
     friend class CallbackUnaryHandler<RequestType, ResponseType>;
 
-    ServerCallbackRpcControllerImpl(ServerContext* ctx, Call* call,
-                                    const RequestType* req,
-                                    std::function<void()> call_requester)
+    ServerCallbackRpcControllerImpl(
+        ServerContext* ctx, Call* call,
+        experimental::RpcAllocatorInfo<RequestType, ResponseType>*
+            allocator_info,
+        experimental::MessageAllocator<RequestType, ResponseType>* allocator,
+        std::function<void()> call_requester)
         : ctx_(ctx),
           call_(*call),
-          req_(req),
+          allocator_info_(allocator_info),
+          allocator_(allocator),
           call_requester_(std::move(call_requester)) {
       ctx_->BeginCompletionOp(call, [this](bool) { MaybeDone(); }, nullptr);
     }
 
-    ~ServerCallbackRpcControllerImpl() { req_->~RequestType(); }
-
-    const RequestType* request() { return req_; }
-    ResponseType* response() { return &resp_; }
+    const RequestType* request() { return allocator_info_->request; }
+    ResponseType* response() { return allocator_info_->response; }
 
     void MaybeDone() {
       if (--callbacks_outstanding_ == 0) {
         grpc_call* call = call_.call();
         auto call_requester = std::move(call_requester_);
+        if (allocator_ != nullptr) {
+          allocator_->DeallocateMessages(allocator_info_);
+        } else {
+          if (allocator_info_->request != nullptr) {
+            allocator_info_->request->~RequestType();
+          }
+          if (allocator_info_->response != nullptr) {
+            allocator_info_->response->~ResponseType();
+          }
+        }
         this->~ServerCallbackRpcControllerImpl();  // explicitly call destructor
         g_core_codegen_interface->grpc_call_unref(call);
         call_requester();
@@ -583,8 +647,8 @@ class CallbackUnaryHandler : public MethodHandler {
 
     ServerContext* ctx_;
     Call call_;
-    const RequestType* req_;
-    ResponseType resp_;
+    experimental::RpcAllocatorInfo<RequestType, ResponseType>* allocator_info_;
+    experimental::MessageAllocator<RequestType, ResponseType>* allocator_;
     std::function<void()> call_requester_;
     std::atomic_int callbacks_outstanding_{
         2};  // reserve for Finish and CompletionOp
@@ -771,8 +835,8 @@ class CallbackServerStreamingHandler : public MethodHandler {
     writer->MaybeDone();
   }
 
-  void* Deserialize(grpc_call* call, grpc_byte_buffer* req,
-                    Status* status) final {
+  void* Deserialize(grpc_call* call, grpc_byte_buffer* req, Status* status,
+                    void** handler_data) final {
     ByteBuffer buf;
     buf.set_buffer(req);
     auto* request = new (g_core_codegen_interface->grpc_call_arena_alloc(

include/grpcpp/impl/codegen/service_type.h

@@ -132,6 +132,11 @@ class Service {
           internal::RpcServiceMethod::ApiType::RAW_CALL_BACK);
     }
 
+    internal::MethodHandler* GetHandler(int index) {
+      size_t idx = static_cast<size_t>(index);
+      return service_->methods_[idx]->handler();
+    }
+
    private:
     Service* service_;
   };

include/grpcpp/security/credentials.h

@@ -98,10 +98,10 @@ class ChannelCredentials : private GrpcLibraryCodegen {
   // This function should have been a pure virtual function, but it is
   // implemented as a virtual function so that it does not break API.
   virtual std::shared_ptr<Channel> CreateChannelWithInterceptors(
-      const grpc::string& target, const ChannelArguments& args,
+      const grpc::string& /* target */, const ChannelArguments& /* args */,
       std::vector<
           std::unique_ptr<experimental::ClientInterceptorFactoryInterface>>
-          interceptor_creators) {
+      /* interceptor_creators */) {
     return nullptr;
   }
 };

include/grpcpp/support/message_allocator.h

@@ -0,0 +1,24 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPCPP_SUPPORT_MESSAGE_ALLOCATOR_H
+#define GRPCPP_SUPPORT_MESSAGE_ALLOCATOR_H
+
+#include <grpcpp/impl/codegen/message_allocator.h>
+
+#endif  // GRPCPP_SUPPORT_MESSAGE_ALLOCATOR_H

src/compiler/cpp_generator.cc

@@ -155,6 +155,10 @@ grpc::string GetHeaderIncludes(grpc_generator::File* file,
                   params.grpc_search_path);
     printer->Print(vars, "\n");
     printer->Print(vars, "namespace grpc {\n");
+    printer->Print(vars, "namespace experimental {\n");
+    printer->Print(vars, "template <typename RequestT, typename ResponseT>\n");
+    printer->Print(vars, "class MessageAllocator;\n");
+    printer->Print(vars, "}  // namespace experimental\n");
     printer->Print(vars, "class CompletionQueue;\n");
     printer->Print(vars, "class Channel;\n");
     printer->Print(vars, "class ServerCompletionQueue;\n");
@@ -607,6 +611,14 @@ void PrintHeaderClientMethodCallbackInterfaces(
                    "virtual void $Method$(::grpc::ClientContext* context, "
                    "const ::grpc::ByteBuffer* request, $Response$* response, "
                    "std::function<void(::grpc::Status)>) = 0;\n");
+    printer->Print(*vars,
+                   "virtual void $Method$(::grpc::ClientContext* context, "
+                   "const $Request$* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) = 0;\n");
+    printer->Print(*vars,
+                   "virtual void $Method$(::grpc::ClientContext* context, "
+                   "const ::grpc::ByteBuffer* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) = 0;\n");
   } else if (ClientOnlyStreaming(method)) {
     printer->Print(*vars,
                    "virtual void $Method$(::grpc::ClientContext* context, "
@@ -673,6 +685,16 @@ void PrintHeaderClientMethodCallback(grpc_generator::Printer* printer,
                    "void $Method$(::grpc::ClientContext* context, "
                    "const ::grpc::ByteBuffer* request, $Response$* response, "
                    "std::function<void(::grpc::Status)>) override;\n");
+    printer->Print(
+        *vars,
+        "void $Method$(::grpc::ClientContext* context, "
+        "const $Request$* request, $Response$* response, "
+        "::grpc::experimental::ClientUnaryReactor* reactor) override;\n");
+    printer->Print(
+        *vars,
+        "void $Method$(::grpc::ClientContext* context, "
+        "const ::grpc::ByteBuffer* request, $Response$* response, "
+        "::grpc::experimental::ClientUnaryReactor* reactor) override;\n");
   } else if (ClientOnlyStreaming(method)) {
     printer->Print(*vars,
                    "void $Method$(::grpc::ClientContext* context, "
@@ -993,7 +1015,15 @@ void PrintHeaderServerMethodCallback(
         "controller) {\n"
         "               return this->$"
         "Method$(context, request, response, controller);\n"
-        "             }));\n");
+        "             }));\n}\n");
+    printer->Print(*vars,
+                   "void SetMessageAllocatorFor_$Method$(\n"
+                   "    ::grpc::experimental::MessageAllocator< "
+                   "$RealRequest$, $RealResponse$>* allocator) {\n"
+                   "  static_cast<::grpc::internal::CallbackUnaryHandler< "
+                   "$RealRequest$, $RealResponse$>*>(\n"
+                   "      ::grpc::Service::experimental().GetHandler($Idx$))\n"
+                   "          ->SetMessageAllocator(allocator);\n");
   } else if (ClientOnlyStreaming(method)) {
     printer->Print(
         *vars,
@@ -1671,7 +1701,7 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer,
                    "const $Request$* request, $Response$* response, "
                    "std::function<void(::grpc::Status)> f) {\n");
     printer->Print(*vars,
-                   "  return ::grpc::internal::CallbackUnaryCall"
+                   "  ::grpc::internal::CallbackUnaryCall"
                    "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
                    "context, request, response, std::move(f));\n}\n\n");
 
@@ -1681,10 +1711,30 @@ void PrintSourceClientMethod(grpc_generator::Printer* printer,
                    "const ::grpc::ByteBuffer* request, $Response$* response, "
                    "std::function<void(::grpc::Status)> f) {\n");
     printer->Print(*vars,
-                   "  return ::grpc::internal::CallbackUnaryCall"
+                   "  ::grpc::internal::CallbackUnaryCall"
                    "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
                    "context, request, response, std::move(f));\n}\n\n");
 
+    printer->Print(*vars,
+                   "void $ns$$Service$::Stub::experimental_async::$Method$("
+                   "::grpc::ClientContext* context, "
+                   "const $Request$* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) {\n");
+    printer->Print(*vars,
+                   "  ::grpc::internal::ClientCallbackUnaryFactory::Create"
+                   "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
+                   "context, request, response, reactor);\n}\n\n");
+
+    printer->Print(*vars,
+                   "void $ns$$Service$::Stub::experimental_async::$Method$("
+                   "::grpc::ClientContext* context, "
+                   "const ::grpc::ByteBuffer* request, $Response$* response, "
+                   "::grpc::experimental::ClientUnaryReactor* reactor) {\n");
+    printer->Print(*vars,
+                   "  ::grpc::internal::ClientCallbackUnaryFactory::Create"
+                   "(stub_->channel_.get(), stub_->rpcmethod_$Method$_, "
+                   "context, request, response, reactor);\n}\n\n");
+
     for (auto async_prefix : async_prefixes) {
       (*vars)["AsyncPrefix"] = async_prefix.prefix;
       (*vars)["AsyncStart"] = async_prefix.start;

src/compiler/protobuf_plugin.h

@@ -55,22 +55,23 @@ class ProtoBufMethod : public grpc_generator::Method {
     return method_->output_type()->file()->name();
   }
 
-  bool get_module_and_message_path_input(grpc::string* str,
-                                         grpc::string generator_file_name,
-                                         bool generate_in_pb2_grpc,
-                                         grpc::string import_prefix) const {
+  // TODO(https://github.com/grpc/grpc/issues/18800): Clean this up.
+  bool get_module_and_message_path_input(
+      grpc::string* str, grpc::string generator_file_name,
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const final {
     return grpc_python_generator::GetModuleAndMessagePath(
         method_->input_type(), str, generator_file_name, generate_in_pb2_grpc,
-        import_prefix);
+        import_prefix, prefixes_to_filter);
   }
 
-  bool get_module_and_message_path_output(grpc::string* str,
-                                          grpc::string generator_file_name,
-                                          bool generate_in_pb2_grpc,
-                                          grpc::string import_prefix) const {
+  bool get_module_and_message_path_output(
+      grpc::string* str, grpc::string generator_file_name,
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const final {
     return grpc_python_generator::GetModuleAndMessagePath(
         method_->output_type(), str, generator_file_name, generate_in_pb2_grpc,
-        import_prefix);
+        import_prefix, prefixes_to_filter);
   }
 
   bool NoStreaming() const {

src/compiler/python_generator.cc

@@ -214,13 +214,15 @@ bool PrivateGenerator::PrintBetaServerFactory(
       grpc::string input_message_module_and_class;
       if (!method->get_module_and_message_path_input(
               &input_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       grpc::string output_message_module_and_class;
       if (!method->get_module_and_message_path_output(
               &output_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       method_implementation_constructors.insert(
@@ -320,13 +322,15 @@ bool PrivateGenerator::PrintBetaStubFactory(
       grpc::string input_message_module_and_class;
       if (!method->get_module_and_message_path_input(
               &input_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       grpc::string output_message_module_and_class;
       if (!method->get_module_and_message_path_output(
               &output_message_module_and_class, generator_file_name,
-              generate_in_pb2_grpc, config.import_prefix)) {
+              generate_in_pb2_grpc, config.import_prefix,
+              config.prefixes_to_filter)) {
         return false;
       }
       method_cardinalities.insert(
@@ -425,13 +429,15 @@ bool PrivateGenerator::PrintStub(
         grpc::string request_module_and_class;
         if (!method->get_module_and_message_path_input(
                 &request_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         grpc::string response_module_and_class;
         if (!method->get_module_and_message_path_output(
                 &response_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         StringMap method_dict;
@@ -516,13 +522,15 @@ bool PrivateGenerator::PrintAddServicerToServer(
         grpc::string request_module_and_class;
         if (!method->get_module_and_message_path_input(
                 &request_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         grpc::string response_module_and_class;
         if (!method->get_module_and_message_path_output(
                 &response_module_and_class, generator_file_name,
-                generate_in_pb2_grpc, config.import_prefix)) {
+                generate_in_pb2_grpc, config.import_prefix,
+                config.prefixes_to_filter)) {
           return false;
         }
         StringMap method_dict;
@@ -589,17 +597,21 @@ bool PrivateGenerator::PrintPreamble(grpc_generator::Printer* out) {
 
         grpc::string input_type_file_name = method->get_input_type_name();
         grpc::string input_module_name =
-            ModuleName(input_type_file_name, config.import_prefix);
+            ModuleName(input_type_file_name, config.import_prefix,
+                       config.prefixes_to_filter);
         grpc::string input_module_alias =
-            ModuleAlias(input_type_file_name, config.import_prefix);
+            ModuleAlias(input_type_file_name, config.import_prefix,
+                        config.prefixes_to_filter);
         imports_set.insert(
             std::make_tuple(input_module_name, input_module_alias));
 
         grpc::string output_type_file_name = method->get_output_type_name();
         grpc::string output_module_name =
-            ModuleName(output_type_file_name, config.import_prefix);
+            ModuleName(output_type_file_name, config.import_prefix,
+                       config.prefixes_to_filter);
         grpc::string output_module_alias =
-            ModuleAlias(output_type_file_name, config.import_prefix);
+            ModuleAlias(output_type_file_name, config.import_prefix,
+                        config.prefixes_to_filter);
         imports_set.insert(
             std::make_tuple(output_module_name, output_module_alias));
       }

src/compiler/python_generator.h

@@ -20,6 +20,7 @@
 #define GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H
 
 #include <utility>
+#include <vector>
 
 #include "src/compiler/config.h"
 #include "src/compiler/schema_interface.h"
@@ -35,6 +36,7 @@ struct GeneratorConfiguration {
   grpc::string beta_package_root;
   // TODO(https://github.com/google/protobuf/issues/888): Drop this.
   grpc::string import_prefix;
+  std::vector<grpc::string> prefixes_to_filter;
 };
 
 class PythonGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {

src/compiler/python_generator_helpers.h

@@ -49,23 +49,39 @@ namespace {
 typedef vector<const Descriptor*> DescriptorVector;
 typedef vector<grpc::string> StringVector;
 
+static grpc::string StripModulePrefixes(
+    const grpc::string& raw_module_name,
+    const std::vector<grpc::string>& prefixes_to_filter) {
+  for (const auto& prefix : prefixes_to_filter) {
+    if (raw_module_name.rfind(prefix, 0) == 0) {
+      return raw_module_name.substr(prefix.size(),
+                                    raw_module_name.size() - prefix.size());
+    }
+  }
+  return raw_module_name;
+}
+
 // TODO(https://github.com/google/protobuf/issues/888):
 // Export `ModuleName` from protobuf's
 // `src/google/protobuf/compiler/python/python_generator.cc` file.
 grpc::string ModuleName(const grpc::string& filename,
-                        const grpc::string& import_prefix) {
+                        const grpc::string& import_prefix,
+                        const std::vector<grpc::string>& prefixes_to_filter) {
   grpc::string basename = StripProto(filename);
   basename = StringReplace(basename, "-", "_");
   basename = StringReplace(basename, "/", ".");
-  return import_prefix + basename + "_pb2";
+  return StripModulePrefixes(import_prefix + basename + "_pb2",
+                             prefixes_to_filter);
 }
 
 // TODO(https://github.com/google/protobuf/issues/888):
 // Export `ModuleAlias` from protobuf's
 // `src/google/protobuf/compiler/python/python_generator.cc` file.
 grpc::string ModuleAlias(const grpc::string& filename,
-                         const grpc::string& import_prefix) {
-  grpc::string module_name = ModuleName(filename, import_prefix);
+                         const grpc::string& import_prefix,
+                         const std::vector<grpc::string>& prefixes_to_filter) {
+  grpc::string module_name =
+      ModuleName(filename, import_prefix, prefixes_to_filter);
   // We can't have dots in the module name, so we replace each with _dot_.
   // But that could lead to a collision between a.b and a_dot_b, so we also
   // duplicate each underscore.
@@ -74,10 +90,10 @@ grpc::string ModuleAlias(const grpc::string& filename,
   return module_name;
 }
 
-bool GetModuleAndMessagePath(const Descriptor* type, grpc::string* out,
-                             grpc::string generator_file_name,
-                             bool generate_in_pb2_grpc,
-                             grpc::string& import_prefix) {
+bool GetModuleAndMessagePath(
+    const Descriptor* type, grpc::string* out, grpc::string generator_file_name,
+    bool generate_in_pb2_grpc, grpc::string& import_prefix,
+    const std::vector<grpc::string>& prefixes_to_filter) {
   const Descriptor* path_elem_type = type;
   DescriptorVector message_path;
   do {
@@ -93,7 +109,7 @@ bool GetModuleAndMessagePath(const Descriptor* type, grpc::string* out,
 
   grpc::string module;
   if (generator_file_name != file_name || generate_in_pb2_grpc) {
-    module = ModuleAlias(file_name, import_prefix) + ".";
+    module = ModuleAlias(file_name, import_prefix, prefixes_to_filter) + ".";
   } else {
     module = "";
   }

src/compiler/schema_interface.h

@@ -57,10 +57,12 @@ struct Method : public CommentHolder {
 
   virtual bool get_module_and_message_path_input(
       grpc::string* str, grpc::string generator_file_name,
-      bool generate_in_pb2_grpc, grpc::string import_prefix) const = 0;
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const = 0;
   virtual bool get_module_and_message_path_output(
       grpc::string* str, grpc::string generator_file_name,
-      bool generate_in_pb2_grpc, grpc::string import_prefix) const = 0;
+      bool generate_in_pb2_grpc, grpc::string import_prefix,
+      const std::vector<grpc::string>& prefixes_to_filter) const = 0;
 
   virtual grpc::string get_input_type_name() const = 0;
   virtual grpc::string get_output_type_name() const = 0;

src/core/ext/filters/client_channel/client_channel.cc

@@ -615,7 +615,7 @@ class CallData {
   grpc_millis deadline_;
   gpr_arena* arena_;
   grpc_call_stack* owning_call_;
-  grpc_call_combiner* call_combiner_;
+  CallCombiner* call_combiner_;
   grpc_call_context_element* call_context_;
 
   RefCountedPtr<ServerRetryThrottleData> retry_throttle_data_;
@@ -3035,7 +3035,7 @@ class CallData::QueuedPickCanceller {
     GRPC_CALL_STACK_REF(calld->owning_call_, "QueuedPickCanceller");
     GRPC_CLOSURE_INIT(&closure_, &CancelLocked, this,
                       grpc_combiner_scheduler(chand->data_plane_combiner()));
-    grpc_call_combiner_set_notify_on_cancel(calld->call_combiner_, &closure_);
+    calld->call_combiner_->SetNotifyOnCancel(&closure_);
   }
 
  private:

src/core/ext/filters/client_channel/health/health_check_client.cc

@@ -37,11 +37,10 @@
 #define HEALTH_CHECK_RECONNECT_MAX_BACKOFF_SECONDS 120
 #define HEALTH_CHECK_RECONNECT_JITTER 0.2
 
-grpc_core::TraceFlag grpc_health_check_client_trace(false,
-                                                    "health_check_client");
-
 namespace grpc_core {
 
+TraceFlag grpc_health_check_client_trace(false, "health_check_client");
+
 //
 // HealthCheckClient
 //
@@ -50,7 +49,7 @@ HealthCheckClient::HealthCheckClient(
     const char* service_name,
     RefCountedPtr<ConnectedSubchannel> connected_subchannel,
     grpc_pollset_set* interested_parties,
-    grpc_core::RefCountedPtr<grpc_core::channelz::SubchannelNode> channelz_node)
+    RefCountedPtr<channelz::SubchannelNode> channelz_node)
     : InternallyRefCounted<HealthCheckClient>(&grpc_health_check_client_trace),
       service_name_(service_name),
       connected_subchannel_(std::move(connected_subchannel)),
@@ -283,9 +282,7 @@ HealthCheckClient::CallState::CallState(
       pollent_(grpc_polling_entity_create_from_pollset_set(interested_parties)),
       arena_(gpr_arena_create(health_check_client_->connected_subchannel_
                                   ->GetInitialCallSizeEstimate(0))),
-      payload_(context_) {
-  grpc_call_combiner_init(&call_combiner_);
-}
+      payload_(context_) {}
 
 HealthCheckClient::CallState::~CallState() {
   if (grpc_health_check_client_trace.enabled()) {
@@ -303,14 +300,13 @@ HealthCheckClient::CallState::~CallState() {
   // holding to the call stack. Also flush the closures on exec_ctx so that
   // filters that schedule cancel notification closures on exec_ctx do not
   // need to take a ref of the call stack to guarantee closure liveness.
-  grpc_call_combiner_set_notify_on_cancel(&call_combiner_, nullptr);
-  grpc_core::ExecCtx::Get()->Flush();
-  grpc_call_combiner_destroy(&call_combiner_);
+  call_combiner_.SetNotifyOnCancel(nullptr);
+  ExecCtx::Get()->Flush();
   gpr_arena_destroy(arena_);
 }
 
 void HealthCheckClient::CallState::Orphan() {
-  grpc_call_combiner_cancel(&call_combiner_, GRPC_ERROR_CANCELLED);
+  call_combiner_.Cancel(GRPC_ERROR_CANCELLED);
   Cancel();
 }
 

src/core/ext/filters/client_channel/health/health_check_client.h

@@ -98,7 +98,7 @@ class HealthCheckClient : public InternallyRefCounted<HealthCheckClient> {
     grpc_polling_entity pollent_;
 
     gpr_arena* arena_;
-    grpc_call_combiner call_combiner_;
+    grpc_core::CallCombiner call_combiner_;
     grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
 
     // The streaming call to the backend. Always non-NULL.

src/core/ext/filters/client_channel/http_connect_handshaker.h

@@ -25,7 +25,7 @@
 
 /// Channel arg indicating HTTP CONNECT headers (string).
 /// Multiple headers are separated by newlines.  Key/value pairs are
-/// seperated by colons.
+/// separated by colons.
 #define GRPC_ARG_HTTP_CONNECT_HEADERS "grpc.http_connect_headers"
 
 /// Registers handshaker factory.

src/core/ext/filters/client_channel/lb_policy.h

@@ -179,6 +179,9 @@ class LoadBalancingPolicy : public InternallyRefCounted<LoadBalancingPolicy> {
 
   /// A proxy object used by the LB policy to communicate with the client
   /// channel.
+  // TODO(juanlishen): Consider adding a mid-layer subclass that helps handle
+  // things like swapping in pending policy when it's ready. Currently, we are
+  // duplicating the logic in many subclasses.
   class ChannelControlHelper {
    public:
     ChannelControlHelper() = default;

src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc

@@ -1142,13 +1142,13 @@ void GrpcLb::BalancerCallState::OnBalancerStatusReceivedLocked(
   // we want to retry connecting. Otherwise, we have deliberately ended this
   // call and no further action is required.
   if (lb_calld == grpclb_policy->lb_calld_.get()) {
-    // If we did not receive a serverlist and the fallback-at-startup checks
-    // are pending, go into fallback mode immediately.  This short-circuits
-    // the timeout for the fallback-at-startup case.
-    if (!lb_calld->seen_serverlist_ &&
-        grpclb_policy->fallback_at_startup_checks_pending_) {
+    // If the fallback-at-startup checks are pending, go into fallback mode
+    // immediately.  This short-circuits the timeout for the fallback-at-startup
+    // case.
+    if (grpclb_policy->fallback_at_startup_checks_pending_) {
+      GPR_ASSERT(!lb_calld->seen_serverlist_);
       gpr_log(GPR_INFO,
-              "[grpclb %p] balancer call finished without receiving "
+              "[grpclb %p] Balancer call finished without receiving "
               "serverlist; entering fallback mode",
               grpclb_policy);
       grpclb_policy->fallback_at_startup_checks_pending_ = false;
@@ -1627,20 +1627,16 @@ void GrpcLb::OnFallbackTimerLocked(void* arg, grpc_error* error) {
 
 grpc_channel_args* GrpcLb::CreateChildPolicyArgsLocked(
     bool is_backend_from_grpclb_load_balancer) {
-  grpc_arg args_to_add[2] = {
-      // A channel arg indicating if the target is a backend inferred from a
-      // grpclb load balancer.
-      grpc_channel_arg_integer_create(
-          const_cast<char*>(
-              GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER),
-          is_backend_from_grpclb_load_balancer),
-  };
-  size_t num_args_to_add = 1;
+  InlinedVector<grpc_arg, 2> args_to_add;
+  args_to_add.emplace_back(grpc_channel_arg_integer_create(
+      const_cast<char*>(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_GRPCLB_LOAD_BALANCER),
+      is_backend_from_grpclb_load_balancer));
   if (is_backend_from_grpclb_load_balancer) {
-    args_to_add[num_args_to_add++] = grpc_channel_arg_integer_create(
-        const_cast<char*>(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1);
+    args_to_add.emplace_back(grpc_channel_arg_integer_create(
+        const_cast<char*>(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1));
   }
-  return grpc_channel_args_copy_and_add(args_, args_to_add, num_args_to_add);
+  return grpc_channel_args_copy_and_add(args_, args_to_add.data(),
+                                        args_to_add.size());
 }
 
 OrphanablePtr<LoadBalancingPolicy> GrpcLb::CreateChildPolicyLocked(

src/core/ext/filters/client_channel/lb_policy/xds/xds.cc

@@ -118,6 +118,7 @@ namespace {
 
 constexpr char kXds[] = "xds_experimental";
 constexpr char kDefaultLocalityName[] = "xds_default_locality";
+constexpr uint32_t kDefaultLocalityWeight = 3;
 
 class ParsedXdsConfig : public ParsedLoadBalancingConfig {
  public:
@@ -266,6 +267,10 @@ class XdsLb : public LoadBalancingPolicy {
     static void OnCallRetryTimerLocked(void* arg, grpc_error* error);
     void StartCallLocked();
 
+    void StartConnectivityWatchLocked();
+    void CancelConnectivityWatchLocked();
+    static void OnConnectivityChangedLocked(void* arg, grpc_error* error);
+
    private:
     // The owning LB policy.
     RefCountedPtr<XdsLb> xdslb_policy_;
@@ -273,6 +278,8 @@ class XdsLb : public LoadBalancingPolicy {
     // The channel and its status.
     grpc_channel* channel_;
     bool shutting_down_ = false;
+    grpc_connectivity_state connectivity_ = GRPC_CHANNEL_IDLE;
+    grpc_closure on_connectivity_changed_;
 
     // The data associated with the current LB call. It holds a ref to this LB
     // channel. It's instantiated every time we query for backends. It's reset
@@ -286,26 +293,73 @@ class XdsLb : public LoadBalancingPolicy {
     bool retry_timer_callback_pending_ = false;
   };
 
+  // Since pickers are UniquePtrs we use this RefCounted wrapper
+  // to control references to it by the xds picker and the locality
+  // entry
+  class PickerRef : public RefCounted<PickerRef> {
+   public:
+    explicit PickerRef(UniquePtr<SubchannelPicker> picker)
+        : picker_(std::move(picker)) {}
+    PickResult Pick(PickArgs* pick, grpc_error** error) {
+      return picker_->Pick(pick, error);
+    }
+
+   private:
+    UniquePtr<SubchannelPicker> picker_;
+  };
+
+  // The picker will use a stateless weighting algorithm to pick the locality to
+  // use for each request.
   class Picker : public SubchannelPicker {
    public:
-    Picker(UniquePtr<SubchannelPicker> child_picker,
-           RefCountedPtr<XdsLbClientStats> client_stats)
-        : child_picker_(std::move(child_picker)),
-          client_stats_(std::move(client_stats)) {}
+    // Maintains a weighted list of pickers from each locality that is in ready
+    // state. The first element in the pair represents the end of a range
+    // proportional to the locality's weight. The start of the range is the
+    // previous value in the vector and is 0 for the first element.
+    using PickerList =
+        InlinedVector<Pair<uint32_t, RefCountedPtr<PickerRef>>, 1>;
+    Picker(RefCountedPtr<XdsLbClientStats> client_stats, PickerList pickers)
+        : client_stats_(std::move(client_stats)),
+          pickers_(std::move(pickers)) {}
 
     PickResult Pick(PickArgs* pick, grpc_error** error) override;
 
    private:
-    UniquePtr<SubchannelPicker> child_picker_;
+    // Calls the picker of the locality that the key falls within
+    PickResult PickFromLocality(const uint32_t key, PickArgs* pick,
+                                grpc_error** error);
     RefCountedPtr<XdsLbClientStats> client_stats_;
+    PickerList pickers_;
+  };
+
+  class FallbackHelper : public ChannelControlHelper {
+   public:
+    explicit FallbackHelper(RefCountedPtr<XdsLb> parent)
+        : parent_(std::move(parent)) {}
+
+    Subchannel* CreateSubchannel(const grpc_channel_args& args) override;
+    grpc_channel* CreateChannel(const char* target,
+                                const grpc_channel_args& args) override;
+    void UpdateState(grpc_connectivity_state state,
+                     UniquePtr<SubchannelPicker> picker) override;
+    void RequestReresolution() override;
+
+    void set_child(LoadBalancingPolicy* child) { child_ = child; }
+
+   private:
+    bool CalledByPendingFallback() const;
+    bool CalledByCurrentFallback() const;
+
+    RefCountedPtr<XdsLb> parent_;
+    LoadBalancingPolicy* child_ = nullptr;
   };
 
   class LocalityMap {
    public:
     class LocalityEntry : public InternallyRefCounted<LocalityEntry> {
      public:
-      explicit LocalityEntry(RefCountedPtr<XdsLb> parent)
-          : parent_(std::move(parent)) {}
+      LocalityEntry(RefCountedPtr<XdsLb> parent, uint32_t locality_weight)
+          : parent_(std::move(parent)), locality_weight_(locality_weight) {}
       ~LocalityEntry() = default;
 
       void UpdateLocked(
@@ -351,6 +405,9 @@ class XdsLb : public LoadBalancingPolicy {
       // pending_child_policy_.
       Mutex child_policy_mu_;
       RefCountedPtr<XdsLb> parent_;
+      RefCountedPtr<PickerRef> picker_ref_;
+      grpc_connectivity_state connectivity_state_;
+      uint32_t locality_weight_;
     };
 
     void UpdateLocked(
@@ -375,7 +432,9 @@ class XdsLb : public LoadBalancingPolicy {
       gpr_free(locality_name);
       xds_grpclb_destroy_serverlist(serverlist);
     }
+
     char* locality_name;
+    uint32_t locality_weight;
     // The deserialized response from the balancer. May be nullptr until one
     // such response has arrived.
     xds_grpclb_serverlist* serverlist;
@@ -400,8 +459,13 @@ class XdsLb : public LoadBalancingPolicy {
                                         : lb_chand_.get();
   }
 
-  // Callback to enter fallback mode.
+  // Methods for dealing with fallback state.
+  void MaybeCancelFallbackAtStartupChecks();
   static void OnFallbackTimerLocked(void* arg, grpc_error* error);
+  void UpdateFallbackPolicyLocked();
+  OrphanablePtr<LoadBalancingPolicy> CreateFallbackPolicyLocked(
+      const char* name, const grpc_channel_args* args);
+  void MaybeExitFallbackMode();
 
   // Who the client is trying to communicate with.
   const char* server_name_ = nullptr;
@@ -426,21 +490,39 @@ class XdsLb : public LoadBalancingPolicy {
   // Timeout in milliseconds for the LB call. 0 means no deadline.
   int lb_call_timeout_ms_ = 0;
 
+  // Whether the checks for fallback at startup are ALL pending. There are
+  // several cases where this can be reset:
+  // 1. The fallback timer fires, we enter fallback mode.
+  // 2. Before the fallback timer fires, the LB channel becomes
+  // TRANSIENT_FAILURE or the LB call fails, we enter fallback mode.
+  // 3. Before the fallback timer fires, we receive a response from the
+  // balancer, we cancel the fallback timer and use the response to update the
+  // locality map.
+  bool fallback_at_startup_checks_pending_ = false;
   // Timeout in milliseconds for before using fallback backend addresses.
   // 0 means not using fallback.
-  RefCountedPtr<ParsedLoadBalancingConfig> fallback_policy_config_;
   int lb_fallback_timeout_ms_ = 0;
   // The backend addresses from the resolver.
-  UniquePtr<ServerAddressList> fallback_backend_addresses_;
+  ServerAddressList fallback_backend_addresses_;
   // Fallback timer.
-  bool fallback_timer_callback_pending_ = false;
   grpc_timer lb_fallback_timer_;
   grpc_closure lb_on_fallback_;
 
+  // The policy to use for the fallback backends.
+  RefCountedPtr<ParsedLoadBalancingConfig> fallback_policy_config_;
+  // Lock held when modifying the value of fallback_policy_ or
+  // pending_fallback_policy_.
+  Mutex fallback_policy_mu_;
+  // Non-null iff we are in fallback mode.
+  OrphanablePtr<LoadBalancingPolicy> fallback_policy_;
+  OrphanablePtr<LoadBalancingPolicy> pending_fallback_policy_;
+
   // The policy to use for the backends.
   RefCountedPtr<ParsedLoadBalancingConfig> child_policy_config_;
   // Map of policies to use in the backend
   LocalityMap locality_map_;
+  // TODO(mhaidry) : Add support for multiple maps of localities
+  // with different priorities
   LocalityList locality_serverlist_;
   // TODO(mhaidry) : Add a pending locality map that may be swapped with the
   // the current one when new localities in the pending map are ready
@@ -453,8 +535,12 @@ class XdsLb : public LoadBalancingPolicy {
 
 XdsLb::PickResult XdsLb::Picker::Pick(PickArgs* pick, grpc_error** error) {
   // TODO(roth): Add support for drop handling.
-  // Forward pick to child policy.
-  PickResult result = child_picker_->Pick(pick, error);
+  // Generate a random number between 0 and the total weight
+  const uint32_t key =
+      (rand() * pickers_[pickers_.size() - 1].first) / RAND_MAX;
+  // Forward pick to whichever locality maps to the range in which the
+  // random number falls in.
+  PickResult result = PickFromLocality(key, pick, error);
   // If pick succeeded, add client stats.
   if (result == PickResult::PICK_COMPLETE &&
       pick->connected_subchannel != nullptr && client_stats_ != nullptr) {
@@ -463,17 +549,113 @@ XdsLb::PickResult XdsLb::Picker::Pick(PickArgs* pick, grpc_error** error) {
   return result;
 }
 
+XdsLb::PickResult XdsLb::Picker::PickFromLocality(const uint32_t key,
+                                                  PickArgs* pick,
+                                                  grpc_error** error) {
+  size_t mid = 0;
+  size_t start_index = 0;
+  size_t end_index = pickers_.size() - 1;
+  size_t index = 0;
+  while (end_index > start_index) {
+    mid = (start_index + end_index) / 2;
+    if (pickers_[mid].first > key) {
+      end_index = mid;
+    } else if (pickers_[mid].first < key) {
+      start_index = mid + 1;
+    } else {
+      index = mid + 1;
+      break;
+    }
+  }
+  if (index == 0) index = start_index;
+  GPR_ASSERT(pickers_[index].first > key);
+  return pickers_[index].second->Pick(pick, error);
+}
+
+//
+// XdsLb::FallbackHelper
+//
+
+bool XdsLb::FallbackHelper::CalledByPendingFallback() const {
+  GPR_ASSERT(child_ != nullptr);
+  return child_ == parent_->pending_fallback_policy_.get();
+}
+
+bool XdsLb::FallbackHelper::CalledByCurrentFallback() const {
+  GPR_ASSERT(child_ != nullptr);
+  return child_ == parent_->fallback_policy_.get();
+}
+
+Subchannel* XdsLb::FallbackHelper::CreateSubchannel(
+    const grpc_channel_args& args) {
+  if (parent_->shutting_down_ ||
+      (!CalledByPendingFallback() && !CalledByCurrentFallback())) {
+    return nullptr;
+  }
+  return parent_->channel_control_helper()->CreateSubchannel(args);
+}
+
+grpc_channel* XdsLb::FallbackHelper::CreateChannel(
+    const char* target, const grpc_channel_args& args) {
+  if (parent_->shutting_down_ ||
+      (!CalledByPendingFallback() && !CalledByCurrentFallback())) {
+    return nullptr;
+  }
+  return parent_->channel_control_helper()->CreateChannel(target, args);
+}
+
+void XdsLb::FallbackHelper::UpdateState(grpc_connectivity_state state,
+                                        UniquePtr<SubchannelPicker> picker) {
+  if (parent_->shutting_down_) return;
+  // If this request is from the pending fallback policy, ignore it until
+  // it reports READY, at which point we swap it into place.
+  if (CalledByPendingFallback()) {
+    if (grpc_lb_xds_trace.enabled()) {
+      gpr_log(
+          GPR_INFO,
+          "[xdslb %p helper %p] pending fallback policy %p reports state=%s",
+          parent_.get(), this, parent_->pending_fallback_policy_.get(),
+          grpc_connectivity_state_name(state));
+    }
+    if (state != GRPC_CHANNEL_READY) return;
+    grpc_pollset_set_del_pollset_set(
+        parent_->fallback_policy_->interested_parties(),
+        parent_->interested_parties());
+    MutexLock lock(&parent_->fallback_policy_mu_);
+    parent_->fallback_policy_ = std::move(parent_->pending_fallback_policy_);
+  } else if (!CalledByCurrentFallback()) {
+    // This request is from an outdated fallback policy, so ignore it.
+    return;
+  }
+  parent_->channel_control_helper()->UpdateState(state, std::move(picker));
+}
+
+void XdsLb::FallbackHelper::RequestReresolution() {
+  if (parent_->shutting_down_) return;
+  const LoadBalancingPolicy* latest_fallback_policy =
+      parent_->pending_fallback_policy_ != nullptr
+          ? parent_->pending_fallback_policy_.get()
+          : parent_->fallback_policy_.get();
+  if (child_ != latest_fallback_policy) return;
+  if (grpc_lb_xds_trace.enabled()) {
+    gpr_log(GPR_INFO,
+            "[xdslb %p] Re-resolution requested from the fallback policy (%p).",
+            parent_.get(), child_);
+  }
+  GPR_ASSERT(parent_->lb_chand_ != nullptr);
+  parent_->channel_control_helper()->RequestReresolution();
+}
+
 //
 // serverlist parsing code
 //
 
 // Returns the backend addresses extracted from the given addresses.
-UniquePtr<ServerAddressList> ExtractBackendAddresses(
-    const ServerAddressList& addresses) {
-  auto backend_addresses = MakeUnique<ServerAddressList>();
+ServerAddressList ExtractBackendAddresses(const ServerAddressList& addresses) {
+  ServerAddressList backend_addresses;
   for (size_t i = 0; i < addresses.size(); ++i) {
     if (!addresses[i].IsBalancer()) {
-      backend_addresses->emplace_back(addresses[i]);
+      backend_addresses.emplace_back(addresses[i]);
     }
   }
   return backend_addresses;
@@ -553,6 +735,9 @@ XdsLb::BalancerChannelState::BalancerChannelState(
               .set_multiplier(GRPC_XDS_RECONNECT_BACKOFF_MULTIPLIER)
               .set_jitter(GRPC_XDS_RECONNECT_JITTER)
               .set_max_backoff(GRPC_XDS_RECONNECT_MAX_BACKOFF_SECONDS * 1000)) {
+  GRPC_CLOSURE_INIT(&on_connectivity_changed_,
+                    &XdsLb::BalancerChannelState::OnConnectivityChangedLocked,
+                    this, grpc_combiner_scheduler(xdslb_policy_->combiner()));
   channel_ = xdslb_policy_->channel_control_helper()->CreateChannel(
       balancer_name, args);
   GPR_ASSERT(channel_ != nullptr);
@@ -621,6 +806,62 @@ void XdsLb::BalancerChannelState::StartCallLocked() {
   lb_calld_->StartQuery();
 }
 
+void XdsLb::BalancerChannelState::StartConnectivityWatchLocked() {
+  grpc_channel_element* client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel_));
+  GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
+  // Ref held by callback.
+  Ref(DEBUG_LOCATION, "watch_lb_channel_connectivity").release();
+  grpc_client_channel_watch_connectivity_state(
+      client_channel_elem,
+      grpc_polling_entity_create_from_pollset_set(
+          xdslb_policy_->interested_parties()),
+      &connectivity_, &on_connectivity_changed_, nullptr);
+}
+
+void XdsLb::BalancerChannelState::CancelConnectivityWatchLocked() {
+  grpc_channel_element* client_channel_elem =
+      grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel_));
+  GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
+  grpc_client_channel_watch_connectivity_state(
+      client_channel_elem,
+      grpc_polling_entity_create_from_pollset_set(
+          xdslb_policy_->interested_parties()),
+      nullptr, &on_connectivity_changed_, nullptr);
+}
+
+void XdsLb::BalancerChannelState::OnConnectivityChangedLocked(
+    void* arg, grpc_error* error) {
+  BalancerChannelState* self = static_cast<BalancerChannelState*>(arg);
+  if (!self->shutting_down_ &&
+      self->xdslb_policy_->fallback_at_startup_checks_pending_) {
+    if (self->connectivity_ != GRPC_CHANNEL_TRANSIENT_FAILURE) {
+      // Not in TRANSIENT_FAILURE.  Renew connectivity watch.
+      grpc_channel_element* client_channel_elem =
+          grpc_channel_stack_last_element(
+              grpc_channel_get_channel_stack(self->channel_));
+      GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter);
+      grpc_client_channel_watch_connectivity_state(
+          client_channel_elem,
+          grpc_polling_entity_create_from_pollset_set(
+              self->xdslb_policy_->interested_parties()),
+          &self->connectivity_, &self->on_connectivity_changed_, nullptr);
+      return;  // Early out so we don't drop the ref below.
+    }
+    // In TRANSIENT_FAILURE.  Cancel the fallback timer and go into
+    // fallback mode immediately.
+    gpr_log(GPR_INFO,
+            "[xdslb %p] Balancer channel in state TRANSIENT_FAILURE; "
+            "entering fallback mode",
+            self);
+    self->xdslb_policy_->fallback_at_startup_checks_pending_ = false;
+    grpc_timer_cancel(&self->xdslb_policy_->lb_fallback_timer_);
+    self->xdslb_policy_->UpdateFallbackPolicyLocked();
+  }
+  // Done watching connectivity state, so drop ref.
+  self->Unref(DEBUG_LOCATION, "watch_lb_channel_connectivity");
+}
+
 //
 // XdsLb::BalancerChannelState::BalancerCallState
 //
@@ -866,6 +1107,14 @@ void XdsLb::BalancerChannelState::BalancerCallState::
       (initial_response = xds_grpclb_initial_response_parse(response_slice)) !=
           nullptr) {
     // Have NOT seen initial response, look for initial response.
+    // TODO(juanlishen): When we convert this to use the xds protocol, the
+    // balancer will send us a fallback timeout such that we should go into
+    // fallback mode if we have lost contact with the balancer after a certain
+    // period of time. We will need to save the timeout value here, and then
+    // when the balancer call ends, we will need to start a timer for the
+    // specified period of time, and if the timer fires, we go into fallback
+    // mode. We will also need to cancel the timer when we receive a serverlist
+    // from the balancer.
     if (initial_response->has_client_stats_report_interval) {
       const grpc_millis interval = xds_grpclb_duration_to_millis(
           &initial_response->client_stats_report_interval);
@@ -907,79 +1156,69 @@ void XdsLb::BalancerChannelState::BalancerCallState::
         gpr_free(ipport);
       }
     }
-    /* update serverlist */
-    // TODO(juanlishen): Don't ingore empty serverlist.
-    if (serverlist->num_servers > 0) {
-      // Pending LB channel receives a serverlist; promote it.
-      // Note that this call can't be on a discarded pending channel, because
-      // such channels don't have any current call but we have checked this call
-      // is a current call.
-      if (!lb_calld->lb_chand_->IsCurrentChannel()) {
-        if (grpc_lb_xds_trace.enabled()) {
-          gpr_log(GPR_INFO,
-                  "[xdslb %p] Promoting pending LB channel %p to replace "
-                  "current LB channel %p",
-                  xdslb_policy, lb_calld->lb_chand_.get(),
-                  lb_calld->xdslb_policy()->lb_chand_.get());
-        }
-        lb_calld->xdslb_policy()->lb_chand_ =
-            std::move(lb_calld->xdslb_policy()->pending_lb_chand_);
-      }
-      // Start sending client load report only after we start using the
-      // serverlist returned from the current LB call.
-      if (lb_calld->client_stats_report_interval_ > 0 &&
-          lb_calld->client_stats_ == nullptr) {
-        lb_calld->client_stats_ = MakeRefCounted<XdsLbClientStats>();
-        // TODO(roth): We currently track this ref manually.  Once the
-        // ClosureRef API is ready, we should pass the RefCountedPtr<> along
-        // with the callback.
-        auto self = lb_calld->Ref(DEBUG_LOCATION, "client_load_report");
-        self.release();
-        lb_calld->ScheduleNextClientLoadReportLocked();
-      }
-      if (!xdslb_policy->locality_serverlist_.empty() &&
-          xds_grpclb_serverlist_equals(
-              xdslb_policy->locality_serverlist_[0]->serverlist, serverlist)) {
-        if (grpc_lb_xds_trace.enabled()) {
-          gpr_log(GPR_INFO,
-                  "[xdslb %p] Incoming server list identical to current, "
-                  "ignoring.",
-                  xdslb_policy);
-        }
-        xds_grpclb_destroy_serverlist(serverlist);
-      } else { /* new serverlist */
-        if (!xdslb_policy->locality_serverlist_.empty()) {
-          /* dispose of the old serverlist */
-          xds_grpclb_destroy_serverlist(
-              xdslb_policy->locality_serverlist_[0]->serverlist);
-        } else {
-          /* or dispose of the fallback */
-          xdslb_policy->fallback_backend_addresses_.reset();
-          if (xdslb_policy->fallback_timer_callback_pending_) {
-            grpc_timer_cancel(&xdslb_policy->lb_fallback_timer_);
-          }
-          /* Initialize locality serverlist, currently the list only handles
-           * one child */
-          xdslb_policy->locality_serverlist_.emplace_back(
-              MakeUnique<LocalityServerlistEntry>());
-          xdslb_policy->locality_serverlist_[0]->locality_name =
-              static_cast<char*>(gpr_strdup(kDefaultLocalityName));
-        }
-        // and update the copy in the XdsLb instance. This
-        // serverlist instance will be destroyed either upon the next
-        // update or when the XdsLb instance is destroyed.
-        xdslb_policy->locality_serverlist_[0]->serverlist = serverlist;
-        xdslb_policy->locality_map_.UpdateLocked(
-            xdslb_policy->locality_serverlist_,
-            xdslb_policy->child_policy_config_, xdslb_policy->args_,
-            xdslb_policy);
+    // Pending LB channel receives a serverlist; promote it.
+    // Note that this call can't be on a discarded pending channel, because
+    // such channels don't have any current call but we have checked this call
+    // is a current call.
+    if (!lb_calld->lb_chand_->IsCurrentChannel()) {
+      if (grpc_lb_xds_trace.enabled()) {
+        gpr_log(GPR_INFO,
+                "[xdslb %p] Promoting pending LB channel %p to replace "
+                "current LB channel %p",
+                xdslb_policy, lb_calld->lb_chand_.get(),
+                lb_calld->xdslb_policy()->lb_chand_.get());
       }
-    } else {
+      lb_calld->xdslb_policy()->lb_chand_ =
+          std::move(lb_calld->xdslb_policy()->pending_lb_chand_);
+    }
+    // Start sending client load report only after we start using the
+    // serverlist returned from the current LB call.
+    if (lb_calld->client_stats_report_interval_ > 0 &&
+        lb_calld->client_stats_ == nullptr) {
+      lb_calld->client_stats_ = MakeRefCounted<XdsLbClientStats>();
+      lb_calld->Ref(DEBUG_LOCATION, "client_load_report").release();
+      lb_calld->ScheduleNextClientLoadReportLocked();
+    }
+    if (!xdslb_policy->locality_serverlist_.empty() &&
+        xds_grpclb_serverlist_equals(
+            xdslb_policy->locality_serverlist_[0]->serverlist, serverlist)) {
       if (grpc_lb_xds_trace.enabled()) {
-        gpr_log(GPR_INFO, "[xdslb %p] Received empty server list, ignoring.",
+        gpr_log(GPR_INFO,
+                "[xdslb %p] Incoming server list identical to current, "
+                "ignoring.",
                 xdslb_policy);
       }
       xds_grpclb_destroy_serverlist(serverlist);
+    } else {  // New serverlist.
+      // If the balancer tells us to drop all the calls, we should exit fallback
+      // mode immediately.
+      // TODO(juanlishen): When we add EDS drop, we should change to check
+      // drop_percentage.
+      if (serverlist->num_servers == 0) xdslb_policy->MaybeExitFallbackMode();
+      if (!xdslb_policy->locality_serverlist_.empty()) {
+        xds_grpclb_destroy_serverlist(
+            xdslb_policy->locality_serverlist_[0]->serverlist);
+      } else {
+        // This is the first serverlist we've received, don't enter fallback
+        // mode.
+        xdslb_policy->MaybeCancelFallbackAtStartupChecks();
+        // Initialize locality serverlist, currently the list only handles
+        // one child.
+        xdslb_policy->locality_serverlist_.emplace_back(
+            MakeUnique<LocalityServerlistEntry>());
+        xdslb_policy->locality_serverlist_[0]->locality_name =
+            static_cast<char*>(gpr_strdup(kDefaultLocalityName));
+        xdslb_policy->locality_serverlist_[0]->locality_weight =
+            kDefaultLocalityWeight;
+      }
+      // Update the serverlist in the XdsLb instance. This serverlist
+      // instance will be destroyed either upon the next update or when the
+      // XdsLb instance is destroyed.
+      xdslb_policy->locality_serverlist_[0]->serverlist = serverlist;
+      xdslb_policy->locality_map_.UpdateLocked(
+          xdslb_policy->locality_serverlist_,
+          xdslb_policy->child_policy_config_, xdslb_policy->args_,
+          xdslb_policy);
     }
   } else {
     // No valid initial response or serverlist found.
@@ -1056,6 +1295,18 @@ void XdsLb::BalancerChannelState::BalancerCallState::
         lb_chand->StartCallRetryTimerLocked();
       }
       xdslb_policy->channel_control_helper()->RequestReresolution();
+      // If the fallback-at-startup checks are pending, go into fallback mode
+      // immediately.  This short-circuits the timeout for the
+      // fallback-at-startup case.
+      if (xdslb_policy->fallback_at_startup_checks_pending_) {
+        gpr_log(GPR_INFO,
+                "[xdslb %p] Balancer call finished; entering fallback mode",
+                xdslb_policy);
+        xdslb_policy->fallback_at_startup_checks_pending_ = false;
+        grpc_timer_cancel(&xdslb_policy->lb_fallback_timer_);
+        lb_chand->CancelConnectivityWatchLocked();
+        xdslb_policy->UpdateFallbackPolicyLocked();
+      }
     }
   }
   lb_calld->Unref(DEBUG_LOCATION, "lb_call_ended");
@@ -1131,7 +1382,7 @@ XdsLb::XdsLb(Args args)
   arg = grpc_channel_args_find(args.args, GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS);
   lb_call_timeout_ms_ = grpc_channel_arg_get_integer(arg, {0, 0, INT_MAX});
   // Record fallback timeout.
-  arg = grpc_channel_args_find(args.args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS);
+  arg = grpc_channel_args_find(args.args, GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS);
   lb_fallback_timeout_ms_ = grpc_channel_arg_get_integer(
       arg, {GRPC_XDS_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX});
 }
@@ -1144,14 +1395,25 @@ XdsLb::~XdsLb() {
 
 void XdsLb::ShutdownLocked() {
   shutting_down_ = true;
-  if (fallback_timer_callback_pending_) {
+  if (fallback_at_startup_checks_pending_) {
     grpc_timer_cancel(&lb_fallback_timer_);
   }
   locality_map_.ShutdownLocked();
-  // We destroy the LB channel here instead of in our destructor because
-  // destroying the channel triggers a last callback to
-  // OnBalancerChannelConnectivityChangedLocked(), and we need to be
-  // alive when that callback is invoked.
+  if (fallback_policy_ != nullptr) {
+    grpc_pollset_set_del_pollset_set(fallback_policy_->interested_parties(),
+                                     interested_parties());
+  }
+  if (pending_fallback_policy_ != nullptr) {
+    grpc_pollset_set_del_pollset_set(
+        pending_fallback_policy_->interested_parties(), interested_parties());
+  }
+  {
+    MutexLock lock(&fallback_policy_mu_);
+    fallback_policy_.reset();
+    pending_fallback_policy_.reset();
+  }
+  // We reset the LB channels here instead of in our destructor because they
+  // hold refs to XdsLb.
   {
     MutexLock lock(&lb_chand_mu_);
     lb_chand_.reset();
@@ -1171,12 +1433,31 @@ void XdsLb::ResetBackoffLocked() {
     grpc_channel_reset_connect_backoff(pending_lb_chand_->channel());
   }
   locality_map_.ResetBackoffLocked();
+  if (fallback_policy_ != nullptr) {
+    fallback_policy_->ResetBackoffLocked();
+  }
+  if (pending_fallback_policy_ != nullptr) {
+    pending_fallback_policy_->ResetBackoffLocked();
+  }
 }
 
 void XdsLb::FillChildRefsForChannelz(channelz::ChildRefsList* child_subchannels,
                                      channelz::ChildRefsList* child_channels) {
-  // Delegate to the child_policy_ to fill the children subchannels.
+  // Delegate to the locality_map_ to fill the children subchannels.
   locality_map_.FillChildRefsForChannelz(child_subchannels, child_channels);
+  {
+    // This must be done holding fallback_policy_mu_, since this method does not
+    // run in the combiner.
+    MutexLock lock(&fallback_policy_mu_);
+    if (fallback_policy_ != nullptr) {
+      fallback_policy_->FillChildRefsForChannelz(child_subchannels,
+                                                 child_channels);
+    }
+    if (pending_fallback_policy_ != nullptr) {
+      pending_fallback_policy_->FillChildRefsForChannelz(child_subchannels,
+                                                         child_channels);
+    }
+  }
   MutexLock lock(&lb_chand_mu_);
   if (lb_chand_ != nullptr) {
     grpc_core::channelz::ChannelNode* channel_node =
@@ -1244,57 +1525,213 @@ void XdsLb::ParseLbConfig(const ParsedXdsConfig* xds_config) {
 void XdsLb::UpdateLocked(UpdateArgs args) {
   const bool is_initial_update = lb_chand_ == nullptr;
   ParseLbConfig(static_cast<const ParsedXdsConfig*>(args.config.get()));
-  // TODO(juanlishen): Pass fallback policy config update after fallback policy
-  // is added.
   if (balancer_name_ == nullptr) {
     gpr_log(GPR_ERROR, "[xdslb %p] LB config parsing fails.", this);
     return;
   }
   ProcessAddressesAndChannelArgsLocked(args.addresses, *args.args);
-  // Update the existing child policy.
-  // Note: We have disabled fallback mode in the code, so this child policy must
-  // have been created from a serverlist.
-  // TODO(vpowar): Handle the fallback_address changes when we add support for
-  // fallback in xDS.
-  locality_map_.UpdateLocked(locality_serverlist_, child_policy_config_, args_,
-                             this);
-  // If this is the initial update, start the fallback timer.
+  locality_map_.UpdateLocked(locality_serverlist_, child_policy_config_,
+                             args_, this);
+  // Update the existing fallback policy. The fallback policy config and/or the
+  // fallback addresses may be new.
+  if (fallback_policy_ != nullptr) UpdateFallbackPolicyLocked();
+  // If this is the initial update, start the fallback-at-startup checks.
   if (is_initial_update) {
-    if (lb_fallback_timeout_ms_ > 0 && locality_serverlist_.empty() &&
-        !fallback_timer_callback_pending_) {
-      grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_;
-      Ref(DEBUG_LOCATION, "on_fallback_timer").release();  // Held by closure
-      GRPC_CLOSURE_INIT(&lb_on_fallback_, &XdsLb::OnFallbackTimerLocked, this,
-                        grpc_combiner_scheduler(combiner()));
-      fallback_timer_callback_pending_ = true;
-      grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_);
-      // TODO(juanlishen): Monitor the connectivity state of the balancer
-      // channel.  If the channel reports TRANSIENT_FAILURE before the
-      // fallback timeout expires, go into fallback mode early.
-    }
+    grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_;
+    Ref(DEBUG_LOCATION, "on_fallback_timer").release();  // Held by closure
+    GRPC_CLOSURE_INIT(&lb_on_fallback_, &XdsLb::OnFallbackTimerLocked, this,
+                      grpc_combiner_scheduler(combiner()));
+    fallback_at_startup_checks_pending_ = true;
+    grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_);
+    // Start watching the channel's connectivity state.  If the channel
+    // goes into state TRANSIENT_FAILURE, we go into fallback mode even if
+    // the fallback timeout has not elapsed.
+    lb_chand_->StartConnectivityWatchLocked();
   }
 }
 
 //
-// code for balancer channel and call
+// fallback-related methods
 //
 
+void XdsLb::MaybeCancelFallbackAtStartupChecks() {
+  if (!fallback_at_startup_checks_pending_) return;
+  gpr_log(GPR_INFO,
+          "[xdslb %p] Cancelling fallback timer and LB channel connectivity "
+          "watch",
+          this);
+  grpc_timer_cancel(&lb_fallback_timer_);
+  lb_chand_->CancelConnectivityWatchLocked();
+  fallback_at_startup_checks_pending_ = false;
+}
+
 void XdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) {
   XdsLb* xdslb_policy = static_cast<XdsLb*>(arg);
-  xdslb_policy->fallback_timer_callback_pending_ = false;
-  // If we receive a serverlist after the timer fires but before this callback
-  // actually runs, don't fall back.
-  if (xdslb_policy->locality_serverlist_.empty() &&
+  // If some fallback-at-startup check is done after the timer fires but before
+  // this callback actually runs, don't fall back.
+  if (xdslb_policy->fallback_at_startup_checks_pending_ &&
       !xdslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) {
     if (grpc_lb_xds_trace.enabled()) {
       gpr_log(GPR_INFO,
-              "[xdslb %p] Fallback timer fired. Not using fallback backends",
+              "[xdslb %p] Child policy not ready after fallback timeout; "
+              "entering fallback mode",
               xdslb_policy);
     }
+    xdslb_policy->fallback_at_startup_checks_pending_ = false;
+    xdslb_policy->UpdateFallbackPolicyLocked();
+    xdslb_policy->lb_chand_->CancelConnectivityWatchLocked();
   }
   xdslb_policy->Unref(DEBUG_LOCATION, "on_fallback_timer");
 }
 
+void XdsLb::UpdateFallbackPolicyLocked() {
+  if (shutting_down_) return;
+  // Construct update args.
+  UpdateArgs update_args;
+  update_args.addresses = fallback_backend_addresses_;
+  update_args.config = fallback_policy_config_ == nullptr
+                           ? nullptr
+                           : fallback_policy_config_->Ref();
+  update_args.args = grpc_channel_args_copy(args_);
+  // If the child policy name changes, we need to create a new child
+  // policy.  When this happens, we leave child_policy_ as-is and store
+  // the new child policy in pending_child_policy_.  Once the new child
+  // policy transitions into state READY, we swap it into child_policy_,
+  // replacing the original child policy.  So pending_child_policy_ is
+  // non-null only between when we apply an update that changes the child
+  // policy name and when the new child reports state READY.
+  //
+  // Updates can arrive at any point during this transition.  We always
+  // apply updates relative to the most recently created child policy,
+  // even if the most recent one is still in pending_child_policy_.  This
+  // is true both when applying the updates to an existing child policy
+  // and when determining whether we need to create a new policy.
+  //
+  // As a result of this, there are several cases to consider here:
+  //
+  // 1. We have no existing child policy (i.e., we have started up but
+  //    have not yet received a serverlist from the balancer or gone
+  //    into fallback mode; in this case, both child_policy_ and
+  //    pending_child_policy_ are null).  In this case, we create a
+  //    new child policy and store it in child_policy_.
+  //
+  // 2. We have an existing child policy and have no pending child policy
+  //    from a previous update (i.e., either there has not been a
+  //    previous update that changed the policy name, or we have already
+  //    finished swapping in the new policy; in this case, child_policy_
+  //    is non-null but pending_child_policy_ is null).  In this case:
+  //    a. If child_policy_->name() equals child_policy_name, then we
+  //       update the existing child policy.
+  //    b. If child_policy_->name() does not equal child_policy_name,
+  //       we create a new policy.  The policy will be stored in
+  //       pending_child_policy_ and will later be swapped into
+  //       child_policy_ by the helper when the new child transitions
+  //       into state READY.
+  //
+  // 3. We have an existing child policy and have a pending child policy
+  //    from a previous update (i.e., a previous update set
+  //    pending_child_policy_ as per case 2b above and that policy has
+  //    not yet transitioned into state READY and been swapped into
+  //    child_policy_; in this case, both child_policy_ and
+  //    pending_child_policy_ are non-null).  In this case:
+  //    a. If pending_child_policy_->name() equals child_policy_name,
+  //       then we update the existing pending child policy.
+  //    b. If pending_child_policy->name() does not equal
+  //       child_policy_name, then we create a new policy.  The new
+  //       policy is stored in pending_child_policy_ (replacing the one
+  //       that was there before, which will be immediately shut down)
+  //       and will later be swapped into child_policy_ by the helper
+  //       when the new child transitions into state READY.
+  const char* fallback_policy_name = fallback_policy_config_ == nullptr
+                                         ? "round_robin"
+                                         : fallback_policy_config_->name();
+  const bool create_policy =
+      // case 1
+      fallback_policy_ == nullptr ||
+      // case 2b
+      (pending_fallback_policy_ == nullptr &&
+       strcmp(fallback_policy_->name(), fallback_policy_name) != 0) ||
+      // case 3b
+      (pending_fallback_policy_ != nullptr &&
+       strcmp(pending_fallback_policy_->name(), fallback_policy_name) != 0);
+  LoadBalancingPolicy* policy_to_update = nullptr;
+  if (create_policy) {
+    // Cases 1, 2b, and 3b: create a new child policy.
+    // If child_policy_ is null, we set it (case 1), else we set
+    // pending_child_policy_ (cases 2b and 3b).
+    if (grpc_lb_xds_trace.enabled()) {
+      gpr_log(GPR_INFO, "[xdslb %p] Creating new %sfallback policy %s", this,
+              fallback_policy_ == nullptr ? "" : "pending ",
+              fallback_policy_name);
+    }
+    auto new_policy =
+        CreateFallbackPolicyLocked(fallback_policy_name, update_args.args);
+    auto& lb_policy = fallback_policy_ == nullptr ? fallback_policy_
+                                                  : pending_fallback_policy_;
+    {
+      MutexLock lock(&fallback_policy_mu_);
+      lb_policy = std::move(new_policy);
+    }
+    policy_to_update = lb_policy.get();
+  } else {
+    // Cases 2a and 3a: update an existing policy.
+    // If we have a pending child policy, send the update to the pending
+    // policy (case 3a), else send it to the current policy (case 2a).
+    policy_to_update = pending_fallback_policy_ != nullptr
+                           ? pending_fallback_policy_.get()
+                           : fallback_policy_.get();
+  }
+  GPR_ASSERT(policy_to_update != nullptr);
+  // Update the policy.
+  if (grpc_lb_xds_trace.enabled()) {
+    gpr_log(
+        GPR_INFO, "[xdslb %p] Updating %sfallback policy %p", this,
+        policy_to_update == pending_fallback_policy_.get() ? "pending " : "",
+        policy_to_update);
+  }
+  policy_to_update->UpdateLocked(std::move(update_args));
+}
+
+OrphanablePtr<LoadBalancingPolicy> XdsLb::CreateFallbackPolicyLocked(
+    const char* name, const grpc_channel_args* args) {
+  FallbackHelper* helper = New<FallbackHelper>(Ref());
+  LoadBalancingPolicy::Args lb_policy_args;
+  lb_policy_args.combiner = combiner();
+  lb_policy_args.args = args;
+  lb_policy_args.channel_control_helper =
+      UniquePtr<ChannelControlHelper>(helper);
+  OrphanablePtr<LoadBalancingPolicy> lb_policy =
+      LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(
+          name, std::move(lb_policy_args));
+  if (GPR_UNLIKELY(lb_policy == nullptr)) {
+    gpr_log(GPR_ERROR, "[xdslb %p] Failure creating fallback policy %s", this,
+            name);
+    return nullptr;
+  }
+  helper->set_child(lb_policy.get());
+  if (grpc_lb_xds_trace.enabled()) {
+    gpr_log(GPR_INFO, "[xdslb %p] Created new fallback policy %s (%p)", this,
+            name, lb_policy.get());
+  }
+  // Add the xDS's interested_parties pollset_set to that of the newly created
+  // child policy. This will make the child policy progress upon activity on xDS
+  // LB, which in turn is tied to the application's call.
+  grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(),
+                                   interested_parties());
+  return lb_policy;
+}
+
+void XdsLb::MaybeExitFallbackMode() {
+  if (fallback_policy_ == nullptr) return;
+  gpr_log(GPR_INFO, "[xdslb %p] Exiting fallback mode", this);
+  fallback_policy_.reset();
+  pending_fallback_policy_.reset();
+}
+
+//
+// XdsLb::LocalityMap
+//
+
 void XdsLb::LocalityMap::PruneLocalities(const LocalityList& locality_list) {
   for (auto iter = map_.begin(); iter != map_.end();) {
     bool found = false;
@@ -1321,8 +1758,8 @@ void XdsLb::LocalityMap::UpdateLocked(
         gpr_strdup(locality_serverlist[i]->locality_name));
     auto iter = map_.find(locality_name);
     if (iter == map_.end()) {
-      OrphanablePtr<LocalityEntry> new_entry =
-          MakeOrphanable<LocalityEntry>(parent->Ref());
+      OrphanablePtr<LocalityEntry> new_entry = MakeOrphanable<LocalityEntry>(
+          parent->Ref(), locality_serverlist[i]->locality_weight);
       MutexLock lock(&child_refs_mu_);
       iter = map_.emplace(std::move(locality_name), std::move(new_entry)).first;
     }
@@ -1334,27 +1771,29 @@ void XdsLb::LocalityMap::UpdateLocked(
   PruneLocalities(locality_serverlist);
 }
 
-void grpc_core::XdsLb::LocalityMap::ShutdownLocked() {
+void XdsLb::LocalityMap::ShutdownLocked() {
   MutexLock lock(&child_refs_mu_);
   map_.clear();
 }
 
-void grpc_core::XdsLb::LocalityMap::ResetBackoffLocked() {
-  for (auto iter = map_.begin(); iter != map_.end(); iter++) {
-    iter->second->ResetBackoffLocked();
+void XdsLb::LocalityMap::ResetBackoffLocked() {
+  for (auto& p : map_) {
+    p.second->ResetBackoffLocked();
   }
 }
 
-void grpc_core::XdsLb::LocalityMap::FillChildRefsForChannelz(
+void XdsLb::LocalityMap::FillChildRefsForChannelz(
     channelz::ChildRefsList* child_subchannels,
     channelz::ChildRefsList* child_channels) {
   MutexLock lock(&child_refs_mu_);
-  for (auto iter = map_.begin(); iter != map_.end(); iter++) {
-    iter->second->FillChildRefsForChannelz(child_subchannels, child_channels);
+  for (auto& p : map_) {
+    p.second->FillChildRefsForChannelz(child_subchannels, child_channels);
   }
 }
 
-// Locality Entry child policy methods
+//
+// XdsLb::LocalityMap::LocalityEntry
+//
 
 grpc_channel_args*
 XdsLb::LocalityMap::LocalityEntry::CreateChildPolicyArgsLocked(
@@ -1409,17 +1848,11 @@ void XdsLb::LocalityMap::LocalityEntry::UpdateLocked(
     RefCountedPtr<ParsedLoadBalancingConfig> child_policy_config,
     const grpc_channel_args* args_in) {
   if (parent_->shutting_down_) return;
-  // This should never be invoked if we do not have serverlist_, as fallback
-  // mode is disabled for xDS plugin.
-  // TODO(juanlishen): Change this as part of implementing fallback mode.
-  GPR_ASSERT(serverlist != nullptr);
-  GPR_ASSERT(serverlist->num_servers > 0);
   // Construct update args.
   UpdateArgs update_args;
   update_args.addresses = ProcessServerlist(serverlist);
   update_args.config = child_policy_config;
   update_args.args = CreateChildPolicyArgsLocked(args_in);
-
   // If the child policy name changes, we need to create a new child
   // policy.  When this happens, we leave child_policy_ as-is and store
   // the new child policy in pending_child_policy_.  Once the new child
@@ -1560,7 +1993,7 @@ void XdsLb::LocalityMap::LocalityEntry::Orphan() {
 }
 
 //
-// LocalityEntry::Helper implementation
+// XdsLb::LocalityEntry::Helper
 //
 
 bool XdsLb::LocalityMap::LocalityEntry::Helper::CalledByPendingChild() const {
@@ -1613,17 +2046,81 @@ void XdsLb::LocalityMap::LocalityEntry::Helper::UpdateState(
     // This request is from an outdated child, so ignore it.
     return;
   }
-  // TODO(juanlishen): When in fallback mode, pass the child picker
-  // through without wrapping it.  (Or maybe use a different helper for
-  // the fallback policy?)
+  // At this point, child_ must be the current child policy.
+  if (state == GRPC_CHANNEL_READY) entry_->parent_->MaybeExitFallbackMode();
+  // If we are in fallback mode, ignore update request from the child policy.
+  if (entry_->parent_->fallback_policy_ != nullptr) return;
   GPR_ASSERT(entry_->parent_->lb_chand_ != nullptr);
   RefCountedPtr<XdsLbClientStats> client_stats =
       entry_->parent_->lb_chand_->lb_calld() == nullptr
           ? nullptr
           : entry_->parent_->lb_chand_->lb_calld()->client_stats();
-  entry_->parent_->channel_control_helper()->UpdateState(
-      state, UniquePtr<SubchannelPicker>(
-                 New<Picker>(std::move(picker), std::move(client_stats))));
+  // Cache the picker and its state in the entry
+  entry_->picker_ref_ = MakeRefCounted<PickerRef>(std::move(picker));
+  entry_->connectivity_state_ = state;
+  // Construct a new xds picker which maintains a map of all locality pickers
+  // that are ready. Each locality is represented by a portion of the range
+  // proportional to its weight, such that the total range is the sum of the
+  // weights of all localities
+  uint32_t end = 0;
+  size_t num_connecting = 0;
+  size_t num_idle = 0;
+  size_t num_transient_failures = 0;
+  auto& locality_map = this->entry_->parent_->locality_map_.map_;
+  Picker::PickerList pickers;
+  for (auto& p : locality_map) {
+    const LocalityEntry* entry = p.second.get();
+    grpc_connectivity_state connectivity_state = entry->connectivity_state_;
+    switch (connectivity_state) {
+      case GRPC_CHANNEL_READY: {
+        end += entry->locality_weight_;
+        pickers.push_back(MakePair(end, entry->picker_ref_));
+        break;
+      }
+      case GRPC_CHANNEL_CONNECTING: {
+        num_connecting++;
+        break;
+      }
+      case GRPC_CHANNEL_IDLE: {
+        num_idle++;
+        break;
+      }
+      case GRPC_CHANNEL_TRANSIENT_FAILURE: {
+        num_transient_failures++;
+        break;
+      }
+      default: {
+        gpr_log(GPR_ERROR, "Invalid locality connectivity state - %d",
+                connectivity_state);
+      }
+    }
+  }
+  // Pass on the constructed xds picker if it has any ready pickers in their map
+  // otherwise pass a QueuePicker if any of the locality pickers are in a
+  // connecting or idle state, finally return a transient failure picker if all
+  // locality pickers are in transient failure
+  if (pickers.size() > 0) {
+    entry_->parent_->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_READY,
+        UniquePtr<LoadBalancingPolicy::SubchannelPicker>(
+            New<Picker>(std::move(client_stats), std::move(pickers))));
+  } else if (num_connecting > 0) {
+    entry_->parent_->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_CONNECTING,
+        UniquePtr<SubchannelPicker>(New<QueuePicker>(this->entry_->parent_)));
+  } else if (num_idle > 0) {
+    entry_->parent_->channel_control_helper()->UpdateState(
+        GRPC_CHANNEL_IDLE,
+        UniquePtr<SubchannelPicker>(New<QueuePicker>(this->entry_->parent_)));
+  } else {
+    GPR_ASSERT(num_transient_failures == locality_map.size());
+    grpc_error* error =
+        grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING(
+                               "connections to all localities failing"),
+                           GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE);
+    entry_->parent_->channel_control_helper()->UpdateState(
+        state, UniquePtr<SubchannelPicker>(New<TransientFailurePicker>(error)));
+  }
 }
 
 void XdsLb::LocalityMap::LocalityEntry::Helper::RequestReresolution() {

src/core/ext/filters/client_channel/subchannel.h

@@ -77,7 +77,7 @@ class ConnectedSubchannel : public RefCounted<ConnectedSubchannel> {
     grpc_millis deadline;
     gpr_arena* arena;
     grpc_call_context_element* context;
-    grpc_call_combiner* call_combiner;
+    grpc_core::CallCombiner* call_combiner;
     size_t parent_data_size;
   };
 

src/core/ext/filters/deadline/deadline_filter.cc

@@ -68,8 +68,7 @@ static void timer_callback(void* arg, grpc_error* error) {
     error = grpc_error_set_int(
         GRPC_ERROR_CREATE_FROM_STATIC_STRING("Deadline Exceeded"),
         GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_DEADLINE_EXCEEDED);
-    grpc_call_combiner_cancel(deadline_state->call_combiner,
-                              GRPC_ERROR_REF(error));
+    deadline_state->call_combiner->Cancel(GRPC_ERROR_REF(error));
     GRPC_CLOSURE_INIT(&deadline_state->timer_callback,
                       send_cancel_op_in_call_combiner, elem,
                       grpc_schedule_on_exec_ctx);
@@ -124,7 +123,7 @@ static void cancel_timer_if_needed(grpc_deadline_state* deadline_state) {
     deadline_state->timer_state = GRPC_DEADLINE_STATE_FINISHED;
     grpc_timer_cancel(&deadline_state->timer);
   } else {
-    // timer was either in STATE_INITAL (nothing to cancel)
+    // timer was either in STATE_INITIAL (nothing to cancel)
     // OR in STATE_FINISHED (again nothing to cancel)
   }
 }
@@ -183,7 +182,7 @@ static void start_timer_after_init(void* arg, grpc_error* error) {
 
 grpc_deadline_state::grpc_deadline_state(grpc_call_element* elem,
                                          grpc_call_stack* call_stack,
-                                         grpc_call_combiner* call_combiner,
+                                         grpc_core::CallCombiner* call_combiner,
                                          grpc_millis deadline)
     : call_stack(call_stack), call_combiner(call_combiner) {
   // Deadline will always be infinite on servers, so the timer will only be

src/core/ext/filters/deadline/deadline_filter.h

@@ -32,12 +32,13 @@ enum grpc_deadline_timer_state {
 // Must be the first field in the filter's call_data.
 struct grpc_deadline_state {
   grpc_deadline_state(grpc_call_element* elem, grpc_call_stack* call_stack,
-                      grpc_call_combiner* call_combiner, grpc_millis deadline);
+                      grpc_core::CallCombiner* call_combiner,
+                      grpc_millis deadline);
   ~grpc_deadline_state();
 
   // We take a reference to the call stack for the timer callback.
   grpc_call_stack* call_stack;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_deadline_timer_state timer_state = GRPC_DEADLINE_STATE_INITIAL;
   grpc_timer timer;
   grpc_closure timer_callback;

src/core/ext/filters/http/client/http_client_filter.cc

@@ -36,7 +36,7 @@
 #define EXPECTED_CONTENT_TYPE "application/grpc"
 #define EXPECTED_CONTENT_TYPE_LENGTH sizeof(EXPECTED_CONTENT_TYPE) - 1
 
-/* default maximum size of payload eligable for GET request */
+/* default maximum size of payload eligible for GET request */
 static constexpr size_t kMaxPayloadSizeForGet = 2048;
 
 static void recv_initial_metadata_ready(void* user_data, grpc_error* error);
@@ -62,7 +62,7 @@ struct call_data {
 
   ~call_data() { GRPC_ERROR_UNREF(recv_initial_metadata_error); }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   // State for handling send_initial_metadata ops.
   grpc_linked_mdelem method;
   grpc_linked_mdelem scheme;
@@ -107,7 +107,8 @@ static grpc_error* client_filter_incoming_metadata(grpc_call_element* elem,
      * https://github.com/grpc/grpc/blob/master/doc/http-grpc-status-mapping.md.
      */
     if (b->idx.named.grpc_status != nullptr ||
-        grpc_mdelem_eq(b->idx.named.status->md, GRPC_MDELEM_STATUS_200)) {
+        grpc_mdelem_static_value_eq(b->idx.named.status->md,
+                                    GRPC_MDELEM_STATUS_200)) {
       grpc_metadata_batch_remove(b, b->idx.named.status);
     } else {
       char* val = grpc_dump_slice(GRPC_MDVALUE(b->idx.named.status->md),
@@ -140,8 +141,9 @@ static grpc_error* client_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.content_type != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.content_type->md,
-                        GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
+    if (!grpc_mdelem_static_value_eq(
+            b->idx.named.content_type->md,
+            GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
       if (grpc_slice_buf_start_eq(GRPC_MDVALUE(b->idx.named.content_type->md),
                                   EXPECTED_CONTENT_TYPE,
                                   EXPECTED_CONTENT_TYPE_LENGTH) &&

src/core/ext/filters/http/client/http_client_filter.h

@@ -25,7 +25,7 @@
 /* Processes metadata on the client side for HTTP2 transports */
 extern const grpc_channel_filter grpc_http_client_filter;
 
-/* Channel arg to determine maximum size of payload eligable for GET request */
+/* Channel arg to determine maximum size of payload eligible for GET request */
 #define GRPC_ARG_MAX_PAYLOAD_SIZE_FOR_GET "grpc.max_payload_size_for_get"
 
 #endif /* GRPC_CORE_EXT_FILTERS_HTTP_CLIENT_HTTP_CLIENT_FILTER_H */

src/core/ext/filters/http/client_authority_filter.cc

@@ -40,7 +40,7 @@ namespace {
 
 struct call_data {
   grpc_linked_mdelem authority_storage;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
 };
 
 struct channel_data {

src/core/ext/filters/http/message_compress/message_compress_filter.cc

@@ -72,7 +72,7 @@ struct call_data {
     GRPC_ERROR_UNREF(cancel_error);
   }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_linked_mdelem compression_algorithm_storage;
   grpc_linked_mdelem stream_compression_algorithm_storage;
   grpc_linked_mdelem accept_encoding_storage;

src/core/ext/filters/http/server/http_server_filter.cc

@@ -61,7 +61,7 @@ struct call_data {
     }
   }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
 
   // Outgoing headers to add to send_initial_metadata.
   grpc_linked_mdelem status;
@@ -131,18 +131,19 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   static const char* error_name = "Failed processing incoming headers";
 
   if (b->idx.named.method != nullptr) {
-    if (grpc_mdelem_eq(b->idx.named.method->md, GRPC_MDELEM_METHOD_POST)) {
+    if (grpc_mdelem_static_value_eq(b->idx.named.method->md,
+                                    GRPC_MDELEM_METHOD_POST)) {
       *calld->recv_initial_metadata_flags &=
           ~(GRPC_INITIAL_METADATA_CACHEABLE_REQUEST |
             GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST);
-    } else if (grpc_mdelem_eq(b->idx.named.method->md,
-                              GRPC_MDELEM_METHOD_PUT)) {
+    } else if (grpc_mdelem_static_value_eq(b->idx.named.method->md,
+                                           GRPC_MDELEM_METHOD_PUT)) {
       *calld->recv_initial_metadata_flags &=
           ~GRPC_INITIAL_METADATA_CACHEABLE_REQUEST;
       *calld->recv_initial_metadata_flags |=
           GRPC_INITIAL_METADATA_IDEMPOTENT_REQUEST;
-    } else if (grpc_mdelem_eq(b->idx.named.method->md,
-                              GRPC_MDELEM_METHOD_GET)) {
+    } else if (grpc_mdelem_static_value_eq(b->idx.named.method->md,
+                                           GRPC_MDELEM_METHOD_GET)) {
       *calld->recv_initial_metadata_flags |=
           GRPC_INITIAL_METADATA_CACHEABLE_REQUEST;
       *calld->recv_initial_metadata_flags &=
@@ -163,7 +164,8 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.te != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.te->md, GRPC_MDELEM_TE_TRAILERS)) {
+    if (!grpc_mdelem_static_value_eq(b->idx.named.te->md,
+                                     GRPC_MDELEM_TE_TRAILERS)) {
       hs_add_error(error_name, &error,
                    grpc_attach_md_to_error(
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
@@ -178,9 +180,12 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.scheme != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTP) &&
-        !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_HTTPS) &&
-        !grpc_mdelem_eq(b->idx.named.scheme->md, GRPC_MDELEM_SCHEME_GRPC)) {
+    if (!grpc_mdelem_static_value_eq(b->idx.named.scheme->md,
+                                     GRPC_MDELEM_SCHEME_HTTP) &&
+        !grpc_mdelem_static_value_eq(b->idx.named.scheme->md,
+                                     GRPC_MDELEM_SCHEME_HTTPS) &&
+        !grpc_mdelem_static_value_eq(b->idx.named.scheme->md,
+                                     GRPC_MDELEM_SCHEME_GRPC)) {
       hs_add_error(error_name, &error,
                    grpc_attach_md_to_error(
                        GRPC_ERROR_CREATE_FROM_STATIC_STRING("Bad header"),
@@ -196,8 +201,9 @@ static grpc_error* hs_filter_incoming_metadata(grpc_call_element* elem,
   }
 
   if (b->idx.named.content_type != nullptr) {
-    if (!grpc_mdelem_eq(b->idx.named.content_type->md,
-                        GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
+    if (!grpc_mdelem_static_value_eq(
+            b->idx.named.content_type->md,
+            GRPC_MDELEM_CONTENT_TYPE_APPLICATION_SLASH_GRPC)) {
       if (grpc_slice_buf_start_eq(GRPC_MDVALUE(b->idx.named.content_type->md),
                                   EXPECTED_CONTENT_TYPE,
                                   EXPECTED_CONTENT_TYPE_LENGTH) &&

src/core/ext/filters/message_size/message_size_filter.cc

@@ -154,7 +154,7 @@ struct call_data {
 
   ~call_data() { GRPC_ERROR_UNREF(error); }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_core::MessageSizeParsedObject::message_size_limits limits;
   // Receive closures are chained: we inject this closure as the
   // recv_message_ready up-call on transport_stream_op, and remember to

src/core/ext/transport/chttp2/alpn/alpn.h

@@ -23,7 +23,7 @@
 
 #include <string.h>
 
-/* Retuns 1 if the version is supported, 0 otherwise. */
+/* Returns 1 if the version is supported, 0 otherwise. */
 int grpc_chttp2_is_alpn_version_supported(const char* version, size_t size);
 
 /* Returns the number of protocol versions to advertise */

src/core/ext/transport/chttp2/transport/chttp2_transport.cc

@@ -1281,8 +1281,8 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t,
 
 static bool contains_non_ok_status(grpc_metadata_batch* batch) {
   if (batch->idx.named.grpc_status != nullptr) {
-    return !grpc_mdelem_eq(batch->idx.named.grpc_status->md,
-                           GRPC_MDELEM_GRPC_STATUS_0);
+    return !grpc_mdelem_static_value_eq(batch->idx.named.grpc_status->md,
+                                        GRPC_MDELEM_GRPC_STATUS_0);
   }
   return false;
 }
@@ -1413,7 +1413,7 @@ static void perform_stream_op_locked(void* stream_op,
   // on_complete will be null if and only if there are no send ops in the batch.
   if (on_complete != nullptr) {
     // This batch has send ops. Use final_data as a barrier until enqueue time;
-    // the inital counter is dropped at the end of this function.
+    // the initial counter is dropped at the end of this function.
     on_complete->next_data.scratch = CLOSURE_BARRIER_FIRST_REF_BIT;
     on_complete->error_data.error = GRPC_ERROR_NONE;
   }

src/core/ext/transport/chttp2/transport/hpack_encoder.cc

@@ -56,7 +56,7 @@
 /* don't consider adding anything bigger than this to the hpack table */
 #define MAX_DECODER_SPACE_USAGE 512
 
-static grpc_slice_refcount terminal_slice_refcount = {nullptr, nullptr};
+static grpc_slice_refcount terminal_slice_refcount;
 static const grpc_slice terminal_slice = {
     &terminal_slice_refcount, /* refcount */
     {{0, nullptr}}            /* data.refcounted */

src/core/ext/transport/chttp2/transport/writing.cc

@@ -163,15 +163,6 @@ static void report_stall(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
   }
 }
 
-static bool stream_ref_if_not_destroyed(gpr_refcount* r) {
-  gpr_atm count;
-  do {
-    count = gpr_atm_acq_load(&r->count);
-    if (count == 0) return false;
-  } while (!gpr_atm_rel_cas(&r->count, count, count + 1));
-  return true;
-}
-
 /* How many bytes would we like to put on the wire during a single syscall */
 static uint32_t target_write_size(grpc_chttp2_transport* t) {
   return 1024 * 1024;
@@ -254,7 +245,7 @@ class WriteContext {
     while (grpc_chttp2_list_pop_stalled_by_transport(t_, &s)) {
       if (t_->closed_with_error == GRPC_ERROR_NONE &&
           grpc_chttp2_list_add_writable_stream(t_, s)) {
-        if (!stream_ref_if_not_destroyed(&s->refcount->refs)) {
+        if (!s->refcount->refs.RefIfNonZero()) {
           grpc_chttp2_list_remove_writable_stream(t_, s);
         }
       }

src/core/ext/transport/cronet/transport/cronet_transport.cc

@@ -29,6 +29,7 @@
 #include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
 #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
 #include "src/core/ext/transport/cronet/transport/cronet_transport.h"
+#include "src/core/lib/debug/trace.h"
 #include "src/core/lib/gpr/host_port.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
@@ -45,14 +46,12 @@
 #define GRPC_HEADER_SIZE_IN_BYTES 5
 #define GRPC_FLUSH_READ_SIZE 4096
 
-#define CRONET_LOG(...)                          \
-  do {                                           \
-    if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \
+grpc_core::TraceFlag grpc_cronet_trace(false, "cronet");
+#define CRONET_LOG(...)                                    \
+  do {                                                     \
+    if (grpc_cronet_trace.enabled()) gpr_log(__VA_ARGS__); \
   } while (0)
 
-/* TODO (makdharma): Hook up into the wider tracing mechanism */
-int grpc_cronet_trace = 0;
-
 enum e_op_result {
   ACTION_TAKEN_WITH_CALLBACK,
   ACTION_TAKEN_NO_CALLBACK,

src/core/lib/channel/channel_stack.h

@@ -70,7 +70,7 @@ typedef struct {
   gpr_timespec start_time;
   grpc_millis deadline;
   gpr_arena* arena;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
 } grpc_call_element_args;
 
 typedef struct {

src/core/lib/channel/connected_channel.cc

@@ -41,12 +41,12 @@ typedef struct connected_channel_channel_data {
 typedef struct {
   grpc_closure closure;
   grpc_closure* original_closure;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   const char* reason;
 } callback_state;
 
 typedef struct connected_channel_call_data {
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   // Closures used for returning results on the call combiner.
   callback_state on_complete[6];  // Max number of pending batches.
   callback_state recv_initial_metadata_ready;

src/core/lib/gprpp/ref_counted.h

@@ -123,6 +123,22 @@ class RefCount {
     RefNonZero();
   }
 
+  bool RefIfNonZero() { return value_.IncrementIfNonzero(); }
+
+  bool RefIfNonZero(const DebugLocation& location, const char* reason) {
+#ifndef NDEBUG
+    if (location.Log() && trace_flag_ != nullptr && trace_flag_->enabled()) {
+      const RefCount::Value old_refs = get();
+      gpr_log(GPR_INFO,
+              "%s:%p %s:%d ref_if_non_zero "
+              "%" PRIdPTR " -> %" PRIdPTR " %s",
+              trace_flag_->name(), this, location.file(), location.line(),
+              old_refs, old_refs + 1, reason);
+    }
+#endif
+    return RefIfNonZero();
+  }
+
   // Decrements the ref-count and returns true if the ref-count reaches 0.
   bool Unref() {
     const Value prior = value_.FetchSub(1, MemoryOrder::ACQ_REL);

src/core/lib/iomgr/call_combiner.cc

@@ -26,23 +26,43 @@
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/profiling/timers.h"
 
-grpc_core::TraceFlag grpc_call_combiner_trace(false, "call_combiner");
+namespace grpc_core {
 
-static grpc_error* decode_cancel_state_error(gpr_atm cancel_state) {
+TraceFlag grpc_call_combiner_trace(false, "call_combiner");
+
+namespace {
+
+grpc_error* DecodeCancelStateError(gpr_atm cancel_state) {
   if (cancel_state & 1) {
     return (grpc_error*)(cancel_state & ~static_cast<gpr_atm>(1));
   }
   return GRPC_ERROR_NONE;
 }
 
-static gpr_atm encode_cancel_state_error(grpc_error* error) {
+gpr_atm EncodeCancelStateError(grpc_error* error) {
   return static_cast<gpr_atm>(1) | (gpr_atm)error;
 }
 
+}  // namespace
+
+CallCombiner::CallCombiner() {
+  gpr_atm_no_barrier_store(&cancel_state_, 0);
+  gpr_atm_no_barrier_store(&size_, 0);
+  gpr_mpscq_init(&queue_);
+#ifdef GRPC_TSAN_ENABLED
+  GRPC_CLOSURE_INIT(&tsan_closure_, TsanClosure, this,
+                    grpc_schedule_on_exec_ctx);
+#endif
+}
+
+CallCombiner::~CallCombiner() {
+  gpr_mpscq_destroy(&queue_);
+  GRPC_ERROR_UNREF(DecodeCancelStateError(cancel_state_));
+}
+
 #ifdef GRPC_TSAN_ENABLED
-static void tsan_closure(void* user_data, grpc_error* error) {
-  grpc_call_combiner* call_combiner =
-      static_cast<grpc_call_combiner*>(user_data);
+void CallCombiner::TsanClosure(void* arg, grpc_error* error) {
+  CallCombiner* self = static_cast<CallCombiner*>(arg);
   // We ref-count the lock, and check if it's already taken.
   // If it was taken, we should do nothing. Otherwise, we will mark it as
   // locked. Note that if two different threads try to do this, only one of
@@ -51,18 +71,18 @@ static void tsan_closure(void* user_data, grpc_error* error) {
   // TSAN will correctly produce an error.
   //
   // TODO(soheil): This only covers the callbacks scheduled by
-  //               grpc_call_combiner_(start|finish). If in the future, a
-  //               callback gets scheduled using other mechanisms, we will need
-  //               to add APIs to externally lock call combiners.
-  grpc_core::RefCountedPtr<grpc_call_combiner::TsanLock> lock =
-      call_combiner->tsan_lock;
+  //               CallCombiner::Start() and CallCombiner::Stop().
+  //               If in the future, a callback gets scheduled using other
+  //               mechanisms, we will need to add APIs to externally lock
+  //               call combiners.
+  RefCountedPtr<TsanLock> lock = self->tsan_lock_;
   bool prev = false;
   if (lock->taken.compare_exchange_strong(prev, true)) {
     TSAN_ANNOTATE_RWLOCK_ACQUIRED(&lock->taken, true);
   } else {
     lock.reset();
   }
-  GRPC_CLOSURE_RUN(call_combiner->original_closure, GRPC_ERROR_REF(error));
+  GRPC_CLOSURE_RUN(self->original_closure_, GRPC_ERROR_REF(error));
   if (lock != nullptr) {
     TSAN_ANNOTATE_RWLOCK_RELEASED(&lock->taken, true);
     bool prev = true;
@@ -71,34 +91,17 @@ static void tsan_closure(void* user_data, grpc_error* error) {
 }
 #endif
 
-static void call_combiner_sched_closure(grpc_call_combiner* call_combiner,
-                                        grpc_closure* closure,
-                                        grpc_error* error) {
+void CallCombiner::ScheduleClosure(grpc_closure* closure, grpc_error* error) {
 #ifdef GRPC_TSAN_ENABLED
-  call_combiner->original_closure = closure;
-  GRPC_CLOSURE_SCHED(&call_combiner->tsan_closure, error);
+  original_closure_ = closure;
+  GRPC_CLOSURE_SCHED(&tsan_closure_, error);
 #else
   GRPC_CLOSURE_SCHED(closure, error);
 #endif
 }
 
-void grpc_call_combiner_init(grpc_call_combiner* call_combiner) {
-  gpr_atm_no_barrier_store(&call_combiner->cancel_state, 0);
-  gpr_atm_no_barrier_store(&call_combiner->size, 0);
-  gpr_mpscq_init(&call_combiner->queue);
-#ifdef GRPC_TSAN_ENABLED
-  GRPC_CLOSURE_INIT(&call_combiner->tsan_closure, tsan_closure, call_combiner,
-                    grpc_schedule_on_exec_ctx);
-#endif
-}
-
-void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner) {
-  gpr_mpscq_destroy(&call_combiner->queue);
-  GRPC_ERROR_UNREF(decode_cancel_state_error(call_combiner->cancel_state));
-}
-
 #ifndef NDEBUG
-#define DEBUG_ARGS , const char *file, int line
+#define DEBUG_ARGS const char *file, int line,
 #define DEBUG_FMT_STR "%s:%d: "
 #define DEBUG_FMT_ARGS , file, line
 #else
@@ -107,20 +110,17 @@ void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner) {
 #define DEBUG_FMT_ARGS
 #endif
 
-void grpc_call_combiner_start(grpc_call_combiner* call_combiner,
-                              grpc_closure* closure,
-                              grpc_error* error DEBUG_ARGS,
-                              const char* reason) {
-  GPR_TIMER_SCOPE("call_combiner_start", 0);
+void CallCombiner::Start(grpc_closure* closure, grpc_error* error,
+                         DEBUG_ARGS const char* reason) {
+  GPR_TIMER_SCOPE("CallCombiner::Start", 0);
   if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_INFO,
-            "==> grpc_call_combiner_start() [%p] closure=%p [" DEBUG_FMT_STR
+            "==> CallCombiner::Start() [%p] closure=%p [" DEBUG_FMT_STR
             "%s] error=%s",
-            call_combiner, closure DEBUG_FMT_ARGS, reason,
-            grpc_error_string(error));
+            this, closure DEBUG_FMT_ARGS, reason, grpc_error_string(error));
   }
-  size_t prev_size = static_cast<size_t>(
-      gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)1));
+  size_t prev_size =
+      static_cast<size_t>(gpr_atm_full_fetch_add(&size_, (gpr_atm)1));
   if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_INFO, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
             prev_size + 1);
@@ -128,34 +128,30 @@ void grpc_call_combiner_start(grpc_call_combiner* call_combiner,
   GRPC_STATS_INC_CALL_COMBINER_LOCKS_SCHEDULED_ITEMS();
   if (prev_size == 0) {
     GRPC_STATS_INC_CALL_COMBINER_LOCKS_INITIATED();
-
     GPR_TIMER_MARK("call_combiner_initiate", 0);
     if (grpc_call_combiner_trace.enabled()) {
       gpr_log(GPR_INFO, "  EXECUTING IMMEDIATELY");
     }
     // Queue was empty, so execute this closure immediately.
-    call_combiner_sched_closure(call_combiner, closure, error);
+    ScheduleClosure(closure, error);
   } else {
     if (grpc_call_combiner_trace.enabled()) {
       gpr_log(GPR_INFO, "  QUEUING");
     }
     // Queue was not empty, so add closure to queue.
     closure->error_data.error = error;
-    gpr_mpscq_push(&call_combiner->queue,
-                   reinterpret_cast<gpr_mpscq_node*>(closure));
+    gpr_mpscq_push(&queue_, reinterpret_cast<gpr_mpscq_node*>(closure));
   }
 }
 
-void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS,
-                             const char* reason) {
-  GPR_TIMER_SCOPE("call_combiner_stop", 0);
+void CallCombiner::Stop(DEBUG_ARGS const char* reason) {
+  GPR_TIMER_SCOPE("CallCombiner::Stop", 0);
   if (grpc_call_combiner_trace.enabled()) {
-    gpr_log(GPR_INFO,
-            "==> grpc_call_combiner_stop() [%p] [" DEBUG_FMT_STR "%s]",
-            call_combiner DEBUG_FMT_ARGS, reason);
+    gpr_log(GPR_INFO, "==> CallCombiner::Stop() [%p] [" DEBUG_FMT_STR "%s]",
+            this DEBUG_FMT_ARGS, reason);
   }
-  size_t prev_size = static_cast<size_t>(
-      gpr_atm_full_fetch_add(&call_combiner->size, (gpr_atm)-1));
+  size_t prev_size =
+      static_cast<size_t>(gpr_atm_full_fetch_add(&size_, (gpr_atm)-1));
   if (grpc_call_combiner_trace.enabled()) {
     gpr_log(GPR_INFO, "  size: %" PRIdPTR " -> %" PRIdPTR, prev_size,
             prev_size - 1);
@@ -168,10 +164,10 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS,
       }
       bool empty;
       grpc_closure* closure = reinterpret_cast<grpc_closure*>(
-          gpr_mpscq_pop_and_check_end(&call_combiner->queue, &empty));
+          gpr_mpscq_pop_and_check_end(&queue_, &empty));
       if (closure == nullptr) {
         // This can happen either due to a race condition within the mpscq
-        // code or because of a race with grpc_call_combiner_start().
+        // code or because of a race with Start().
         if (grpc_call_combiner_trace.enabled()) {
           gpr_log(GPR_INFO, "  queue returned no result; checking again");
         }
@@ -181,8 +177,7 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS,
         gpr_log(GPR_INFO, "  EXECUTING FROM QUEUE: closure=%p error=%s",
                 closure, grpc_error_string(closure->error_data.error));
       }
-      call_combiner_sched_closure(call_combiner, closure,
-                                  closure->error_data.error);
+      ScheduleClosure(closure, closure->error_data.error);
       break;
     }
   } else if (grpc_call_combiner_trace.enabled()) {
@@ -190,13 +185,12 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner DEBUG_ARGS,
   }
 }
 
-void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner,
-                                             grpc_closure* closure) {
+void CallCombiner::SetNotifyOnCancel(grpc_closure* closure) {
   GRPC_STATS_INC_CALL_COMBINER_SET_NOTIFY_ON_CANCEL();
   while (true) {
     // Decode original state.
-    gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
-    grpc_error* original_error = decode_cancel_state_error(original_state);
+    gpr_atm original_state = gpr_atm_acq_load(&cancel_state_);
+    grpc_error* original_error = DecodeCancelStateError(original_state);
     // If error is set, invoke the cancellation closure immediately.
     // Otherwise, store the new closure.
     if (original_error != GRPC_ERROR_NONE) {
@@ -204,16 +198,15 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner,
         gpr_log(GPR_INFO,
                 "call_combiner=%p: scheduling notify_on_cancel callback=%p "
                 "for pre-existing cancellation",
-                call_combiner, closure);
+                this, closure);
       }
       GRPC_CLOSURE_SCHED(closure, GRPC_ERROR_REF(original_error));
       break;
     } else {
-      if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
-                           (gpr_atm)closure)) {
+      if (gpr_atm_full_cas(&cancel_state_, original_state, (gpr_atm)closure)) {
         if (grpc_call_combiner_trace.enabled()) {
           gpr_log(GPR_INFO, "call_combiner=%p: setting notify_on_cancel=%p",
-                  call_combiner, closure);
+                  this, closure);
         }
         // If we replaced an earlier closure, invoke the original
         // closure with GRPC_ERROR_NONE.  This allows callers to clean
@@ -222,8 +215,8 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner,
           closure = (grpc_closure*)original_state;
           if (grpc_call_combiner_trace.enabled()) {
             gpr_log(GPR_INFO,
-                    "call_combiner=%p: scheduling old cancel callback=%p",
-                    call_combiner, closure);
+                    "call_combiner=%p: scheduling old cancel callback=%p", this,
+                    closure);
           }
           GRPC_CLOSURE_SCHED(closure, GRPC_ERROR_NONE);
         }
@@ -234,24 +227,23 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner,
   }
 }
 
-void grpc_call_combiner_cancel(grpc_call_combiner* call_combiner,
-                               grpc_error* error) {
+void CallCombiner::Cancel(grpc_error* error) {
   GRPC_STATS_INC_CALL_COMBINER_CANCELLED();
   while (true) {
-    gpr_atm original_state = gpr_atm_acq_load(&call_combiner->cancel_state);
-    grpc_error* original_error = decode_cancel_state_error(original_state);
+    gpr_atm original_state = gpr_atm_acq_load(&cancel_state_);
+    grpc_error* original_error = DecodeCancelStateError(original_state);
     if (original_error != GRPC_ERROR_NONE) {
       GRPC_ERROR_UNREF(error);
       break;
     }
-    if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
-                         encode_cancel_state_error(error))) {
+    if (gpr_atm_full_cas(&cancel_state_, original_state,
+                         EncodeCancelStateError(error))) {
       if (original_state != 0) {
         grpc_closure* notify_on_cancel = (grpc_closure*)original_state;
         if (grpc_call_combiner_trace.enabled()) {
           gpr_log(GPR_INFO,
                   "call_combiner=%p: scheduling notify_on_cancel callback=%p",
-                  call_combiner, notify_on_cancel);
+                  this, notify_on_cancel);
         }
         GRPC_CLOSURE_SCHED(notify_on_cancel, GRPC_ERROR_REF(error));
       }
@@ -260,3 +252,5 @@ void grpc_call_combiner_cancel(grpc_call_combiner* call_combiner,
     // cas failed, try again.
   }
 }
+
+}  // namespace grpc_core

src/core/lib/iomgr/call_combiner.h

@@ -41,15 +41,78 @@
 // when it is done with the action that was kicked off by the original
 // callback.
 
-extern grpc_core::TraceFlag grpc_call_combiner_trace;
+namespace grpc_core {
+
+extern TraceFlag grpc_call_combiner_trace;
+
+class CallCombiner {
+ public:
+  CallCombiner();
+  ~CallCombiner();
+
+#ifndef NDEBUG
+#define GRPC_CALL_COMBINER_START(call_combiner, closure, error, reason) \
+  (call_combiner)->Start((closure), (error), __FILE__, __LINE__, (reason))
+#define GRPC_CALL_COMBINER_STOP(call_combiner, reason) \
+  (call_combiner)->Stop(__FILE__, __LINE__, (reason))
+  /// Starts processing \a closure.
+  void Start(grpc_closure* closure, grpc_error* error, const char* file,
+             int line, const char* reason);
+  /// Yields the call combiner to the next closure in the queue, if any.
+  void Stop(const char* file, int line, const char* reason);
+#else
+#define GRPC_CALL_COMBINER_START(call_combiner, closure, error, reason) \
+  (call_combiner)->Start((closure), (error), (reason))
+#define GRPC_CALL_COMBINER_STOP(call_combiner, reason) \
+  (call_combiner)->Stop((reason))
+  /// Starts processing \a closure.
+  void Start(grpc_closure* closure, grpc_error* error, const char* reason);
+  /// Yields the call combiner to the next closure in the queue, if any.
+  void Stop(const char* reason);
+#endif
+
+  /// Registers \a closure to be invoked when Cancel() is called.
+  ///
+  /// Once a closure is registered, it will always be scheduled exactly
+  /// once; this allows the closure to hold references that will be freed
+  /// regardless of whether or not the call was cancelled.  If a cancellation
+  /// does occur, the closure will be scheduled with the cancellation error;
+  /// otherwise, it will be scheduled with GRPC_ERROR_NONE.
+  ///
+  /// The closure will be scheduled in the following cases:
+  /// - If Cancel() was called prior to registering the closure, it will be
+  ///   scheduled immediately with the cancelation error.
+  /// - If Cancel() is called after registering the closure, the closure will
+  ///   be scheduled with the cancellation error.
+  /// - If SetNotifyOnCancel() is called again to register a new cancellation
+  ///   closure, the previous cancellation closure will be scheduled with
+  ///   GRPC_ERROR_NONE.
+  ///
+  /// If \a closure is NULL, then no closure will be invoked on
+  /// cancellation; this effectively unregisters the previously set closure.
+  /// However, most filters will not need to explicitly unregister their
+  /// callbacks, as this is done automatically when the call is destroyed.
+  /// Filters that schedule the cancellation closure on ExecCtx do not need
+  /// to take a ref on the call stack to guarantee closure liveness. This is
+  /// done by explicitly flushing ExecCtx after the unregistration during
+  /// call destruction.
+  void SetNotifyOnCancel(grpc_closure* closure);
+
+  /// Indicates that the call has been cancelled.
+  void Cancel(grpc_error* error);
+
+ private:
+  void ScheduleClosure(grpc_closure* closure, grpc_error* error);
+#ifdef GRPC_TSAN_ENABLED
+  static void TsanClosure(void* arg, grpc_error* error);
+#endif
 
-struct grpc_call_combiner {
-  gpr_atm size = 0;  // size_t, num closures in queue or currently executing
-  gpr_mpscq queue;
+  gpr_atm size_ = 0;  // size_t, num closures in queue or currently executing
+  gpr_mpscq queue_;
   // Either 0 (if not cancelled and no cancellation closure set),
   // a grpc_closure* (if the lowest bit is 0),
   // or a grpc_error* (if the lowest bit is 1).
-  gpr_atm cancel_state = 0;
+  gpr_atm cancel_state_ = 0;
 #ifdef GRPC_TSAN_ENABLED
   // A fake ref-counted lock that is kept alive after the destruction of
   // grpc_call_combiner, when we are running the original closure.
@@ -58,90 +121,20 @@ struct grpc_call_combiner {
   // callback is called. However, original_closure is free to trigger
   // anything on the call combiner (including destruction of grpc_call).
   // Thus, we need a ref-counted structure that can outlive the call combiner.
-  struct TsanLock
-      : public grpc_core::RefCounted<TsanLock,
-                                     grpc_core::NonPolymorphicRefCount> {
+  struct TsanLock : public RefCounted<TsanLock, NonPolymorphicRefCount> {
     TsanLock() { TSAN_ANNOTATE_RWLOCK_CREATE(&taken); }
     ~TsanLock() { TSAN_ANNOTATE_RWLOCK_DESTROY(&taken); }
-
     // To avoid double-locking by the same thread, we should acquire/release
     // the lock only when taken is false. On each acquire taken must be set to
     // true.
     std::atomic<bool> taken{false};
   };
-  grpc_core::RefCountedPtr<TsanLock> tsan_lock =
-      grpc_core::MakeRefCounted<TsanLock>();
-  grpc_closure tsan_closure;
-  grpc_closure* original_closure;
+  RefCountedPtr<TsanLock> tsan_lock_ = MakeRefCounted<TsanLock>();
+  grpc_closure tsan_closure_;
+  grpc_closure* original_closure_;
 #endif
 };
 
-// Assumes memory was initialized to zero.
-void grpc_call_combiner_init(grpc_call_combiner* call_combiner);
-
-void grpc_call_combiner_destroy(grpc_call_combiner* call_combiner);
-
-#ifndef NDEBUG
-#define GRPC_CALL_COMBINER_START(call_combiner, closure, error, reason)   \
-  grpc_call_combiner_start((call_combiner), (closure), (error), __FILE__, \
-                           __LINE__, (reason))
-#define GRPC_CALL_COMBINER_STOP(call_combiner, reason) \
-  grpc_call_combiner_stop((call_combiner), __FILE__, __LINE__, (reason))
-/// Starts processing \a closure on \a call_combiner.
-void grpc_call_combiner_start(grpc_call_combiner* call_combiner,
-                              grpc_closure* closure, grpc_error* error,
-                              const char* file, int line, const char* reason);
-/// Yields the call combiner to the next closure in the queue, if any.
-void grpc_call_combiner_stop(grpc_call_combiner* call_combiner,
-                             const char* file, int line, const char* reason);
-#else
-#define GRPC_CALL_COMBINER_START(call_combiner, closure, error, reason) \
-  grpc_call_combiner_start((call_combiner), (closure), (error), (reason))
-#define GRPC_CALL_COMBINER_STOP(call_combiner, reason) \
-  grpc_call_combiner_stop((call_combiner), (reason))
-/// Starts processing \a closure on \a call_combiner.
-void grpc_call_combiner_start(grpc_call_combiner* call_combiner,
-                              grpc_closure* closure, grpc_error* error,
-                              const char* reason);
-/// Yields the call combiner to the next closure in the queue, if any.
-void grpc_call_combiner_stop(grpc_call_combiner* call_combiner,
-                             const char* reason);
-#endif
-
-/// Registers \a closure to be invoked by \a call_combiner when
-/// grpc_call_combiner_cancel() is called.
-///
-/// Once a closure is registered, it will always be scheduled exactly
-/// once; this allows the closure to hold references that will be freed
-/// regardless of whether or not the call was cancelled.  If a cancellation
-/// does occur, the closure will be scheduled with the cancellation error;
-/// otherwise, it will be scheduled with GRPC_ERROR_NONE.
-///
-/// The closure will be scheduled in the following cases:
-/// - If grpc_call_combiner_cancel() was called prior to registering the
-///   closure, it will be scheduled immediately with the cancelation error.
-/// - If grpc_call_combiner_cancel() is called after registering the
-///   closure, the closure will be scheduled with the cancellation error.
-/// - If grpc_call_combiner_set_notify_on_cancel() is called again to
-///   register a new cancellation closure, the previous cancellation
-///   closure will be scheduled with GRPC_ERROR_NONE.
-///
-/// If \a closure is NULL, then no closure will be invoked on
-/// cancellation; this effectively unregisters the previously set closure.
-/// However, most filters will not need to explicitly unregister their
-/// callbacks, as this is done automatically when the call is destroyed. Filters
-/// that schedule the cancellation closure on ExecCtx do not need to take a ref
-/// on the call stack to guarantee closure liveness. This is done by explicitly
-/// flushing ExecCtx after the unregistration during call destruction.
-void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner,
-                                             grpc_closure* closure);
-
-/// Indicates that the call has been cancelled.
-void grpc_call_combiner_cancel(grpc_call_combiner* call_combiner,
-                               grpc_error* error);
-
-namespace grpc_core {
-
 // Helper for running a list of closures in a call combiner.
 //
 // Each callback running in the call combiner will eventually be
@@ -166,7 +159,7 @@ class CallCombinerClosureList {
   // scheduled via GRPC_CLOSURE_SCHED(), which will eventually result in
   // yielding the call combiner.  If the list is empty, then the call
   // combiner will be yielded immediately.
-  void RunClosures(grpc_call_combiner* call_combiner) {
+  void RunClosures(CallCombiner* call_combiner) {
     if (closures_.empty()) {
       GRPC_CALL_COMBINER_STOP(call_combiner, "no closures to schedule");
       return;
@@ -190,7 +183,7 @@ class CallCombinerClosureList {
 
   // Runs all closures in the call combiner, but does NOT yield the call
   // combiner.  All closures will be scheduled via GRPC_CALL_COMBINER_START().
-  void RunClosuresWithoutYielding(grpc_call_combiner* call_combiner) {
+  void RunClosuresWithoutYielding(CallCombiner* call_combiner) {
     for (size_t i = 0; i < closures_.size(); ++i) {
       auto& closure = closures_[i];
       GRPC_CALL_COMBINER_START(call_combiner, closure.closure, closure.error,

src/core/lib/iomgr/cfstream_handle.h

@@ -37,8 +37,8 @@ class CFStreamHandle final {
   static CFStreamHandle* CreateStreamHandle(CFReadStreamRef read_stream,
                                             CFWriteStreamRef write_stream);
   ~CFStreamHandle();
-  CFStreamHandle(const CFReadStreamRef& ref) = delete;
-  CFStreamHandle(CFReadStreamRef&& ref) = delete;
+  CFStreamHandle(const CFStreamHandle& ref) = delete;
+  CFStreamHandle(CFStreamHandle&& ref) = delete;
   CFStreamHandle& operator=(const CFStreamHandle& rhs) = delete;
 
   void NotifyOnOpen(grpc_closure* closure);

src/core/lib/iomgr/resource_quota.cc

@@ -32,6 +32,7 @@
 
 #include "src/core/lib/gpr/useful.h"
 #include "src/core/lib/iomgr/combiner.h"
+#include "src/core/lib/slice/slice_internal.h"
 
 grpc_core::TraceFlag grpc_resource_quota_trace(false, "resource_quota");
 
@@ -430,41 +431,43 @@ static bool rq_reclaim(grpc_resource_quota* resource_quota, bool destructive) {
  * ru_slice: a slice implementation that is backed by a grpc_resource_user
  */
 
-typedef struct {
-  grpc_slice_refcount base;
-  gpr_refcount refs;
-  grpc_resource_user* resource_user;
-  size_t size;
-} ru_slice_refcount;
+namespace grpc_core {
 
-static void ru_slice_ref(void* p) {
-  ru_slice_refcount* rc = static_cast<ru_slice_refcount*>(p);
-  gpr_ref(&rc->refs);
-}
-
-static void ru_slice_unref(void* p) {
-  ru_slice_refcount* rc = static_cast<ru_slice_refcount*>(p);
-  if (gpr_unref(&rc->refs)) {
-    grpc_resource_user_free(rc->resource_user, rc->size);
+class RuSliceRefcount {
+ public:
+  static void Destroy(void* p) {
+    auto* rc = static_cast<RuSliceRefcount*>(p);
+    rc->~RuSliceRefcount();
     gpr_free(rc);
   }
-}
+  RuSliceRefcount(grpc_resource_user* resource_user, size_t size)
+      : base_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this,
+              &base_),
+        resource_user_(resource_user),
+        size_(size) {
+    // Nothing to do here.
+  }
+  ~RuSliceRefcount() { grpc_resource_user_free(resource_user_, size_); }
+
+  grpc_slice_refcount* base_refcount() { return &base_; }
 
-static const grpc_slice_refcount_vtable ru_slice_vtable = {
-    ru_slice_ref, ru_slice_unref, grpc_slice_default_eq_impl,
-    grpc_slice_default_hash_impl};
+ private:
+  grpc_slice_refcount base_;
+  RefCount refs_;
+  grpc_resource_user* resource_user_;
+  size_t size_;
+};
+
+}  // namespace grpc_core
 
 static grpc_slice ru_slice_create(grpc_resource_user* resource_user,
                                   size_t size) {
-  ru_slice_refcount* rc = static_cast<ru_slice_refcount*>(
-      gpr_malloc(sizeof(ru_slice_refcount) + size));
-  rc->base.vtable = &ru_slice_vtable;
-  rc->base.sub_refcount = &rc->base;
-  gpr_ref_init(&rc->refs, 1);
-  rc->resource_user = resource_user;
-  rc->size = size;
+  auto* rc = static_cast<grpc_core::RuSliceRefcount*>(
+      gpr_malloc(sizeof(grpc_core::RuSliceRefcount) + size));
+  new (rc) grpc_core::RuSliceRefcount(resource_user, size);
   grpc_slice slice;
-  slice.refcount = &rc->base;
+
+  slice.refcount = rc->base_refcount();
   slice.data.refcounted.bytes = reinterpret_cast<uint8_t*>(rc + 1);
   slice.data.refcounted.length = size;
   return slice;

src/core/lib/iomgr/tcp_windows.cc

@@ -74,20 +74,6 @@ static grpc_error* set_dualstack(SOCKET sock) {
              : GRPC_WSA_ERROR(WSAGetLastError(), "setsockopt(IPV6_V6ONLY)");
 }
 
-static grpc_error* enable_loopback_fast_path(SOCKET sock) {
-  int status;
-  uint32_t param = 1;
-  DWORD ret;
-  status = WSAIoctl(sock, /*SIO_LOOPBACK_FAST_PATH==*/_WSAIOW(IOC_VENDOR, 16),
-                    &param, sizeof(param), NULL, 0, &ret, 0, 0);
-  if (status == SOCKET_ERROR) {
-    status = WSAGetLastError();
-  }
-  return status == 0 || status == WSAEOPNOTSUPP
-             ? GRPC_ERROR_NONE
-             : GRPC_WSA_ERROR(status, "WSAIoctl(SIO_LOOPBACK_FAST_PATH)");
-}
-
 static grpc_error* enable_socket_low_latency(SOCKET sock) {
   int status;
   BOOL param = TRUE;
@@ -106,8 +92,6 @@ grpc_error* grpc_tcp_prepare_socket(SOCKET sock) {
   if (err != GRPC_ERROR_NONE) return err;
   err = set_dualstack(sock);
   if (err != GRPC_ERROR_NONE) return err;
-  err = enable_loopback_fast_path(sock);
-  if (err != GRPC_ERROR_NONE) return err;
   err = enable_socket_low_latency(sock);
   if (err != GRPC_ERROR_NONE) return err;
   return GRPC_ERROR_NONE;

src/core/lib/security/transport/client_auth_filter.cc

@@ -92,7 +92,7 @@ struct call_data {
   }
 
   grpc_call_stack* owning_call;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_core::RefCountedPtr<grpc_call_credentials> creds;
   grpc_slice host = grpc_empty_slice();
   grpc_slice method = grpc_empty_slice();
@@ -270,11 +270,9 @@ static void send_security_metadata(grpc_call_element* elem,
     GRPC_ERROR_UNREF(error);
   } else {
     // Async return; register cancellation closure with call combiner.
-    grpc_call_combiner_set_notify_on_cancel(
-        calld->call_combiner,
-        GRPC_CLOSURE_INIT(&calld->get_request_metadata_cancel_closure,
-                          cancel_get_request_metadata, elem,
-                          grpc_schedule_on_exec_ctx));
+    calld->call_combiner->SetNotifyOnCancel(GRPC_CLOSURE_INIT(
+        &calld->get_request_metadata_cancel_closure,
+        cancel_get_request_metadata, elem, grpc_schedule_on_exec_ctx));
   }
 }
 
@@ -345,11 +343,9 @@ static void auth_start_transport_stream_op_batch(
         GRPC_ERROR_UNREF(error);
       } else {
         // Async return; register cancellation closure with call combiner.
-        grpc_call_combiner_set_notify_on_cancel(
-            calld->call_combiner,
-            GRPC_CLOSURE_INIT(&calld->check_call_host_cancel_closure,
-                              cancel_check_call_host, elem,
-                              grpc_schedule_on_exec_ctx));
+        calld->call_combiner->SetNotifyOnCancel(GRPC_CLOSURE_INIT(
+            &calld->check_call_host_cancel_closure, cancel_check_call_host,
+            elem, grpc_schedule_on_exec_ctx));
       }
       gpr_free(call_host);
       return; /* early exit */

src/core/lib/security/transport/server_auth_filter.cc

@@ -74,7 +74,7 @@ struct call_data {
 
   ~call_data() { GRPC_ERROR_UNREF(recv_initial_metadata_error); }
 
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_call_stack* owning_call;
   grpc_transport_stream_op_batch* recv_initial_metadata_batch;
   grpc_closure* original_recv_initial_metadata_ready;
@@ -219,8 +219,7 @@ static void recv_initial_metadata_ready(void* arg, grpc_error* error) {
       // to drop the call combiner early if we get cancelled.
       GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem,
                         grpc_schedule_on_exec_ctx);
-      grpc_call_combiner_set_notify_on_cancel(calld->call_combiner,
-                                              &calld->cancel_closure);
+      calld->call_combiner->SetNotifyOnCancel(&calld->cancel_closure);
       GRPC_CALL_STACK_REF(calld->owning_call, "server_auth_metadata");
       calld->md = metadata_batch_to_md_array(
           batch->payload->recv_initial_metadata.recv_initial_metadata);

src/core/lib/slice/slice.cc

@@ -26,6 +26,7 @@
 
 #include <string.h>
 
+#include "src/core/lib/gprpp/memory.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 
@@ -67,17 +68,10 @@ void grpc_slice_unref(grpc_slice slice) {
 
 /* grpc_slice_from_static_string support structure - a refcount that does
    nothing */
-static void noop_ref(void* unused) {}
-static void noop_unref(void* unused) {}
-
-static const grpc_slice_refcount_vtable noop_refcount_vtable = {
-    noop_ref, noop_unref, grpc_slice_default_eq_impl,
-    grpc_slice_default_hash_impl};
-static grpc_slice_refcount noop_refcount = {&noop_refcount_vtable,
-                                            &noop_refcount};
+static grpc_slice_refcount NoopRefcount;
 
 size_t grpc_slice_memory_usage(grpc_slice s) {
-  if (s.refcount == nullptr || s.refcount == &noop_refcount) {
+  if (s.refcount == nullptr || s.refcount == &NoopRefcount) {
     return 0;
   } else {
     return s.data.refcounted.length;
@@ -86,7 +80,7 @@ size_t grpc_slice_memory_usage(grpc_slice s) {
 
 grpc_slice grpc_slice_from_static_buffer(const void* s, size_t len) {
   grpc_slice slice;
-  slice.refcount = &noop_refcount;
+  slice.refcount = &NoopRefcount;
   slice.data.refcounted.bytes = (uint8_t*)s;
   slice.data.refcounted.length = len;
   return slice;
@@ -96,45 +90,43 @@ grpc_slice grpc_slice_from_static_string(const char* s) {
   return grpc_slice_from_static_buffer(s, strlen(s));
 }
 
+namespace grpc_core {
+
 /* grpc_slice_new support structures - we create a refcount object extended
    with the user provided data pointer & destroy function */
-typedef struct new_slice_refcount {
-  grpc_slice_refcount rc;
-  gpr_refcount refs;
-  void (*user_destroy)(void*);
-  void* user_data;
-} new_slice_refcount;
-
-static void new_slice_ref(void* p) {
-  new_slice_refcount* r = static_cast<new_slice_refcount*>(p);
-  gpr_ref(&r->refs);
-}
-
-static void new_slice_unref(void* p) {
-  new_slice_refcount* r = static_cast<new_slice_refcount*>(p);
-  if (gpr_unref(&r->refs)) {
-    r->user_destroy(r->user_data);
-    gpr_free(r);
+class NewSliceRefcount {
+ public:
+  static void Destroy(void* arg) {
+    Delete(static_cast<NewSliceRefcount*>(arg));
   }
-}
 
-static const grpc_slice_refcount_vtable new_slice_vtable = {
-    new_slice_ref, new_slice_unref, grpc_slice_default_eq_impl,
-    grpc_slice_default_hash_impl};
+  NewSliceRefcount(void (*destroy)(void*), void* user_data)
+      : rc_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this, &rc_),
+        user_destroy_(destroy),
+        user_data_(user_data) {}
+
+  GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE
+
+  grpc_slice_refcount* base_refcount() { return &rc_; }
+
+ private:
+  ~NewSliceRefcount() { user_destroy_(user_data_); }
+
+  grpc_slice_refcount rc_;
+  RefCount refs_;
+  void (*user_destroy_)(void*);
+  void* user_data_;
+};
+
+}  // namespace grpc_core
 
 grpc_slice grpc_slice_new_with_user_data(void* p, size_t len,
                                          void (*destroy)(void*),
                                          void* user_data) {
   grpc_slice slice;
-  new_slice_refcount* rc =
-      static_cast<new_slice_refcount*>(gpr_malloc(sizeof(new_slice_refcount)));
-  gpr_ref_init(&rc->refs, 1);
-  rc->rc.vtable = &new_slice_vtable;
-  rc->rc.sub_refcount = &rc->rc;
-  rc->user_destroy = destroy;
-  rc->user_data = user_data;
-
-  slice.refcount = &rc->rc;
+  slice.refcount =
+      grpc_core::New<grpc_core::NewSliceRefcount>(destroy, user_data)
+          ->base_refcount();
   slice.data.refcounted.bytes = static_cast<uint8_t*>(p);
   slice.data.refcounted.length = len;
   return slice;
@@ -145,46 +137,45 @@ grpc_slice grpc_slice_new(void* p, size_t len, void (*destroy)(void*)) {
   return grpc_slice_new_with_user_data(p, len, destroy, p);
 }
 
+namespace grpc_core {
 /* grpc_slice_new_with_len support structures - we create a refcount object
    extended with the user provided data pointer & destroy function */
-typedef struct new_with_len_slice_refcount {
-  grpc_slice_refcount rc;
-  gpr_refcount refs;
-  void* user_data;
-  size_t user_length;
-  void (*user_destroy)(void*, size_t);
-} new_with_len_slice_refcount;
-
-static void new_with_len_ref(void* p) {
-  new_with_len_slice_refcount* r = static_cast<new_with_len_slice_refcount*>(p);
-  gpr_ref(&r->refs);
-}
 
-static void new_with_len_unref(void* p) {
-  new_with_len_slice_refcount* r = static_cast<new_with_len_slice_refcount*>(p);
-  if (gpr_unref(&r->refs)) {
-    r->user_destroy(r->user_data, r->user_length);
-    gpr_free(r);
+class NewWithLenSliceRefcount {
+ public:
+  static void Destroy(void* arg) {
+    Delete(static_cast<NewWithLenSliceRefcount*>(arg));
   }
-}
 
-static const grpc_slice_refcount_vtable new_with_len_vtable = {
-    new_with_len_ref, new_with_len_unref, grpc_slice_default_eq_impl,
-    grpc_slice_default_hash_impl};
+  NewWithLenSliceRefcount(void (*destroy)(void*, size_t), void* user_data,
+                          size_t user_length)
+      : rc_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this, &rc_),
+        user_data_(user_data),
+        user_length_(user_length),
+        user_destroy_(destroy) {}
+
+  GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE
+
+  grpc_slice_refcount* base_refcount() { return &rc_; }
+
+ private:
+  ~NewWithLenSliceRefcount() { user_destroy_(user_data_, user_length_); }
+
+  grpc_slice_refcount rc_;
+  RefCount refs_;
+  void* user_data_;
+  size_t user_length_;
+  void (*user_destroy_)(void*, size_t);
+};
+
+}  // namespace grpc_core
 
 grpc_slice grpc_slice_new_with_len(void* p, size_t len,
                                    void (*destroy)(void*, size_t)) {
   grpc_slice slice;
-  new_with_len_slice_refcount* rc = static_cast<new_with_len_slice_refcount*>(
-      gpr_malloc(sizeof(new_with_len_slice_refcount)));
-  gpr_ref_init(&rc->refs, 1);
-  rc->rc.vtable = &new_with_len_vtable;
-  rc->rc.sub_refcount = &rc->rc;
-  rc->user_destroy = destroy;
-  rc->user_data = p;
-  rc->user_length = len;
-
-  slice.refcount = &rc->rc;
+  slice.refcount =
+      grpc_core::New<grpc_core::NewWithLenSliceRefcount>(destroy, p, len)
+          ->base_refcount();
   slice.data.refcounted.bytes = static_cast<uint8_t*>(p);
   slice.data.refcounted.length = len;
   return slice;
@@ -203,39 +194,28 @@ grpc_slice grpc_slice_from_copied_string(const char* source) {
 
 namespace {
 
-struct MallocRefCount {
-  MallocRefCount(const grpc_slice_refcount_vtable* vtable) {
-    base.vtable = vtable;
-    base.sub_refcount = &base;
+class MallocRefCount {
+ public:
+  static void Destroy(void* arg) {
+    MallocRefCount* r = static_cast<MallocRefCount*>(arg);
+    r->~MallocRefCount();
+    gpr_free(r);
   }
 
-  void Ref() { refs.Ref(); }
-  void Unref() {
-    if (refs.Unref()) {
-      gpr_free(this);
-    }
-  }
+  MallocRefCount()
+      : base_(grpc_slice_refcount::Type::REGULAR, &refs_, Destroy, this,
+              &base_) {}
+  ~MallocRefCount() = default;
+
+  grpc_slice_refcount* base_refcount() { return &base_; }
 
-  grpc_slice_refcount base;
-  grpc_core::RefCount refs;
+ private:
+  grpc_slice_refcount base_;
+  grpc_core::RefCount refs_;
 };
 
 }  // namespace
 
-static void malloc_ref(void* p) {
-  MallocRefCount* r = static_cast<MallocRefCount*>(p);
-  r->Ref();
-}
-
-static void malloc_unref(void* p) {
-  MallocRefCount* r = static_cast<MallocRefCount*>(p);
-  r->Unref();
-}
-
-static const grpc_slice_refcount_vtable malloc_vtable = {
-    malloc_ref, malloc_unref, grpc_slice_default_eq_impl,
-    grpc_slice_default_hash_impl};
-
 grpc_slice grpc_slice_malloc_large(size_t length) {
   grpc_slice slice;
 
@@ -248,14 +228,16 @@ grpc_slice grpc_slice_malloc_large(size_t length) {
      refcount is a malloc_refcount
      bytes is an array of bytes of the requested length
      Both parts are placed in the same allocation returned from gpr_malloc */
-  void* data =
+  auto* rc =
       static_cast<MallocRefCount*>(gpr_malloc(sizeof(MallocRefCount) + length));
 
-  auto* rc = new (data) MallocRefCount(&malloc_vtable);
+  /* Initial refcount on rc is 1 - and it's up to the caller to release
+     this reference. */
+  new (rc) MallocRefCount();
 
   /* Build up the slice to be returned. */
   /* The slices refcount points back to the allocated block. */
-  slice.refcount = &rc->base;
+  slice.refcount = rc->base_refcount();
   /* The data bytes are placed immediately after the refcount struct */
   slice.data.refcounted.bytes = reinterpret_cast<uint8_t*>(rc + 1);
   /* And the length of the block is set to the requested length */
@@ -286,7 +268,7 @@ grpc_slice grpc_slice_sub_no_ref(grpc_slice source, size_t begin, size_t end) {
     GPR_ASSERT(source.data.refcounted.length >= end);
 
     /* Build the result */
-    subset.refcount = source.refcount->sub_refcount;
+    subset.refcount = source.refcount->sub_refcount();
     /* Point into the source array */
     subset.data.refcounted.bytes = source.data.refcounted.bytes + begin;
     subset.data.refcounted.length = end - begin;
@@ -312,7 +294,7 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) {
   } else {
     subset = grpc_slice_sub_no_ref(source, begin, end);
     /* Bump the refcount */
-    subset.refcount->vtable->ref(subset.refcount);
+    subset.refcount->Ref();
   }
   return subset;
 }
@@ -340,23 +322,23 @@ grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice* source, size_t split,
       tail.data.inlined.length = static_cast<uint8_t>(tail_length);
       memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split,
              tail_length);
-      source->refcount = source->refcount->sub_refcount;
+      source->refcount = source->refcount->sub_refcount();
     } else {
       /* Build the result */
       switch (ref_whom) {
         case GRPC_SLICE_REF_TAIL:
-          tail.refcount = source->refcount->sub_refcount;
-          source->refcount = &noop_refcount;
+          tail.refcount = source->refcount->sub_refcount();
+          source->refcount = &NoopRefcount;
           break;
         case GRPC_SLICE_REF_HEAD:
-          tail.refcount = &noop_refcount;
-          source->refcount = source->refcount->sub_refcount;
+          tail.refcount = &NoopRefcount;
+          source->refcount = source->refcount->sub_refcount();
           break;
         case GRPC_SLICE_REF_BOTH:
-          tail.refcount = source->refcount->sub_refcount;
-          source->refcount = source->refcount->sub_refcount;
+          tail.refcount = source->refcount->sub_refcount();
+          source->refcount = source->refcount->sub_refcount();
           /* Bump the refcount */
-          tail.refcount->vtable->ref(tail.refcount);
+          tail.refcount->Ref();
           break;
       }
       /* Point into the source array */
@@ -392,20 +374,20 @@ grpc_slice grpc_slice_split_head(grpc_slice* source, size_t split) {
     head.refcount = nullptr;
     head.data.inlined.length = static_cast<uint8_t>(split);
     memcpy(head.data.inlined.bytes, source->data.refcounted.bytes, split);
-    source->refcount = source->refcount->sub_refcount;
+    source->refcount = source->refcount->sub_refcount();
     source->data.refcounted.bytes += split;
     source->data.refcounted.length -= split;
   } else {
     GPR_ASSERT(source->data.refcounted.length >= split);
 
     /* Build the result */
-    head.refcount = source->refcount->sub_refcount;
+    head.refcount = source->refcount->sub_refcount();
     /* Bump the refcount */
-    head.refcount->vtable->ref(head.refcount);
+    head.refcount->Ref();
     /* Point into the source array */
     head.data.refcounted.bytes = source->data.refcounted.bytes;
     head.data.refcounted.length = split;
-    source->refcount = source->refcount->sub_refcount;
+    source->refcount = source->refcount->sub_refcount();
     source->data.refcounted.bytes += split;
     source->data.refcounted.length -= split;
   }
@@ -421,8 +403,9 @@ int grpc_slice_default_eq_impl(grpc_slice a, grpc_slice b) {
 }
 
 int grpc_slice_eq(grpc_slice a, grpc_slice b) {
-  if (a.refcount && b.refcount && a.refcount->vtable == b.refcount->vtable) {
-    return a.refcount->vtable->eq(a, b);
+  if (a.refcount && b.refcount &&
+      a.refcount->GetType() == b.refcount->GetType()) {
+    return a.refcount->Eq(a, b);
   }
   return grpc_slice_default_eq_impl(a, b);
 }

src/core/lib/slice/slice_intern.cc

@@ -27,6 +27,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/iomgr/iomgr_internal.h" /* for iomgr_abort_on_leaks() */
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_string_helpers.h"
@@ -39,24 +40,17 @@
 #define TABLE_IDX(hash, capacity) (((hash) >> LOG2_SHARD_COUNT) % (capacity))
 #define SHARD_IDX(hash) ((hash) & ((1 << LOG2_SHARD_COUNT) - 1))
 
-typedef struct interned_slice_refcount {
-  grpc_slice_refcount base;
-  grpc_slice_refcount sub;
-  size_t length;
-  gpr_atm refcnt;
-  uint32_t hash;
-  struct interned_slice_refcount* bucket_next;
-} interned_slice_refcount;
+using grpc_core::InternedSliceRefcount;
 
 typedef struct slice_shard {
   gpr_mu mu;
-  interned_slice_refcount** strs;
+  InternedSliceRefcount** strs;
   size_t count;
   size_t capacity;
 } slice_shard;
 
 /* hash seed: decided at initialization time */
-static uint32_t g_hash_seed;
+uint32_t g_hash_seed;
 static int g_forced_hash_seed = 0;
 
 static slice_shard g_shards[SHARD_COUNT];
@@ -69,73 +63,35 @@ typedef struct {
 static static_metadata_hash_ent
     static_metadata_hash[4 * GRPC_STATIC_MDSTR_COUNT];
 static uint32_t max_static_metadata_hash_probe;
-static uint32_t static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
+uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
 
-static void interned_slice_ref(void* p) {
-  interned_slice_refcount* s = static_cast<interned_slice_refcount*>(p);
-  GPR_ASSERT(gpr_atm_no_barrier_fetch_add(&s->refcnt, 1) > 0);
-}
+namespace grpc_core {
 
-static void interned_slice_destroy(interned_slice_refcount* s) {
-  slice_shard* shard = &g_shards[SHARD_IDX(s->hash)];
-  gpr_mu_lock(&shard->mu);
-  GPR_ASSERT(0 == gpr_atm_no_barrier_load(&s->refcnt));
-  interned_slice_refcount** prev_next;
-  interned_slice_refcount* cur;
-  for (prev_next = &shard->strs[TABLE_IDX(s->hash, shard->capacity)],
+InternedSliceRefcount::~InternedSliceRefcount() {
+  slice_shard* shard = &g_shards[SHARD_IDX(this->hash)];
+  MutexLock lock(&shard->mu);
+  InternedSliceRefcount** prev_next;
+  InternedSliceRefcount* cur;
+  for (prev_next = &shard->strs[TABLE_IDX(this->hash, shard->capacity)],
       cur = *prev_next;
-       cur != s; prev_next = &cur->bucket_next, cur = cur->bucket_next)
+       cur != this; prev_next = &cur->bucket_next, cur = cur->bucket_next)
     ;
   *prev_next = cur->bucket_next;
   shard->count--;
-  gpr_free(s);
-  gpr_mu_unlock(&shard->mu);
-}
-
-static void interned_slice_unref(void* p) {
-  interned_slice_refcount* s = static_cast<interned_slice_refcount*>(p);
-  if (1 == gpr_atm_full_fetch_add(&s->refcnt, -1)) {
-    interned_slice_destroy(s);
-  }
-}
-
-static void interned_slice_sub_ref(void* p) {
-  interned_slice_ref((static_cast<char*>(p)) -
-                     offsetof(interned_slice_refcount, sub));
-}
-
-static void interned_slice_sub_unref(void* p) {
-  interned_slice_unref((static_cast<char*>(p)) -
-                       offsetof(interned_slice_refcount, sub));
-}
-
-static uint32_t interned_slice_hash(grpc_slice slice) {
-  interned_slice_refcount* s =
-      reinterpret_cast<interned_slice_refcount*>(slice.refcount);
-  return s->hash;
-}
-
-static int interned_slice_eq(grpc_slice a, grpc_slice b) {
-  return a.refcount == b.refcount;
 }
 
-static const grpc_slice_refcount_vtable interned_slice_vtable = {
-    interned_slice_ref, interned_slice_unref, interned_slice_eq,
-    interned_slice_hash};
-static const grpc_slice_refcount_vtable interned_slice_sub_vtable = {
-    interned_slice_sub_ref, interned_slice_sub_unref,
-    grpc_slice_default_eq_impl, grpc_slice_default_hash_impl};
+}  // namespace grpc_core
 
 static void grow_shard(slice_shard* shard) {
   GPR_TIMER_SCOPE("grow_strtab", 0);
 
   size_t capacity = shard->capacity * 2;
   size_t i;
-  interned_slice_refcount** strtab;
-  interned_slice_refcount *s, *next;
+  InternedSliceRefcount** strtab;
+  InternedSliceRefcount *s, *next;
 
-  strtab = static_cast<interned_slice_refcount**>(
-      gpr_zalloc(sizeof(interned_slice_refcount*) * capacity));
+  strtab = static_cast<InternedSliceRefcount**>(
+      gpr_zalloc(sizeof(InternedSliceRefcount*) * capacity));
 
   for (i = 0; i < shard->capacity; i++) {
     for (s = shard->strs[i]; s; s = next) {
@@ -150,7 +106,7 @@ static void grow_shard(slice_shard* shard) {
   shard->capacity = capacity;
 }
 
-static grpc_slice materialize(interned_slice_refcount* s) {
+static grpc_slice materialize(InternedSliceRefcount* s) {
   grpc_slice slice;
   slice.refcount = &s->base;
   slice.data.refcounted.bytes = reinterpret_cast<uint8_t*>(s + 1);
@@ -164,7 +120,7 @@ uint32_t grpc_slice_default_hash_impl(grpc_slice s) {
 }
 
 uint32_t grpc_static_slice_hash(grpc_slice s) {
-  return static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(s)];
+  return grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(s)];
 }
 
 int grpc_static_slice_eq(grpc_slice a, grpc_slice b) {
@@ -173,7 +129,7 @@ int grpc_static_slice_eq(grpc_slice a, grpc_slice b) {
 
 uint32_t grpc_slice_hash(grpc_slice s) {
   return s.refcount == nullptr ? grpc_slice_default_hash_impl(s)
-                               : s.refcount->vtable->hash(s);
+                               : s.refcount->Hash(s);
 }
 
 grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
@@ -197,8 +153,9 @@ grpc_slice grpc_slice_maybe_static_intern(grpc_slice slice,
 }
 
 bool grpc_slice_is_interned(const grpc_slice& slice) {
-  return (slice.refcount && slice.refcount->vtable == &interned_slice_vtable) ||
-         GRPC_IS_STATIC_METADATA_STRING(slice);
+  return (slice.refcount &&
+          (slice.refcount->GetType() == grpc_slice_refcount::Type::INTERNED ||
+           GRPC_IS_STATIC_METADATA_STRING(slice)));
 }
 
 grpc_slice grpc_slice_intern(grpc_slice slice) {
@@ -208,6 +165,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
   }
 
   uint32_t hash = grpc_slice_hash(slice);
+
   for (uint32_t i = 0; i <= max_static_metadata_hash_probe; i++) {
     static_metadata_hash_ent ent =
         static_metadata_hash[(hash + i) % GPR_ARRAY_SIZE(static_metadata_hash)];
@@ -217,7 +175,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
     }
   }
 
-  interned_slice_refcount* s;
+  InternedSliceRefcount* s;
   slice_shard* shard = &g_shards[SHARD_IDX(hash)];
 
   gpr_mu_lock(&shard->mu);
@@ -226,14 +184,7 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
   size_t idx = TABLE_IDX(hash, shard->capacity);
   for (s = shard->strs[idx]; s; s = s->bucket_next) {
     if (s->hash == hash && grpc_slice_eq(slice, materialize(s))) {
-      if (gpr_atm_no_barrier_fetch_add(&s->refcnt, 1) == 0) {
-        /* If we get here, we've added a ref to something that was about to
-         * die - drop it immediately.
-         * The *only* possible path here (given the shard mutex) should be to
-         * drop from one ref back to zero - assert that with a CAS */
-        GPR_ASSERT(gpr_atm_rel_cas(&s->refcnt, 1, 0));
-        /* and treat this as if we were never here... sshhh */
-      } else {
+      if (s->refcnt.RefIfNonZero()) {
         gpr_mu_unlock(&shard->mu);
         return materialize(s);
       }
@@ -242,27 +193,20 @@ grpc_slice grpc_slice_intern(grpc_slice slice) {
 
   /* not found: create a new string */
   /* string data goes after the internal_string header */
-  s = static_cast<interned_slice_refcount*>(
+  s = static_cast<InternedSliceRefcount*>(
       gpr_malloc(sizeof(*s) + GRPC_SLICE_LENGTH(slice)));
-  gpr_atm_rel_store(&s->refcnt, 1);
-  s->length = GRPC_SLICE_LENGTH(slice);
-  s->hash = hash;
-  s->base.vtable = &interned_slice_vtable;
-  s->base.sub_refcount = &s->sub;
-  s->sub.vtable = &interned_slice_sub_vtable;
-  s->sub.sub_refcount = &s->sub;
-  s->bucket_next = shard->strs[idx];
-  shard->strs[idx] = s;
-  memcpy(s + 1, GRPC_SLICE_START_PTR(slice), GRPC_SLICE_LENGTH(slice));
 
+  new (s) grpc_core::InternedSliceRefcount(GRPC_SLICE_LENGTH(slice), hash,
+                                           shard->strs[idx]);
+  memcpy(reinterpret_cast<char*>(s + 1), GRPC_SLICE_START_PTR(slice),
+         GRPC_SLICE_LENGTH(slice));
+  shard->strs[idx] = s;
   shard->count++;
-
   if (shard->count > shard->capacity * 2) {
     grow_shard(shard);
   }
 
   gpr_mu_unlock(&shard->mu);
-
   return materialize(s);
 }
 
@@ -280,7 +224,7 @@ void grpc_slice_intern_init(void) {
     gpr_mu_init(&shard->mu);
     shard->count = 0;
     shard->capacity = INITIAL_SHARD_CAPACITY;
-    shard->strs = static_cast<interned_slice_refcount**>(
+    shard->strs = static_cast<InternedSliceRefcount**>(
         gpr_zalloc(sizeof(*shard->strs) * shard->capacity));
   }
   for (size_t i = 0; i < GPR_ARRAY_SIZE(static_metadata_hash); i++) {
@@ -289,13 +233,13 @@ void grpc_slice_intern_init(void) {
   }
   max_static_metadata_hash_probe = 0;
   for (size_t i = 0; i < GRPC_STATIC_MDSTR_COUNT; i++) {
-    static_metadata_hash_values[i] =
+    grpc_static_metadata_hash_values[i] =
         grpc_slice_default_hash_impl(grpc_static_slice_table[i]);
     for (size_t j = 0; j < GPR_ARRAY_SIZE(static_metadata_hash); j++) {
-      size_t slot = (static_metadata_hash_values[i] + j) %
+      size_t slot = (grpc_static_metadata_hash_values[i] + j) %
                     GPR_ARRAY_SIZE(static_metadata_hash);
       if (static_metadata_hash[slot].idx == GRPC_STATIC_MDSTR_COUNT) {
-        static_metadata_hash[slot].hash = static_metadata_hash_values[i];
+        static_metadata_hash[slot].hash = grpc_static_metadata_hash_values[i];
         static_metadata_hash[slot].idx = static_cast<uint32_t>(i);
         if (j > max_static_metadata_hash_probe) {
           max_static_metadata_hash_probe = static_cast<uint32_t>(j);
@@ -315,8 +259,7 @@ void grpc_slice_intern_shutdown(void) {
       gpr_log(GPR_DEBUG, "WARNING: %" PRIuPTR " metadata strings were leaked",
               shard->count);
       for (size_t j = 0; j < shard->capacity; j++) {
-        for (interned_slice_refcount* s = shard->strs[j]; s;
-             s = s->bucket_next) {
+        for (InternedSliceRefcount* s = shard->strs[j]; s; s = s->bucket_next) {
           char* text =
               grpc_dump_slice(materialize(s), GPR_DUMP_HEX | GPR_DUMP_ASCII);
           gpr_log(GPR_DEBUG, "LEAKED: %s", text);

src/core/lib/slice/slice_internal.h

@@ -23,17 +23,215 @@
 
 #include <grpc/slice.h>
 #include <grpc/slice_buffer.h>
+#include <string.h>
 
-inline const grpc_slice& grpc_slice_ref_internal(const grpc_slice& slice) {
+#include "src/core/lib/gpr/murmur_hash.h"
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/transport/static_metadata.h"
+
+// Interned slices have specific fast-path operations for hashing. To inline
+// these operations, we need to forward declare them here.
+extern uint32_t grpc_static_metadata_hash_values[GRPC_STATIC_MDSTR_COUNT];
+extern uint32_t g_hash_seed;
+
+// grpc_slice_refcount : A reference count for grpc_slice.
+//
+// Non-inlined grpc_slice objects are refcounted. Historically this was
+// implemented via grpc_slice_refcount, a C-style polymorphic class using a
+// manually managed vtable of operations. Subclasses would define their own
+// vtable; the 'virtual' methods (ref, unref, equals and hash) would simply call
+// the function pointers in the vtable as necessary.
+//
+// Unfortunately, this leads to some inefficiencies in the generated code that
+// can be improved upon. For example, equality checking for interned slices is a
+// simple equality check on the refcount pointer. With the vtable approach, this
+// would translate to roughly the following (high-level) instructions:
+//
+// grpc_slice_equals(slice1, slice2):
+//   load vtable->eq -> eq_func
+//   call eq_func(slice1, slice2)
+//
+// interned_slice_equals(slice1, slice2)
+//   load slice1.ref -> r1
+//   load slice2.ref -> r2
+//   cmp r1, r2 -> retval
+//   ret retval
+//
+// This leads to a function call for a function defined in another translation
+// unit, which imposes memory barriers, which reduces the compiler's ability to
+// optimize (in addition to the added overhead of call/ret). Additionally, it
+// may be harder to reason about branch prediction when we're jumping to
+// essentially arbitrarily provided function pointers.
+//
+// In addition, it is arguable that while virtualization was helpful for
+// Equals()/Hash() methods, that it was fundamentally unnecessary for
+// Ref()/Unref().
+//
+// Instead, grpc_slice_refcount provides the same functionality as the C-style
+// virtual class, but in a de-virtualized manner - Eq(), Hash(), Ref() and
+// Unref() are provided within this header file. Fastpaths for Eq()/Hash()
+// (interned and static metadata slices), as well as the Ref() operation, can
+// all be inlined without any memory barriers.
+//
+// It does this by:
+// 1. Using grpc_core::RefCount<> (header-only) for Ref/Unref. Two special cases
+//    need support: No-op ref/unref (eg. static metadata slices) and stream
+//    slice references (where all the slices share the streamref). This is in
+//    addition to the normal case of '1 slice, 1 ref'.
+//    To support these cases, we explicitly track a nullable pointer to the
+//    underlying RefCount<>. No-op ref/unref is used by checking the pointer for
+//    null, and doing nothing if it is. Both stream slice refs and 'normal'
+//    slices use the same path for Ref/Unref (by targeting the non-null
+//    pointer).
+//
+// 2. introducing the notion of grpc_slice_refcount::Type. This describes if a
+//    slice ref is used by a static metadata slice, an interned slice, or other
+//    slices. We switch on the slice ref type in order to provide fastpaths for
+//    Equals() and Hash().
+//
+// In total, this saves us roughly 1-2% latency for unary calls, with smaller
+// calls benefitting. The effect is present, but not as useful, for larger calls
+// where the cost of sending the data dominates.
+struct grpc_slice_refcount {
+ public:
+  enum class Type {
+    STATIC,    // Refcount for a static metadata slice.
+    INTERNED,  // Refcount for an interned slice.
+    REGULAR    // Refcount for non-static-metadata, non-interned slices.
+  };
+  typedef void (*DestroyerFn)(void*);
+
+  grpc_slice_refcount() = default;
+
+  explicit grpc_slice_refcount(grpc_slice_refcount* sub) : sub_refcount_(sub) {}
+  // Regular constructor for grpc_slice_refcount.
+  //
+  // Parameters:
+  //  1. grpc_slice_refcount::Type type
+  //  Whether we are the refcount for a static
+  //  metadata slice, an interned slice, or any other kind of slice.
+  //
+  //  2. RefCount* ref
+  //  The pointer to the actual underlying grpc_core::RefCount. Rather than
+  //  performing struct offset computations as in the original implementation to
+  //  get to the refcount, which requires a virtual method, we devirtualize by
+  //  using a nullable pointer to allow a single pair of Ref/Unref methods.
+  //
+  //  3. DestroyerFn destroyer_fn
+  //  Called when the refcount goes to 0, with destroyer_arg as parameter.
+  //
+  //  4. void* destroyer_arg
+  //  Argument for the virtualized destructor.
+  //
+  //  5. grpc_slice_refcount* sub
+  //  Argument used for interned slices.
+  grpc_slice_refcount(grpc_slice_refcount::Type type, grpc_core::RefCount* ref,
+                      DestroyerFn destroyer_fn, void* destroyer_arg,
+                      grpc_slice_refcount* sub)
+      : ref_(ref),
+        ref_type_(type),
+        sub_refcount_(sub),
+        dest_fn_(destroyer_fn),
+        destroy_fn_arg_(destroyer_arg) {}
+  // Initializer for static refcounts.
+  grpc_slice_refcount(grpc_slice_refcount* sub, Type type)
+      : ref_type_(type), sub_refcount_(sub) {}
+
+  Type GetType() const { return ref_type_; }
+
+  int Eq(const grpc_slice& a, const grpc_slice& b);
+
+  uint32_t Hash(const grpc_slice& slice);
+  void Ref() {
+    if (ref_ == nullptr) return;
+    ref_->RefNonZero();
+  }
+  void Unref() {
+    if (ref_ == nullptr) return;
+    if (ref_->Unref()) {
+      dest_fn_(destroy_fn_arg_);
+    }
+  }
+
+  grpc_slice_refcount* sub_refcount() const { return sub_refcount_; }
+
+ private:
+  grpc_core::RefCount* ref_ = nullptr;
+  const Type ref_type_ = Type::REGULAR;
+  grpc_slice_refcount* sub_refcount_ = this;
+  DestroyerFn dest_fn_ = nullptr;
+  void* destroy_fn_arg_ = nullptr;
+};
+
+namespace grpc_core {
+
+struct InternedSliceRefcount {
+  static void Destroy(void* arg) {
+    auto* rc = static_cast<InternedSliceRefcount*>(arg);
+    rc->~InternedSliceRefcount();
+    gpr_free(rc);
+  }
+
+  InternedSliceRefcount(size_t length, uint32_t hash,
+                        InternedSliceRefcount* bucket_next)
+      : base(grpc_slice_refcount::Type::INTERNED, &refcnt, Destroy, this, &sub),
+        sub(grpc_slice_refcount::Type::REGULAR, &refcnt, Destroy, this, &sub),
+        length(length),
+        hash(hash),
+        bucket_next(bucket_next) {}
+
+  ~InternedSliceRefcount();
+
+  grpc_slice_refcount base;
+  grpc_slice_refcount sub;
+  const size_t length;
+  RefCount refcnt;
+  const uint32_t hash;
+  InternedSliceRefcount* bucket_next;
+};
+
+}  // namespace grpc_core
+
+inline int grpc_slice_refcount::Eq(const grpc_slice& a, const grpc_slice& b) {
+  switch (ref_type_) {
+    case Type::STATIC:
+      return GRPC_STATIC_METADATA_INDEX(a) == GRPC_STATIC_METADATA_INDEX(b);
+    case Type::INTERNED:
+      return a.refcount == b.refcount;
+    case Type::REGULAR:
+      break;
+  }
+  if (GRPC_SLICE_LENGTH(a) != GRPC_SLICE_LENGTH(b)) return false;
+  if (GRPC_SLICE_LENGTH(a) == 0) return true;
+  return 0 == memcmp(GRPC_SLICE_START_PTR(a), GRPC_SLICE_START_PTR(b),
+                     GRPC_SLICE_LENGTH(a));
+}
+
+inline uint32_t grpc_slice_refcount::Hash(const grpc_slice& slice) {
+  switch (ref_type_) {
+    case Type::STATIC:
+      return ::grpc_static_metadata_hash_values[GRPC_STATIC_METADATA_INDEX(
+          slice)];
+    case Type::INTERNED:
+      return reinterpret_cast<grpc_core::InternedSliceRefcount*>(slice.refcount)
+          ->hash;
+    case Type::REGULAR:
+      break;
+  }
+  return gpr_murmur_hash3(GRPC_SLICE_START_PTR(slice), GRPC_SLICE_LENGTH(slice),
+                          g_hash_seed);
+}
+
+inline grpc_slice grpc_slice_ref_internal(const grpc_slice& slice) {
   if (slice.refcount) {
-    slice.refcount->vtable->ref(slice.refcount);
+    slice.refcount->Ref();
   }
   return slice;
 }
 
 inline void grpc_slice_unref_internal(const grpc_slice& slice) {
   if (slice.refcount) {
-    slice.refcount->vtable->unref(slice.refcount);
+    slice.refcount->Unref();
   }
 }
 

src/core/lib/surface/call.cc

@@ -131,7 +131,6 @@ struct grpc_call {
         is_client(args.server_transport_data == nullptr),
         stream_op_payload(context) {
     gpr_ref_init(&ext_ref, 1);
-    grpc_call_combiner_init(&call_combiner);
     for (int i = 0; i < 2; i++) {
       for (int j = 0; j < 2; j++) {
         metadata_batch[i][j].deadline = GRPC_MILLIS_INF_FUTURE;
@@ -141,12 +140,11 @@ struct grpc_call {
 
   ~grpc_call() {
     gpr_free(static_cast<void*>(const_cast<char*>(final_info.error_string)));
-    grpc_call_combiner_destroy(&call_combiner);
   }
 
   gpr_refcount ext_ref;
   gpr_arena* arena;
-  grpc_call_combiner call_combiner;
+  grpc_core::CallCombiner call_combiner;
   grpc_completion_queue* cq;
   grpc_polling_entity pollent;
   grpc_channel* channel;
@@ -589,7 +587,7 @@ void grpc_call_unref(grpc_call* c) {
     // holding to the call stack. Also flush the closures on exec_ctx so that
     // filters that schedule cancel notification closures on exec_ctx do not
     // need to take a ref of the call stack to guarantee closure liveness.
-    grpc_call_combiner_set_notify_on_cancel(&c->call_combiner, nullptr);
+    c->call_combiner.SetNotifyOnCancel(nullptr);
     grpc_core::ExecCtx::Get()->Flush();
   }
   GRPC_CALL_INTERNAL_UNREF(c, "destroy");
@@ -685,7 +683,7 @@ static void cancel_with_error(grpc_call* c, grpc_error* error) {
   // any in-flight asynchronous actions that may be holding the call
   // combiner.  This ensures that the cancel_stream batch can be sent
   // down the filter stack in a timely manner.
-  grpc_call_combiner_cancel(&c->call_combiner, GRPC_ERROR_REF(error));
+  c->call_combiner.Cancel(GRPC_ERROR_REF(error));
   cancel_state* state = static_cast<cancel_state*>(gpr_malloc(sizeof(*state)));
   state->call = c;
   GRPC_CLOSURE_INIT(&state->finish_batch, done_termination, state,

src/core/lib/surface/lame_client.cc

@@ -39,7 +39,7 @@ namespace grpc_core {
 namespace {
 
 struct CallData {
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
   grpc_linked_mdelem status;
   grpc_linked_mdelem details;
   grpc_core::Atomic<bool> filled_metadata;

src/core/lib/surface/server.cc

@@ -123,7 +123,7 @@ typedef struct shutdown_tag {
 typedef enum {
   /* waiting for metadata */
   NOT_STARTED,
-  /* inital metadata read, not flow controlled in yet */
+  /* initial metadata read, not flow controlled in yet */
   PENDING,
   /* flow controlled in, on completion queue */
   ACTIVATED,
@@ -190,7 +190,7 @@ struct call_data {
   grpc_closure publish;
 
   call_data* pending_next = nullptr;
-  grpc_call_combiner* call_combiner;
+  grpc_core::CallCombiner* call_combiner;
 };
 
 struct request_matcher {

src/core/lib/transport/metadata.h

@@ -124,7 +124,18 @@ grpc_mdelem grpc_mdelem_create(
     const grpc_slice& key, const grpc_slice& value,
     grpc_mdelem_data* compatible_external_backing_store);
 
+#define GRPC_MDKEY(md) (GRPC_MDELEM_DATA(md)->key)
+#define GRPC_MDVALUE(md) (GRPC_MDELEM_DATA(md)->value)
+
 bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b);
+/* Often we compare metadata where we know a-priori that the second parameter is
+ * static, and that the keys match. This most commonly happens when processing
+ * metadata batch callouts in initial/trailing filters. In this case, fastpath
+ * grpc_mdelem_eq and remove unnecessary checks. */
+inline bool grpc_mdelem_static_value_eq(grpc_mdelem a, grpc_mdelem b_static) {
+  if (a.payload == b_static.payload) return true;
+  return grpc_slice_eq(GRPC_MDVALUE(a), GRPC_MDVALUE(b_static));
+}
 
 /* Mutator and accessor for grpc_mdelem user data. The destructor function
    is used as a type tag and is checked during user_data fetch. */
@@ -144,9 +155,6 @@ grpc_mdelem grpc_mdelem_ref(grpc_mdelem md);
 void grpc_mdelem_unref(grpc_mdelem md);
 #endif
 
-#define GRPC_MDKEY(md) (GRPC_MDELEM_DATA(md)->key)
-#define GRPC_MDVALUE(md) (GRPC_MDELEM_DATA(md)->value)
-
 #define GRPC_MDNULL GRPC_MAKE_MDELEM(NULL, GRPC_MDELEM_STORAGE_EXTERNAL)
 #define GRPC_MDISNULL(md) (GRPC_MDELEM_DATA(md) == NULL)
 

src/core/lib/transport/static_metadata.cc

@@ -116,123 +116,115 @@ static uint8_t g_bytes[] = {
     103, 122, 105, 112, 105, 100, 101, 110, 116, 105, 116, 121, 44,  100, 101,
     102, 108, 97,  116, 101, 44,  103, 122, 105, 112};
 
-static void static_ref(void* unused) {}
-static void static_unref(void* unused) {}
-static const grpc_slice_refcount_vtable static_sub_vtable = {
-    static_ref, static_unref, grpc_slice_default_eq_impl,
-    grpc_slice_default_hash_impl};
-const grpc_slice_refcount_vtable grpc_static_metadata_vtable = {
-    static_ref, static_unref, grpc_static_slice_eq, grpc_static_slice_hash};
-static grpc_slice_refcount static_sub_refcnt = {&static_sub_vtable,
-                                                &static_sub_refcnt};
+static grpc_slice_refcount static_sub_refcnt;
 grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = {
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
-    {&grpc_static_metadata_vtable, &static_sub_refcnt},
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
+    grpc_slice_refcount(&static_sub_refcnt, grpc_slice_refcount::Type::STATIC),
 };
 
 const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = {

src/core/lib/transport/static_metadata.h

@@ -256,12 +256,11 @@ extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \
   (grpc_static_slice_table[106])
 
-extern const grpc_slice_refcount_vtable grpc_static_metadata_vtable;
 extern grpc_slice_refcount
     grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT];
 #define GRPC_IS_STATIC_METADATA_STRING(slice) \
   ((slice).refcount != NULL &&                \
-   (slice).refcount->vtable == &grpc_static_metadata_vtable)
+   (slice).refcount->GetType() == grpc_slice_refcount::Type::STATIC)
 
 #define GRPC_STATIC_METADATA_INDEX(static_slice) \
   ((int)((static_slice).refcount - grpc_static_metadata_refcounts))

src/core/lib/transport/status_metadata.cc

@@ -31,13 +31,13 @@
 static void destroy_status(void* ignored) {}
 
 grpc_status_code grpc_get_status_code_from_metadata(grpc_mdelem md) {
-  if (grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) {
+  if (grpc_mdelem_static_value_eq(md, GRPC_MDELEM_GRPC_STATUS_0)) {
     return GRPC_STATUS_OK;
   }
-  if (grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_1)) {
+  if (grpc_mdelem_static_value_eq(md, GRPC_MDELEM_GRPC_STATUS_1)) {
     return GRPC_STATUS_CANCELLED;
   }
-  if (grpc_mdelem_eq(md, GRPC_MDELEM_GRPC_STATUS_2)) {
+  if (grpc_mdelem_static_value_eq(md, GRPC_MDELEM_GRPC_STATUS_2)) {
     return GRPC_STATUS_UNKNOWN;
   }
   void* user_data = grpc_mdelem_get_user_data(md, destroy_status);

src/core/lib/transport/transport.cc

@@ -39,72 +39,39 @@
 grpc_core::DebugOnlyTraceFlag grpc_trace_stream_refcount(false,
                                                          "stream_refcount");
 
-#ifndef NDEBUG
-void grpc_stream_ref(grpc_stream_refcount* refcount, const char* reason) {
-  if (grpc_trace_stream_refcount.enabled()) {
-    gpr_atm val = gpr_atm_no_barrier_load(&refcount->refs.count);
-    gpr_log(GPR_DEBUG, "%s %p:%p   REF %" PRIdPTR "->%" PRIdPTR " %s",
-            refcount->object_type, refcount, refcount->destroy.cb_arg, val,
-            val + 1, reason);
+void grpc_stream_destroy(grpc_stream_refcount* refcount) {
+  if (!grpc_iomgr_is_any_background_poller_thread() &&
+      (grpc_core::ExecCtx::Get()->flags() &
+       GRPC_EXEC_CTX_FLAG_THREAD_RESOURCE_LOOP)) {
+    /* Ick.
+       The thread we're running on MAY be owned (indirectly) by a call-stack.
+       If that's the case, destroying the call-stack MAY try to destroy the
+       thread, which is a tangled mess that we just don't want to ever have to
+       cope with.
+       Throw this over to the executor (on a core-owned thread) and process it
+       there. */
+    refcount->destroy.scheduler =
+        grpc_core::Executor::Scheduler(grpc_core::ExecutorJobType::SHORT);
   }
-#else
-void grpc_stream_ref(grpc_stream_refcount* refcount) {
-#endif
-  gpr_ref_non_zero(&refcount->refs);
+  GRPC_CLOSURE_SCHED(&refcount->destroy, GRPC_ERROR_NONE);
 }
 
-#ifndef NDEBUG
-void grpc_stream_unref(grpc_stream_refcount* refcount, const char* reason) {
-  if (grpc_trace_stream_refcount.enabled()) {
-    gpr_atm val = gpr_atm_no_barrier_load(&refcount->refs.count);
-    gpr_log(GPR_DEBUG, "%s %p:%p UNREF %" PRIdPTR "->%" PRIdPTR " %s",
-            refcount->object_type, refcount, refcount->destroy.cb_arg, val,
-            val - 1, reason);
-  }
-#else
-void grpc_stream_unref(grpc_stream_refcount* refcount) {
-#endif
-  if (gpr_unref(&refcount->refs)) {
-    if (!grpc_iomgr_is_any_background_poller_thread() &&
-        (grpc_core::ExecCtx::Get()->flags() &
-         GRPC_EXEC_CTX_FLAG_THREAD_RESOURCE_LOOP)) {
-      /* Ick.
-         The thread we're running on MAY be owned (indirectly) by a call-stack.
-         If that's the case, destroying the call-stack MAY try to destroy the
-         thread, which is a tangled mess that we just don't want to ever have to
-         cope with.
-         Throw this over to the executor (on a core-owned thread) and process it
-         there. */
-      refcount->destroy.scheduler =
-          grpc_core::Executor::Scheduler(grpc_core::ExecutorJobType::SHORT);
-    }
-    GRPC_CLOSURE_SCHED(&refcount->destroy, GRPC_ERROR_NONE);
-  }
+void slice_stream_destroy(void* arg) {
+  grpc_stream_destroy(static_cast<grpc_stream_refcount*>(arg));
 }
 
 #define STREAM_REF_FROM_SLICE_REF(p)       \
   ((grpc_stream_refcount*)(((uint8_t*)p) - \
                            offsetof(grpc_stream_refcount, slice_refcount)))
 
-static void slice_stream_ref(void* p) {
-#ifndef NDEBUG
-  grpc_stream_ref(STREAM_REF_FROM_SLICE_REF(p), "slice");
-#else
-  grpc_stream_ref(STREAM_REF_FROM_SLICE_REF(p));
-#endif
-}
-
-static void slice_stream_unref(void* p) {
+grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount* refcount,
+                                               void* buffer, size_t length) {
 #ifndef NDEBUG
-  grpc_stream_unref(STREAM_REF_FROM_SLICE_REF(p), "slice");
+  grpc_stream_ref(STREAM_REF_FROM_SLICE_REF(&refcount->slice_refcount),
+                  "slice");
 #else
-  grpc_stream_unref(STREAM_REF_FROM_SLICE_REF(p));
+  grpc_stream_ref(STREAM_REF_FROM_SLICE_REF(&refcount->slice_refcount));
 #endif
-}
-
-grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount* refcount,
-                                               void* buffer, size_t length) {
-  slice_stream_ref(&refcount->slice_refcount);
   grpc_slice res;
   res.refcount = &refcount->slice_refcount;
   res.data.refcounted.bytes = static_cast<uint8_t*>(buffer);
@@ -112,13 +79,6 @@ grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount* refcount,
   return res;
 }
 
-static const grpc_slice_refcount_vtable stream_ref_slice_vtable = {
-    slice_stream_ref,            /* ref */
-    slice_stream_unref,          /* unref */
-    grpc_slice_default_eq_impl,  /* eq */
-    grpc_slice_default_hash_impl /* hash */
-};
-
 #ifndef NDEBUG
 void grpc_stream_ref_init(grpc_stream_refcount* refcount, int initial_refs,
                           grpc_iomgr_cb_func cb, void* cb_arg,
@@ -128,10 +88,12 @@ void grpc_stream_ref_init(grpc_stream_refcount* refcount, int initial_refs,
 void grpc_stream_ref_init(grpc_stream_refcount* refcount, int initial_refs,
                           grpc_iomgr_cb_func cb, void* cb_arg) {
 #endif
-  gpr_ref_init(&refcount->refs, initial_refs);
   GRPC_CLOSURE_INIT(&refcount->destroy, cb, cb_arg, grpc_schedule_on_exec_ctx);
-  refcount->slice_refcount.vtable = &stream_ref_slice_vtable;
-  refcount->slice_refcount.sub_refcount = &refcount->slice_refcount;
+
+  new (&refcount->refs) grpc_core::RefCount();
+  new (&refcount->slice_refcount) grpc_slice_refcount(
+      grpc_slice_refcount::Type::REGULAR, &refcount->refs, slice_stream_destroy,
+      refcount, &refcount->slice_refcount);
 }
 
 static void move64(uint64_t* from, uint64_t* to) {
@@ -212,7 +174,7 @@ grpc_endpoint* grpc_transport_get_endpoint(grpc_transport* transport) {
 // it's grpc_transport_stream_op_batch_finish_with_failure
 void grpc_transport_stream_op_batch_finish_with_failure(
     grpc_transport_stream_op_batch* batch, grpc_error* error,
-    grpc_call_combiner* call_combiner) {
+    grpc_core::CallCombiner* call_combiner) {
   if (batch->send_message) {
     batch->payload->send_message.send_message.reset();
   }

src/core/lib/transport/transport.h

@@ -30,6 +30,7 @@
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/transport/byte_stream.h"
 #include "src/core/lib/transport/metadata_batch.h"
 
@@ -51,7 +52,7 @@ typedef struct grpc_stream grpc_stream;
 extern grpc_core::DebugOnlyTraceFlag grpc_trace_stream_refcount;
 
 typedef struct grpc_stream_refcount {
-  gpr_refcount refs;
+  grpc_core::RefCount refs;
   grpc_closure destroy;
 #ifndef NDEBUG
   const char* object_type;
@@ -63,19 +64,45 @@ typedef struct grpc_stream_refcount {
 void grpc_stream_ref_init(grpc_stream_refcount* refcount, int initial_refs,
                           grpc_iomgr_cb_func cb, void* cb_arg,
                           const char* object_type);
-void grpc_stream_ref(grpc_stream_refcount* refcount, const char* reason);
-void grpc_stream_unref(grpc_stream_refcount* refcount, const char* reason);
 #define GRPC_STREAM_REF_INIT(rc, ir, cb, cb_arg, objtype) \
   grpc_stream_ref_init(rc, ir, cb, cb_arg, objtype)
 #else
 void grpc_stream_ref_init(grpc_stream_refcount* refcount, int initial_refs,
                           grpc_iomgr_cb_func cb, void* cb_arg);
-void grpc_stream_ref(grpc_stream_refcount* refcount);
-void grpc_stream_unref(grpc_stream_refcount* refcount);
 #define GRPC_STREAM_REF_INIT(rc, ir, cb, cb_arg, objtype) \
   grpc_stream_ref_init(rc, ir, cb, cb_arg)
 #endif
 
+#ifndef NDEBUG
+inline void grpc_stream_ref(grpc_stream_refcount* refcount,
+                            const char* reason) {
+  if (grpc_trace_stream_refcount.enabled()) {
+    gpr_log(GPR_DEBUG, "%s %p:%p REF %s", refcount->object_type, refcount,
+            refcount->destroy.cb_arg, reason);
+  }
+#else
+inline void grpc_stream_ref(grpc_stream_refcount* refcount) {
+#endif
+  refcount->refs.RefNonZero();
+}
+
+void grpc_stream_destroy(grpc_stream_refcount* refcount);
+
+#ifndef NDEBUG
+inline void grpc_stream_unref(grpc_stream_refcount* refcount,
+                              const char* reason) {
+  if (grpc_trace_stream_refcount.enabled()) {
+    gpr_log(GPR_DEBUG, "%s %p:%p UNREF %s", refcount->object_type, refcount,
+            refcount->destroy.cb_arg, reason);
+  }
+#else
+inline void grpc_stream_unref(grpc_stream_refcount* refcount) {
+#endif
+  if (refcount->refs.Unref()) {
+    grpc_stream_destroy(refcount);
+  }
+}
+
 /* Wrap a buffer that is owned by some stream object into a slice that shares
    the same refcount */
 grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount* refcount,
@@ -352,7 +379,7 @@ void grpc_transport_destroy_stream(grpc_transport* transport,
 
 void grpc_transport_stream_op_batch_finish_with_failure(
     grpc_transport_stream_op_batch* op, grpc_error* error,
-    grpc_call_combiner* call_combiner);
+    grpc_core::CallCombiner* call_combiner);
 
 char* grpc_transport_stream_op_batch_string(grpc_transport_stream_op_batch* op);
 char* grpc_transport_op_string(grpc_transport_op* op);

src/cpp/server/server_cc.cc

@@ -281,7 +281,7 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
         auto* handler = resources_ ? method_->handler()
                                    : server_->resource_exhausted_handler_.get();
         request_ = handler->Deserialize(call_.call(), request_payload_,
-                                        &request_status_);
+                                        &request_status_, nullptr);
 
         request_payload_ = nullptr;
         interceptor_methods_.AddInterceptionHookPoint(
@@ -305,7 +305,7 @@ class Server::SyncRequest final : public internal::CompletionQueueTag {
         auto* handler = resources_ ? method_->handler()
                                    : server_->resource_exhausted_handler_.get();
         handler->RunHandler(internal::MethodHandler::HandlerParameter(
-            &call_, &ctx_, request_, request_status_, nullptr));
+            &call_, &ctx_, request_, request_status_, nullptr, nullptr));
         request_ = nullptr;
         global_callbacks_->PostSynchronousRequest(&ctx_);
 
@@ -512,7 +512,8 @@ class Server::CallbackRequest final : public Server::CallbackRequestBase {
       if (req_->has_request_payload_) {
         // Set interception point for RECV MESSAGE
         req_->request_ = req_->method_->handler()->Deserialize(
-            req_->call_, req_->request_payload_, &req_->request_status_);
+            req_->call_, req_->request_payload_, &req_->request_status_,
+            &req_->handler_data_);
         req_->request_payload_ = nullptr;
         req_->interceptor_methods_.AddInterceptionHookPoint(
             experimental::InterceptionHookPoints::POST_RECV_MESSAGE);
@@ -532,7 +533,8 @@ class Server::CallbackRequest final : public Server::CallbackRequestBase {
                           ? req_->method_->handler()
                           : req_->server_->generic_handler_.get();
       handler->RunHandler(internal::MethodHandler::HandlerParameter(
-          call_, &req_->ctx_, req_->request_, req_->request_status_, [this] {
+          call_, &req_->ctx_, req_->request_, req_->request_status_,
+          req_->handler_data_, [this] {
             // Recycle this request if there aren't too many outstanding.
             // Note that we don't have to worry about a case where there
             // are no requests waiting to match for this method since that
@@ -577,6 +579,7 @@ class Server::CallbackRequest final : public Server::CallbackRequestBase {
     ctx_.Setup(gpr_inf_future(GPR_CLOCK_REALTIME));
     request_payload_ = nullptr;
     request_ = nullptr;
+    handler_data_ = nullptr;
     request_status_ = Status();
   }
 
@@ -587,6 +590,7 @@ class Server::CallbackRequest final : public Server::CallbackRequestBase {
   const bool has_request_payload_;
   grpc_byte_buffer* request_payload_;
   void* request_;
+  void* handler_data_;
   Status request_status_;
   grpc_call_details* call_details_ = nullptr;
   grpc_call* call_;

src/objective-c/!ProtoCompiler-gRPCPlugin.podspec

@@ -101,7 +101,7 @@ Pod::Spec.new do |s|
   s.preserve_paths = plugin
 
   # Restrict the protoc version to the one supported by this plugin.
-  s.dependency '!ProtoCompiler', '3.6.1'
+  s.dependency '!ProtoCompiler', '3.7.0'
   # For the Protobuf dependency not to complain:
   s.ios.deployment_target = '7.0'
   s.osx.deployment_target = '10.9'

src/objective-c/!ProtoCompiler.podspec

@@ -36,7 +36,7 @@ Pod::Spec.new do |s|
   # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed
   # before them.
   s.name     = '!ProtoCompiler'
-  v = '3.6.1'
+  v = '3.7.0'
   s.version  = v
   s.summary  = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files'
   s.description = <<-DESC

src/objective-c/manual_tests/GrpcIosTestUITests/GrpcIosTestUITests.m

@@ -90,6 +90,15 @@ int const kNumIterations = 1;
   XCTAssert([testApp.staticTexts[@"Call failed"] waitForExistenceWithTimeout:kWaitTime]);
 }
 
+- (void)expectCallSuccessOrFailed {
+  NSDate *startTime = [NSDate date];
+  while (![testApp.staticTexts[@"Call done"] exists] &&
+         ![testApp.staticTexts[@"Call failed"] exists]) {
+    XCTAssertLessThan([[NSDate date] timeIntervalSinceDate:startTime], kWaitTime);
+    [NSThread sleepForTimeInterval:1];
+  }
+}
+
 - (void)setAirplaneMode:(BOOL)to {
   [settingsApp activate];
   XCUIElement *mySwitch = settingsApp.tables.element.cells.switches[@"Airplane Mode"];
@@ -118,13 +127,6 @@ int const kNumIterations = 1;
   [backButton tap];
 }
 
-- (void)typeText:(NSString *)text inApp:(XCUIApplication *)app {
-  [app typeText:text];
-  // Wait until all events in run loop have been processed
-  while (CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, true) == kCFRunLoopRunHandledSource)
-    ;
-}
-
 - (int)getRandomNumberBetween:(int)min max:(int)max {
   return min + arc4random_uniform((max - min + 1));
 }
@@ -216,24 +218,17 @@ int const kNumIterations = 1;
   // Send test app to background
   [XCUIDevice.sharedDevice pressButton:XCUIDeviceButtonHome];
 
-  // Open safari and goto a URL
-  XCUIApplication *safari =
-      [[XCUIApplication alloc] initWithBundleIdentifier:@"com.apple.mobilesafari"];
-  [safari activate];
-  // Ensure that safari is running in the foreground
-  XCTAssert([safari waitForState:XCUIApplicationStateRunningForeground timeout:5]);
-  // Move cursor to address bar
-  [safari.buttons[@"URL"] tap];
-  // Wait for keyboard to appear
-  [NSThread sleepForTimeInterval:2];
-  // Enter URL
-  [self typeText:@"http://maps.google.com" inApp:safari];
-  // Presses return key
-  [self typeText:@"\n" inApp:safari];
+  // Open stocks app
+  XCUIApplication *stocksApp =
+      [[XCUIApplication alloc] initWithBundleIdentifier:@"com.apple.stocks"];
+  [stocksApp activate];
+  // Ensure that stocks app is running in the foreground
+  XCTAssert([stocksApp waitForState:XCUIApplicationStateRunningForeground timeout:5]);
   // Wait a bit
   int sleepTime = [self getRandomNumberBetween:5 max:10];
   NSLog(@"Sleeping for %d seconds", sleepTime);
   [NSThread sleepForTimeInterval:sleepTime];
+  [stocksApp terminate];
 
   // Make another unary call
   [self doUnaryCall];
@@ -256,8 +251,13 @@ int const kNumIterations = 1;
   [self setWifi:YES];
 
   [testApp activate];
-  // We expect the call to have failed because the network flapped
-  [self expectCallFailed];
+  [self pressButton:@"Stop streaming call"];
+  // The call will fail if the stream gets a read error, else the call will succeed.
+  [self expectCallSuccessOrFailed];
+
+  // Make another unary call, it should succeed
+  [self doUnaryCall];
+  [self expectCallSuccess];
 }
 
 - (void)testConcurrentCalls {

src/objective-c/tests/InteropTests.m

@@ -341,6 +341,90 @@ BOOL isRemoteInteropTest(NSString *host) {
   [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
 }
 
+- (void)testConcurrentRPCsWithErrorsWithV2API {
+  NSMutableArray *completeExpectations = [NSMutableArray array];
+  NSMutableArray *calls = [NSMutableArray array];
+  int num_rpcs = 10;
+  for (int i = 0; i < num_rpcs; ++i) {
+    [completeExpectations
+        addObject:[self expectationWithDescription:
+                            [NSString stringWithFormat:@"Received trailer for RPC %d", i]]];
+
+    RMTSimpleRequest *request = [RMTSimpleRequest message];
+    request.responseType = RMTPayloadType_Compressable;
+    request.responseSize = 314159;
+    request.payload.body = [NSMutableData dataWithLength:271828];
+    if (i % 3 == 0) {
+      request.responseStatus.code = GRPC_STATUS_UNAVAILABLE;
+    } else if (i % 7 == 0) {
+      request.responseStatus.code = GRPC_STATUS_CANCELLED;
+    }
+    GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+    options.transportType = [[self class] transportType];
+    options.PEMRootCertificates = [[self class] PEMRootCertificates];
+    options.hostNameOverride = [[self class] hostNameOverride];
+
+    GRPCUnaryProtoCall *call = [_service
+        unaryCallWithMessage:request
+             responseHandler:[[InteropTestsBlockCallbacks alloc] initWithInitialMetadataCallback:nil
+                                 messageCallback:^(id message) {
+                                   if (message) {
+                                     RMTSimpleResponse *expectedResponse =
+                                         [RMTSimpleResponse message];
+                                     expectedResponse.payload.type = RMTPayloadType_Compressable;
+                                     expectedResponse.payload.body =
+                                         [NSMutableData dataWithLength:314159];
+                                     XCTAssertEqualObjects(message, expectedResponse);
+                                   }
+                                 }
+                                 closeCallback:^(NSDictionary *trailingMetadata, NSError *error) {
+                                   [completeExpectations[i] fulfill];
+                                 }]
+                 callOptions:options];
+    [calls addObject:call];
+  }
+
+  for (int i = 0; i < num_rpcs; ++i) {
+    GRPCUnaryProtoCall *call = calls[i];
+    [call start];
+  }
+  [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
+}
+
+- (void)testConcurrentRPCsWithErrors {
+  NSMutableArray *completeExpectations = [NSMutableArray array];
+  int num_rpcs = 10;
+  for (int i = 0; i < num_rpcs; ++i) {
+    [completeExpectations
+        addObject:[self expectationWithDescription:
+                            [NSString stringWithFormat:@"Received trailer for RPC %d", i]]];
+
+    RMTSimpleRequest *request = [RMTSimpleRequest message];
+    request.responseType = RMTPayloadType_Compressable;
+    request.responseSize = 314159;
+    request.payload.body = [NSMutableData dataWithLength:271828];
+    if (i % 3 == 0) {
+      request.responseStatus.code = GRPC_STATUS_UNAVAILABLE;
+    } else if (i % 7 == 0) {
+      request.responseStatus.code = GRPC_STATUS_CANCELLED;
+    }
+
+    [_service unaryCallWithRequest:request
+                           handler:^(RMTSimpleResponse *response, NSError *error) {
+                             if (error == nil) {
+                               RMTSimpleResponse *expectedResponse = [RMTSimpleResponse message];
+                               expectedResponse.payload.type = RMTPayloadType_Compressable;
+                               expectedResponse.payload.body =
+                                   [NSMutableData dataWithLength:314159];
+                               XCTAssertEqualObjects(response, expectedResponse);
+                             }
+                             [completeExpectations[i] fulfill];
+                           }];
+  }
+
+  [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
+}
+
 - (void)testPacketCoalescing {
   XCTAssertNotNil([[self class] host]);
   __weak XCTestExpectation *expectation = [self expectationWithDescription:@"LargeUnary"];

src/objective-c/tests/MacTests/StressTests.h

@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#import <XCTest/XCTest.h>
+
+#import <GRPCClient/GRPCCallOptions.h>
+
+@interface StressTests : XCTestCase
+/**
+ * Host to send the RPCs to. The base implementation returns nil, which would make all tests to
+ * fail.
+ * Override in a subclass to perform these tests against a specific address.
+ */
++ (NSString *)host;
+
+/**
+ * Bytes of overhead of test proto responses due to encoding. This is used to excercise the behavior
+ * when responses are just above or below the max response size. For some reason, the local and
+ * remote servers enconde responses with different overhead (?), so this is defined per-subclass.
+ */
+- (int32_t)encodingOverhead;
+
+/**
+ * The type of transport to be used. The base implementation returns default. Subclasses should
+ * override to appropriate settings.
+ */
++ (GRPCTransportType)transportType;
+
+/**
+ * The root certificates to be used. The base implementation returns nil. Subclasses should override
+ * to appropriate settings.
+ */
++ (NSString *)PEMRootCertificates;
+
+/**
+ * The root certificates to be used. The base implementation returns nil. Subclasses should override
+ * to appropriate settings.
+ */
++ (NSString *)hostNameOverride;
+
+@end

src/objective-c/tests/MacTests/StressTests.m

@@ -0,0 +1,237 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "StressTests.h"
+
+#import <GRPCClient/GRPCCall+ChannelArg.h>
+#import <GRPCClient/GRPCCall+Tests.h>
+#import <GRPCClient/internal_testing/GRPCCall+InternalTests.h>
+#import <ProtoRPC/ProtoRPC.h>
+#import <RemoteTest/Messages.pbobjc.h>
+#import <RemoteTest/Test.pbobjc.h>
+#import <RemoteTest/Test.pbrpc.h>
+#import <RxLibrary/GRXBufferedPipe.h>
+#import <RxLibrary/GRXWriter+Immediate.h>
+#import <grpc/grpc.h>
+#import <grpc/support/log.h>
+
+#define TEST_TIMEOUT 32
+
+extern const char *kCFStreamVarName;
+
+// Convenience class to use blocks as callbacks
+@interface MacTestsBlockCallbacks : NSObject<GRPCProtoResponseHandler>
+
+- (instancetype)initWithInitialMetadataCallback:(void (^)(NSDictionary *))initialMetadataCallback
+                                messageCallback:(void (^)(id))messageCallback
+                                  closeCallback:(void (^)(NSDictionary *, NSError *))closeCallback;
+
+@end
+
+@implementation MacTestsBlockCallbacks {
+  void (^_initialMetadataCallback)(NSDictionary *);
+  void (^_messageCallback)(id);
+  void (^_closeCallback)(NSDictionary *, NSError *);
+  dispatch_queue_t _dispatchQueue;
+}
+
+- (instancetype)initWithInitialMetadataCallback:(void (^)(NSDictionary *))initialMetadataCallback
+                                messageCallback:(void (^)(id))messageCallback
+                                  closeCallback:(void (^)(NSDictionary *, NSError *))closeCallback {
+  if ((self = [super init])) {
+    _initialMetadataCallback = initialMetadataCallback;
+    _messageCallback = messageCallback;
+    _closeCallback = closeCallback;
+    _dispatchQueue = dispatch_queue_create(nil, DISPATCH_QUEUE_SERIAL);
+  }
+  return self;
+}
+
+- (void)didReceiveInitialMetadata:(NSDictionary *)initialMetadata {
+  if (_initialMetadataCallback) {
+    _initialMetadataCallback(initialMetadata);
+  }
+}
+
+- (void)didReceiveProtoMessage:(GPBMessage *)message {
+  if (_messageCallback) {
+    _messageCallback(message);
+  }
+}
+
+- (void)didCloseWithTrailingMetadata:(NSDictionary *)trailingMetadata error:(NSError *)error {
+  if (_closeCallback) {
+    _closeCallback(trailingMetadata, error);
+  }
+}
+
+- (dispatch_queue_t)dispatchQueue {
+  return _dispatchQueue;
+}
+
+@end
+
+@implementation StressTests {
+  RMTTestService *_service;
+}
+
++ (NSString *)host {
+  return nil;
+}
+
++ (NSString *)hostAddress {
+  return nil;
+}
+
++ (NSString *)PEMRootCertificates {
+  return nil;
+}
+
++ (NSString *)hostNameOverride {
+  return nil;
+}
+
+- (int32_t)encodingOverhead {
+  return 0;
+}
+
++ (void)setUp {
+  setenv(kCFStreamVarName, "1", 1);
+}
+
+- (void)setUp {
+  self.continueAfterFailure = NO;
+
+  [GRPCCall resetHostSettings];
+
+  GRPCMutableCallOptions *options = [[GRPCMutableCallOptions alloc] init];
+  options.transportType = [[self class] transportType];
+  options.PEMRootCertificates = [[self class] PEMRootCertificates];
+  options.hostNameOverride = [[self class] hostNameOverride];
+  _service = [RMTTestService serviceWithHost:[[self class] host] callOptions:options];
+  system([[NSString stringWithFormat:@"sudo ifconfig lo0 alias %@", [[self class] hostAddress]]
+      UTF8String]);
+}
+
+- (void)tearDown {
+  system([[NSString stringWithFormat:@"sudo ifconfig lo0 -alias %@", [[self class] hostAddress]]
+      UTF8String]);
+}
+
++ (GRPCTransportType)transportType {
+  return GRPCTransportTypeChttp2BoringSSL;
+}
+
+- (void)testNetworkFlapWithV2API {
+  NSMutableArray *completeExpectations = [NSMutableArray array];
+  NSMutableArray *calls = [NSMutableArray array];
+  int num_rpcs = 100;
+  __block BOOL address_removed = FALSE;
+  __block BOOL address_readded = FALSE;
+  for (int i = 0; i < num_rpcs; ++i) {
+    [completeExpectations
+        addObject:[self expectationWithDescription:
+                            [NSString stringWithFormat:@"Received trailer for RPC %d", i]]];
+
+    RMTSimpleRequest *request = [RMTSimpleRequest message];
+    request.responseType = RMTPayloadType_Compressable;
+    request.responseSize = 314159;
+    request.payload.body = [NSMutableData dataWithLength:271828];
+
+    GRPCUnaryProtoCall *call = [_service
+        unaryCallWithMessage:request
+             responseHandler:[[MacTestsBlockCallbacks alloc] initWithInitialMetadataCallback:nil
+                                 messageCallback:^(id message) {
+                                   if (message) {
+                                     RMTSimpleResponse *expectedResponse =
+                                         [RMTSimpleResponse message];
+                                     expectedResponse.payload.type = RMTPayloadType_Compressable;
+                                     expectedResponse.payload.body =
+                                         [NSMutableData dataWithLength:314159];
+                                     XCTAssertEqualObjects(message, expectedResponse);
+                                   }
+                                 }
+                                 closeCallback:^(NSDictionary *trailingMetadata, NSError *error) {
+
+                                   @synchronized(self) {
+                                     if (error == nil && !address_removed) {
+                                       system([[NSString
+                                           stringWithFormat:@"sudo ifconfig lo0 -alias %@",
+                                                            [[self class] hostAddress]]
+                                           UTF8String]);
+                                       address_removed = YES;
+                                     } else if (error != nil && !address_readded) {
+                                       system([
+                                           [NSString stringWithFormat:@"sudo ifconfig lo0 alias %@",
+                                                                      [[self class] hostAddress]]
+                                           UTF8String]);
+                                       address_readded = YES;
+                                     }
+                                   }
+                                   [completeExpectations[i] fulfill];
+                                 }]
+                 callOptions:nil];
+    [calls addObject:call];
+  }
+
+  for (int i = 0; i < num_rpcs; ++i) {
+    GRPCUnaryProtoCall *call = calls[i];
+    [call start];
+    [NSThread sleepForTimeInterval:0.1f];
+  }
+  [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
+}
+
+- (void)testNetworkFlapWithV1API {
+  NSMutableArray *completeExpectations = [NSMutableArray array];
+  int num_rpcs = 100;
+  __block BOOL address_removed = FALSE;
+  __block BOOL address_readded = FALSE;
+  for (int i = 0; i < num_rpcs; ++i) {
+    [completeExpectations
+        addObject:[self expectationWithDescription:
+                            [NSString stringWithFormat:@"Received response for RPC %d", i]]];
+
+    RMTSimpleRequest *request = [RMTSimpleRequest message];
+    request.responseType = RMTPayloadType_Compressable;
+    request.responseSize = 314159;
+    request.payload.body = [NSMutableData dataWithLength:271828];
+
+    [_service unaryCallWithRequest:request
+                           handler:^(RMTSimpleResponse *response, NSError *error) {
+                             @synchronized(self) {
+                               if (error == nil && !address_removed) {
+                                 system([[NSString stringWithFormat:@"sudo ifconfig lo0 -alias %@",
+                                                                    [[self class] hostAddress]]
+                                     UTF8String]);
+                                 address_removed = YES;
+                               } else if (error != nil && !address_readded) {
+                                 system([[NSString stringWithFormat:@"sudo ifconfig lo0 alias %@",
+                                                                    [[self class] hostAddress]]
+                                     UTF8String]);
+                                 address_readded = YES;
+                               }
+                             }
+
+                             [completeExpectations[i] fulfill];
+                           }];
+
+    [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil];
+  }
+}
+
+@end

src/objective-c/tests/MacTests/StressTestsCleartext.m

@@ -0,0 +1,68 @@
+
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#import <GRPCClient/GRPCCall+Tests.h>
+#import <GRPCClient/internal_testing/GRPCCall+InternalTests.h>
+
+#import "StressTests.h"
+
+static NSString *const kHostAddress = @"10.0.0.1";
+
+// The Protocol Buffers encoding overhead of local interop server. Acquired
+// by experiment. Adjust this when server's proto file changes.
+static int32_t kLocalInteropServerOverhead = 10;
+
+/** Tests in InteropTests.m, sending the RPCs to a local cleartext server. */
+@interface StressTestsCleartext : StressTests
+@end
+
+@implementation StressTestsCleartext
+
++ (NSString *)host {
+  return [NSString stringWithFormat:@"%@:5050", kHostAddress];
+}
+
++ (NSString *)hostAddress {
+  return kHostAddress;
+}
+
++ (NSString *)PEMRootCertificates {
+  return nil;
+}
+
++ (NSString *)hostNameOverride {
+  return nil;
+}
+
+- (int32_t)encodingOverhead {
+  return kLocalInteropServerOverhead;  // bytes
+}
+
+- (void)setUp {
+  [super setUp];
+
+  // Register test server as non-SSL.
+  [GRPCCall useInsecureConnectionsForHost:[[self class] host]];
+}
+
++ (GRPCTransportType)transportType {
+  return GRPCTransportTypeInsecure;
+}
+
+@end

src/objective-c/tests/MacTests/StressTestsSSL.m

@@ -0,0 +1,71 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#import <GRPCClient/GRPCCall+Tests.h>
+#import <GRPCClient/internal_testing/GRPCCall+InternalTests.h>
+
+#include "StressTests.h"
+
+static NSString *const kHostAddress = @"10.0.0.1";
+// The Protocol Buffers encoding overhead of local interop server. Acquired
+// by experiment. Adjust this when server's proto file changes.
+static int32_t kLocalInteropServerOverhead = 10;
+
+@interface StressTestsSSL : StressTests
+@end
+
+@implementation StressTestsSSL
+
++ (NSString *)host {
+  return [NSString stringWithFormat:@"%@:5051", kHostAddress];
+}
+
++ (NSString *)hostAddress {
+  return kHostAddress;
+}
+
++ (NSString *)PEMRootCertificates {
+  NSBundle *bundle = [NSBundle bundleForClass:[self class]];
+  NSString *certsPath =
+      [bundle pathForResource:@"TestCertificates.bundle/test-certificates" ofType:@"pem"];
+  NSError *error;
+  return [NSString stringWithContentsOfFile:certsPath encoding:NSUTF8StringEncoding error:&error];
+}
+
++ (NSString *)hostNameOverride {
+  return @"foo.test.google.fr";
+}
+
+- (int32_t)encodingOverhead {
+  return kLocalInteropServerOverhead;  // bytes
+}
+
++ (GRPCTransportType)transportType {
+  return GRPCTransportTypeChttp2BoringSSL;
+}
+
+- (void)setUp {
+  [super setUp];
+
+  // Register test server certificates and name.
+  NSBundle *bundle = [NSBundle bundleForClass:[self class]];
+  NSString *certsPath =
+      [bundle pathForResource:@"TestCertificates.bundle/test-certificates" ofType:@"pem"];
+  [GRPCCall useTestCertsPath:certsPath testName:@"foo.test.google.fr" forHost:[[self class] host]];
+}
+@end

src/objective-c/tests/Podfile

@@ -127,7 +127,7 @@ post_install do |installer|
         # GPR_UNREACHABLE_CODE causes "Control may reach end of non-void
         # function" warning
         config.build_settings['GCC_WARN_ABOUT_RETURN_TYPE'] = 'NO'
-        config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = '$(inherited) COCOAPODS=1 GRPC_CRONET_WITH_PACKET_COALESCING=1'
+        config.build_settings['GCC_PREPROCESSOR_DEFINITIONS'] = '$(inherited) COCOAPODS=1 GRPC_CRONET_WITH_PACKET_COALESCING=1 GRPC_CFSTREAM=1'
       end
     end
 

src/objective-c/tests/RemoteTestClient/RemoteTest.podspec

@@ -14,20 +14,34 @@ Pod::Spec.new do |s|
   s.dependency "!ProtoCompiler-gRPCPlugin"
 
   repo_root = '../../../..'
-  bin_dir = "#{repo_root}/bins/$CONFIG"
+  config = ENV['CONFIG'] || 'opt'
+  bin_dir = "#{repo_root}/bins/#{config}"
 
   protoc = "#{bin_dir}/protobuf/protoc"
   well_known_types_dir = "#{repo_root}/third_party/protobuf/src"
   plugin = "#{bin_dir}/grpc_objective_c_plugin"
 
   s.prepare_command = <<-CMD
-    #{protoc} \
-        --plugin=protoc-gen-grpc=#{plugin} \
-        --objc_out=. \
-        --grpc_out=. \
-        -I . \
-        -I #{well_known_types_dir} \
-        *.proto
+    if [ -f #{protoc} ]; then
+      #{protoc} \
+          --plugin=protoc-gen-grpc=#{plugin} \
+          --objc_out=. \
+          --grpc_out=. \
+          -I . \
+          -I #{well_known_types_dir} \
+          *.proto
+    else
+      # protoc was not found bin_dir, use installed version instead
+      (>&2 echo "\nWARNING: Using installed version of protoc. It might be incompatible with gRPC")
+
+      protoc \
+          --plugin=protoc-gen-grpc=#{plugin} \
+          --objc_out=. \
+          --grpc_out=. \
+          -I . \
+          -I #{well_known_types_dir} \
+          *.proto
+    fi
   CMD
 
   s.subspec "Messages" do |ms|

src/objective-c/tests/Tests.xcodeproj/project.pbxproj

@@ -68,6 +68,7 @@
 		91D4B3C85B6D8562F409CB48 /* libPods-InteropTestsLocalSSLCFStream.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F3AB031E0E26AC8EF30A2A2A /* libPods-InteropTestsLocalSSLCFStream.a */; };
 		953CD2942A3A6D6CE695BE87 /* libPods-MacTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 276873A05AC5479B60DF6079 /* libPods-MacTests.a */; };
 		98478C9F42329DF769A45B6C /* libPods-APIv2Tests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = B6AD69CACF67505B0F028E92 /* libPods-APIv2Tests.a */; };
+		B071230B22669EED004B64A1 /* StressTests.m in Sources */ = {isa = PBXBuildFile; fileRef = B071230A22669EED004B64A1 /* StressTests.m */; };
 		B0BB3F02225E7A3C008DA580 /* InteropTestsLocalSSL.m in Sources */ = {isa = PBXBuildFile; fileRef = 63E240CD1B6C4E2B005F3B0E /* InteropTestsLocalSSL.m */; };
 		B0BB3F03225E7A44008DA580 /* InteropTestsLocalCleartext.m in Sources */ = {isa = PBXBuildFile; fileRef = 63715F551B780C020029CB0B /* InteropTestsLocalCleartext.m */; };
 		B0BB3F04225E7A8D008DA580 /* RxLibraryUnitTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 63423F501B151B77006CF63C /* RxLibraryUnitTests.m */; };
@@ -77,6 +78,8 @@
 		B0BB3F08225E7ABA008DA580 /* UnitTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 5E0282E8215AA697007AC99D /* UnitTests.m */; };
 		B0BB3F0A225EA511008DA580 /* TestCertificates.bundle in Resources */ = {isa = PBXBuildFile; fileRef = 63E240CF1B6C63DC005F3B0E /* TestCertificates.bundle */; };
 		B0BB3F0B225EB110008DA580 /* InteropTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 635ED2EB1B1A3BC400FDE5C3 /* InteropTests.m */; };
+		B0D39B9A2266F3CB00A4078D /* StressTestsSSL.m in Sources */ = {isa = PBXBuildFile; fileRef = B0D39B992266F3CB00A4078D /* StressTestsSSL.m */; };
+		B0D39B9C2266FF9800A4078D /* StressTestsCleartext.m in Sources */ = {isa = PBXBuildFile; fileRef = B0D39B9B2266FF9800A4078D /* StressTestsCleartext.m */; };
 		BC111C80CBF7068B62869352 /* libPods-InteropTestsRemoteCFStream.a in Frameworks */ = {isa = PBXBuildFile; fileRef = F44AC3F44E3491A8C0D890FE /* libPods-InteropTestsRemoteCFStream.a */; };
 		C3D6F4270A2FFF634D8849ED /* libPods-InteropTestsLocalCleartextCFStream.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 0BDA4BA011779D5D25B5618C /* libPods-InteropTestsLocalCleartextCFStream.a */; };
 		CCF5C0719EF608276AE16374 /* libPods-UnitTests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 22A3EBB488699C8CEA19707B /* libPods-UnitTests.a */; };
@@ -288,8 +291,12 @@
 		AA7CB64B4DD9915AE7C03163 /* Pods-InteropTestsLocalCleartext.cronet.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsLocalCleartext.cronet.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsLocalCleartext/Pods-InteropTestsLocalCleartext.cronet.xcconfig"; sourceTree = "<group>"; };
 		AC414EF7A6BF76ED02B6E480 /* Pods-InteropTestsRemoteWithCronet.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-InteropTestsRemoteWithCronet.release.xcconfig"; path = "Pods/Target Support Files/Pods-InteropTestsRemoteWithCronet/Pods-InteropTestsRemoteWithCronet.release.xcconfig"; sourceTree = "<group>"; };
 		AF3FC2CFFE7B0961823BC740 /* libPods-InteropTestsCallOptions.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-InteropTestsCallOptions.a"; sourceTree = BUILT_PRODUCTS_DIR; };
+		B071230A22669EED004B64A1 /* StressTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = StressTests.m; sourceTree = "<group>"; };
 		B0BB3EF7225E795F008DA580 /* MacTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = MacTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
 		B0BB3EFB225E795F008DA580 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		B0C5FC172267C77200F192BE /* StressTests.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = StressTests.h; sourceTree = "<group>"; };
+		B0D39B992266F3CB00A4078D /* StressTestsSSL.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = StressTestsSSL.m; sourceTree = "<group>"; };
+		B0D39B9B2266FF9800A4078D /* StressTestsCleartext.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = StressTestsCleartext.m; sourceTree = "<group>"; };
 		B226619DC4E709E0FFFF94B8 /* Pods-CronetUnitTests.test.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-CronetUnitTests.test.xcconfig"; path = "Pods/Target Support Files/Pods-CronetUnitTests/Pods-CronetUnitTests.test.xcconfig"; sourceTree = "<group>"; };
 		B6AD69CACF67505B0F028E92 /* libPods-APIv2Tests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-APIv2Tests.a"; sourceTree = BUILT_PRODUCTS_DIR; };
 		B94C27C06733CF98CE1B2757 /* Pods-AllTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AllTests.debug.xcconfig"; path = "Pods/Target Support Files/Pods-AllTests/Pods-AllTests.debug.xcconfig"; sourceTree = "<group>"; };
@@ -728,6 +735,10 @@
 			isa = PBXGroup;
 			children = (
 				B0BB3EFB225E795F008DA580 /* Info.plist */,
+				B071230A22669EED004B64A1 /* StressTests.m */,
+				B0D39B992266F3CB00A4078D /* StressTestsSSL.m */,
+				B0D39B9B2266FF9800A4078D /* StressTestsCleartext.m */,
+				B0C5FC172267C77200F192BE /* StressTests.h */,
 			);
 			path = MacTests;
 			sourceTree = "<group>";
@@ -2149,9 +2160,12 @@
 			files = (
 				B0BB3F07225E7AB5008DA580 /* APIv2Tests.m in Sources */,
 				B0BB3F08225E7ABA008DA580 /* UnitTests.m in Sources */,
+				B071230B22669EED004B64A1 /* StressTests.m in Sources */,
 				B0BB3F05225E7A9F008DA580 /* InteropTestsRemote.m in Sources */,
+				B0D39B9A2266F3CB00A4078D /* StressTestsSSL.m in Sources */,
 				B0BB3F03225E7A44008DA580 /* InteropTestsLocalCleartext.m in Sources */,
 				B0BB3F04225E7A8D008DA580 /* RxLibraryUnitTests.m in Sources */,
+				B0D39B9C2266FF9800A4078D /* StressTestsCleartext.m in Sources */,
 				B0BB3F0B225EB110008DA580 /* InteropTests.m in Sources */,
 				B0BB3F02225E7A3C008DA580 /* InteropTestsLocalSSL.m in Sources */,
 				B0BB3F06225E7AAD008DA580 /* GRPCClientTests.m in Sources */,

src/objective-c/tests/Tests.xcodeproj/xcshareddata/xcschemes/MacTests.xcscheme

@@ -41,6 +41,9 @@
                <Test
                   Identifier = "InteropTests">
                </Test>
+               <Test
+                  Identifier = "StressTests">
+               </Test>
             </SkippedTests>
          </TestableReference>
       </Testables>

src/php/bin/run_tests.sh

@@ -22,16 +22,17 @@ cd src/php/bin
 source ./determine_extension_dir.sh
 # in some jenkins macos machine, somehow the PHP build script can't find libgrpc.dylib
 export DYLD_LIBRARY_PATH=$root/libs/$CONFIG
-php $extension_dir -d max_execution_time=300 $(which phpunit) -v --debug \
+$(which php) $extension_dir -d max_execution_time=300 $(which phpunit) -v --debug \
   --exclude-group persistent_list_bound_tests ../tests/unit_tests
 
-php $extension_dir -d max_execution_time=300 $(which phpunit) -v --debug \
+$(which php) $extension_dir -d max_execution_time=300 $(which phpunit) -v --debug \
   ../tests/unit_tests/PersistentChannelTests
 
 export ZEND_DONT_UNLOAD_MODULES=1
 export USE_ZEND_ALLOC=0
 # Detect whether valgrind is executable
 if [ -x "$(command -v valgrind)" ]; then
-  valgrind --error-exitcode=10 --leak-check=yes php $extension_dir -d max_execution_time=300 \
+  $(which valgrind) --error-exitcode=10 --leak-check=yes \
+    $(which php) $extension_dir -d max_execution_time=300 \
     ../tests/MemoryLeakTest/MemoryLeakTest.php
 fi

src/php/ext/grpc/php_grpc.c

@@ -205,11 +205,15 @@ void register_fork_handlers() {
 
 void apply_ini_settings() {
   if (GRPC_G(enable_fork_support)) {
-    setenv("GRPC_ENABLE_FORK_SUPPORT", "1", 1 /* overwrite? */);
+    putenv("GRPC_ENABLE_FORK_SUPPORT=1");
   }
 
   if (GRPC_G(poll_strategy)) {
-    setenv("GRPC_POLL_STRATEGY", GRPC_G(poll_strategy), 1 /* overwrite? */);
+    char *poll_str = malloc(sizeof("GRPC_POLL_STRATEGY=") +
+                            strlen(GRPC_G(poll_strategy)));
+    strcpy(poll_str, "GRPC_POLL_STRATEGY=");
+    strcat(poll_str, GRPC_G(poll_strategy));
+    putenv(poll_str);
   }
 }
 

src/proto/grpc/testing/echo_messages.proto

@@ -17,6 +17,8 @@ syntax = "proto3";
 
 package grpc.testing;
 
+option cc_enable_arenas = true;
+
 // Message to be echoed back serialized in trailer.
 message DebugInfo {
   repeated string stack_entries = 1;
@@ -47,6 +49,7 @@ message RequestParams {
   ErrorStatus expected_error = 14;
   int32 server_sleep_us = 15; // Amount to sleep when invoking server
   int32 backend_channel_idx = 16; // which backend to send request to
+  bool echo_metadata_initially = 17;
 }
 
 message EchoRequest {

src/python/grpcio_tests/tests/stress/client.py

@@ -34,12 +34,12 @@ def _args():
         description='gRPC Python stress test client')
     parser.add_argument(
         '--server_addresses',
-        help='comma seperated list of hostname:port to run servers on',
+        help='comma separated list of hostname:port to run servers on',
         default='localhost:8080',
         type=str)
     parser.add_argument(
         '--test_cases',
-        help='comma seperated list of testcase:weighting of tests to run',
+        help='comma separated list of testcase:weighting of tests to run',
         default='large_unary:100',
         type=str)
     parser.add_argument(

templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template

@@ -103,7 +103,7 @@
     s.preserve_paths = plugin
 
     # Restrict the protoc version to the one supported by this plugin.
-    s.dependency '!ProtoCompiler', '3.6.1'
+    s.dependency '!ProtoCompiler', '3.7.0'
     # For the Protobuf dependency not to complain:
     s.ios.deployment_target = '7.0'
     s.osx.deployment_target = '10.9'

test/core/slice/slice_test.cc

@@ -51,13 +51,6 @@ static void test_slice_malloc_returns_something_sensible(void) {
     }
     /* Returned slice length must be what was requested. */
     GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == length);
-    /* If the slice has a refcount, it must be destroyable. */
-    if (slice.refcount) {
-      GPR_ASSERT(slice.refcount->vtable != nullptr);
-      GPR_ASSERT(slice.refcount->vtable->ref != nullptr);
-      GPR_ASSERT(slice.refcount->vtable->unref != nullptr);
-      GPR_ASSERT(slice.refcount->vtable->hash != nullptr);
-    }
     /* We must be able to write to every byte of the data */
     for (i = 0; i < length; i++) {
       GRPC_SLICE_START_PTR(slice)[i] = static_cast<uint8_t>(i);

test/core/transport/stream_owned_slice_test.cc

@@ -32,14 +32,11 @@ int main(int argc, char** argv) {
   uint8_t buffer[] = "abc123";
   grpc_stream_refcount r;
   GRPC_STREAM_REF_INIT(&r, 1, do_nothing, nullptr, "test");
-  GPR_ASSERT(r.refs.count == 1);
   grpc_slice slice =
       grpc_slice_from_stream_owned_buffer(&r, buffer, sizeof(buffer));
   GPR_ASSERT(GRPC_SLICE_START_PTR(slice) == buffer);
   GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == sizeof(buffer));
-  GPR_ASSERT(r.refs.count == 2);
   grpc_slice_unref(slice);
-  GPR_ASSERT(r.refs.count == 1);
 
   grpc_shutdown();
   return 0;

test/cpp/codegen/compiler_test_golden

@@ -41,6 +41,10 @@
 #include <grpcpp/impl/codegen/sync_stream.h>
 
 namespace grpc {
+namespace experimental {
+template <typename RequestT, typename ResponseT>
+class MessageAllocator;
+}  // namespace experimental
 class CompletionQueue;
 class Channel;
 class ServerCompletionQueue;
@@ -114,6 +118,8 @@ class ServiceA final {
       // MethodA1 leading comment 1
       virtual void MethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) = 0;
       virtual void MethodA1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) = 0;
+      virtual void MethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0;
+      virtual void MethodA1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0;
       // MethodA1 trailing comment 1
       // MethodA2 detached leading comment 1
       //
@@ -184,6 +190,8 @@ class ServiceA final {
      public:
       void MethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) override;
       void MethodA1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) override;
+      void MethodA1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) override;
+      void MethodA1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) override;
       void MethodA2(::grpc::ClientContext* context, ::grpc::testing::Response* response, ::grpc::experimental::ClientWriteReactor< ::grpc::testing::Request>* reactor) override;
       void MethodA3(::grpc::ClientContext* context, ::grpc::testing::Request* request, ::grpc::experimental::ClientReadReactor< ::grpc::testing::Response>* reactor) override;
       void MethodA4(::grpc::ClientContext* context, ::grpc::experimental::ClientBidiReactor< ::grpc::testing::Request,::grpc::testing::Response>* reactor) override;
@@ -332,6 +340,12 @@ class ServiceA final {
                    return this->MethodA1(context, request, response, controller);
                  }));
     }
+    void SetMessageAllocatorFor_MethodA1(
+        ::grpc::experimental::MessageAllocator< ::grpc::testing::Request, ::grpc::testing::Response>* allocator) {
+      static_cast<::grpc::internal::CallbackUnaryHandler< ::grpc::testing::Request, ::grpc::testing::Response>*>(
+          ::grpc::Service::experimental().GetHandler(0))
+              ->SetMessageAllocator(allocator);
+    }
     ~ExperimentalWithCallbackMethod_MethodA1() override {
       BaseClassMustBeDerivedFromService(this);
     }
@@ -717,6 +731,8 @@ class ServiceB final {
       // MethodB1 leading comment 1
       virtual void MethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) = 0;
       virtual void MethodB1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) = 0;
+      virtual void MethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0;
+      virtual void MethodB1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0;
       // MethodB1 trailing comment 1
     };
     virtual class experimental_async_interface* experimental_async() { return nullptr; }
@@ -739,6 +755,8 @@ class ServiceB final {
      public:
       void MethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) override;
       void MethodB1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, std::function<void(::grpc::Status)>) override;
+      void MethodB1(::grpc::ClientContext* context, const ::grpc::testing::Request* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) override;
+      void MethodB1(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::grpc::testing::Response* response, ::grpc::experimental::ClientUnaryReactor* reactor) override;
      private:
       friend class Stub;
       explicit experimental_async(Stub* stub): stub_(stub) { }
@@ -800,6 +818,12 @@ class ServiceB final {
                    return this->MethodB1(context, request, response, controller);
                  }));
     }
+    void SetMessageAllocatorFor_MethodB1(
+        ::grpc::experimental::MessageAllocator< ::grpc::testing::Request, ::grpc::testing::Response>* allocator) {
+      static_cast<::grpc::internal::CallbackUnaryHandler< ::grpc::testing::Request, ::grpc::testing::Response>*>(
+          ::grpc::Service::experimental().GetHandler(0))
+              ->SetMessageAllocator(allocator);
+    }
     ~ExperimentalWithCallbackMethod_MethodB1() override {
       BaseClassMustBeDerivedFromService(this);
     }

test/cpp/end2end/BUILD

@@ -675,3 +675,23 @@ grpc_cc_test(
         "//test/cpp/util:test_util",
     ],
 )
+
+grpc_cc_test(
+    name = "message_allocator_end2end_test",
+    srcs = ["message_allocator_end2end_test.cc"],
+    external_deps = [
+        "gtest",
+    ],
+    deps = [
+        ":test_service_impl",
+        "//:grpc",
+        "//:gpr",
+        "//:grpc++",
+        "//src/proto/grpc/testing:echo_messages_proto",
+        "//src/proto/grpc/testing:echo_proto",
+        "//src/proto/grpc/testing:simple_messages_proto",
+        "//test/core/util:grpc_test_util",
+        "//test/cpp/util:test_util",
+    ],
+)
+

test/cpp/end2end/cfstream_test.cc

@@ -45,6 +45,7 @@
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
 #include "test/cpp/end2end/test_service_impl.h"
+#include "test/cpp/util/test_credentials_provider.h"
 
 #ifdef GRPC_CFSTREAM
 using grpc::ClientAsyncResponseReader;
@@ -57,13 +58,19 @@ namespace grpc {
 namespace testing {
 namespace {
 
-class CFStreamTest : public ::testing::Test {
+struct TestScenario {
+  TestScenario(const grpc::string& creds_type, const grpc::string& content)
+      : credentials_type(creds_type), message_content(content) {}
+  const grpc::string credentials_type;
+  const grpc::string message_content;
+};
+
+class CFStreamTest : public ::testing::TestWithParam<TestScenario> {
  protected:
   CFStreamTest()
       : server_host_("grpctest"),
         interface_("lo0"),
-        ipv4_address_("10.0.0.1"),
-        kRequestMessage_("🖖") {}
+        ipv4_address_("10.0.0.1") {}
 
   void DNSUp() {
     std::ostringstream cmd;
@@ -118,7 +125,7 @@ class CFStreamTest : public ::testing::Test {
 
   void StartServer() {
     port_ = grpc_pick_unused_port_or_die();
-    server_.reset(new ServerData(port_));
+    server_.reset(new ServerData(port_, GetParam().credentials_type));
     server_->Start(server_host_);
   }
   void StopServer() { server_->Shutdown(); }
@@ -131,8 +138,10 @@ class CFStreamTest : public ::testing::Test {
   std::shared_ptr<Channel> BuildChannel() {
     std::ostringstream server_address;
     server_address << server_host_ << ":" << port_;
-    return CreateCustomChannel(
-        server_address.str(), InsecureChannelCredentials(), ChannelArguments());
+    ChannelArguments args;
+    auto channel_creds = GetCredentialsProvider()->GetChannelCredentials(
+        GetParam().credentials_type, &args);
+    return CreateCustomChannel(server_address.str(), channel_creds, args);
   }
 
   void SendRpc(
@@ -140,11 +149,12 @@ class CFStreamTest : public ::testing::Test {
       bool expect_success = false) {
     auto response = std::unique_ptr<EchoResponse>(new EchoResponse());
     EchoRequest request;
-    request.set_message(kRequestMessage_);
+    auto& msg = GetParam().message_content;
+    request.set_message(msg);
     ClientContext context;
     Status status = stub->Echo(&context, request, response.get());
     if (status.ok()) {
-      gpr_log(GPR_DEBUG, "RPC returned %s\n", response->message().c_str());
+      EXPECT_EQ(msg, response->message());
     } else {
       gpr_log(GPR_DEBUG, "RPC failed: %s", status.error_message().c_str());
     }
@@ -156,9 +166,7 @@ class CFStreamTest : public ::testing::Test {
       const std::unique_ptr<grpc::testing::EchoTestService::Stub>& stub,
       RequestParams param = RequestParams()) {
     EchoRequest request;
-    auto msg = std::to_string(ctr.load());
-    request.set_message(msg);
-    ctr++;
+    request.set_message(GetParam().message_content);
     *request.mutable_param() = std::move(param);
     AsyncClientCall* call = new AsyncClientCall;
 
@@ -166,7 +174,6 @@ class CFStreamTest : public ::testing::Test {
         stub->PrepareAsyncEcho(&call->context, request, &cq_);
 
     call->response_reader->StartCall();
-    gpr_log(GPR_DEBUG, "Sending request: %s", msg.c_str());
     call->response_reader->Finish(&call->reply, &call->status, (void*)call);
   }
 
@@ -206,12 +213,14 @@ class CFStreamTest : public ::testing::Test {
  private:
   struct ServerData {
     int port_;
+    const grpc::string creds_;
     std::unique_ptr<Server> server_;
     TestServiceImpl service_;
     std::unique_ptr<std::thread> thread_;
     bool server_ready_ = false;
 
-    explicit ServerData(int port) { port_ = port; }
+    ServerData(int port, const grpc::string& creds)
+        : port_(port), creds_(creds) {}
 
     void Start(const grpc::string& server_host) {
       gpr_log(GPR_INFO, "starting server on port %d", port_);
@@ -230,8 +239,9 @@ class CFStreamTest : public ::testing::Test {
       std::ostringstream server_address;
       server_address << server_host << ":" << port_;
       ServerBuilder builder;
-      builder.AddListeningPort(server_address.str(),
-                               InsecureServerCredentials());
+      auto server_creds =
+          GetCredentialsProvider()->GetServerCredentials(creds_);
+      builder.AddListeningPort(server_address.str(), server_creds);
       builder.RegisterService(&service_);
       server_ = builder.BuildAndStart();
       std::lock_guard<std::mutex> lock(*mu);
@@ -251,13 +261,44 @@ class CFStreamTest : public ::testing::Test {
   const grpc::string ipv4_address_;
   std::unique_ptr<ServerData> server_;
   int port_;
-  const grpc::string kRequestMessage_;
-  std::atomic_int ctr{0};
 };
 
+std::vector<TestScenario> CreateTestScenarios() {
+  std::vector<TestScenario> scenarios;
+  std::vector<grpc::string> credentials_types;
+  std::vector<grpc::string> messages;
+
+  credentials_types.push_back(kInsecureCredentialsType);
+  auto sec_list = GetCredentialsProvider()->GetSecureCredentialsTypeList();
+  for (auto sec = sec_list.begin(); sec != sec_list.end(); sec++) {
+    credentials_types.push_back(*sec);
+  }
+
+  messages.push_back("🖖");
+  for (size_t k = 1; k < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH / 1024; k *= 32) {
+    grpc::string big_msg;
+    for (size_t i = 0; i < k * 1024; ++i) {
+      char c = 'a' + (i % 26);
+      big_msg += c;
+    }
+    messages.push_back(big_msg);
+  }
+  for (auto cred = credentials_types.begin(); cred != credentials_types.end();
+       ++cred) {
+    for (auto msg = messages.begin(); msg != messages.end(); msg++) {
+      scenarios.emplace_back(*cred, *msg);
+    }
+  }
+
+  return scenarios;
+}
+
+INSTANTIATE_TEST_CASE_P(CFStreamTest, CFStreamTest,
+                        ::testing::ValuesIn(CreateTestScenarios()));
+
 // gRPC should automatically detech network flaps (without enabling keepalives)
 //  when CFStream is enabled
-TEST_F(CFStreamTest, NetworkTransition) {
+TEST_P(CFStreamTest, NetworkTransition) {
   auto channel = BuildChannel();
   auto stub = BuildStub(channel);
   // Channel should be in READY state after we send an RPC
@@ -293,7 +334,7 @@ TEST_F(CFStreamTest, NetworkTransition) {
 }
 
 // Network flaps while RPCs are in flight
-TEST_F(CFStreamTest, NetworkFlapRpcsInFlight) {
+TEST_P(CFStreamTest, NetworkFlapRpcsInFlight) {
   auto channel = BuildChannel();
   auto stub = BuildStub(channel);
   std::atomic_int rpcs_sent{0};
@@ -318,9 +359,7 @@ TEST_F(CFStreamTest, NetworkFlapRpcsInFlight) {
       ++total_completions;
       GPR_ASSERT(ok);
       AsyncClientCall* call = static_cast<AsyncClientCall*>(got_tag);
-      if (call->status.ok()) {
-        gpr_log(GPR_DEBUG, "RPC response: %s", call->reply.message().c_str());
-      } else {
+      if (!call->status.ok()) {
         gpr_log(GPR_DEBUG, "RPC failed with error: %s",
                 call->status.error_message().c_str());
         // Bring network up when RPCs start failing
@@ -347,7 +386,7 @@ TEST_F(CFStreamTest, NetworkFlapRpcsInFlight) {
 
 // Send a bunch of RPCs, some of which are expected to fail.
 // We should get back a response for all RPCs
-TEST_F(CFStreamTest, ConcurrentRpc) {
+TEST_P(CFStreamTest, ConcurrentRpc) {
   auto channel = BuildChannel();
   auto stub = BuildStub(channel);
   std::atomic_int rpcs_sent{0};
@@ -361,9 +400,7 @@ TEST_F(CFStreamTest, ConcurrentRpc) {
       ++total_completions;
       GPR_ASSERT(ok);
       AsyncClientCall* call = static_cast<AsyncClientCall*>(got_tag);
-      if (call->status.ok()) {
-        gpr_log(GPR_DEBUG, "RPC response: %s", call->reply.message().c_str());
-      } else {
+      if (!call->status.ok()) {
         gpr_log(GPR_DEBUG, "RPC failed: %s",
                 call->status.error_message().c_str());
         // Bring network up when RPCs start failing

test/cpp/end2end/client_callback_end2end_test.cc

@@ -696,6 +696,65 @@ TEST_P(ClientCallbackEnd2endTest, RequestStreamServerCancelAfterReads) {
   }
 }
 
+TEST_P(ClientCallbackEnd2endTest, UnaryReactor) {
+  MAYBE_SKIP_TEST;
+  ResetStub();
+  class UnaryClient : public grpc::experimental::ClientUnaryReactor {
+   public:
+    UnaryClient(grpc::testing::EchoTestService::Stub* stub) {
+      cli_ctx_.AddMetadata("key1", "val1");
+      cli_ctx_.AddMetadata("key2", "val2");
+      request_.mutable_param()->set_echo_metadata_initially(true);
+      request_.set_message("Hello metadata");
+      stub->experimental_async()->Echo(&cli_ctx_, &request_, &response_, this);
+      StartCall();
+    }
+    void OnReadInitialMetadataDone(bool ok) override {
+      EXPECT_TRUE(ok);
+      EXPECT_EQ(1u, cli_ctx_.GetServerInitialMetadata().count("key1"));
+      EXPECT_EQ(
+          "val1",
+          ToString(cli_ctx_.GetServerInitialMetadata().find("key1")->second));
+      EXPECT_EQ(1u, cli_ctx_.GetServerInitialMetadata().count("key2"));
+      EXPECT_EQ(
+          "val2",
+          ToString(cli_ctx_.GetServerInitialMetadata().find("key2")->second));
+      initial_metadata_done_ = true;
+    }
+    void OnDone(const Status& s) override {
+      EXPECT_TRUE(initial_metadata_done_);
+      EXPECT_EQ(0u, cli_ctx_.GetServerTrailingMetadata().size());
+      EXPECT_TRUE(s.ok());
+      EXPECT_EQ(request_.message(), response_.message());
+      std::unique_lock<std::mutex> l(mu_);
+      done_ = true;
+      cv_.notify_one();
+    }
+    void Await() {
+      std::unique_lock<std::mutex> l(mu_);
+      while (!done_) {
+        cv_.wait(l);
+      }
+    }
+
+   private:
+    EchoRequest request_;
+    EchoResponse response_;
+    ClientContext cli_ctx_;
+    std::mutex mu_;
+    std::condition_variable cv_;
+    bool done_{false};
+    bool initial_metadata_done_{false};
+  };
+
+  UnaryClient test{stub_.get()};
+  test.Await();
+  // Make sure that the server interceptors were not notified of a cancel
+  if (GetParam().use_interceptors) {
+    EXPECT_EQ(0, DummyInterceptor::GetNumTimesCancel());
+  }
+}
+
 class ReadClient : public grpc::experimental::ClientReadReactor<EchoResponse> {
  public:
   ReadClient(grpc::testing::EchoTestService::Stub* stub,