Merge branch 'master' into node_electron_build

.gitignore

@@ -96,7 +96,7 @@ DerivedData
 Pods/
 
 # Artifacts directory
-artifacts/
+/artifacts/
 
 # Git generated files for conflicting
 *.orig

.gitmodules

@@ -17,6 +17,6 @@
 [submodule "third_party/thrift"]
 	path = third_party/thrift
 	url = https://github.com/apache/thrift.git
-[submodule "third_party/google_benchmark"]
-	path = third_party/google_benchmark
+[submodule "third_party/benchmark"]
+	path = third_party/benchmark
 	url = https://github.com/google/benchmark

Makefile

@@ -1260,9 +1260,9 @@ pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc
 pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc
 
 ifeq ($(EMBED_OPENSSL),true)
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libboringssl_test_util.a $(LIBDIR)/$(CONFIG)/libboringssl_aes_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_asn1_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_base64_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bio_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bn_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_bytestring_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_aead_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cipher_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_cmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ed25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x25519_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_dh_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_digest_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ec_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ecdsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_err_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_extra_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_evp_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pbkdf_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_hmac_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs12_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_pkcs8_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_poly1305_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_rsa_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_x509_test_lib.a $(LIBDIR)/$(CONFIG)/libboringssl_ssl_test_lib.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 else
-privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+privatelibs_cxx:  $(LIBDIR)/$(CONFIG)/libgrpc++_proto_reflection_desc_db.a $(LIBDIR)/$(CONFIG)/libgrpc++_test.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libinterop_client_helper.a $(LIBDIR)/$(CONFIG)/libinterop_client_main.a $(LIBDIR)/$(CONFIG)/libinterop_server_helper.a $(LIBDIR)/$(CONFIG)/libinterop_server_lib.a $(LIBDIR)/$(CONFIG)/libinterop_server_main.a $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libbenchmark.a
 endif
 
 
@@ -6998,43 +6998,43 @@ ifneq ($(NO_DEPS),true)
 endif
 
 
-LIBGOOGLE_BENCHMARK_SRC = \
-    third_party/google_benchmark/src/benchmark.cc \
-    third_party/google_benchmark/src/benchmark_register.cc \
-    third_party/google_benchmark/src/colorprint.cc \
-    third_party/google_benchmark/src/commandlineflags.cc \
-    third_party/google_benchmark/src/complexity.cc \
-    third_party/google_benchmark/src/console_reporter.cc \
-    third_party/google_benchmark/src/csv_reporter.cc \
-    third_party/google_benchmark/src/json_reporter.cc \
-    third_party/google_benchmark/src/reporter.cc \
-    third_party/google_benchmark/src/sleep.cc \
-    third_party/google_benchmark/src/string_util.cc \
-    third_party/google_benchmark/src/sysinfo.cc \
-    third_party/google_benchmark/src/timers.cc \
+LIBBENCHMARK_SRC = \
+    third_party/benchmark/src/benchmark.cc \
+    third_party/benchmark/src/benchmark_register.cc \
+    third_party/benchmark/src/colorprint.cc \
+    third_party/benchmark/src/commandlineflags.cc \
+    third_party/benchmark/src/complexity.cc \
+    third_party/benchmark/src/console_reporter.cc \
+    third_party/benchmark/src/csv_reporter.cc \
+    third_party/benchmark/src/json_reporter.cc \
+    third_party/benchmark/src/reporter.cc \
+    third_party/benchmark/src/sleep.cc \
+    third_party/benchmark/src/string_util.cc \
+    third_party/benchmark/src/sysinfo.cc \
+    third_party/benchmark/src/timers.cc \
 
 PUBLIC_HEADERS_CXX += \
 
-LIBGOOGLE_BENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGOOGLE_BENCHMARK_SRC))))
+LIBBENCHMARK_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBBENCHMARK_SRC))))
 
-$(LIBGOOGLE_BENCHMARK_OBJS): CPPFLAGS += -Ithird_party/google_benchmark/include -DHAVE_POSIX_REGEX
+$(LIBBENCHMARK_OBJS): CPPFLAGS += -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX
 
 ifeq ($(NO_PROTOBUF),true)
 
 # You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
 
-$(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a: protobuf_dep_error
+$(LIBDIR)/$(CONFIG)/libbenchmark.a: protobuf_dep_error
 
 
 else
 
-$(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(LIBGOOGLE_BENCHMARK_OBJS) 
+$(LIBDIR)/$(CONFIG)/libbenchmark.a: $(ZLIB_DEP)  $(PROTOBUF_DEP) $(LIBBENCHMARK_OBJS) 
 	$(E) "[AR]      Creating $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
-	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBGOOGLE_BENCHMARK_OBJS) 
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libbenchmark.a
+	$(Q) $(AR) $(AROPTS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBBENCHMARK_OBJS) 
 ifeq ($(SYSTEM),Darwin)
-	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+	$(Q) ranlib -no_warning_for_no_symbols $(LIBDIR)/$(CONFIG)/libbenchmark.a
 endif
 
 
@@ -7043,7 +7043,7 @@ endif
 endif
 
 ifneq ($(NO_DEPS),true)
--include $(LIBGOOGLE_BENCHMARK_OBJS:.o=.dep)
+-include $(LIBBENCHMARK_OBJS:.o=.dep)
 endif
 
 
@@ -11736,16 +11736,16 @@ $(BINDIR)/$(CONFIG)/bm_fullstack: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(BINDIR)/$(CONFIG)/bm_fullstack: $(PROTOBUF_DEP) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack
+	$(Q) $(LDXX) $(LDFLAGS) $(BM_FULLSTACK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_fullstack
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o:  $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_fullstack.o:  $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
 
 deps_bm_fullstack: $(BM_FULLSTACK_OBJS:.o=.dep)
 
@@ -13192,16 +13192,16 @@ $(BINDIR)/$(CONFIG)/noop-benchmark: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/noop-benchmark: $(PROTOBUF_DEP) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+$(BINDIR)/$(CONFIG)/noop-benchmark: $(PROTOBUF_DEP) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a
 	$(E) "[LD]      Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/noop-benchmark
+	$(Q) $(LDXX) $(LDFLAGS) $(NOOP-BENCHMARK_OBJS) $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/noop-benchmark
 
 endif
 
 endif
 
-$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/noop-benchmark.o:  $(LIBDIR)/$(CONFIG)/libgoogle_benchmark.a
+$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/noop-benchmark.o:  $(LIBDIR)/$(CONFIG)/libbenchmark.a
 
 deps_noop-benchmark: $(NOOP-BENCHMARK_OBJS:.o=.dep)
 

build.yaml

@@ -2850,7 +2850,7 @@ targets:
   src:
   - test/cpp/microbenchmarks/bm_fullstack.cc
   deps:
-  - google_benchmark
+  - benchmark
   - grpc++_test_util
   - grpc_test_util
   - grpc++
@@ -3300,7 +3300,7 @@ targets:
   src:
   - test/cpp/microbenchmarks/noop-benchmark.cc
   deps:
-  - google_benchmark
+  - benchmark
 - name: proto_server_reflection_test
   gtest: true
   build: test
@@ -3786,6 +3786,8 @@ configs:
       UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
     timeout_multiplier: 1.5
 defaults:
+  benchmark:
+    CPPFLAGS: -Ithird_party/benchmark/include -DHAVE_POSIX_REGEX
   boringssl:
     CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-unknown-pragmas
       -Wno-implicit-function-declaration -Wno-unused-variable -Wno-sign-compare $(NO_W_EXTRA_SEMI)
@@ -3794,8 +3796,6 @@ defaults:
   global:
     CPPFLAGS: -g -Wall -Wextra -Werror -Wno-long-long -Wno-unused-parameter
     LDFLAGS: -g
-  google_benchmark:
-    CPPFLAGS: -Ithird_party/google_benchmark/include -DHAVE_POSIX_REGEX
   zlib:
     CFLAGS: -Wno-sign-conversion -Wno-conversion -Wno-unused-value -Wno-implicit-function-declaration
       $(W_NO_SHIFT_NEGATIVE_VALUE) -fvisibility=hidden

doc/connectivity-semantics-and-api.md

@@ -16,7 +16,7 @@ reconnect, or in the case of HTTP/2 GO_AWAY, re-resolve the name and reconnect.
 
 To hide the details of all this activity from the user of the gRPC API (i.e.,
 application code) while exposing meaningful information about the state of a
-channel, we use a state machine with four states, defined below:
+channel, we use a state machine with five states, defined below:
 
 CONNECTING: The channel is trying to establish a connection and is waiting to
 make progress on one of the steps involved in name resolution, TCP connection
@@ -116,7 +116,7 @@ Channel State API
 
 All gRPC libraries will expose a channel-level API method to poll the current
 state of a channel. In C++, this method is called GetCurrentState and returns
-an enum for one of the four legal states.
+an enum for one of the five legal states.
 
 All libraries should also expose an API that enables the application (user of
 the gRPC API) to be notified when the channel state changes. Since state

doc/negative-http2-interop-test-descriptions.md

@@ -0,0 +1,194 @@
+Negative HTTP/2 Interop Test Case Descriptions
+=======================================
+
+Client and server use
+[test.proto](../src/proto/grpc/testing/test.proto).
+
+Server
+------
+The code for the custom http2 server can be found
+[here](https://github.com/grpc/grpc/tree/master/test/http2_test).
+It is responsible for handling requests and sending responses, and also for 
+fulfilling the behavior of each particular test case.
+
+Server should accept these arguments:
+* --port=PORT
+  * The port the server will run on. For example, "8080"
+* --test_case=TESTCASE
+  * The name of the test case to execute. For example, "goaway"
+
+Client
+------
+
+Clients implement test cases that test certain functionality. Each client is
+provided the test case it is expected to run as a command-line parameter. Names
+should be lowercase and without spaces.
+
+Clients should accept these arguments:
+* --server_host=HOSTNAME
+    * The server host to connect to. For example, "localhost" or "127.0.0.1"
+* --server_port=PORT
+    * The server port to connect to. For example, "8080"
+* --test_case=TESTCASE
+    * The name of the test case to execute. For example, "goaway"
+
+Note
+-----
+
+Note that the server and client must be invoked with the same test case or else
+the test will be meaningless. For convenience, we provide a shell script wrapper
+that invokes both server and client at the same time, with the same test_case.
+This is the preferred way to run these tests.
+
+## Test Cases
+
+### goaway
+
+This test verifies that the client correctly responds to a goaway sent by the
+server. The client should handle the goaway by switching to a new stream without
+the user application having to do a thing.
+
+Client Procedure:
+ 1. Client sends two UnaryCall requests with:
+ 
+    ```
+    {
+      response_size: 314159
+      payload:{
+        body: 271828 bytes of zeros
+      }
+    }
+    ```
+
+Client asserts:
+* Call was successful.
+* Response payload body is 314159 bytes in size.
+
+Server Procedure:
+  1. Server sends a GOAWAY after receiving the first UnaryCall.
+
+Server asserts:
+* The second UnaryCall has a different stream_id than the first one.
+
+### rst_after_header
+
+This test verifies that the client fails correctly when the server sends a
+RST_STREAM immediately after sending headers to the client.
+
+Procedure:
+ 1. Client sends UnaryCall with:
+ 
+    ```
+    {
+      response_size: 314159
+      payload:{
+        body: 271828 bytes of zeros
+      }
+    }
+    ```
+
+Client asserts:
+* Call was not successful.
+
+Server Procedure:
+  1. Server sends a RST_STREAM with error code 0 after sending headers to the client.
+  
+*At the moment the error code and message returned are not standardized throughout all
+languages. Those checks will be added once all client languages behave the same way. [#9142](https://github.com/grpc/grpc/issues/9142) is in flight.*
+
+### rst_during_data
+
+This test verifies that the client fails "correctly" when the server sends a
+RST_STREAM halfway through sending data to the client.
+
+Procedure:
+ 1. Client sends UnaryCall with:
+ 
+    ```
+    {
+      response_size: 314159
+      payload:{
+        body: 271828 bytes of zeros
+      }
+    }
+    ```
+
+Client asserts:
+* Call was not successful.
+
+Server Procedure:
+  1. Server sends a RST_STREAM with error code 0 after sending half of 
+     the requested data to the client.
+
+### rst_after_data
+
+This test verifies that the client fails "correctly" when the server sends a
+RST_STREAM after sending all of the data to the client.
+
+Procedure:
+ 1. Client sends UnaryCall with:
+ 
+    ```
+    {
+      response_size: 314159
+      payload:{
+        body: 271828 bytes of zeros
+      }
+    }
+    ```
+
+Client asserts:
+* Call was not successful.
+
+Server Procedure:
+  1. Server sends a RST_STREAM with error code 0 after sending all of the
+  data to the client.
+
+*Certain client languages allow the data to be accessed even though a RST_STREAM
+was encountered. Once all client languages behave this way, checks will be added on
+the incoming data.*
+
+### ping
+
+This test verifies that the client correctly acknowledges all pings it gets from the
+server.
+
+Procedure:
+ 1. Client sends UnaryCall with:
+ 
+    ```
+    {
+      response_size: 314159
+      payload:{
+        body: 271828 bytes of zeros
+      }
+    }
+    ```
+  
+Client asserts:
+* call was successful.
+* response payload body is 314159 bytes in size.
+
+Server Procedure:
+  1. Server tracks the number of outstanding pings (i.e. +1 when it sends a ping, and -1 
+  when it receives an ack from the client).
+  2. Server sends pings before and after sending headers, also before and after sending data.
+  
+Server Asserts:
+* Number of outstanding pings is 0 when the connection is lost.
+
+### max_streams
+
+This test verifies that the client observes the MAX_CONCURRENT_STREAMS limit set by the server.
+
+Client Procedure:
+  1. Client sends initial UnaryCall to allow the server to update its MAX_CONCURRENT_STREAMS settings.
+  2. Client concurrently sends 10 UnaryCalls.
+  
+Client Asserts:
+* All UnaryCalls were successful, and had the correct type and payload size.
+ 
+Server Procedure:
+  1. Sets MAX_CONCURRENT_STREAMS to one after the connection is made.
+
+*The assertion that the MAX_CONCURRENT_STREAMS limit is upheld occurs in the http2 library we used.*

examples/cpp/helloworld/greeter_client.cc

@@ -51,7 +51,7 @@ class GreeterClient {
   GreeterClient(std::shared_ptr<Channel> channel)
       : stub_(Greeter::NewStub(channel)) {}
 
-  // Assambles the client's payload, sends it and presents the response back
+  // Assembles the client's payload, sends it and presents the response back
   // from the server.
   std::string SayHello(const std::string& user) {
     // Data we are sending to the server.

examples/python/helloworld/greeter_client.py

@@ -34,11 +34,12 @@ from __future__ import print_function
 import grpc
 
 import helloworld_pb2
+import helloworld_pb2_grpc
 
 
 def run():
   channel = grpc.insecure_channel('localhost:50051')
-  stub = helloworld_pb2.GreeterStub(channel)
+  stub = helloworld_pb2_grpc.GreeterStub(channel)
   response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
   print("Greeter client received: " + response.message)
 

examples/python/helloworld/greeter_server.py

@@ -35,11 +35,12 @@ import time
 import grpc
 
 import helloworld_pb2
+import helloworld_pb2_grpc
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
 
 
-class Greeter(helloworld_pb2.GreeterServicer):
+class Greeter(helloworld_pb2_grpc.GreeterServicer):
 
   def SayHello(self, request, context):
     return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
@@ -47,7 +48,7 @@ class Greeter(helloworld_pb2.GreeterServicer):
 
 def serve():
   server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-  helloworld_pb2.add_GreeterServicer_to_server(Greeter(), server)
+  helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
   server.add_insecure_port('[::]:50051')
   server.start()
   try:

examples/python/helloworld/helloworld_pb2.py

@@ -107,98 +107,123 @@ _sym_db.RegisterMessage(HelloReply)
 
 DESCRIPTOR.has_options = True
 DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW'))
-import grpc
-from grpc.beta import implementations as beta_implementations
-from grpc.beta import interfaces as beta_interfaces
-from grpc.framework.common import cardinality
-from grpc.framework.interfaces.face import utilities as face_utilities
+try:
+  # THESE ELEMENTS WILL BE DEPRECATED.
+  # Please use the generated *_pb2_grpc.py files instead.
+  import grpc
+  from grpc.framework.common import cardinality
+  from grpc.framework.interfaces.face import utilities as face_utilities
+  from grpc.beta import implementations as beta_implementations
+  from grpc.beta import interfaces as beta_interfaces
+
+
+  class GreeterStub(object):
+    """The greeting service definition.
+    """
 
+    def __init__(self, channel):
+      """Constructor.
 
-class GreeterStub(object):
-  """The greeting service definition.
-  """
+      Args:
+        channel: A grpc.Channel.
+      """
+      self.SayHello = channel.unary_unary(
+          '/helloworld.Greeter/SayHello',
+          request_serializer=HelloRequest.SerializeToString,
+          response_deserializer=HelloReply.FromString,
+          )
 
-  def __init__(self, channel):
-    """Constructor.
 
-    Args:
-      channel: A grpc.Channel.
+  class GreeterServicer(object):
+    """The greeting service definition.
     """
-    self.SayHello = channel.unary_unary(
-        '/helloworld.Greeter/SayHello',
-        request_serializer=HelloRequest.SerializeToString,
-        response_deserializer=HelloReply.FromString,
-        )
-
-
-class GreeterServicer(object):
-  """The greeting service definition.
-  """
 
-  def SayHello(self, request, context):
-    """Sends a greeting
+    def SayHello(self, request, context):
+      """Sends a greeting
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+
+  def add_GreeterServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        'SayHello': grpc.unary_unary_rpc_method_handler(
+            servicer.SayHello,
+            request_deserializer=HelloRequest.FromString,
+            response_serializer=HelloReply.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        'helloworld.Greeter', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+  class BetaGreeterServicer(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The greeting service definition.
     """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_GreeterServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'SayHello': grpc.unary_unary_rpc_method_handler(
-          servicer.SayHello,
-          request_deserializer=HelloRequest.FromString,
-          response_serializer=HelloReply.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'helloworld.Greeter', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BetaGreeterServicer(object):
-  """The greeting service definition.
-  """
-  def SayHello(self, request, context):
-    """Sends a greeting
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def SayHello(self, request, context):
+      """Sends a greeting
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
 
+  class BetaGreeterStub(object):
+    """The Beta API is deprecated for 0.15.0 and later.
 
-class BetaGreeterStub(object):
-  """The greeting service definition.
-  """
-  def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
-    """Sends a greeting
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The greeting service definition.
     """
-    raise NotImplementedError()
-  SayHello.future = None
-
-
-def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-  request_deserializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString,
-  }
-  response_serializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString,
-  }
-  method_implementations = {
-    ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello),
-  }
-  server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-  return beta_implementations.server(method_implementations, options=server_options)
-
-
-def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-  request_serializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString,
-  }
-  response_deserializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloReply.FromString,
-  }
-  cardinalities = {
-    'SayHello': cardinality.Cardinality.UNARY_UNARY,
-  }
-  stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-  return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options)
+    def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Sends a greeting
+      """
+      raise NotImplementedError()
+    SayHello.future = None
+
+
+  def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_deserializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString,
+    }
+    response_serializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString,
+    }
+    method_implementations = {
+      ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello),
+    }
+    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+    return beta_implementations.server(method_implementations, options=server_options)
+
+
+  def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_serializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString,
+    }
+    response_deserializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloReply.FromString,
+    }
+    cardinalities = {
+      'SayHello': cardinality.Cardinality.UNARY_UNARY,
+    }
+    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+    return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options)
+except ImportError:
+  pass
 # @@protoc_insertion_point(module_scope)

examples/python/helloworld/helloworld_pb2_grpc.py

@@ -0,0 +1,47 @@
+import grpc
+from grpc.framework.common import cardinality
+from grpc.framework.interfaces.face import utilities as face_utilities
+
+import helloworld_pb2 as helloworld__pb2
+
+
+class GreeterStub(object):
+  """The greeting service definition.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.SayHello = channel.unary_unary(
+        '/helloworld.Greeter/SayHello',
+        request_serializer=helloworld__pb2.HelloRequest.SerializeToString,
+        response_deserializer=helloworld__pb2.HelloReply.FromString,
+        )
+
+
+class GreeterServicer(object):
+  """The greeting service definition.
+  """
+
+  def SayHello(self, request, context):
+    """Sends a greeting
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_GreeterServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'SayHello': grpc.unary_unary_rpc_method_handler(
+          servicer.SayHello,
+          request_deserializer=helloworld__pb2.HelloRequest.FromString,
+          response_serializer=helloworld__pb2.HelloReply.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'helloworld.Greeter', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

examples/python/multiplex/helloworld_pb2.py

@@ -107,98 +107,123 @@ _sym_db.RegisterMessage(HelloReply)
 
 DESCRIPTOR.has_options = True
 DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW'))
-import grpc
-from grpc.beta import implementations as beta_implementations
-from grpc.beta import interfaces as beta_interfaces
-from grpc.framework.common import cardinality
-from grpc.framework.interfaces.face import utilities as face_utilities
+try:
+  # THESE ELEMENTS WILL BE DEPRECATED.
+  # Please use the generated *_pb2_grpc.py files instead.
+  import grpc
+  from grpc.framework.common import cardinality
+  from grpc.framework.interfaces.face import utilities as face_utilities
+  from grpc.beta import implementations as beta_implementations
+  from grpc.beta import interfaces as beta_interfaces
+
+
+  class GreeterStub(object):
+    """The greeting service definition.
+    """
 
+    def __init__(self, channel):
+      """Constructor.
 
-class GreeterStub(object):
-  """The greeting service definition.
-  """
+      Args:
+        channel: A grpc.Channel.
+      """
+      self.SayHello = channel.unary_unary(
+          '/helloworld.Greeter/SayHello',
+          request_serializer=HelloRequest.SerializeToString,
+          response_deserializer=HelloReply.FromString,
+          )
 
-  def __init__(self, channel):
-    """Constructor.
 
-    Args:
-      channel: A grpc.Channel.
+  class GreeterServicer(object):
+    """The greeting service definition.
     """
-    self.SayHello = channel.unary_unary(
-        '/helloworld.Greeter/SayHello',
-        request_serializer=HelloRequest.SerializeToString,
-        response_deserializer=HelloReply.FromString,
-        )
-
-
-class GreeterServicer(object):
-  """The greeting service definition.
-  """
 
-  def SayHello(self, request, context):
-    """Sends a greeting
+    def SayHello(self, request, context):
+      """Sends a greeting
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+
+  def add_GreeterServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        'SayHello': grpc.unary_unary_rpc_method_handler(
+            servicer.SayHello,
+            request_deserializer=HelloRequest.FromString,
+            response_serializer=HelloReply.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        'helloworld.Greeter', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+  class BetaGreeterServicer(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The greeting service definition.
     """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_GreeterServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'SayHello': grpc.unary_unary_rpc_method_handler(
-          servicer.SayHello,
-          request_deserializer=HelloRequest.FromString,
-          response_serializer=HelloReply.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'helloworld.Greeter', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BetaGreeterServicer(object):
-  """The greeting service definition.
-  """
-  def SayHello(self, request, context):
-    """Sends a greeting
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def SayHello(self, request, context):
+      """Sends a greeting
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
 
+  class BetaGreeterStub(object):
+    """The Beta API is deprecated for 0.15.0 and later.
 
-class BetaGreeterStub(object):
-  """The greeting service definition.
-  """
-  def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
-    """Sends a greeting
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """The greeting service definition.
     """
-    raise NotImplementedError()
-  SayHello.future = None
-
-
-def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-  request_deserializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString,
-  }
-  response_serializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString,
-  }
-  method_implementations = {
-    ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello),
-  }
-  server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-  return beta_implementations.server(method_implementations, options=server_options)
-
-
-def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-  request_serializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString,
-  }
-  response_deserializers = {
-    ('helloworld.Greeter', 'SayHello'): HelloReply.FromString,
-  }
-  cardinalities = {
-    'SayHello': cardinality.Cardinality.UNARY_UNARY,
-  }
-  stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-  return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options)
+    def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """Sends a greeting
+      """
+      raise NotImplementedError()
+    SayHello.future = None
+
+
+  def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_deserializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString,
+    }
+    response_serializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString,
+    }
+    method_implementations = {
+      ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello),
+    }
+    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+    return beta_implementations.server(method_implementations, options=server_options)
+
+
+  def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_serializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString,
+    }
+    response_deserializers = {
+      ('helloworld.Greeter', 'SayHello'): HelloReply.FromString,
+    }
+    cardinalities = {
+      'SayHello': cardinality.Cardinality.UNARY_UNARY,
+    }
+    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+    return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options)
+except ImportError:
+  pass
 # @@protoc_insertion_point(module_scope)

examples/python/multiplex/helloworld_pb2_grpc.py

@@ -0,0 +1,47 @@
+import grpc
+from grpc.framework.common import cardinality
+from grpc.framework.interfaces.face import utilities as face_utilities
+
+import helloworld_pb2 as helloworld__pb2
+
+
+class GreeterStub(object):
+  """The greeting service definition.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.SayHello = channel.unary_unary(
+        '/helloworld.Greeter/SayHello',
+        request_serializer=helloworld__pb2.HelloRequest.SerializeToString,
+        response_deserializer=helloworld__pb2.HelloReply.FromString,
+        )
+
+
+class GreeterServicer(object):
+  """The greeting service definition.
+  """
+
+  def SayHello(self, request, context):
+    """Sends a greeting
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_GreeterServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'SayHello': grpc.unary_unary_rpc_method_handler(
+          servicer.SayHello,
+          request_deserializer=helloworld__pb2.HelloRequest.FromString,
+          response_serializer=helloworld__pb2.HelloReply.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'helloworld.Greeter', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

examples/python/multiplex/multiplex_client.py

@@ -37,7 +37,9 @@ import time
 import grpc
 
 import helloworld_pb2
+import helloworld_pb2_grpc
 import route_guide_pb2
+import route_guide_pb2_grpc
 import route_guide_resources
 
 
@@ -120,8 +122,8 @@ def guide_route_chat(route_guide_stub):
 
 def run():
   channel = grpc.insecure_channel('localhost:50051')
-  greeter_stub = helloworld_pb2.GreeterStub(channel)
-  route_guide_stub = route_guide_pb2.RouteGuideStub(channel)
+  greeter_stub = helloworld_pb2_grpc.GreeterStub(channel)
+  route_guide_stub = route_guide_pb2_grpc.RouteGuideStub(channel)
   greeter_response = greeter_stub.SayHello(
       helloworld_pb2.HelloRequest(name='you'))
   print("Greeter client received: " + greeter_response.message)

examples/python/multiplex/multiplex_server.py

@@ -36,7 +36,9 @@ import math
 import grpc
 
 import helloworld_pb2
+import helloworld_pb2_grpc
 import route_guide_pb2
+import route_guide_pb2_grpc
 import route_guide_resources
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
@@ -70,13 +72,13 @@ def _get_distance(start, end):
   return R * c;
 
 
-class _GreeterServicer(helloworld_pb2.GreeterServicer):
+class _GreeterServicer(helloworld_pb2_grpc.GreeterServicer):
 
   def SayHello(self, request, context):
     return helloworld_pb2.HelloReply(message='Hello, {}!'.format(request.name))
 
 
-class _RouteGuideServicer(route_guide_pb2.RouteGuideServicer):
+class _RouteGuideServicer(route_guide_pb2_grpc.RouteGuideServicer):
   """Provides methods that implement functionality of route guide server."""
 
   def __init__(self):
@@ -133,8 +135,8 @@ class _RouteGuideServicer(route_guide_pb2.RouteGuideServicer):
 
 def serve():
   server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-  helloworld_pb2.add_GreeterServicer_to_server(_GreeterServicer(), server)
-  route_guide_pb2.add_RouteGuideServicer_to_server(
+  helloworld_pb2_grpc.add_GreeterServicer_to_server(_GreeterServicer(), server)
+  route_guide_pb2_grpc.add_RouteGuideServicer_to_server(
       _RouteGuideServicer(), server)
   server.add_insecure_port('[::]:50051')
   server.start()

examples/python/multiplex/route_guide_pb2.py

@@ -277,240 +277,265 @@ _sym_db.RegisterMessage(RouteSummary)
 
 DESCRIPTOR.has_options = True
 DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG'))
-import grpc
-from grpc.beta import implementations as beta_implementations
-from grpc.beta import interfaces as beta_interfaces
-from grpc.framework.common import cardinality
-from grpc.framework.interfaces.face import utilities as face_utilities
-
-
-class RouteGuideStub(object):
-  """Interface exported by the server.
-  """
-
-  def __init__(self, channel):
-    """Constructor.
-
-    Args:
-      channel: A grpc.Channel.
+try:
+  # THESE ELEMENTS WILL BE DEPRECATED.
+  # Please use the generated *_pb2_grpc.py files instead.
+  import grpc
+  from grpc.framework.common import cardinality
+  from grpc.framework.interfaces.face import utilities as face_utilities
+  from grpc.beta import implementations as beta_implementations
+  from grpc.beta import interfaces as beta_interfaces
+
+
+  class RouteGuideStub(object):
+    """Interface exported by the server.
     """
-    self.GetFeature = channel.unary_unary(
-        '/routeguide.RouteGuide/GetFeature',
-        request_serializer=Point.SerializeToString,
-        response_deserializer=Feature.FromString,
-        )
-    self.ListFeatures = channel.unary_stream(
-        '/routeguide.RouteGuide/ListFeatures',
-        request_serializer=Rectangle.SerializeToString,
-        response_deserializer=Feature.FromString,
-        )
-    self.RecordRoute = channel.stream_unary(
-        '/routeguide.RouteGuide/RecordRoute',
-        request_serializer=Point.SerializeToString,
-        response_deserializer=RouteSummary.FromString,
-        )
-    self.RouteChat = channel.stream_stream(
-        '/routeguide.RouteGuide/RouteChat',
-        request_serializer=RouteNote.SerializeToString,
-        response_deserializer=RouteNote.FromString,
-        )
-
-
-class RouteGuideServicer(object):
-  """Interface exported by the server.
-  """
-
-  def GetFeature(self, request, context):
-    """A simple RPC.
-
-    Obtains the feature at a given position.
-
-    A feature with an empty name is returned if there's no feature at the given
-    position.
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-  def ListFeatures(self, request, context):
-    """A server-to-client streaming RPC.
 
-    Obtains the Features available within the given Rectangle.  Results are
-    streamed rather than returned at once (e.g. in a response message with a
-    repeated field), as the rectangle may cover a large area and contain a
-    huge number of features.
+    def __init__(self, channel):
+      """Constructor.
+
+      Args:
+        channel: A grpc.Channel.
+      """
+      self.GetFeature = channel.unary_unary(
+          '/routeguide.RouteGuide/GetFeature',
+          request_serializer=Point.SerializeToString,
+          response_deserializer=Feature.FromString,
+          )
+      self.ListFeatures = channel.unary_stream(
+          '/routeguide.RouteGuide/ListFeatures',
+          request_serializer=Rectangle.SerializeToString,
+          response_deserializer=Feature.FromString,
+          )
+      self.RecordRoute = channel.stream_unary(
+          '/routeguide.RouteGuide/RecordRoute',
+          request_serializer=Point.SerializeToString,
+          response_deserializer=RouteSummary.FromString,
+          )
+      self.RouteChat = channel.stream_stream(
+          '/routeguide.RouteGuide/RouteChat',
+          request_serializer=RouteNote.SerializeToString,
+          response_deserializer=RouteNote.FromString,
+          )
+
+
+  class RouteGuideServicer(object):
+    """Interface exported by the server.
     """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-  def RecordRoute(self, request_iterator, context):
-    """A client-to-server streaming RPC.
 
-    Accepts a stream of Points on a route being traversed, returning a
-    RouteSummary when traversal is completed.
+    def GetFeature(self, request, context):
+      """A simple RPC.
+
+      Obtains the feature at a given position.
+
+      A feature with an empty name is returned if there's no feature at the given
+      position.
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def ListFeatures(self, request, context):
+      """A server-to-client streaming RPC.
+
+      Obtains the Features available within the given Rectangle.  Results are
+      streamed rather than returned at once (e.g. in a response message with a
+      repeated field), as the rectangle may cover a large area and contain a
+      huge number of features.
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def RecordRoute(self, request_iterator, context):
+      """A client-to-server streaming RPC.
+
+      Accepts a stream of Points on a route being traversed, returning a
+      RouteSummary when traversal is completed.
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def RouteChat(self, request_iterator, context):
+      """A Bidirectional streaming RPC.
+
+      Accepts a stream of RouteNotes sent while a route is being traversed,
+      while receiving other RouteNotes (e.g. from other users).
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+
+  def add_RouteGuideServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        'GetFeature': grpc.unary_unary_rpc_method_handler(
+            servicer.GetFeature,
+            request_deserializer=Point.FromString,
+            response_serializer=Feature.SerializeToString,
+        ),
+        'ListFeatures': grpc.unary_stream_rpc_method_handler(
+            servicer.ListFeatures,
+            request_deserializer=Rectangle.FromString,
+            response_serializer=Feature.SerializeToString,
+        ),
+        'RecordRoute': grpc.stream_unary_rpc_method_handler(
+            servicer.RecordRoute,
+            request_deserializer=Point.FromString,
+            response_serializer=RouteSummary.SerializeToString,
+        ),
+        'RouteChat': grpc.stream_stream_rpc_method_handler(
+            servicer.RouteChat,
+            request_deserializer=RouteNote.FromString,
+            response_serializer=RouteNote.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        'routeguide.RouteGuide', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+  class BetaRouteGuideServicer(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """Interface exported by the server.
     """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-  def RouteChat(self, request_iterator, context):
-    """A Bidirectional streaming RPC.
-
-    Accepts a stream of RouteNotes sent while a route is being traversed,
-    while receiving other RouteNotes (e.g. from other users).
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_RouteGuideServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'GetFeature': grpc.unary_unary_rpc_method_handler(
-          servicer.GetFeature,
-          request_deserializer=Point.FromString,
-          response_serializer=Feature.SerializeToString,
-      ),
-      'ListFeatures': grpc.unary_stream_rpc_method_handler(
-          servicer.ListFeatures,
-          request_deserializer=Rectangle.FromString,
-          response_serializer=Feature.SerializeToString,
-      ),
-      'RecordRoute': grpc.stream_unary_rpc_method_handler(
-          servicer.RecordRoute,
-          request_deserializer=Point.FromString,
-          response_serializer=RouteSummary.SerializeToString,
-      ),
-      'RouteChat': grpc.stream_stream_rpc_method_handler(
-          servicer.RouteChat,
-          request_deserializer=RouteNote.FromString,
-          response_serializer=RouteNote.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'routeguide.RouteGuide', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BetaRouteGuideServicer(object):
-  """Interface exported by the server.
-  """
-  def GetFeature(self, request, context):
-    """A simple RPC.
-
-    Obtains the feature at a given position.
-
-    A feature with an empty name is returned if there's no feature at the given
-    position.
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-  def ListFeatures(self, request, context):
-    """A server-to-client streaming RPC.
-
-    Obtains the Features available within the given Rectangle.  Results are
-    streamed rather than returned at once (e.g. in a response message with a
-    repeated field), as the rectangle may cover a large area and contain a
-    huge number of features.
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-  def RecordRoute(self, request_iterator, context):
-    """A client-to-server streaming RPC.
-
-    Accepts a stream of Points on a route being traversed, returning a
-    RouteSummary when traversal is completed.
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-  def RouteChat(self, request_iterator, context):
-    """A Bidirectional streaming RPC.
-
-    Accepts a stream of RouteNotes sent while a route is being traversed,
-    while receiving other RouteNotes (e.g. from other users).
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
-class BetaRouteGuideStub(object):
-  """Interface exported by the server.
-  """
-  def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A simple RPC.
-
-    Obtains the feature at a given position.
-
-    A feature with an empty name is returned if there's no feature at the given
-    position.
-    """
-    raise NotImplementedError()
-  GetFeature.future = None
-  def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A server-to-client streaming RPC.
-
-    Obtains the Features available within the given Rectangle.  Results are
-    streamed rather than returned at once (e.g. in a response message with a
-    repeated field), as the rectangle may cover a large area and contain a
-    huge number of features.
-    """
-    raise NotImplementedError()
-  def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A client-to-server streaming RPC.
-
-    Accepts a stream of Points on a route being traversed, returning a
-    RouteSummary when traversal is completed.
-    """
-    raise NotImplementedError()
-  RecordRoute.future = None
-  def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A Bidirectional streaming RPC.
-
-    Accepts a stream of RouteNotes sent while a route is being traversed,
-    while receiving other RouteNotes (e.g. from other users).
+    def GetFeature(self, request, context):
+      """A simple RPC.
+
+      Obtains the feature at a given position.
+
+      A feature with an empty name is returned if there's no feature at the given
+      position.
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def ListFeatures(self, request, context):
+      """A server-to-client streaming RPC.
+
+      Obtains the Features available within the given Rectangle.  Results are
+      streamed rather than returned at once (e.g. in a response message with a
+      repeated field), as the rectangle may cover a large area and contain a
+      huge number of features.
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def RecordRoute(self, request_iterator, context):
+      """A client-to-server streaming RPC.
+
+      Accepts a stream of Points on a route being traversed, returning a
+      RouteSummary when traversal is completed.
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def RouteChat(self, request_iterator, context):
+      """A Bidirectional streaming RPC.
+
+      Accepts a stream of RouteNotes sent while a route is being traversed,
+      while receiving other RouteNotes (e.g. from other users).
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
+
+  class BetaRouteGuideStub(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """Interface exported by the server.
     """
-    raise NotImplementedError()
-
-
-def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-  request_deserializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Point.FromString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString,
-    ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
-  }
-  response_serializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString,
-    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
-  }
-  method_implementations = {
-    ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature),
-    ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures),
-    ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute),
-    ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat),
-  }
-  server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-  return beta_implementations.server(method_implementations, options=server_options)
-
-
-def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-  request_serializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString,
-    ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
-  }
-  response_deserializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString,
-    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
-  }
-  cardinalities = {
-    'GetFeature': cardinality.Cardinality.UNARY_UNARY,
-    'ListFeatures': cardinality.Cardinality.UNARY_STREAM,
-    'RecordRoute': cardinality.Cardinality.STREAM_UNARY,
-    'RouteChat': cardinality.Cardinality.STREAM_STREAM,
-  }
-  stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-  return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options)
+    def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A simple RPC.
+
+      Obtains the feature at a given position.
+
+      A feature with an empty name is returned if there's no feature at the given
+      position.
+      """
+      raise NotImplementedError()
+    GetFeature.future = None
+    def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A server-to-client streaming RPC.
+
+      Obtains the Features available within the given Rectangle.  Results are
+      streamed rather than returned at once (e.g. in a response message with a
+      repeated field), as the rectangle may cover a large area and contain a
+      huge number of features.
+      """
+      raise NotImplementedError()
+    def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A client-to-server streaming RPC.
+
+      Accepts a stream of Points on a route being traversed, returning a
+      RouteSummary when traversal is completed.
+      """
+      raise NotImplementedError()
+    RecordRoute.future = None
+    def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A Bidirectional streaming RPC.
+
+      Accepts a stream of RouteNotes sent while a route is being traversed,
+      while receiving other RouteNotes (e.g. from other users).
+      """
+      raise NotImplementedError()
+
+
+  def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_deserializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Point.FromString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString,
+      ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
+    }
+    response_serializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString,
+      ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
+    }
+    method_implementations = {
+      ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature),
+      ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures),
+      ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute),
+      ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat),
+    }
+    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+    return beta_implementations.server(method_implementations, options=server_options)
+
+
+  def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_serializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString,
+      ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
+    }
+    response_deserializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString,
+      ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
+    }
+    cardinalities = {
+      'GetFeature': cardinality.Cardinality.UNARY_UNARY,
+      'ListFeatures': cardinality.Cardinality.UNARY_STREAM,
+      'RecordRoute': cardinality.Cardinality.STREAM_UNARY,
+      'RouteChat': cardinality.Cardinality.STREAM_STREAM,
+    }
+    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+    return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options)
+except ImportError:
+  pass
 # @@protoc_insertion_point(module_scope)

examples/python/multiplex/route_guide_pb2_grpc.py

@@ -0,0 +1,114 @@
+import grpc
+from grpc.framework.common import cardinality
+from grpc.framework.interfaces.face import utilities as face_utilities
+
+import route_guide_pb2 as route__guide__pb2
+
+
+class RouteGuideStub(object):
+  """Interface exported by the server.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.GetFeature = channel.unary_unary(
+        '/routeguide.RouteGuide/GetFeature',
+        request_serializer=route__guide__pb2.Point.SerializeToString,
+        response_deserializer=route__guide__pb2.Feature.FromString,
+        )
+    self.ListFeatures = channel.unary_stream(
+        '/routeguide.RouteGuide/ListFeatures',
+        request_serializer=route__guide__pb2.Rectangle.SerializeToString,
+        response_deserializer=route__guide__pb2.Feature.FromString,
+        )
+    self.RecordRoute = channel.stream_unary(
+        '/routeguide.RouteGuide/RecordRoute',
+        request_serializer=route__guide__pb2.Point.SerializeToString,
+        response_deserializer=route__guide__pb2.RouteSummary.FromString,
+        )
+    self.RouteChat = channel.stream_stream(
+        '/routeguide.RouteGuide/RouteChat',
+        request_serializer=route__guide__pb2.RouteNote.SerializeToString,
+        response_deserializer=route__guide__pb2.RouteNote.FromString,
+        )
+
+
+class RouteGuideServicer(object):
+  """Interface exported by the server.
+  """
+
+  def GetFeature(self, request, context):
+    """A simple RPC.
+
+    Obtains the feature at a given position.
+
+    A feature with an empty name is returned if there's no feature at the given
+    position.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def ListFeatures(self, request, context):
+    """A server-to-client streaming RPC.
+
+    Obtains the Features available within the given Rectangle.  Results are
+    streamed rather than returned at once (e.g. in a response message with a
+    repeated field), as the rectangle may cover a large area and contain a
+    huge number of features.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RecordRoute(self, request_iterator, context):
+    """A client-to-server streaming RPC.
+
+    Accepts a stream of Points on a route being traversed, returning a
+    RouteSummary when traversal is completed.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RouteChat(self, request_iterator, context):
+    """A Bidirectional streaming RPC.
+
+    Accepts a stream of RouteNotes sent while a route is being traversed,
+    while receiving other RouteNotes (e.g. from other users).
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_RouteGuideServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'GetFeature': grpc.unary_unary_rpc_method_handler(
+          servicer.GetFeature,
+          request_deserializer=route__guide__pb2.Point.FromString,
+          response_serializer=route__guide__pb2.Feature.SerializeToString,
+      ),
+      'ListFeatures': grpc.unary_stream_rpc_method_handler(
+          servicer.ListFeatures,
+          request_deserializer=route__guide__pb2.Rectangle.FromString,
+          response_serializer=route__guide__pb2.Feature.SerializeToString,
+      ),
+      'RecordRoute': grpc.stream_unary_rpc_method_handler(
+          servicer.RecordRoute,
+          request_deserializer=route__guide__pb2.Point.FromString,
+          response_serializer=route__guide__pb2.RouteSummary.SerializeToString,
+      ),
+      'RouteChat': grpc.stream_stream_rpc_method_handler(
+          servicer.RouteChat,
+          request_deserializer=route__guide__pb2.RouteNote.FromString,
+          response_serializer=route__guide__pb2.RouteNote.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'routeguide.RouteGuide', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

examples/python/multiplex/run_codegen.py

@@ -29,7 +29,7 @@
 
 """Generates protocol messages and gRPC stubs."""
 
-from grpc.tools import protoc
+from grpc_tools import protoc
 
 protoc.main(
     (

examples/python/route_guide/route_guide_client.py

@@ -37,6 +37,7 @@ import time
 import grpc
 
 import route_guide_pb2
+import route_guide_pb2_grpc
 import route_guide_resources
 
 
@@ -116,7 +117,7 @@ def guide_route_chat(stub):
 
 def run():
   channel = grpc.insecure_channel('localhost:50051')
-  stub = route_guide_pb2.RouteGuideStub(channel)
+  stub = route_guide_pb2_grpc.RouteGuideStub(channel)
   print("-------------- GetFeature --------------")
   guide_get_feature(stub)
   print("-------------- ListFeatures --------------")

examples/python/route_guide/route_guide_pb2.py

@@ -277,240 +277,265 @@ _sym_db.RegisterMessage(RouteSummary)
 
 DESCRIPTOR.has_options = True
 DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG'))
-import grpc
-from grpc.beta import implementations as beta_implementations
-from grpc.beta import interfaces as beta_interfaces
-from grpc.framework.common import cardinality
-from grpc.framework.interfaces.face import utilities as face_utilities
-
-
-class RouteGuideStub(object):
-  """Interface exported by the server.
-  """
-
-  def __init__(self, channel):
-    """Constructor.
-
-    Args:
-      channel: A grpc.Channel.
+try:
+  # THESE ELEMENTS WILL BE DEPRECATED.
+  # Please use the generated *_pb2_grpc.py files instead.
+  import grpc
+  from grpc.framework.common import cardinality
+  from grpc.framework.interfaces.face import utilities as face_utilities
+  from grpc.beta import implementations as beta_implementations
+  from grpc.beta import interfaces as beta_interfaces
+
+
+  class RouteGuideStub(object):
+    """Interface exported by the server.
     """
-    self.GetFeature = channel.unary_unary(
-        '/routeguide.RouteGuide/GetFeature',
-        request_serializer=Point.SerializeToString,
-        response_deserializer=Feature.FromString,
-        )
-    self.ListFeatures = channel.unary_stream(
-        '/routeguide.RouteGuide/ListFeatures',
-        request_serializer=Rectangle.SerializeToString,
-        response_deserializer=Feature.FromString,
-        )
-    self.RecordRoute = channel.stream_unary(
-        '/routeguide.RouteGuide/RecordRoute',
-        request_serializer=Point.SerializeToString,
-        response_deserializer=RouteSummary.FromString,
-        )
-    self.RouteChat = channel.stream_stream(
-        '/routeguide.RouteGuide/RouteChat',
-        request_serializer=RouteNote.SerializeToString,
-        response_deserializer=RouteNote.FromString,
-        )
-
-
-class RouteGuideServicer(object):
-  """Interface exported by the server.
-  """
-
-  def GetFeature(self, request, context):
-    """A simple RPC.
-
-    Obtains the feature at a given position.
-
-    A feature with an empty name is returned if there's no feature at the given
-    position.
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-  def ListFeatures(self, request, context):
-    """A server-to-client streaming RPC.
 
-    Obtains the Features available within the given Rectangle.  Results are
-    streamed rather than returned at once (e.g. in a response message with a
-    repeated field), as the rectangle may cover a large area and contain a
-    huge number of features.
+    def __init__(self, channel):
+      """Constructor.
+
+      Args:
+        channel: A grpc.Channel.
+      """
+      self.GetFeature = channel.unary_unary(
+          '/routeguide.RouteGuide/GetFeature',
+          request_serializer=Point.SerializeToString,
+          response_deserializer=Feature.FromString,
+          )
+      self.ListFeatures = channel.unary_stream(
+          '/routeguide.RouteGuide/ListFeatures',
+          request_serializer=Rectangle.SerializeToString,
+          response_deserializer=Feature.FromString,
+          )
+      self.RecordRoute = channel.stream_unary(
+          '/routeguide.RouteGuide/RecordRoute',
+          request_serializer=Point.SerializeToString,
+          response_deserializer=RouteSummary.FromString,
+          )
+      self.RouteChat = channel.stream_stream(
+          '/routeguide.RouteGuide/RouteChat',
+          request_serializer=RouteNote.SerializeToString,
+          response_deserializer=RouteNote.FromString,
+          )
+
+
+  class RouteGuideServicer(object):
+    """Interface exported by the server.
     """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-  def RecordRoute(self, request_iterator, context):
-    """A client-to-server streaming RPC.
 
-    Accepts a stream of Points on a route being traversed, returning a
-    RouteSummary when traversal is completed.
+    def GetFeature(self, request, context):
+      """A simple RPC.
+
+      Obtains the feature at a given position.
+
+      A feature with an empty name is returned if there's no feature at the given
+      position.
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def ListFeatures(self, request, context):
+      """A server-to-client streaming RPC.
+
+      Obtains the Features available within the given Rectangle.  Results are
+      streamed rather than returned at once (e.g. in a response message with a
+      repeated field), as the rectangle may cover a large area and contain a
+      huge number of features.
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def RecordRoute(self, request_iterator, context):
+      """A client-to-server streaming RPC.
+
+      Accepts a stream of Points on a route being traversed, returning a
+      RouteSummary when traversal is completed.
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+    def RouteChat(self, request_iterator, context):
+      """A Bidirectional streaming RPC.
+
+      Accepts a stream of RouteNotes sent while a route is being traversed,
+      while receiving other RouteNotes (e.g. from other users).
+      """
+      context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+      context.set_details('Method not implemented!')
+      raise NotImplementedError('Method not implemented!')
+
+
+  def add_RouteGuideServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+        'GetFeature': grpc.unary_unary_rpc_method_handler(
+            servicer.GetFeature,
+            request_deserializer=Point.FromString,
+            response_serializer=Feature.SerializeToString,
+        ),
+        'ListFeatures': grpc.unary_stream_rpc_method_handler(
+            servicer.ListFeatures,
+            request_deserializer=Rectangle.FromString,
+            response_serializer=Feature.SerializeToString,
+        ),
+        'RecordRoute': grpc.stream_unary_rpc_method_handler(
+            servicer.RecordRoute,
+            request_deserializer=Point.FromString,
+            response_serializer=RouteSummary.SerializeToString,
+        ),
+        'RouteChat': grpc.stream_stream_rpc_method_handler(
+            servicer.RouteChat,
+            request_deserializer=RouteNote.FromString,
+            response_serializer=RouteNote.SerializeToString,
+        ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+        'routeguide.RouteGuide', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+  class BetaRouteGuideServicer(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """Interface exported by the server.
     """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-  def RouteChat(self, request_iterator, context):
-    """A Bidirectional streaming RPC.
-
-    Accepts a stream of RouteNotes sent while a route is being traversed,
-    while receiving other RouteNotes (e.g. from other users).
-    """
-    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-    context.set_details('Method not implemented!')
-    raise NotImplementedError('Method not implemented!')
-
-
-def add_RouteGuideServicer_to_server(servicer, server):
-  rpc_method_handlers = {
-      'GetFeature': grpc.unary_unary_rpc_method_handler(
-          servicer.GetFeature,
-          request_deserializer=Point.FromString,
-          response_serializer=Feature.SerializeToString,
-      ),
-      'ListFeatures': grpc.unary_stream_rpc_method_handler(
-          servicer.ListFeatures,
-          request_deserializer=Rectangle.FromString,
-          response_serializer=Feature.SerializeToString,
-      ),
-      'RecordRoute': grpc.stream_unary_rpc_method_handler(
-          servicer.RecordRoute,
-          request_deserializer=Point.FromString,
-          response_serializer=RouteSummary.SerializeToString,
-      ),
-      'RouteChat': grpc.stream_stream_rpc_method_handler(
-          servicer.RouteChat,
-          request_deserializer=RouteNote.FromString,
-          response_serializer=RouteNote.SerializeToString,
-      ),
-  }
-  generic_handler = grpc.method_handlers_generic_handler(
-      'routeguide.RouteGuide', rpc_method_handlers)
-  server.add_generic_rpc_handlers((generic_handler,))
-
-
-class BetaRouteGuideServicer(object):
-  """Interface exported by the server.
-  """
-  def GetFeature(self, request, context):
-    """A simple RPC.
-
-    Obtains the feature at a given position.
-
-    A feature with an empty name is returned if there's no feature at the given
-    position.
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-  def ListFeatures(self, request, context):
-    """A server-to-client streaming RPC.
-
-    Obtains the Features available within the given Rectangle.  Results are
-    streamed rather than returned at once (e.g. in a response message with a
-    repeated field), as the rectangle may cover a large area and contain a
-    huge number of features.
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-  def RecordRoute(self, request_iterator, context):
-    """A client-to-server streaming RPC.
-
-    Accepts a stream of Points on a route being traversed, returning a
-    RouteSummary when traversal is completed.
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-  def RouteChat(self, request_iterator, context):
-    """A Bidirectional streaming RPC.
-
-    Accepts a stream of RouteNotes sent while a route is being traversed,
-    while receiving other RouteNotes (e.g. from other users).
-    """
-    context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
-
-
-class BetaRouteGuideStub(object):
-  """Interface exported by the server.
-  """
-  def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A simple RPC.
-
-    Obtains the feature at a given position.
-
-    A feature with an empty name is returned if there's no feature at the given
-    position.
-    """
-    raise NotImplementedError()
-  GetFeature.future = None
-  def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A server-to-client streaming RPC.
-
-    Obtains the Features available within the given Rectangle.  Results are
-    streamed rather than returned at once (e.g. in a response message with a
-    repeated field), as the rectangle may cover a large area and contain a
-    huge number of features.
-    """
-    raise NotImplementedError()
-  def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A client-to-server streaming RPC.
-
-    Accepts a stream of Points on a route being traversed, returning a
-    RouteSummary when traversal is completed.
-    """
-    raise NotImplementedError()
-  RecordRoute.future = None
-  def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
-    """A Bidirectional streaming RPC.
-
-    Accepts a stream of RouteNotes sent while a route is being traversed,
-    while receiving other RouteNotes (e.g. from other users).
+    def GetFeature(self, request, context):
+      """A simple RPC.
+
+      Obtains the feature at a given position.
+
+      A feature with an empty name is returned if there's no feature at the given
+      position.
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def ListFeatures(self, request, context):
+      """A server-to-client streaming RPC.
+
+      Obtains the Features available within the given Rectangle.  Results are
+      streamed rather than returned at once (e.g. in a response message with a
+      repeated field), as the rectangle may cover a large area and contain a
+      huge number of features.
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def RecordRoute(self, request_iterator, context):
+      """A client-to-server streaming RPC.
+
+      Accepts a stream of Points on a route being traversed, returning a
+      RouteSummary when traversal is completed.
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+    def RouteChat(self, request_iterator, context):
+      """A Bidirectional streaming RPC.
+
+      Accepts a stream of RouteNotes sent while a route is being traversed,
+      while receiving other RouteNotes (e.g. from other users).
+      """
+      context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)
+
+
+  class BetaRouteGuideStub(object):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This class was generated
+    only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0."""
+    """Interface exported by the server.
     """
-    raise NotImplementedError()
-
-
-def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
-  request_deserializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Point.FromString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString,
-    ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
-  }
-  response_serializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString,
-    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
-  }
-  method_implementations = {
-    ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature),
-    ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures),
-    ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute),
-    ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat),
-  }
-  server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
-  return beta_implementations.server(method_implementations, options=server_options)
-
-
-def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
-  request_serializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString,
-    ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
-  }
-  response_deserializers = {
-    ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString,
-    ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString,
-    ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString,
-    ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
-  }
-  cardinalities = {
-    'GetFeature': cardinality.Cardinality.UNARY_UNARY,
-    'ListFeatures': cardinality.Cardinality.UNARY_STREAM,
-    'RecordRoute': cardinality.Cardinality.STREAM_UNARY,
-    'RouteChat': cardinality.Cardinality.STREAM_STREAM,
-  }
-  stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
-  return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options)
+    def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A simple RPC.
+
+      Obtains the feature at a given position.
+
+      A feature with an empty name is returned if there's no feature at the given
+      position.
+      """
+      raise NotImplementedError()
+    GetFeature.future = None
+    def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A server-to-client streaming RPC.
+
+      Obtains the Features available within the given Rectangle.  Results are
+      streamed rather than returned at once (e.g. in a response message with a
+      repeated field), as the rectangle may cover a large area and contain a
+      huge number of features.
+      """
+      raise NotImplementedError()
+    def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A client-to-server streaming RPC.
+
+      Accepts a stream of Points on a route being traversed, returning a
+      RouteSummary when traversal is completed.
+      """
+      raise NotImplementedError()
+    RecordRoute.future = None
+    def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None):
+      """A Bidirectional streaming RPC.
+
+      Accepts a stream of RouteNotes sent while a route is being traversed,
+      while receiving other RouteNotes (e.g. from other users).
+      """
+      raise NotImplementedError()
+
+
+  def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_deserializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Point.FromString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString,
+      ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
+    }
+    response_serializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString,
+      ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
+    }
+    method_implementations = {
+      ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature),
+      ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures),
+      ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute),
+      ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat),
+    }
+    server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout)
+    return beta_implementations.server(method_implementations, options=server_options)
+
+
+  def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None):
+    """The Beta API is deprecated for 0.15.0 and later.
+
+    It is recommended to use the GA API (classes and functions in this
+    file not marked beta) for all further purposes. This function was
+    generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"""
+    request_serializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString,
+      ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString,
+    }
+    response_deserializers = {
+      ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString,
+      ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString,
+      ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString,
+      ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString,
+    }
+    cardinalities = {
+      'GetFeature': cardinality.Cardinality.UNARY_UNARY,
+      'ListFeatures': cardinality.Cardinality.UNARY_STREAM,
+      'RecordRoute': cardinality.Cardinality.STREAM_UNARY,
+      'RouteChat': cardinality.Cardinality.STREAM_STREAM,
+    }
+    stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size)
+    return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options)
+except ImportError:
+  pass
 # @@protoc_insertion_point(module_scope)

examples/python/route_guide/route_guide_pb2_grpc.py

@@ -0,0 +1,114 @@
+import grpc
+from grpc.framework.common import cardinality
+from grpc.framework.interfaces.face import utilities as face_utilities
+
+import route_guide_pb2 as route__guide__pb2
+
+
+class RouteGuideStub(object):
+  """Interface exported by the server.
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.GetFeature = channel.unary_unary(
+        '/routeguide.RouteGuide/GetFeature',
+        request_serializer=route__guide__pb2.Point.SerializeToString,
+        response_deserializer=route__guide__pb2.Feature.FromString,
+        )
+    self.ListFeatures = channel.unary_stream(
+        '/routeguide.RouteGuide/ListFeatures',
+        request_serializer=route__guide__pb2.Rectangle.SerializeToString,
+        response_deserializer=route__guide__pb2.Feature.FromString,
+        )
+    self.RecordRoute = channel.stream_unary(
+        '/routeguide.RouteGuide/RecordRoute',
+        request_serializer=route__guide__pb2.Point.SerializeToString,
+        response_deserializer=route__guide__pb2.RouteSummary.FromString,
+        )
+    self.RouteChat = channel.stream_stream(
+        '/routeguide.RouteGuide/RouteChat',
+        request_serializer=route__guide__pb2.RouteNote.SerializeToString,
+        response_deserializer=route__guide__pb2.RouteNote.FromString,
+        )
+
+
+class RouteGuideServicer(object):
+  """Interface exported by the server.
+  """
+
+  def GetFeature(self, request, context):
+    """A simple RPC.
+
+    Obtains the feature at a given position.
+
+    A feature with an empty name is returned if there's no feature at the given
+    position.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def ListFeatures(self, request, context):
+    """A server-to-client streaming RPC.
+
+    Obtains the Features available within the given Rectangle.  Results are
+    streamed rather than returned at once (e.g. in a response message with a
+    repeated field), as the rectangle may cover a large area and contain a
+    huge number of features.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RecordRoute(self, request_iterator, context):
+    """A client-to-server streaming RPC.
+
+    Accepts a stream of Points on a route being traversed, returning a
+    RouteSummary when traversal is completed.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def RouteChat(self, request_iterator, context):
+    """A Bidirectional streaming RPC.
+
+    Accepts a stream of RouteNotes sent while a route is being traversed,
+    while receiving other RouteNotes (e.g. from other users).
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_RouteGuideServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'GetFeature': grpc.unary_unary_rpc_method_handler(
+          servicer.GetFeature,
+          request_deserializer=route__guide__pb2.Point.FromString,
+          response_serializer=route__guide__pb2.Feature.SerializeToString,
+      ),
+      'ListFeatures': grpc.unary_stream_rpc_method_handler(
+          servicer.ListFeatures,
+          request_deserializer=route__guide__pb2.Rectangle.FromString,
+          response_serializer=route__guide__pb2.Feature.SerializeToString,
+      ),
+      'RecordRoute': grpc.stream_unary_rpc_method_handler(
+          servicer.RecordRoute,
+          request_deserializer=route__guide__pb2.Point.FromString,
+          response_serializer=route__guide__pb2.RouteSummary.SerializeToString,
+      ),
+      'RouteChat': grpc.stream_stream_rpc_method_handler(
+          servicer.RouteChat,
+          request_deserializer=route__guide__pb2.RouteNote.FromString,
+          response_serializer=route__guide__pb2.RouteNote.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'routeguide.RouteGuide', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))

examples/python/route_guide/route_guide_server.py

@@ -36,6 +36,7 @@ import math
 import grpc
 
 import route_guide_pb2
+import route_guide_pb2_grpc
 import route_guide_resources
 
 _ONE_DAY_IN_SECONDS = 60 * 60 * 24
@@ -68,7 +69,7 @@ def get_distance(start, end):
   R = 6371000; # metres
   return R * c;
 
-class RouteGuideServicer(route_guide_pb2.RouteGuideServicer):
+class RouteGuideServicer(route_guide_pb2_grpc.RouteGuideServicer):
   """Provides methods that implement functionality of route guide server."""
 
   def __init__(self):
@@ -125,7 +126,7 @@ class RouteGuideServicer(route_guide_pb2.RouteGuideServicer):
 
 def serve():
   server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-  route_guide_pb2.add_RouteGuideServicer_to_server(
+  route_guide_pb2_grpc.add_RouteGuideServicer_to_server(
       RouteGuideServicer(), server)
   server.add_insecure_port('[::]:50051')
   server.start()

examples/python/route_guide/run_codegen.py

@@ -29,7 +29,7 @@
 
 """Runs protoc with the gRPC plugin to generate messages and gRPC stubs."""
 
-from grpc.tools import protoc
+from grpc_tools import protoc
 
 protoc.main(
     (

gRPC-Core.podspec

@@ -35,7 +35,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '1.0.1'
+  version = '1.0.2'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'http://www.grpc.io'
@@ -44,7 +44,9 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "v#{version}",
+    # TODO(mxyan): Change back to "v#{version}" for next release
+    #:tag => "v#{version}",
+    :tag => "objective-c-v#{version}",
     # TODO(jcanizales): Depend explicitly on the nanopb pod, and disable submodules.
     :submodules => true,
   }

gRPC-ProtoRPC.podspec

@@ -30,7 +30,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '1.0.1'
+  version = '1.0.2'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'http://www.grpc.io'
@@ -39,7 +39,7 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "v#{version}",
+    :tag => "objective-c-v#{version}",
   }
 
   s.ios.deployment_target = '7.1'

gRPC-RxLibrary.podspec

@@ -30,7 +30,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '1.0.1'
+  version = '1.0.2'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'http://www.grpc.io'
@@ -39,7 +39,7 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "v#{version}",
+    :tag => "objective-c-v#{version}",
   }
 
   s.ios.deployment_target = '7.1'

gRPC.podspec

@@ -30,7 +30,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '1.0.1'
+  version = '1.0.2'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'http://www.grpc.io'
@@ -39,7 +39,7 @@ Pod::Spec.new do |s|
 
   s.source = {
     :git => 'https://github.com/grpc/grpc.git',
-    :tag => "v#{version}",
+    :tag => "objective-c-v#{version}",
   }
 
   s.ios.deployment_target = '7.1'

grpc.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
   s.require_paths = %w( src/ruby/bin src/ruby/lib src/ruby/pb )
   s.platform      = Gem::Platform::RUBY
 
-  s.add_dependency 'google-protobuf', '~> 3.0.2'
+  s.add_dependency 'google-protobuf', '~> 3.1.0'
   s.add_dependency 'googleauth',      '~> 0.5.1'
 
   s.add_development_dependency 'bundler',            '~> 1.9'

include/grpc++/impl/codegen/completion_queue.h

@@ -52,6 +52,7 @@
 #include <grpc++/impl/codegen/grpc_library.h>
 #include <grpc++/impl/codegen/status.h>
 #include <grpc++/impl/codegen/time.h>
+#include <grpc/impl/codegen/atm.h>
 
 struct grpc_completion_queue;
 
@@ -101,6 +102,7 @@ class CompletionQueue : private GrpcLibraryCodegen {
   /// instance.
   CompletionQueue() {
     cq_ = g_core_codegen_interface->grpc_completion_queue_create(nullptr);
+    InitialAvalanching();  // reserve this for the future shutdown
   }
 
   /// Wrap \a take, taking ownership of the instance.
@@ -151,7 +153,8 @@ class CompletionQueue : private GrpcLibraryCodegen {
 
   /// Request the shutdown of the queue.
   ///
-  /// \warning This method must be called at some point. Once invoked, \a Next
+  /// \warning This method must be called at some point if this completion queue
+  /// is accessed with Next or AsyncNext. Once invoked, \a Next
   /// will start to return false and \a AsyncNext will return \a
   /// NextStatus::SHUTDOWN. Only once either one of these methods does that
   /// (that is, once the queue has been \em drained) can an instance of this
@@ -165,6 +168,21 @@ class CompletionQueue : private GrpcLibraryCodegen {
   /// owership is performed.
   grpc_completion_queue* cq() { return cq_; }
 
+  /// Manage state of avalanching operations : completion queue tags that
+  /// trigger other completion queue operations. The underlying core completion
+  /// queue should not really shutdown until all avalanching operations have
+  /// been finalized. Note that we maintain the requirement that an avalanche
+  /// registration must take place before CQ shutdown (which must be maintained
+  /// elsehwere)
+  void InitialAvalanching() {
+    gpr_atm_rel_store(&avalanches_in_flight_, static_cast<gpr_atm>(1));
+  }
+  void RegisterAvalanching() {
+    gpr_atm_no_barrier_fetch_add(&avalanches_in_flight_,
+                                 static_cast<gpr_atm>(1));
+  };
+  void CompleteAvalanching();
+
  private:
   // Friend synchronous wrappers so that they can access Pluck(), which is
   // a semi-private API geared towards the synchronous implementation.
@@ -229,6 +247,8 @@ class CompletionQueue : private GrpcLibraryCodegen {
   }
 
   grpc_completion_queue* cq_;  // owned
+
+  gpr_atm avalanches_in_flight_;
 };
 
 /// A specific type of completion queue used by the processing of notifications

include/grpc++/impl/codegen/server_interface.h

@@ -140,7 +140,7 @@ class ServerInterface : public CallHook {
                      ServerAsyncStreamingInterface* stream,
                      CompletionQueue* call_cq, void* tag,
                      bool delete_on_finalize);
-    virtual ~BaseAsyncRequest() {}
+    virtual ~BaseAsyncRequest();
 
     bool FinalizeResult(void** tag, bool* status) override;
 

include/grpc++/resource_quota.h

@@ -37,6 +37,7 @@
 struct grpc_resource_quota;
 
 #include <grpc++/impl/codegen/config.h>
+#include <grpc++/impl/codegen/grpc_library.h>
 
 namespace grpc {
 
@@ -44,7 +45,7 @@ namespace grpc {
 /// A ResourceQuota can be attached to a server (via ServerBuilder), or a client
 /// channel (via ChannelArguments). gRPC will attempt to keep memory used by
 /// all attached entities below the ResourceQuota bound.
-class ResourceQuota final {
+class ResourceQuota final : private GrpcLibraryCodegen {
  public:
   explicit ResourceQuota(const grpc::string& name);
   ResourceQuota();

include/grpc++/server_builder.h

@@ -187,7 +187,7 @@ class ServerBuilder {
 
   struct SyncServerSettings {
     SyncServerSettings()
-        : num_cqs(GPR_MAX(gpr_cpu_num_cores(), 4)),
+        : num_cqs(1),
           min_pollers(1),
           max_pollers(INT_MAX),
           cq_timeout_msec(1000) {}

include/grpc/grpc.h

@@ -202,9 +202,15 @@ GRPCAPI grpc_call *grpc_channel_create_registered_call(
     completion of type 'tag' to the completion queue bound to the call.
     The order of ops specified in the batch has no significance.
     Only one operation of each type can be active at once in any given
-    batch. You must call grpc_completion_queue_next or
-    grpc_completion_queue_pluck on the completion queue associated with 'call'
-    for work to be performed.
+    batch.
+    If a call to grpc_call_start_batch returns GRPC_CALL_OK you must call
+    grpc_completion_queue_next or grpc_completion_queue_pluck on the completion
+    queue associated with 'call' for work to be performed. If a call to
+    grpc_call_start_batch returns any value other than GRPC_CALL_OK it is
+    guaranteed that no state associated with 'call' is changed and it is not
+    appropriate to call grpc_completion_queue_next or
+    grpc_completion_queue_pluck consequent to the failed grpc_call_start_batch
+    call.
     THREAD SAFETY: access to grpc_call_start_batch in multi-threaded environment
     needs to be synchronized. As an optimization, you may synchronize batches
     containing just send operations independently from batches containing just

include/grpc/impl/codegen/grpc_types.h

@@ -206,18 +206,12 @@ typedef struct {
 /** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */
 #define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport"
 /** If non-zero, a pointer to a buffer pool (use grpc_resource_quota_arg_vtable
-   to fetch an appropriate pointer arg vtable */
+   to fetch an appropriate pointer arg vtable) */
 #define GRPC_ARG_RESOURCE_QUOTA "grpc.resource_quota"
-/** Service config data, to be passed to subchannels.
-    Not intended for external use. */
+/** Service config data in JSON form. Not intended for use outside of tests. */
 #define GRPC_ARG_SERVICE_CONFIG "grpc.service_config"
 /** LB policy name. */
 #define GRPC_ARG_LB_POLICY_NAME "grpc.lb_policy_name"
-/** Server name. Not intended for external use. */
-#define GRPC_ARG_SERVER_NAME "grpc.server_name"
-/** Resolved addresses in a form used by the LB policy.
-    Not intended for external use. */
-#define GRPC_ARG_LB_ADDRESSES "grpc.lb_addresses"
 /** The grpc_socket_mutator instance that set the socket options. A pointer. */
 #define GRPC_ARG_SOCKET_MUTATOR "grpc.socket_mutator"
 /** \} */

setup.py

@@ -218,15 +218,18 @@ SETUP_REQUIRES = INSTALL_REQUIRES + (
     'six>=1.10',
   ) if ENABLE_DOCUMENTATION_BUILD else ()
 
-if BUILD_WITH_CYTHON:
-  sys.stderr.write(
-    "You requested a Cython build via GRPC_PYTHON_BUILD_WITH_CYTHON, "
-    "but do not have Cython installed. We won't stop you from using "
-    "other commands, but the extension files will fail to build.\n")
-elif need_cython:
-  sys.stderr.write(
-      'We could not find Cython. Setup may take 10-20 minutes.\n')
-  SETUP_REQUIRES += ('cython>=0.23',)
+try:
+  import Cython
+except ImportError:
+  if BUILD_WITH_CYTHON:
+    sys.stderr.write(
+      "You requested a Cython build via GRPC_PYTHON_BUILD_WITH_CYTHON, "
+      "but do not have Cython installed. We won't stop you from using "
+      "other commands, but the extension files will fail to build.\n")
+  elif need_cython:
+    sys.stderr.write(
+        'We could not find Cython. Setup may take 10-20 minutes.\n')
+    SETUP_REQUIRES += ('cython>=0.23',)
 
 COMMAND_CLASS = {
     'doc': commands.SphinxDocumentation,

src/google_benchmark/gen_build_yaml.py → src/benchmark/gen_build_yaml.py

@@ -39,15 +39,15 @@ os.chdir(os.path.dirname(sys.argv[0])+'/../..')
 out = {}
 
 out['libs'] = [{
-    'name': 'google_benchmark',
+    'name': 'benchmark',
     'build': 'private',
     'language': 'c++',
     'secure': 'no',
-    'defaults': 'google_benchmark',
-    'src': sorted(glob.glob('third_party/google_benchmark/src/*.cc')),
+    'defaults': 'benchmark',
+    'src': sorted(glob.glob('third_party/benchmark/src/*.cc')),
     'headers': sorted(
-        glob.glob('third_party/google_benchmark/src/*.h') +
-        glob.glob('third_party/google_benchmark/include/benchmark/*.h')),
+        glob.glob('third_party/benchmark/src/*.h') +
+        glob.glob('third_party/benchmark/include/benchmark/*.h')),
 }]
 
 print yaml.dump(out)

src/compiler/csharp_generator.cc

@@ -68,13 +68,13 @@ namespace {
 // Currently, we cannot easily reuse the functionality as
 // google/protobuf/compiler/csharp/csharp_doc_comment.h is not a public header.
 // TODO(jtattermusch): reuse the functionality from google/protobuf.
-void GenerateDocCommentBodyImpl(grpc::protobuf::io::Printer *printer,
+bool GenerateDocCommentBodyImpl(grpc::protobuf::io::Printer *printer,
                                 grpc::protobuf::SourceLocation location) {
   grpc::string comments = location.leading_comments.empty()
                               ? location.trailing_comments
                               : location.leading_comments;
   if (comments.empty()) {
-    return;
+    return false;
   }
   // XML escaping... no need for apostrophes etc as the whole text is going to
   // be a child
@@ -107,18 +107,84 @@ void GenerateDocCommentBodyImpl(grpc::protobuf::io::Printer *printer,
         printer->Print("///\n");
       }
       last_was_empty = false;
-      printer->Print("/// $line$\n", "line", *it);
+      printer->Print("///$line$\n", "line", *it);
     }
   }
   printer->Print("/// </summary>\n");
+  return true;
 }
 
 template <typename DescriptorType>
-void GenerateDocCommentBody(grpc::protobuf::io::Printer *printer,
+bool GenerateDocCommentBody(grpc::protobuf::io::Printer *printer,
                             const DescriptorType *descriptor) {
   grpc::protobuf::SourceLocation location;
-  if (descriptor->GetSourceLocation(&location)) {
-    GenerateDocCommentBodyImpl(printer, location);
+  if (!descriptor->GetSourceLocation(&location)) {
+    return false;
+  }
+  return GenerateDocCommentBodyImpl(printer, location);
+}
+
+void GenerateDocCommentServerMethod(grpc::protobuf::io::Printer *printer,
+                                    const MethodDescriptor *method) {
+  if (GenerateDocCommentBody(printer, method)) {
+    if (method->client_streaming()) {
+      printer->Print(
+          "/// <param name=\"requestStream\">Used for reading requests from "
+          "the client.</param>\n");
+    } else {
+      printer->Print(
+          "/// <param name=\"request\">The request received from the "
+          "client.</param>\n");
+    }
+    if (method->server_streaming()) {
+      printer->Print(
+          "/// <param name=\"responseStream\">Used for sending responses back "
+          "to the client.</param>\n");
+    }
+    printer->Print(
+        "/// <param name=\"context\">The context of the server-side call "
+        "handler being invoked.</param>\n");
+    if (method->server_streaming()) {
+      printer->Print(
+          "/// <returns>A task indicating completion of the "
+          "handler.</returns>\n");
+    } else {
+      printer->Print(
+          "/// <returns>The response to send back to the client (wrapped by a "
+          "task).</returns>\n");
+    }
+  }
+}
+
+void GenerateDocCommentClientMethod(grpc::protobuf::io::Printer *printer,
+                                    const MethodDescriptor *method,
+                                    bool is_sync, bool use_call_options) {
+  if (GenerateDocCommentBody(printer, method)) {
+    if (!method->client_streaming()) {
+      printer->Print(
+          "/// <param name=\"request\">The request to send to the "
+          "server.</param>\n");
+    }
+    if (!use_call_options) {
+      printer->Print(
+          "/// <param name=\"headers\">The initial metadata to send with the "
+          "call. This parameter is optional.</param>\n");
+      printer->Print(
+          "/// <param name=\"deadline\">An optional deadline for the call. The "
+          "call will be cancelled if deadline is hit.</param>\n");
+      printer->Print(
+          "/// <param name=\"cancellationToken\">An optional token for "
+          "canceling the call.</param>\n");
+    } else {
+      printer->Print(
+          "/// <param name=\"options\">The options for the call.</param>\n");
+    }
+    if (is_sync) {
+      printer->Print(
+          "/// <returns>The response received from the server.</returns>\n");
+    } else {
+      printer->Print("/// <returns>The call object.</returns>\n");
+    }
   }
 }
 
@@ -319,7 +385,7 @@ void GenerateServerClass(Printer *out, const ServiceDescriptor *service) {
   out->Indent();
   for (int i = 0; i < service->method_count(); i++) {
     const MethodDescriptor *method = service->method(i);
-    GenerateDocCommentBody(out, method);
+    GenerateDocCommentServerMethod(out, method);
     out->Print(
         "public virtual $returntype$ "
         "$methodname$($request$$response_stream_maybe$, "
@@ -393,7 +459,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
 
     if (method_type == METHODTYPE_NO_STREAMING) {
       // unary calls have an extra synchronous stub method
-      GenerateDocCommentBody(out, method);
+      GenerateDocCommentClientMethod(out, method, true, false);
       out->Print(
           "public virtual $response$ $methodname$($request$ request, Metadata "
           "headers = null, DateTime? deadline = null, CancellationToken "
@@ -411,7 +477,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
       out->Print("}\n");
 
       // overload taking CallOptions as a param
-      GenerateDocCommentBody(out, method);
+      GenerateDocCommentClientMethod(out, method, true, true);
       out->Print(
           "public virtual $response$ $methodname$($request$ request, "
           "CallOptions options)\n",
@@ -432,7 +498,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
     if (method_type == METHODTYPE_NO_STREAMING) {
       method_name += "Async";  // prevent name clash with synchronous method.
     }
-    GenerateDocCommentBody(out, method);
+    GenerateDocCommentClientMethod(out, method, false, false);
     out->Print(
         "public virtual $returntype$ $methodname$($request_maybe$Metadata "
         "headers = null, DateTime? deadline = null, CancellationToken "
@@ -452,7 +518,7 @@ void GenerateClientStub(Printer *out, const ServiceDescriptor *service) {
     out->Print("}\n");
 
     // overload taking CallOptions as a param
-    GenerateDocCommentBody(out, method);
+    GenerateDocCommentClientMethod(out, method, false, true);
     out->Print(
         "public virtual $returntype$ $methodname$($request_maybe$CallOptions "
         "options)\n",
@@ -517,6 +583,9 @@ void GenerateBindServiceMethod(Printer *out, const ServiceDescriptor *service) {
   out->Print(
       "/// <summary>Creates service definition that can be registered with a "
       "server</summary>\n");
+  out->Print(
+      "/// <param name=\"serviceImpl\">An object implementing the server-side"
+      " handling logic.</param>\n");
   out->Print(
       "public static ServerServiceDefinition BindService($implclass$ "
       "serviceImpl)\n",

src/compiler/python_generator.cc

@@ -40,6 +40,7 @@
 #include <map>
 #include <memory>
 #include <ostream>
+#include <set>
 #include <sstream>
 #include <tuple>
 #include <vector>
@@ -64,7 +65,9 @@ using std::make_pair;
 using std::map;
 using std::pair;
 using std::replace;
+using std::tuple;
 using std::vector;
+using std::set;
 
 namespace grpc_python_generator {
 
@@ -73,6 +76,8 @@ namespace {
 typedef vector<const Descriptor*> DescriptorVector;
 typedef map<grpc::string, grpc::string> StringMap;
 typedef vector<grpc::string> StringVector;
+typedef tuple<grpc::string, grpc::string> StringPair;
+typedef set<StringPair> StringPairSet;
 
 // Provides RAII indentation handling. Use as:
 // {
@@ -651,6 +656,7 @@ bool PrivateGenerator::PrintPreamble() {
       "face_utilities\n");
   if (generate_in_pb2_grpc) {
     out->Print("\n");
+    StringPairSet imports_set;
     for (int i = 0; i < file->service_count(); ++i) {
       const ServiceDescriptor* service = file->service(i);
       for (int j = 0; j < service->method_count(); ++j) {
@@ -662,11 +668,15 @@ bool PrivateGenerator::PrintPreamble() {
           grpc::string type_file_name = type->file()->name();
           grpc::string module_name = ModuleName(type_file_name);
           grpc::string module_alias = ModuleAlias(type_file_name);
-          out->Print("import $ModuleName$ as $ModuleAlias$\n", "ModuleName",
-                     module_name, "ModuleAlias", module_alias);
+          imports_set.insert(std::make_tuple(module_name, module_alias));
         }
       }
     }
+    for (StringPairSet::iterator it = imports_set.begin();
+         it != imports_set.end(); ++it) {
+      out->Print("import $ModuleName$ as $ModuleAlias$\n", "ModuleName",
+                 std::get<0>(*it), "ModuleAlias", std::get<1>(*it));
+    }
   }
   return true;
 }
@@ -714,6 +724,9 @@ pair<bool, grpc::string> PrivateGenerator::GetGrpcServices() {
     out = &out_printer;
 
     if (generate_in_pb2_grpc) {
+      out->Print(
+          "# Generated by the gRPC Python protocol compiler plugin. "
+          "DO NOT EDIT!\n");
       if (!PrintPreamble()) {
         return make_pair(false, "");
       }

src/core/ext/census/grpc_filter.c

@@ -154,7 +154,8 @@ static grpc_error *server_init_call_elem(grpc_exec_ctx *exec_ctx,
   memset(d, 0, sizeof(*d));
   d->start_ts = args->start_time;
   /* TODO(hongyu): call census_tracing_start_op here. */
-  grpc_closure_init(&d->finish_recv, server_on_done_recv, elem);
+  grpc_closure_init(&d->finish_recv, server_on_done_recv, elem,
+                    grpc_schedule_on_exec_ctx);
   return GRPC_ERROR_NONE;
 }
 
@@ -167,11 +168,12 @@ static void server_destroy_call_elem(grpc_exec_ctx *exec_ctx,
   /* TODO(hongyu): record rpc server stats and census_tracing_end_op here */
 }
 
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
+static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_channel_element *elem,
+                                     grpc_channel_element_args *args) {
   channel_data *chand = elem->channel_data;
   GPR_ASSERT(chand != NULL);
+  return GRPC_ERROR_NONE;
 }
 
 static void destroy_channel_elem(grpc_exec_ctx *exec_ctx,

src/core/ext/client_channel/channel_connectivity.c

@@ -198,7 +198,8 @@ void grpc_channel_watch_connectivity_state(
   grpc_cq_begin_op(cq, tag);
 
   gpr_mu_init(&w->mu);
-  grpc_closure_init(&w->on_complete, watch_complete, w);
+  grpc_closure_init(&w->on_complete, watch_complete, w,
+                    grpc_schedule_on_exec_ctx);
   w->phase = WAITING;
   w->state = last_observed_state;
   w->cq = cq;

src/core/ext/client_channel/client_channel.c

@@ -44,6 +44,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/ext/client_channel/lb_policy_registry.h"
+#include "src/core/ext/client_channel/resolver_registry.h"
 #include "src/core/ext/client_channel/subchannel.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
@@ -248,7 +249,8 @@ static void watch_lb_policy(grpc_exec_ctx *exec_ctx, channel_data *chand,
   GRPC_CHANNEL_STACK_REF(chand->owning_stack, "watch_lb_policy");
 
   w->chand = chand;
-  grpc_closure_init(&w->on_changed, on_lb_policy_state_changed, w);
+  grpc_closure_init(&w->on_changed, on_lb_policy_state_changed, w,
+                    grpc_schedule_on_exec_ctx);
   w->state = current_state;
   w->lb_policy = lb_policy;
   grpc_lb_policy_notify_on_state_change(exec_ctx, lb_policy, &w->state,
@@ -360,14 +362,12 @@ static void on_resolver_result_changed(grpc_exec_ctx *exec_ctx, void *arg,
   }
   chand->method_params_table = method_params_table;
   if (lb_policy != NULL) {
-    grpc_exec_ctx_enqueue_list(exec_ctx, &chand->waiting_for_config_closures,
-                               NULL);
+    grpc_closure_list_sched(exec_ctx, &chand->waiting_for_config_closures);
   } else if (chand->resolver == NULL /* disconnected */) {
     grpc_closure_list_fail_all(
         &chand->waiting_for_config_closures,
         GRPC_ERROR_CREATE_REFERENCING("Channel disconnected", &error, 1));
-    grpc_exec_ctx_enqueue_list(exec_ctx, &chand->waiting_for_config_closures,
-                               NULL);
+    grpc_closure_list_sched(exec_ctx, &chand->waiting_for_config_closures);
   }
   if (lb_policy != NULL && chand->exit_idle_when_lb_policy_arrives) {
     GRPC_LB_POLICY_REF(lb_policy, "exit_idle");
@@ -424,7 +424,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
                                   grpc_transport_op *op) {
   channel_data *chand = elem->channel_data;
 
-  grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
+  grpc_closure_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE);
 
   GPR_ASSERT(op->set_accept_stream == false);
   if (op->bind_pollset != NULL) {
@@ -443,9 +443,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
 
   if (op->send_ping != NULL) {
     if (chand->lb_policy == NULL) {
-      grpc_exec_ctx_sched(exec_ctx, op->send_ping,
-                          GRPC_ERROR_CREATE("Ping with no load balancing"),
-                          NULL);
+      grpc_closure_sched(exec_ctx, op->send_ping,
+                         GRPC_ERROR_CREATE("Ping with no load balancing"));
     } else {
       grpc_lb_policy_ping_one(exec_ctx, chand->lb_policy, op->send_ping);
       op->bind_pollset = NULL;
@@ -464,8 +463,7 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
       if (!chand->started_resolving) {
         grpc_closure_list_fail_all(&chand->waiting_for_config_closures,
                                    GRPC_ERROR_REF(op->disconnect_with_error));
-        grpc_exec_ctx_enqueue_list(exec_ctx,
-                                   &chand->waiting_for_config_closures, NULL);
+        grpc_closure_list_sched(exec_ctx, &chand->waiting_for_config_closures);
       }
       if (chand->lb_policy != NULL) {
         grpc_pollset_set_del_pollset_set(exec_ctx,
@@ -499,24 +497,40 @@ static void cc_get_channel_info(grpc_exec_ctx *exec_ctx,
 }
 
 /* Constructor for channel_data */
-static void cc_init_channel_elem(grpc_exec_ctx *exec_ctx,
-                                 grpc_channel_element *elem,
-                                 grpc_channel_element_args *args) {
+static grpc_error *cc_init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                        grpc_channel_element *elem,
+                                        grpc_channel_element_args *args) {
   channel_data *chand = elem->channel_data;
-
   memset(chand, 0, sizeof(*chand));
-
   GPR_ASSERT(args->is_last);
   GPR_ASSERT(elem->filter == &grpc_client_channel_filter);
-
+  // Initialize data members.
   gpr_mu_init(&chand->mu);
-  grpc_closure_init(&chand->on_resolver_result_changed,
-                    on_resolver_result_changed, chand);
   chand->owning_stack = args->channel_stack;
-
+  grpc_closure_init(&chand->on_resolver_result_changed,
+                    on_resolver_result_changed, chand,
+                    grpc_schedule_on_exec_ctx);
+  chand->interested_parties = grpc_pollset_set_create();
   grpc_connectivity_state_init(&chand->state_tracker, GRPC_CHANNEL_IDLE,
                                "client_channel");
-  chand->interested_parties = grpc_pollset_set_create();
+  // Record client channel factory.
+  const grpc_arg *arg = grpc_channel_args_find(args->channel_args,
+                                               GRPC_ARG_CLIENT_CHANNEL_FACTORY);
+  GPR_ASSERT(arg != NULL);
+  GPR_ASSERT(arg->type == GRPC_ARG_POINTER);
+  grpc_client_channel_factory_ref(arg->value.pointer.p);
+  chand->client_channel_factory = arg->value.pointer.p;
+  // Instantiate resolver.
+  arg = grpc_channel_args_find(args->channel_args, GRPC_ARG_SERVER_URI);
+  GPR_ASSERT(arg != NULL);
+  GPR_ASSERT(arg->type == GRPC_ARG_STRING);
+  chand->resolver =
+      grpc_resolver_create(exec_ctx, arg->value.string, args->channel_args,
+                           chand->interested_parties);
+  if (chand->resolver == NULL) {
+    return GRPC_ERROR_CREATE("resolver creation failed");
+  }
+  return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel_data */
@@ -662,8 +676,9 @@ static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) {
   calld->waiting_ops_count = 0;
   calld->waiting_ops_capacity = 0;
   GRPC_SUBCHANNEL_CALL_REF(a->call, "retry_ops");
-  grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(retry_ops, a),
-                      GRPC_ERROR_NONE, NULL);
+  grpc_closure_sched(
+      exec_ctx, grpc_closure_create(retry_ops, a, grpc_schedule_on_exec_ctx),
+      GRPC_ERROR_NONE);
 }
 
 static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
@@ -683,9 +698,15 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg,
                                      "Failed to create subchannel", &error, 1));
   } else if (GET_CALL(calld) == CANCELLED_CALL) {
     /* already cancelled before subchannel became ready */
-    fail_locked(exec_ctx, calld,
-                GRPC_ERROR_CREATE_REFERENCING(
-                    "Cancelled before creating subchannel", &error, 1));
+    grpc_error *cancellation_error = GRPC_ERROR_CREATE_REFERENCING(
+        "Cancelled before creating subchannel", &error, 1);
+    /* if due to deadline, attach the deadline exceeded status to the error */
+    if (gpr_time_cmp(calld->deadline, gpr_now(GPR_CLOCK_MONOTONIC)) < 0) {
+      cancellation_error =
+          grpc_error_set_int(cancellation_error, GRPC_ERROR_INT_GRPC_STATUS,
+                             GRPC_STATUS_DEADLINE_EXCEEDED);
+    }
+    fail_locked(exec_ctx, calld, cancellation_error);
   } else {
     /* Create call on subchannel. */
     grpc_subchannel_call *subchannel_call = NULL;
@@ -739,14 +760,14 @@ static void continue_picking(grpc_exec_ctx *exec_ctx, void *arg,
   if (cpa->connected_subchannel == NULL) {
     /* cancelled, do nothing */
   } else if (error != GRPC_ERROR_NONE) {
-    grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error), NULL);
+    grpc_closure_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_REF(error));
   } else {
     call_data *calld = cpa->elem->call_data;
     gpr_mu_lock(&calld->mu);
     if (pick_subchannel(exec_ctx, cpa->elem, cpa->initial_metadata,
                         cpa->initial_metadata_flags, cpa->connected_subchannel,
                         cpa->on_ready, GRPC_ERROR_NONE)) {
-      grpc_exec_ctx_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE, NULL);
+      grpc_closure_sched(exec_ctx, cpa->on_ready, GRPC_ERROR_NONE);
     }
     gpr_mu_unlock(&calld->mu);
   }
@@ -778,9 +799,9 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
       cpa = closure->cb_arg;
       if (cpa->connected_subchannel == connected_subchannel) {
         cpa->connected_subchannel = NULL;
-        grpc_exec_ctx_sched(
+        grpc_closure_sched(
             exec_ctx, cpa->on_ready,
-            GRPC_ERROR_CREATE_REFERENCING("Pick cancelled", &error, 1), NULL);
+            GRPC_ERROR_CREATE_REFERENCING("Pick cancelled", &error, 1));
       }
     }
     gpr_mu_unlock(&chand->mu);
@@ -809,7 +830,6 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
         initial_metadata_flags &= ~GRPC_INITIAL_METADATA_WAIT_FOR_READY;
       }
     }
-    // TODO(dgq): make this deadline configurable somehow.
     const grpc_lb_policy_pick_args inputs = {
         initial_metadata, initial_metadata_flags, &calld->lb_token_mdelem,
         gpr_inf_future(GPR_CLOCK_MONOTONIC)};
@@ -832,12 +852,12 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
     cpa->connected_subchannel = connected_subchannel;
     cpa->on_ready = on_ready;
     cpa->elem = elem;
-    grpc_closure_init(&cpa->closure, continue_picking, cpa);
+    grpc_closure_init(&cpa->closure, continue_picking, cpa,
+                      grpc_schedule_on_exec_ctx);
     grpc_closure_list_append(&chand->waiting_for_config_closures, &cpa->closure,
                              GRPC_ERROR_NONE);
   } else {
-    grpc_exec_ctx_sched(exec_ctx, on_ready, GRPC_ERROR_CREATE("Disconnected"),
-                        NULL);
+    grpc_closure_sched(exec_ctx, on_ready, GRPC_ERROR_CREATE("Disconnected"));
   }
   gpr_mu_unlock(&chand->mu);
 
@@ -922,7 +942,8 @@ retry:
       calld->connected_subchannel == NULL &&
       op->send_initial_metadata != NULL) {
     calld->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_PICKING_SUBCHANNEL;
-    grpc_closure_init(&calld->next_step, subchannel_ready, elem);
+    grpc_closure_init(&calld->next_step, subchannel_ready, elem,
+                      grpc_schedule_on_exec_ctx);
     GRPC_CALL_STACK_REF(calld->owning_call, "pick_subchannel");
     /* If a subchannel is not available immediately, the polling entity from
        call_data should be provided to channel_data's interested_parties, so
@@ -1068,7 +1089,8 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx,
     // get the service config data once the resolver returns.
     // Take a reference to the call stack to be owned by the callback.
     GRPC_CALL_STACK_REF(calld->owning_call, "read_service_config");
-    grpc_closure_init(&calld->read_service_config, read_service_config, elem);
+    grpc_closure_init(&calld->read_service_config, read_service_config, elem,
+                      grpc_schedule_on_exec_ctx);
     grpc_closure_list_append(&chand->waiting_for_config_closures,
                              &calld->read_service_config, GRPC_ERROR_NONE);
     gpr_mu_unlock(&chand->mu);
@@ -1130,30 +1152,6 @@ const grpc_channel_filter grpc_client_channel_filter = {
     "client-channel",
 };
 
-void grpc_client_channel_finish_initialization(
-    grpc_exec_ctx *exec_ctx, grpc_channel_stack *channel_stack,
-    grpc_resolver *resolver,
-    grpc_client_channel_factory *client_channel_factory) {
-  /* post construction initialization: set the transport setup pointer */
-  GPR_ASSERT(client_channel_factory != NULL);
-  grpc_channel_element *elem = grpc_channel_stack_last_element(channel_stack);
-  channel_data *chand = elem->channel_data;
-  gpr_mu_lock(&chand->mu);
-  GPR_ASSERT(!chand->resolver);
-  chand->resolver = resolver;
-  GRPC_RESOLVER_REF(resolver, "channel");
-  if (!grpc_closure_list_empty(chand->waiting_for_config_closures) ||
-      chand->exit_idle_when_lb_policy_arrives) {
-    chand->started_resolving = true;
-    GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
-    grpc_resolver_next(exec_ctx, resolver, &chand->resolver_result,
-                       &chand->on_resolver_result_changed);
-  }
-  chand->client_channel_factory = client_channel_factory;
-  grpc_client_channel_factory_ref(client_channel_factory);
-  gpr_mu_unlock(&chand->mu);
-}
-
 grpc_connectivity_state grpc_client_channel_check_connectivity_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect) {
   channel_data *chand = elem->channel_data;
@@ -1205,7 +1203,8 @@ void grpc_client_channel_watch_connectivity_state(
   w->pollset = pollset;
   w->on_complete = on_complete;
   grpc_pollset_set_add_pollset(exec_ctx, chand->interested_parties, pollset);
-  grpc_closure_init(&w->my_closure, on_external_watch_complete, w);
+  grpc_closure_init(&w->my_closure, on_external_watch_complete, w,
+                    grpc_schedule_on_exec_ctx);
   GRPC_CHANNEL_STACK_REF(w->chand->owning_stack,
                          "external_connectivity_watcher");
   gpr_mu_lock(&chand->mu);

src/core/ext/client_channel/client_channel.h

@@ -38,6 +38,9 @@
 #include "src/core/ext/client_channel/resolver.h"
 #include "src/core/lib/channel/channel_stack.h"
 
+// Channel arg key for server URI string.
+#define GRPC_ARG_SERVER_URI "grpc.server_uri"
+
 /* A client channel is a channel that begins disconnected, and can connect
    to some endpoint on demand. If that endpoint disconnects, it will be
    connected to again later.
@@ -47,13 +50,6 @@
 
 extern const grpc_channel_filter grpc_client_channel_filter;
 
-/* Post-construction initializer to give the client channel its resolver
-   and factory. */
-void grpc_client_channel_finish_initialization(
-    grpc_exec_ctx *exec_ctx, grpc_channel_stack *channel_stack,
-    grpc_resolver *resolver,
-    grpc_client_channel_factory *client_channel_factory);
-
 grpc_connectivity_state grpc_client_channel_check_connectivity_state(
     grpc_exec_ctx *exec_ctx, grpc_channel_element *elem, int try_to_connect);
 

src/core/ext/client_channel/client_channel_factory.c

@@ -55,3 +55,35 @@ grpc_channel* grpc_client_channel_factory_create_channel(
   return factory->vtable->create_client_channel(exec_ctx, factory, target, type,
                                                 args);
 }
+
+static void* factory_arg_copy(void* factory) {
+  grpc_client_channel_factory_ref(factory);
+  return factory;
+}
+
+static void factory_arg_destroy(void* factory) {
+  // TODO(roth): Remove local exec_ctx when
+  // https://github.com/grpc/grpc/pull/8705 is merged.
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_client_channel_factory_unref(&exec_ctx, factory);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static int factory_arg_cmp(void* factory1, void* factory2) {
+  if (factory1 < factory2) return -1;
+  if (factory1 > factory2) return 1;
+  return 0;
+}
+
+static const grpc_arg_pointer_vtable factory_arg_vtable = {
+    factory_arg_copy, factory_arg_destroy, factory_arg_cmp};
+
+grpc_arg grpc_client_channel_factory_create_channel_arg(
+    grpc_client_channel_factory* factory) {
+  grpc_arg arg;
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_ARG_CLIENT_CHANNEL_FACTORY;
+  arg.value.pointer.p = factory;
+  arg.value.pointer.vtable = &factory_arg_vtable;
+  return arg;
+}

src/core/ext/client_channel/client_channel_factory.h

@@ -39,6 +39,9 @@
 #include "src/core/ext/client_channel/subchannel.h"
 #include "src/core/lib/channel/channel_stack.h"
 
+// Channel arg key for client channel factory.
+#define GRPC_ARG_CLIENT_CHANNEL_FACTORY "grpc.client_channel_factory"
+
 typedef struct grpc_client_channel_factory grpc_client_channel_factory;
 typedef struct grpc_client_channel_factory_vtable
     grpc_client_channel_factory_vtable;
@@ -83,4 +86,7 @@ grpc_channel *grpc_client_channel_factory_create_channel(
     const char *target, grpc_client_channel_type type,
     const grpc_channel_args *args);
 
+grpc_arg grpc_client_channel_factory_create_channel_arg(
+    grpc_client_channel_factory *factory);
+
 #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_CLIENT_CHANNEL_FACTORY_H */

src/core/ext/client_channel/http_connect_handshaker.c

@@ -40,6 +40,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#include "src/core/ext/client_channel/client_channel.h"
+#include "src/core/ext/client_channel/resolver_registry.h"
 #include "src/core/ext/client_channel/uri_parser.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/http/format_request.h"
@@ -51,7 +53,6 @@ typedef struct http_connect_handshaker {
   grpc_handshaker base;
 
   char* proxy_server;
-  char* server_name;
 
   gpr_refcount refcount;
   gpr_mu mu;
@@ -86,7 +87,6 @@ static void http_connect_handshaker_unref(grpc_exec_ctx* exec_ctx,
       gpr_free(handshaker->read_buffer_to_destroy);
     }
     gpr_free(handshaker->proxy_server);
-    gpr_free(handshaker->server_name);
     grpc_slice_buffer_destroy(&handshaker->write_buffer);
     grpc_http_parser_destroy(&handshaker->http_parser);
     grpc_http_response_destroy(&handshaker->http_response);
@@ -131,7 +131,7 @@ static void handshake_failed_locked(grpc_exec_ctx* exec_ctx,
     handshaker->shutdown = true;
   }
   // Invoke callback.
-  grpc_exec_ctx_sched(exec_ctx, handshaker->on_handshake_done, error, NULL);
+  grpc_closure_sched(exec_ctx, handshaker->on_handshake_done, error);
 }
 
 // Callback invoked when finished writing HTTP CONNECT request.
@@ -229,7 +229,7 @@ static void on_read_done(grpc_exec_ctx* exec_ctx, void* arg,
     goto done;
   }
   // Success.  Invoke handshake-done callback.
-  grpc_exec_ctx_sched(exec_ctx, handshaker->on_handshake_done, error, NULL);
+  grpc_closure_sched(exec_ctx, handshaker->on_handshake_done, error);
 done:
   // Set shutdown to true so that subsequent calls to
   // http_connect_handshaker_shutdown() do nothing.
@@ -265,18 +265,27 @@ static void http_connect_handshaker_do_handshake(
     grpc_tcp_server_acceptor* acceptor, grpc_closure* on_handshake_done,
     grpc_handshaker_args* args) {
   http_connect_handshaker* handshaker = (http_connect_handshaker*)handshaker_in;
-  gpr_mu_lock(&handshaker->mu);
+  // Get server name from channel args.
+  const grpc_arg* arg = grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI);
+  GPR_ASSERT(arg != NULL);
+  GPR_ASSERT(arg->type == GRPC_ARG_STRING);
+  char* canonical_uri =
+      grpc_resolver_factory_add_default_prefix_if_needed(arg->value.string);
+  grpc_uri* uri = grpc_uri_parse(canonical_uri, 1);
+  char* server_name = uri->path;
+  if (server_name[0] == '/') ++server_name;
   // Save state in the handshaker object.
+  gpr_mu_lock(&handshaker->mu);
   handshaker->args = args;
   handshaker->on_handshake_done = on_handshake_done;
   // Send HTTP CONNECT request.
-  gpr_log(GPR_INFO, "Connecting to server %s via HTTP proxy %s",
-          handshaker->server_name, handshaker->proxy_server);
+  gpr_log(GPR_INFO, "Connecting to server %s via HTTP proxy %s", server_name,
+          handshaker->proxy_server);
   grpc_httpcli_request request;
   memset(&request, 0, sizeof(request));
-  request.host = handshaker->proxy_server;
+  request.host = server_name;
   request.http.method = "CONNECT";
-  request.http.path = handshaker->server_name;
+  request.http.path = server_name;
   request.handshaker = &grpc_httpcli_plaintext;
   grpc_slice request_slice = grpc_httpcli_format_connect_request(&request);
   grpc_slice_buffer_add(&handshaker->write_buffer, request_slice);
@@ -285,28 +294,28 @@ static void http_connect_handshaker_do_handshake(
   grpc_endpoint_write(exec_ctx, args->endpoint, &handshaker->write_buffer,
                       &handshaker->request_done_closure);
   gpr_mu_unlock(&handshaker->mu);
+  // Clean up.
+  gpr_free(canonical_uri);
+  grpc_uri_destroy(uri);
 }
 
 static const grpc_handshaker_vtable http_connect_handshaker_vtable = {
     http_connect_handshaker_destroy, http_connect_handshaker_shutdown,
     http_connect_handshaker_do_handshake};
 
-grpc_handshaker* grpc_http_connect_handshaker_create(const char* proxy_server,
-                                                     const char* server_name) {
+grpc_handshaker* grpc_http_connect_handshaker_create(const char* proxy_server) {
   GPR_ASSERT(proxy_server != NULL);
-  GPR_ASSERT(server_name != NULL);
   http_connect_handshaker* handshaker = gpr_malloc(sizeof(*handshaker));
   memset(handshaker, 0, sizeof(*handshaker));
   grpc_handshaker_init(&http_connect_handshaker_vtable, &handshaker->base);
   gpr_mu_init(&handshaker->mu);
   gpr_ref_init(&handshaker->refcount, 1);
   handshaker->proxy_server = gpr_strdup(proxy_server);
-  handshaker->server_name = gpr_strdup(server_name);
   grpc_slice_buffer_init(&handshaker->write_buffer);
   grpc_closure_init(&handshaker->request_done_closure, on_write_done,
-                    handshaker);
+                    handshaker, grpc_schedule_on_exec_ctx);
   grpc_closure_init(&handshaker->response_read_closure, on_read_done,
-                    handshaker);
+                    handshaker, grpc_schedule_on_exec_ctx);
   grpc_http_parser_init(&handshaker->http_parser, GRPC_HTTP_RESPONSE,
                         &handshaker->http_response);
   return &handshaker->base;

src/core/ext/client_channel/http_connect_handshaker.h

@@ -36,9 +36,8 @@
 
 #include "src/core/lib/channel/handshaker.h"
 
-/// Does NOT take ownership of \a proxy_server or \a server_name.
-grpc_handshaker* grpc_http_connect_handshaker_create(const char* proxy_server,
-                                                     const char* server_name);
+/// Does NOT take ownership of \a proxy_server.
+grpc_handshaker* grpc_http_connect_handshaker_create(const char* proxy_server);
 
 /// Returns the name of the proxy to use, or NULL if no proxy is configured.
 /// Caller takes ownership of result.

src/core/ext/client_channel/lb_policy_factory.h

@@ -40,6 +40,9 @@
 #include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 
+// Channel arg key for grpc_lb_addresses.
+#define GRPC_ARG_LB_ADDRESSES "grpc.lb_addresses"
+
 typedef struct grpc_lb_policy_factory grpc_lb_policy_factory;
 typedef struct grpc_lb_policy_factory_vtable grpc_lb_policy_factory_vtable;
 

src/core/ext/client_channel/resolver_factory.c

@@ -43,9 +43,10 @@ void grpc_resolver_factory_unref(grpc_resolver_factory* factory) {
 
 /** Create a resolver instance for a name */
 grpc_resolver* grpc_resolver_factory_create_resolver(
-    grpc_resolver_factory* factory, grpc_resolver_args* args) {
+    grpc_exec_ctx* exec_ctx, grpc_resolver_factory* factory,
+    grpc_resolver_args* args) {
   if (factory == NULL) return NULL;
-  return factory->vtable->create_resolver(factory, args);
+  return factory->vtable->create_resolver(exec_ctx, factory, args);
 }
 
 char* grpc_resolver_factory_get_default_authority(

src/core/ext/client_channel/resolver_factory.h

@@ -37,6 +37,7 @@
 #include "src/core/ext/client_channel/client_channel_factory.h"
 #include "src/core/ext/client_channel/resolver.h"
 #include "src/core/ext/client_channel/uri_parser.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 
 typedef struct grpc_resolver_factory grpc_resolver_factory;
 typedef struct grpc_resolver_factory_vtable grpc_resolver_factory_vtable;
@@ -48,6 +49,7 @@ struct grpc_resolver_factory {
 typedef struct grpc_resolver_args {
   grpc_uri *uri;
   const grpc_channel_args *args;
+  grpc_pollset_set *pollset_set;
 } grpc_resolver_args;
 
 struct grpc_resolver_factory_vtable {
@@ -55,7 +57,8 @@ struct grpc_resolver_factory_vtable {
   void (*unref)(grpc_resolver_factory *factory);
 
   /** Implementation of grpc_resolver_factory_create_resolver */
-  grpc_resolver *(*create_resolver)(grpc_resolver_factory *factory,
+  grpc_resolver *(*create_resolver)(grpc_exec_ctx *exec_ctx,
+                                    grpc_resolver_factory *factory,
                                     grpc_resolver_args *args);
 
   /** Implementation of grpc_resolver_factory_get_default_authority */
@@ -70,7 +73,8 @@ void grpc_resolver_factory_unref(grpc_resolver_factory *resolver);
 
 /** Create a resolver instance for a name */
 grpc_resolver *grpc_resolver_factory_create_resolver(
-    grpc_resolver_factory *factory, grpc_resolver_args *args);
+    grpc_exec_ctx *exec_ctx, grpc_resolver_factory *factory,
+    grpc_resolver_args *args);
 
 /** Return a (freshly allocated with gpr_malloc) string representing
     the default authority to use for this scheme. */

src/core/ext/client_channel/resolver_registry.c

@@ -109,8 +109,8 @@ static grpc_resolver_factory *lookup_factory_by_uri(grpc_uri *uri) {
 }
 
 static grpc_resolver_factory *resolve_factory(const char *target,
-                                              grpc_uri **uri) {
-  char *tmp;
+                                              grpc_uri **uri,
+                                              char **canonical_target) {
   grpc_resolver_factory *factory = NULL;
 
   GPR_ASSERT(uri != NULL);
@@ -118,37 +118,54 @@ static grpc_resolver_factory *resolve_factory(const char *target,
   factory = lookup_factory_by_uri(*uri);
   if (factory == NULL) {
     grpc_uri_destroy(*uri);
-    gpr_asprintf(&tmp, "%s%s", g_default_resolver_prefix, target);
-    *uri = grpc_uri_parse(tmp, 1);
+    gpr_asprintf(canonical_target, "%s%s", g_default_resolver_prefix, target);
+    *uri = grpc_uri_parse(*canonical_target, 1);
     factory = lookup_factory_by_uri(*uri);
     if (factory == NULL) {
       grpc_uri_destroy(grpc_uri_parse(target, 0));
-      grpc_uri_destroy(grpc_uri_parse(tmp, 0));
-      gpr_log(GPR_ERROR, "don't know how to resolve '%s' or '%s'", target, tmp);
+      grpc_uri_destroy(grpc_uri_parse(*canonical_target, 0));
+      gpr_log(GPR_ERROR, "don't know how to resolve '%s' or '%s'", target,
+              *canonical_target);
     }
-    gpr_free(tmp);
   }
   return factory;
 }
 
-grpc_resolver *grpc_resolver_create(const char *target,
-                                    const grpc_channel_args *args) {
+grpc_resolver *grpc_resolver_create(grpc_exec_ctx *exec_ctx, const char *target,
+                                    const grpc_channel_args *args,
+                                    grpc_pollset_set *pollset_set) {
   grpc_uri *uri = NULL;
-  grpc_resolver_factory *factory = resolve_factory(target, &uri);
+  char *canonical_target = NULL;
+  grpc_resolver_factory *factory =
+      resolve_factory(target, &uri, &canonical_target);
   grpc_resolver *resolver;
   grpc_resolver_args resolver_args;
   memset(&resolver_args, 0, sizeof(resolver_args));
   resolver_args.uri = uri;
   resolver_args.args = args;
-  resolver = grpc_resolver_factory_create_resolver(factory, &resolver_args);
+  resolver_args.pollset_set = pollset_set;
+  resolver =
+      grpc_resolver_factory_create_resolver(exec_ctx, factory, &resolver_args);
   grpc_uri_destroy(uri);
+  gpr_free(canonical_target);
   return resolver;
 }
 
 char *grpc_get_default_authority(const char *target) {
   grpc_uri *uri = NULL;
-  grpc_resolver_factory *factory = resolve_factory(target, &uri);
+  char *canonical_target = NULL;
+  grpc_resolver_factory *factory =
+      resolve_factory(target, &uri, &canonical_target);
   char *authority = grpc_resolver_factory_get_default_authority(factory, uri);
   grpc_uri_destroy(uri);
+  gpr_free(canonical_target);
   return authority;
 }
+
+char *grpc_resolver_factory_add_default_prefix_if_needed(const char *target) {
+  grpc_uri *uri = NULL;
+  char *canonical_target = NULL;
+  resolve_factory(target, &uri, &canonical_target);
+  grpc_uri_destroy(uri);
+  return canonical_target == NULL ? gpr_strdup(target) : canonical_target;
+}

src/core/ext/client_channel/resolver_registry.h

@@ -35,6 +35,7 @@
 #define GRPC_CORE_EXT_CLIENT_CHANNEL_RESOLVER_REGISTRY_H
 
 #include "src/core/ext/client_channel/resolver_factory.h"
+#include "src/core/lib/iomgr/pollset_set.h"
 
 void grpc_resolver_registry_init();
 void grpc_resolver_registry_shutdown(void);
@@ -60,8 +61,9 @@ void grpc_register_resolver_type(grpc_resolver_factory *factory);
     If a resolver factory was not found, return NULL.
     \a args is a set of channel arguments to be included in the result
     (typically the set of arguments passed in from the client API). */
-grpc_resolver *grpc_resolver_create(const char *target,
-                                    const grpc_channel_args *args);
+grpc_resolver *grpc_resolver_create(grpc_exec_ctx *exec_ctx, const char *target,
+                                    const grpc_channel_args *args,
+                                    grpc_pollset_set *pollset_set);
 
 /** Find a resolver factory given a name and return an (owned-by-the-caller)
  *  reference to it */
@@ -71,4 +73,8 @@ grpc_resolver_factory *grpc_resolver_factory_lookup(const char *name);
     representing the default authority to pass from a client. */
 char *grpc_get_default_authority(const char *target);
 
+/** Returns a newly allocated string containing \a target, adding the
+    default prefix if needed. */
+char *grpc_resolver_factory_add_default_prefix_if_needed(const char *target);
+
 #endif /* GRPC_CORE_EXT_CLIENT_CHANNEL_RESOLVER_REGISTRY_H */

src/core/ext/client_channel/subchannel.c

@@ -293,8 +293,9 @@ void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
   gpr_atm old_refs;
   old_refs = ref_mutate(c, -(gpr_atm)1, 1 REF_MUTATE_PURPOSE("WEAK_UNREF"));
   if (old_refs == 1) {
-    grpc_exec_ctx_sched(exec_ctx, grpc_closure_create(subchannel_destroy, c),
-                        GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, grpc_closure_create(subchannel_destroy, c,
+                                                     grpc_schedule_on_exec_ctx),
+                       GRPC_ERROR_NONE);
   }
 }
 
@@ -330,7 +331,8 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
   c->args = grpc_channel_args_copy(args->args);
   c->root_external_state_watcher.next = c->root_external_state_watcher.prev =
       &c->root_external_state_watcher;
-  grpc_closure_init(&c->connected, subchannel_connected, c);
+  grpc_closure_init(&c->connected, subchannel_connected, c,
+                    grpc_schedule_on_exec_ctx);
   grpc_connectivity_state_init(&c->state_tracker, GRPC_CHANNEL_IDLE,
                                "subchannel");
   int initial_backoff_ms =
@@ -505,7 +507,8 @@ void grpc_subchannel_notify_on_state_change(
     w->subchannel = c;
     w->pollset_set = interested_parties;
     w->notify = notify;
-    grpc_closure_init(&w->closure, on_external_state_watcher_done, w);
+    grpc_closure_init(&w->closure, on_external_state_watcher_done, w,
+                      grpc_schedule_on_exec_ctx);
     if (interested_parties != NULL) {
       grpc_pollset_set_add_pollset_set(exec_ctx, c->pollset_set,
                                        interested_parties);
@@ -604,14 +607,20 @@ static void publish_transport_locked(grpc_exec_ctx *exec_ctx,
   grpc_channel_stack_builder_set_transport(builder,
                                            c->connecting_result.transport);
 
-  if (grpc_channel_init_create_stack(exec_ctx, builder,
-                                     GRPC_CLIENT_SUBCHANNEL)) {
-    con = grpc_channel_stack_builder_finish(exec_ctx, builder, 0, 1,
-                                            connection_destroy, NULL);
-  } else {
+  if (!grpc_channel_init_create_stack(exec_ctx, builder,
+                                      GRPC_CLIENT_SUBCHANNEL)) {
     grpc_channel_stack_builder_destroy(builder);
     abort(); /* TODO(ctiller): what to do here (previously we just crashed) */
   }
+  grpc_error *error = grpc_channel_stack_builder_finish(
+      exec_ctx, builder, 0, 1, connection_destroy, NULL, (void **)&con);
+  if (error != GRPC_ERROR_NONE) {
+    const char *msg = grpc_error_string(error);
+    gpr_log(GPR_ERROR, "error initializing subchannel stack: %s", msg);
+    grpc_error_free_string(msg);
+    GRPC_ERROR_UNREF(error);
+    abort(); /* TODO(ctiller): what to do here? */
+  }
   stk = CHANNEL_STACK_FROM_CONNECTION(con);
   memset(&c->connecting_result, 0, sizeof(c->connecting_result));
 
@@ -620,7 +629,7 @@ static void publish_transport_locked(grpc_exec_ctx *exec_ctx,
   sw_subchannel->subchannel = c;
   sw_subchannel->connectivity_state = GRPC_CHANNEL_READY;
   grpc_closure_init(&sw_subchannel->closure, subchannel_on_child_state_changed,
-                    sw_subchannel);
+                    sw_subchannel, grpc_schedule_on_exec_ctx);
 
   if (c->disconnected) {
     gpr_free(sw_subchannel);

src/core/ext/client_channel/subchannel.h

@@ -164,8 +164,6 @@ struct grpc_subchannel_args {
   size_t filter_count;
   /** Channel arguments to be supplied to the newly created channel */
   const grpc_channel_args *args;
-  /** Server name */
-  const char *server_name;
   /** Address to connect to */
   grpc_resolved_address *addr;
 };

src/core/ext/client_channel/subchannel_index.c

@@ -86,7 +86,6 @@ static grpc_subchannel_key *create_key(
   } else {
     k->args.filters = NULL;
   }
-  k->args.server_name = gpr_strdup(args->server_name);
   k->args.addr = gpr_malloc(sizeof(grpc_resolved_address));
   k->args.addr->len = args->addr->len;
   if (k->args.addr->len > 0) {
@@ -113,8 +112,6 @@ static int subchannel_key_compare(grpc_subchannel_key *a,
   if (c != 0) return c;
   c = GPR_ICMP(a->args.filter_count, b->args.filter_count);
   if (c != 0) return c;
-  c = strcmp(a->args.server_name, b->args.server_name);
-  if (c != 0) return c;
   if (a->args.addr->len) {
     c = memcmp(a->args.addr->addr, b->args.addr->addr, a->args.addr->len);
     if (c != 0) return c;
@@ -132,7 +129,6 @@ void grpc_subchannel_key_destroy(grpc_exec_ctx *exec_ctx,
   grpc_connector_unref(exec_ctx, k->connector);
   gpr_free((grpc_channel_args *)k->args.filters);
   grpc_channel_args_destroy((grpc_channel_args *)k->args.args);
-  gpr_free((void *)k->args.server_name);
   gpr_free(k->args.addr);
   gpr_free(k);
 }

src/core/ext/lb_policy/grpclb/grpclb.c

@@ -106,6 +106,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
+#include "src/core/ext/client_channel/client_channel.h"
 #include "src/core/ext/client_channel/client_channel_factory.h"
 #include "src/core/ext/client_channel/lb_policy_factory.h"
 #include "src/core/ext/client_channel/lb_policy_registry.h"
@@ -179,8 +180,7 @@ static void wrapped_rr_closure(grpc_exec_ctx *exec_ctx, void *arg,
   wrapped_rr_closure_arg *wc_arg = arg;
 
   GPR_ASSERT(wc_arg->wrapped_closure != NULL);
-  grpc_exec_ctx_sched(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_REF(error),
-                      NULL);
+  grpc_closure_sched(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_REF(error));
 
   if (wc_arg->rr_policy != NULL) {
     /* if *target is NULL, no pick has been made by the RR policy (eg, all
@@ -247,7 +247,8 @@ static void add_pending_pick(pending_pick **root,
       pick_args->lb_token_mdelem_storage;
   pp->wrapped_on_complete_arg.free_when_done = pp;
   grpc_closure_init(&pp->wrapped_on_complete_arg.wrapper_closure,
-                    wrapped_rr_closure, &pp->wrapped_on_complete_arg);
+                    wrapped_rr_closure, &pp->wrapped_on_complete_arg,
+                    grpc_schedule_on_exec_ctx);
   *root = pp;
 }
 
@@ -267,7 +268,8 @@ static void add_pending_ping(pending_ping **root, grpc_closure *notify) {
   pping->wrapped_notify_arg.free_when_done = pping;
   pping->next = *root;
   grpc_closure_init(&pping->wrapped_notify_arg.wrapper_closure,
-                    wrapped_rr_closure, &pping->wrapped_notify_arg);
+                    wrapped_rr_closure, &pping->wrapped_notify_arg,
+                    grpc_schedule_on_exec_ctx);
   *root = pping;
 }
 
@@ -666,7 +668,7 @@ static void rr_handover_locked(grpc_exec_ctx *exec_ctx,
       gpr_malloc(sizeof(rr_connectivity_data));
   memset(rr_connectivity, 0, sizeof(rr_connectivity_data));
   grpc_closure_init(&rr_connectivity->on_change, glb_rr_connectivity_changed,
-                    rr_connectivity);
+                    rr_connectivity, grpc_schedule_on_exec_ctx);
   rr_connectivity->glb_policy = glb_policy;
   rr_connectivity->state = new_rr_state;
 
@@ -743,12 +745,6 @@ static void glb_rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
 static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
                                   grpc_lb_policy_factory *factory,
                                   grpc_lb_policy_args *args) {
-  /* Get server name. */
-  const grpc_arg *arg =
-      grpc_channel_args_find(args->args, GRPC_ARG_SERVER_NAME);
-  const char *server_name =
-      arg != NULL && arg->type == GRPC_ARG_STRING ? arg->value.string : NULL;
-
   /* Count the number of gRPC-LB addresses. There must be at least one.
    * TODO(roth): For now, we ignore non-balancer addresses, but in the
    * future, we may change the behavior such that we fall back to using
@@ -756,7 +752,8 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
    * time, this should be changed to allow a list with no balancer addresses,
    * since the resolver might fail to return a balancer address even when
    * this is the right LB policy to use. */
-  arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
+  const grpc_arg *arg =
+      grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
   GPR_ASSERT(arg != NULL && arg->type == GRPC_ARG_POINTER);
   grpc_lb_addresses *addresses = arg->value.pointer.p;
   size_t num_grpclb_addrs = 0;
@@ -768,13 +765,25 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
   glb_lb_policy *glb_policy = gpr_malloc(sizeof(*glb_policy));
   memset(glb_policy, 0, sizeof(*glb_policy));
 
+  /* Get server name. */
+  arg = grpc_channel_args_find(args->args, GRPC_ARG_SERVER_URI);
+  GPR_ASSERT(arg != NULL);
+  GPR_ASSERT(arg->type == GRPC_ARG_STRING);
+  grpc_uri *uri = grpc_uri_parse(arg->value.string, true);
+  GPR_ASSERT(uri->path[0] != '\0');
+  glb_policy->server_name =
+      gpr_strdup(uri->path[0] == '/' ? uri->path + 1 : uri->path);
+  if (grpc_lb_glb_trace) {
+    gpr_log(GPR_INFO, "Will use '%s' as the server name for LB request.",
+            glb_policy->server_name);
+  }
+  grpc_uri_destroy(uri);
+
   /* All input addresses in addresses come from a resolver that claims
    * they are LB services. It's the resolver's responsibility to make sure
-   * this
-   * policy is only instantiated and used in that case.
+   * this policy is only instantiated and used in that case.
    *
    * Create a client channel over them to communicate with a LB service */
-  glb_policy->server_name = gpr_strdup(server_name);
   glb_policy->cc_factory = args->client_channel_factory;
   glb_policy->args = grpc_channel_args_copy(args->args);
   GPR_ASSERT(glb_policy->cc_factory != NULL);
@@ -818,9 +827,14 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
    * We need the LB channel to return addresses with is_balancer=false
    * so that it does not wind up recursively using the grpclb LB policy,
    * as per the special case logic in client_channel.c.
+   *
+   * Finally, we also strip out the channel arg for the server URI,
+   * since that will be different for the LB channel than for the parent
+   * channel.  (The client channel factory will re-add this arg with
+   * the right value.)
    */
-  static const char *keys_to_remove[] = {GRPC_ARG_LB_POLICY_NAME,
-                                         GRPC_ARG_LB_ADDRESSES};
+  static const char *keys_to_remove[] = {
+      GRPC_ARG_LB_POLICY_NAME, GRPC_ARG_LB_ADDRESSES, GRPC_ARG_SERVER_URI};
   grpc_channel_args *new_args = grpc_channel_args_copy_and_remove(
       args->args, keys_to_remove, GPR_ARRAY_SIZE(keys_to_remove));
   glb_policy->lb_channel = grpc_client_channel_factory_create_channel(
@@ -895,15 +909,15 @@ static void glb_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   while (pp != NULL) {
     pending_pick *next = pp->next;
     *pp->target = NULL;
-    grpc_exec_ctx_sched(exec_ctx, &pp->wrapped_on_complete_arg.wrapper_closure,
-                        GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, &pp->wrapped_on_complete_arg.wrapper_closure,
+                       GRPC_ERROR_NONE);
     pp = next;
   }
 
   while (pping != NULL) {
     pending_ping *next = pping->next;
-    grpc_exec_ctx_sched(exec_ctx, &pping->wrapped_notify_arg.wrapper_closure,
-                        GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, &pping->wrapped_notify_arg.wrapper_closure,
+                       GRPC_ERROR_NONE);
     pping = next;
   }
 }
@@ -919,9 +933,9 @@ static void glb_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if (pp->target == target) {
       *target = NULL;
-      grpc_exec_ctx_sched(
+      grpc_closure_sched(
           exec_ctx, &pp->wrapped_on_complete_arg.wrapper_closure,
-          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1), NULL);
+          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1));
     } else {
       pp->next = glb_policy->pending_picks;
       glb_policy->pending_picks = pp;
@@ -944,9 +958,9 @@ static void glb_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if ((pp->pick_args.initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      grpc_exec_ctx_sched(
+      grpc_closure_sched(
           exec_ctx, &pp->wrapped_on_complete_arg.wrapper_closure,
-          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1), NULL);
+          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1));
     } else {
       pp->next = glb_policy->pending_picks;
       glb_policy->pending_picks = pp;
@@ -981,11 +995,10 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
                     grpc_closure *on_complete) {
   if (pick_args->lb_token_mdelem_storage == NULL) {
     *target = NULL;
-    grpc_exec_ctx_sched(
+    grpc_closure_sched(
         exec_ctx, on_complete,
         GRPC_ERROR_CREATE("No mdelem storage for the LB token. Load reporting "
-                          "won't work without it. Failing"),
-        NULL);
+                          "won't work without it. Failing"));
     return 0;
   }
 
@@ -1004,7 +1017,8 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     wrapped_rr_closure_arg *wc_arg = gpr_malloc(sizeof(wrapped_rr_closure_arg));
     memset(wc_arg, 0, sizeof(wrapped_rr_closure_arg));
 
-    grpc_closure_init(&wc_arg->wrapper_closure, wrapped_rr_closure, wc_arg);
+    grpc_closure_init(&wc_arg->wrapper_closure, wrapped_rr_closure, wc_arg,
+                      grpc_schedule_on_exec_ctx);
     wc_arg->rr_policy = glb_policy->rr_policy;
     wc_arg->target = target;
     wc_arg->wrapped_closure = on_complete;
@@ -1104,9 +1118,11 @@ static void lb_call_init_locked(glb_lb_policy *glb_policy) {
   glb_policy->lb_call_status_details_capacity = 0;
 
   grpc_closure_init(&glb_policy->lb_on_server_status_received,
-                    lb_on_server_status_received, glb_policy);
+                    lb_on_server_status_received, glb_policy,
+                    grpc_schedule_on_exec_ctx);
   grpc_closure_init(&glb_policy->lb_on_response_received,
-                    lb_on_response_received, glb_policy);
+                    lb_on_response_received, glb_policy,
+                    grpc_schedule_on_exec_ctx);
 
   gpr_backoff_init(&glb_policy->lb_call_backoff_state,
                    GRPC_GRPCLB_INITIAL_CONNECT_BACKOFF_SECONDS,

src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c

@@ -1,35 +1,3 @@
-/*
- *
- * Copyright 2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
 /* Automatically generated nanopb constant definitions */
 /* Generated by nanopb-0.3.7-dev */
 

src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h

@@ -1,35 +1,3 @@
-/*
- *
- * Copyright 2016, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
 /* Automatically generated nanopb header */
 /* Generated by nanopb-0.3.7-dev */
 

src/core/ext/lb_policy/pick_first/pick_first.c

@@ -120,7 +120,7 @@ static void pf_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   while (pp != NULL) {
     pending_pick *next = pp->next;
     *pp->target = NULL;
-    grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE);
     gpr_free(pp);
     pp = next;
   }
@@ -138,9 +138,9 @@ static void pf_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if (pp->target == target) {
       *target = NULL;
-      grpc_exec_ctx_sched(
+      grpc_closure_sched(
           exec_ctx, pp->on_complete,
-          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1), NULL);
+          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1));
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -165,9 +165,9 @@ static void pf_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
-      grpc_exec_ctx_sched(
+      grpc_closure_sched(
           exec_ctx, pp->on_complete,
-          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1), NULL);
+          GRPC_ERROR_CREATE_REFERENCING("Pick Cancelled", &error, 1));
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -306,14 +306,15 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         /* drop the pick list: we are connected now */
         GRPC_LB_POLICY_WEAK_REF(&p->base, "destroy_subchannels");
         gpr_atm_rel_store(&p->selected, (gpr_atm)selected);
-        grpc_exec_ctx_sched(exec_ctx,
-                            grpc_closure_create(destroy_subchannels, p),
-                            GRPC_ERROR_NONE, NULL);
+        grpc_closure_sched(exec_ctx,
+                           grpc_closure_create(destroy_subchannels, p,
+                                               grpc_schedule_on_exec_ctx),
+                           GRPC_ERROR_NONE);
         /* update any calls that were waiting for a pick */
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
           *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(selected, "picked");
-          grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
+          grpc_closure_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE);
           gpr_free(pp);
         }
         grpc_connected_subchannel_notify_on_state_change(
@@ -366,8 +367,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
           while ((pp = p->pending_picks)) {
             p->pending_picks = pp->next;
             *pp->target = NULL;
-            grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE,
-                                NULL);
+            grpc_closure_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE);
             gpr_free(pp);
           }
           GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &p->base,
@@ -419,8 +419,7 @@ static void pf_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   if (selected) {
     grpc_connected_subchannel_ping(exec_ctx, selected, closure);
   } else {
-    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("Not connected"),
-                        NULL);
+    grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_CREATE("Not connected"));
   }
 }
 
@@ -438,15 +437,10 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx,
                                          grpc_lb_policy_args *args) {
   GPR_ASSERT(args->client_channel_factory != NULL);
 
-  /* Get server name. */
-  const grpc_arg *arg =
-      grpc_channel_args_find(args->args, GRPC_ARG_SERVER_NAME);
-  const char *server_name =
-      arg != NULL && arg->type == GRPC_ARG_STRING ? arg->value.string : NULL;
-
   /* Find the number of backend addresses. We ignore balancer
    * addresses, since we don't know how to handle them. */
-  arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
+  const grpc_arg *arg =
+      grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
   GPR_ASSERT(arg != NULL && arg->type == GRPC_ARG_POINTER);
   grpc_lb_addresses *addresses = arg->value.pointer.p;
   size_t num_addrs = 0;
@@ -472,9 +466,6 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx,
     }
 
     memset(&sc_args, 0, sizeof(grpc_subchannel_args));
-    /* server_name will be copied as part of the subchannel creation. This makes
-     * the copying of server_name (a borrowed pointer) OK. */
-    sc_args.server_name = server_name;
     sc_args.addr = &addresses->addresses[i].address;
     sc_args.args = args->args;
 
@@ -493,7 +484,8 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx,
   p->num_subchannels = subchannel_idx;
 
   grpc_lb_policy_init(&p->base, &pick_first_lb_policy_vtable);
-  grpc_closure_init(&p->connectivity_changed, pf_connectivity_changed, p);
+  grpc_closure_init(&p->connectivity_changed, pf_connectivity_changed, p,
+                    grpc_schedule_on_exec_ctx);
   gpr_mu_init(&p->mu);
   return &p->base;
 }

src/core/ext/lb_policy/round_robin/round_robin.c

@@ -321,8 +321,8 @@ static void rr_shutdown(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
   while ((pp = p->pending_picks)) {
     p->pending_picks = pp->next;
     *pp->target = NULL;
-    grpc_exec_ctx_sched(exec_ctx, pp->on_complete,
-                        GRPC_ERROR_CREATE("Channel Shutdown"), NULL);
+    grpc_closure_sched(exec_ctx, pp->on_complete,
+                       GRPC_ERROR_CREATE("Channel Shutdown"));
     gpr_free(pp);
   }
   grpc_connectivity_state_set(
@@ -348,9 +348,9 @@ static void rr_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     pending_pick *next = pp->next;
     if (pp->target == target) {
       *target = NULL;
-      grpc_exec_ctx_sched(
+      grpc_closure_sched(
           exec_ctx, pp->on_complete,
-          GRPC_ERROR_CREATE_REFERENCING("Pick cancelled", &error, 1), NULL);
+          GRPC_ERROR_CREATE_REFERENCING("Pick cancelled", &error, 1));
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -376,9 +376,9 @@ static void rr_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     if ((pp->initial_metadata_flags & initial_metadata_flags_mask) ==
         initial_metadata_flags_eq) {
       *pp->target = NULL;
-      grpc_exec_ctx_sched(
+      grpc_closure_sched(
           exec_ctx, pp->on_complete,
-          GRPC_ERROR_CREATE_REFERENCING("Pick cancelled", &error, 1), NULL);
+          GRPC_ERROR_CREATE_REFERENCING("Pick cancelled", &error, 1));
       gpr_free(pp);
     } else {
       pp->next = p->pending_picks;
@@ -581,7 +581,7 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
                   "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)",
                   (void *)selected->subchannel, (void *)selected);
         }
-        grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
+        grpc_closure_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE);
         gpr_free(pp);
       }
       update_lb_connectivity_status(exec_ctx, sd, error);
@@ -634,7 +634,7 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
           *pp->target = NULL;
-          grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
+          grpc_closure_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE);
           gpr_free(pp);
         }
       }
@@ -684,8 +684,8 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
     GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, target, "rr_picked");
   } else {
     gpr_mu_unlock(&p->mu);
-    grpc_exec_ctx_sched(exec_ctx, closure,
-                        GRPC_ERROR_CREATE("Round Robin not connected"), NULL);
+    grpc_closure_sched(exec_ctx, closure,
+                       GRPC_ERROR_CREATE("Round Robin not connected"));
   }
 }
 
@@ -703,15 +703,10 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
                                           grpc_lb_policy_args *args) {
   GPR_ASSERT(args->client_channel_factory != NULL);
 
-  /* Get server name. */
-  const grpc_arg *arg =
-      grpc_channel_args_find(args->args, GRPC_ARG_SERVER_NAME);
-  const char *server_name =
-      arg != NULL && arg->type == GRPC_ARG_STRING ? arg->value.string : NULL;
-
   /* Find the number of backend addresses. We ignore balancer
    * addresses, since we don't know how to handle them. */
-  arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
+  const grpc_arg *arg =
+      grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
   GPR_ASSERT(arg != NULL && arg->type == GRPC_ARG_POINTER);
   grpc_lb_addresses *addresses = arg->value.pointer.p;
   size_t num_addrs = 0;
@@ -734,9 +729,6 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
     if (addresses->addresses[i].is_balancer) continue;
 
     memset(&sc_args, 0, sizeof(grpc_subchannel_args));
-    /* server_name will be copied as part of the subchannel creation. This makes
-     * the copying of server_name (a borrowed pointer) OK. */
-    sc_args.server_name = server_name;
     sc_args.addr = &addresses->addresses[i].address;
     sc_args.args = args->args;
 
@@ -757,7 +749,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx,
       }
       ++subchannel_idx;
       grpc_closure_init(&sd->connectivity_changed_closure,
-                        rr_connectivity_changed, sd);
+                        rr_connectivity_changed, sd, grpc_schedule_on_exec_ctx);
     }
   }
   if (subchannel_idx == 0) {

src/core/ext/load_reporting/load_reporting_filter.c

@@ -114,7 +114,8 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   memset(calld, 0, sizeof(call_data));
 
   calld->id = (intptr_t)args->call_stack;
-  grpc_closure_init(&calld->on_initial_md_ready, on_initial_md_ready, elem);
+  grpc_closure_init(&calld->on_initial_md_ready, on_initial_md_ready, elem,
+                    grpc_schedule_on_exec_ctx);
 
   /* TODO(dgq): do something with the data
   channel_data *chand = elem->channel_data;
@@ -152,9 +153,9 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 }
 
 /* Constructor for channel_data */
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
+static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_channel_element *elem,
+                                     grpc_channel_element_args *args) {
   GPR_ASSERT(!args->is_last);
 
   channel_data *chand = elem->channel_data;
@@ -171,6 +172,8 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
                                                 NULL,
                                                 NULL};
                                                 */
+
+  return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel data */

src/core/ext/resolver/dns/native/dns_resolver.c

@@ -61,6 +61,8 @@ typedef struct {
   char *default_port;
   /** channel args. */
   grpc_channel_args *channel_args;
+  /** pollset_set to drive the name resolution process */
+  grpc_pollset_set *interested_parties;
 
   /** mutex guarding the rest of the state */
   gpr_mu mu;
@@ -110,8 +112,8 @@ static void dns_shutdown(grpc_exec_ctx *exec_ctx, grpc_resolver *resolver) {
   }
   if (r->next_completion != NULL) {
     *r->target_result = NULL;
-    grpc_exec_ctx_sched(exec_ctx, r->next_completion,
-                        GRPC_ERROR_CREATE("Resolver Shutdown"), NULL);
+    grpc_closure_sched(exec_ctx, r->next_completion,
+                       GRPC_ERROR_CREATE("Resolver Shutdown"));
     r->next_completion = NULL;
   }
   gpr_mu_unlock(&r->mu);
@@ -217,8 +219,10 @@ static void dns_start_resolving_locked(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(!r->resolving);
   r->resolving = true;
   r->addresses = NULL;
-  grpc_resolve_address(exec_ctx, r->name_to_resolve, r->default_port,
-                       grpc_closure_create(dns_on_resolved, r), &r->addresses);
+  grpc_resolve_address(
+      exec_ctx, r->name_to_resolve, r->default_port, r->interested_parties,
+      grpc_closure_create(dns_on_resolved, r, grpc_schedule_on_exec_ctx),
+      &r->addresses);
 }
 
 static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
@@ -228,7 +232,7 @@ static void dns_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
     *r->target_result = r->resolved_result == NULL
                             ? NULL
                             : grpc_channel_args_copy(r->resolved_result);
-    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE);
     r->next_completion = NULL;
     r->published_version = r->resolved_version;
   }
@@ -240,13 +244,15 @@ static void dns_destroy(grpc_exec_ctx *exec_ctx, grpc_resolver *gr) {
   if (r->resolved_result != NULL) {
     grpc_channel_args_destroy(r->resolved_result);
   }
+  grpc_pollset_set_destroy(r->interested_parties);
   gpr_free(r->name_to_resolve);
   gpr_free(r->default_port);
   grpc_channel_args_destroy(r->channel_args);
   gpr_free(r);
 }
 
-static grpc_resolver *dns_create(grpc_resolver_args *args,
+static grpc_resolver *dns_create(grpc_exec_ctx *exec_ctx,
+                                 grpc_resolver_args *args,
                                  const char *default_port) {
   if (0 != strcmp(args->uri->authority, "")) {
     gpr_log(GPR_ERROR, "authority based dns uri's not supported");
@@ -264,12 +270,12 @@ static grpc_resolver *dns_create(grpc_resolver_args *args,
   grpc_resolver_init(&r->base, &dns_resolver_vtable);
   r->name_to_resolve = proxy_name == NULL ? gpr_strdup(path) : proxy_name;
   r->default_port = gpr_strdup(default_port);
-  grpc_arg server_name_arg;
-  server_name_arg.type = GRPC_ARG_STRING;
-  server_name_arg.key = GRPC_ARG_SERVER_NAME;
-  server_name_arg.value.string = (char *)path;
-  r->channel_args =
-      grpc_channel_args_copy_and_add(args->args, &server_name_arg, 1);
+  r->channel_args = grpc_channel_args_copy(args->args);
+  r->interested_parties = grpc_pollset_set_create();
+  if (args->pollset_set != NULL) {
+    grpc_pollset_set_add_pollset_set(exec_ctx, r->interested_parties,
+                                     args->pollset_set);
+  }
   gpr_backoff_init(&r->backoff_state, GRPC_DNS_INITIAL_CONNECT_BACKOFF_SECONDS,
                    GRPC_DNS_RECONNECT_BACKOFF_MULTIPLIER,
                    GRPC_DNS_RECONNECT_JITTER,
@@ -287,8 +293,9 @@ static void dns_factory_ref(grpc_resolver_factory *factory) {}
 static void dns_factory_unref(grpc_resolver_factory *factory) {}
 
 static grpc_resolver *dns_factory_create_resolver(
-    grpc_resolver_factory *factory, grpc_resolver_args *args) {
-  return dns_create(args, "https");
+    grpc_exec_ctx *exec_ctx, grpc_resolver_factory *factory,
+    grpc_resolver_args *args) {
+  return dns_create(exec_ctx, args, "https");
 }
 
 static char *dns_factory_get_default_host_name(grpc_resolver_factory *factory,

src/core/ext/resolver/sockaddr/sockaddr_resolver.c

@@ -89,7 +89,7 @@ static void sockaddr_shutdown(grpc_exec_ctx *exec_ctx,
   gpr_mu_lock(&r->mu);
   if (r->next_completion != NULL) {
     *r->target_result = NULL;
-    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE);
     r->next_completion = NULL;
   }
   gpr_mu_unlock(&r->mu);
@@ -123,7 +123,7 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx,
     grpc_arg arg = grpc_lb_addresses_create_channel_arg(r->addresses);
     *r->target_result =
         grpc_channel_args_copy_and_add(r->channel_args, &arg, 1);
-    grpc_exec_ctx_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE);
     r->next_completion = NULL;
   }
 }
@@ -198,12 +198,7 @@ static grpc_resolver *sockaddr_create(grpc_resolver_args *args,
   sockaddr_resolver *r = gpr_malloc(sizeof(sockaddr_resolver));
   memset(r, 0, sizeof(*r));
   r->addresses = addresses;
-  grpc_arg server_name_arg;
-  server_name_arg.type = GRPC_ARG_STRING;
-  server_name_arg.key = GRPC_ARG_SERVER_NAME;
-  server_name_arg.value.string = args->uri->path;
-  r->channel_args =
-      grpc_channel_args_copy_and_add(args->args, &server_name_arg, 1);
+  r->channel_args = grpc_channel_args_copy(args->args);
   gpr_mu_init(&r->mu);
   grpc_resolver_init(&r->base, &sockaddr_resolver_vtable);
   return &r->base;
@@ -219,7 +214,8 @@ static void sockaddr_factory_unref(grpc_resolver_factory *factory) {}
 
 #define DECL_FACTORY(name)                                                  \
   static grpc_resolver *name##_factory_create_resolver(                     \
-      grpc_resolver_factory *factory, grpc_resolver_args *args) {           \
+      grpc_exec_ctx *exec_ctx, grpc_resolver_factory *factory,              \
+      grpc_resolver_args *args) {                                           \
     return sockaddr_create(args, parse_##name);                             \
   }                                                                         \
   static const grpc_resolver_factory_vtable name##_factory_vtable = {       \

src/core/ext/transport/chttp2/client/chttp2_connector.c

@@ -56,10 +56,10 @@ typedef struct {
   gpr_refcount refs;
 
   bool shutdown;
+  bool connecting;
 
-  char *server_name;
-  grpc_chttp2_create_handshakers_func create_handshakers;
-  void *create_handshakers_user_data;
+  grpc_chttp2_add_handshakers_func add_handshakers;
+  void *add_handshakers_user_data;
 
   grpc_closure *notify;
   grpc_connect_in_args args;
@@ -88,7 +88,6 @@ static void chttp2_connector_unref(grpc_exec_ctx *exec_ctx,
     // If handshaking is not yet in progress, destroy the endpoint.
     // Otherwise, the handshaker will do this for us.
     if (c->endpoint != NULL) grpc_endpoint_destroy(exec_ctx, c->endpoint);
-    gpr_free(c->server_name);
     gpr_free(c);
   }
 }
@@ -103,7 +102,9 @@ static void chttp2_connector_shutdown(grpc_exec_ctx *exec_ctx,
   }
   // If handshaking is not yet in progress, shutdown the endpoint.
   // Otherwise, the handshaker will do this for us.
-  if (c->endpoint != NULL) grpc_endpoint_shutdown(exec_ctx, c->endpoint);
+  if (!c->connecting && c->endpoint != NULL) {
+    grpc_endpoint_shutdown(exec_ctx, c->endpoint);
+  }
   gpr_mu_unlock(&c->mu);
 }
 
@@ -140,7 +141,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
   }
   grpc_closure *notify = c->notify;
   c->notify = NULL;
-  grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+  grpc_closure_sched(exec_ctx, notify, error);
   grpc_handshake_manager_destroy(exec_ctx, c->handshake_mgr);
   c->handshake_mgr = NULL;
   gpr_mu_unlock(&c->mu);
@@ -152,14 +153,13 @@ static void start_handshake_locked(grpc_exec_ctx *exec_ctx,
   c->handshake_mgr = grpc_handshake_manager_create();
   char *proxy_name = grpc_get_http_proxy_server();
   if (proxy_name != NULL) {
-    grpc_handshake_manager_add(
-        c->handshake_mgr,
-        grpc_http_connect_handshaker_create(proxy_name, c->server_name));
+    grpc_handshake_manager_add(c->handshake_mgr,
+                               grpc_http_connect_handshaker_create(proxy_name));
     gpr_free(proxy_name);
   }
-  if (c->create_handshakers != NULL) {
-    c->create_handshakers(exec_ctx, c->create_handshakers_user_data,
-                          c->handshake_mgr);
+  if (c->add_handshakers != NULL) {
+    c->add_handshakers(exec_ctx, c->add_handshakers_user_data,
+                       c->handshake_mgr);
   }
   grpc_handshake_manager_do_handshake(
       exec_ctx, c->handshake_mgr, c->endpoint, c->args.channel_args,
@@ -180,7 +180,7 @@ static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
     memset(c->result, 0, sizeof(*c->result));
     grpc_closure *notify = c->notify;
     c->notify = NULL;
-    grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+    grpc_closure_sched(exec_ctx, notify, error);
     gpr_mu_unlock(&c->mu);
     chttp2_connector_unref(exec_ctx, arg);
   } else {
@@ -192,6 +192,8 @@ static void on_initial_connect_string_sent(grpc_exec_ctx *exec_ctx, void *arg,
 static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   chttp2_connector *c = arg;
   gpr_mu_lock(&c->mu);
+  GPR_ASSERT(c->connecting);
+  c->connecting = false;
   if (error != GRPC_ERROR_NONE || c->shutdown) {
     if (error == GRPC_ERROR_NONE) {
       error = GRPC_ERROR_CREATE("connector shutdown");
@@ -201,14 +203,15 @@ static void connected(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
     memset(c->result, 0, sizeof(*c->result));
     grpc_closure *notify = c->notify;
     c->notify = NULL;
-    grpc_exec_ctx_sched(exec_ctx, notify, error, NULL);
+    grpc_closure_sched(exec_ctx, notify, error);
+    if (c->endpoint != NULL) grpc_endpoint_shutdown(exec_ctx, c->endpoint);
     gpr_mu_unlock(&c->mu);
     chttp2_connector_unref(exec_ctx, arg);
   } else {
     GPR_ASSERT(c->endpoint != NULL);
     if (!GRPC_SLICE_IS_EMPTY(c->args.initial_connect_string)) {
       grpc_closure_init(&c->initial_string_sent, on_initial_connect_string_sent,
-                        c);
+                        c, grpc_schedule_on_exec_ctx);
       grpc_slice_buffer_init(&c->initial_string_buffer);
       grpc_slice_buffer_add(&c->initial_string_buffer,
                             c->args.initial_connect_string);
@@ -234,7 +237,9 @@ static void chttp2_connector_connect(grpc_exec_ctx *exec_ctx,
   c->result = result;
   GPR_ASSERT(c->endpoint == NULL);
   chttp2_connector_ref(con);  // Ref taken for callback.
-  grpc_closure_init(&c->connected, connected, c);
+  grpc_closure_init(&c->connected, connected, c, grpc_schedule_on_exec_ctx);
+  GPR_ASSERT(!c->connecting);
+  c->connecting = true;
   grpc_tcp_client_connect(exec_ctx, &c->connected, &c->endpoint,
                           args->interested_parties, args->channel_args,
                           args->addr, args->deadline);
@@ -246,16 +251,14 @@ static const grpc_connector_vtable chttp2_connector_vtable = {
     chttp2_connector_connect};
 
 grpc_connector *grpc_chttp2_connector_create(
-    grpc_exec_ctx *exec_ctx, const char *server_name,
-    grpc_chttp2_create_handshakers_func create_handshakers,
-    void *create_handshakers_user_data) {
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_add_handshakers_func add_handshakers,
+    void *add_handshakers_user_data) {
   chttp2_connector *c = gpr_malloc(sizeof(*c));
   memset(c, 0, sizeof(*c));
   c->base.vtable = &chttp2_connector_vtable;
   gpr_mu_init(&c->mu);
   gpr_ref_init(&c->refs, 1);
-  c->server_name = gpr_strdup(server_name);
-  c->create_handshakers = create_handshakers;
-  c->create_handshakers_user_data = create_handshakers_user_data;
+  c->add_handshakers = add_handshakers;
+  c->add_handshakers_user_data = add_handshakers_user_data;
   return &c->base;
 }

src/core/ext/transport/chttp2/client/chttp2_connector.h

@@ -38,15 +38,14 @@
 #include "src/core/lib/channel/handshaker.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 
-typedef void (*grpc_chttp2_create_handshakers_func)(
+typedef void (*grpc_chttp2_add_handshakers_func)(
     grpc_exec_ctx* exec_ctx, void* user_data,
     grpc_handshake_manager* handshake_mgr);
 
-/// If \a create_handshakers is non-NULL, it will be called with
-/// \a create_handshakers_user_data to add handshakers.
+/// If \a add_handshakers is non-NULL, it will be called with
+/// \a add_handshakers_user_data to add handshakers.
 grpc_connector* grpc_chttp2_connector_create(
-    grpc_exec_ctx* exec_ctx, const char* server_name,
-    grpc_chttp2_create_handshakers_func create_handshakers,
-    void* create_handshakers_user_data);
+    grpc_exec_ctx* exec_ctx, grpc_chttp2_add_handshakers_func add_handshakers,
+    void* add_handshakers_user_data);
 
 #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_CLIENT_CHTTP2_CONNECTOR_H */

src/core/ext/transport/chttp2/client/insecure/channel_create.c

@@ -39,8 +39,8 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/client_channel/client_channel.h"
-#include "src/core/ext/client_channel/resolver_registry.h"
 #include "src/core/ext/transport/chttp2/client/chttp2_connector.h"
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
 
@@ -54,8 +54,7 @@ static grpc_subchannel *client_channel_factory_create_subchannel(
     grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory,
     const grpc_subchannel_args *args) {
   grpc_connector *connector = grpc_chttp2_connector_create(
-      exec_ctx, args->server_name, NULL /* create_handshakers */,
-      NULL /* user_data */);
+      exec_ctx, NULL /* add_handshakers */, NULL /* user_data */);
   grpc_subchannel *s = grpc_subchannel_create(exec_ctx, connector, args);
   grpc_connector_unref(exec_ctx, connector);
   return s;
@@ -65,17 +64,15 @@ static grpc_channel *client_channel_factory_create_channel(
     grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory,
     const char *target, grpc_client_channel_type type,
     const grpc_channel_args *args) {
-  grpc_channel *channel =
-      grpc_channel_create(exec_ctx, target, args, GRPC_CLIENT_CHANNEL, NULL);
-  grpc_resolver *resolver = grpc_resolver_create(target, args);
-  if (resolver == NULL) {
-    GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel,
-                                "client_channel_factory_create_channel");
-    return NULL;
-  }
-  grpc_client_channel_finish_initialization(
-      exec_ctx, grpc_channel_get_channel_stack(channel), resolver, cc_factory);
-  GRPC_RESOLVER_UNREF(exec_ctx, resolver, "create_channel");
+  // Add channel arg containing the server URI.
+  grpc_arg arg;
+  arg.type = GRPC_ARG_STRING;
+  arg.key = GRPC_ARG_SERVER_URI;
+  arg.value.string = (char *)target;
+  grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1);
+  grpc_channel *channel = grpc_channel_create(exec_ctx, target, new_args,
+                                              GRPC_CLIENT_CHANNEL, NULL);
+  grpc_channel_args_destroy(new_args);
   return channel;
 }
 
@@ -101,8 +98,14 @@ grpc_channel *grpc_insecure_channel_create(const char *target,
   GPR_ASSERT(reserved == NULL);
   grpc_client_channel_factory *factory =
       (grpc_client_channel_factory *)&client_channel_factory;
+  // Add channel arg containing the client channel factory.
+  grpc_arg arg = grpc_client_channel_factory_create_channel_arg(factory);
+  grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1);
+  // Create channel.
   grpc_channel *channel = client_channel_factory_create_channel(
-      &exec_ctx, factory, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, args);
+      &exec_ctx, factory, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, new_args);
+  // Clean up.
+  grpc_channel_args_destroy(new_args);
   grpc_client_channel_factory_unref(&exec_ctx, factory);
   grpc_exec_ctx_finish(&exec_ctx);
   return channel != NULL ? channel : grpc_lame_client_channel_create(

src/core/ext/transport/chttp2/client/secure/secure_channel_create.c

@@ -39,7 +39,6 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/client_channel/client_channel.h"
-#include "src/core/ext/client_channel/resolver_registry.h"
 #include "src/core/ext/transport/chttp2/client/chttp2_connector.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/security/credentials/credentials.h"
@@ -69,11 +68,10 @@ static void client_channel_factory_unref(
   }
 }
 
-static void create_handshakers(grpc_exec_ctx *exec_ctx,
-                               void *security_connector,
-                               grpc_handshake_manager *handshake_mgr) {
-  grpc_channel_security_connector_create_handshakers(
-      exec_ctx, security_connector, handshake_mgr);
+static void add_handshakers(grpc_exec_ctx *exec_ctx, void *security_connector,
+                            grpc_handshake_manager *handshake_mgr) {
+  grpc_channel_security_connector_add_handshakers(exec_ctx, security_connector,
+                                                  handshake_mgr);
 }
 
 static grpc_subchannel *client_channel_factory_create_subchannel(
@@ -81,7 +79,7 @@ static grpc_subchannel *client_channel_factory_create_subchannel(
     const grpc_subchannel_args *args) {
   client_channel_factory *f = (client_channel_factory *)cc_factory;
   grpc_connector *connector = grpc_chttp2_connector_create(
-      exec_ctx, args->server_name, create_handshakers, f->security_connector);
+      exec_ctx, add_handshakers, f->security_connector);
   grpc_subchannel *s = grpc_subchannel_create(exec_ctx, connector, args);
   grpc_connector_unref(exec_ctx, connector);
   return s;
@@ -91,18 +89,15 @@ static grpc_channel *client_channel_factory_create_channel(
     grpc_exec_ctx *exec_ctx, grpc_client_channel_factory *cc_factory,
     const char *target, grpc_client_channel_type type,
     const grpc_channel_args *args) {
-  client_channel_factory *f = (client_channel_factory *)cc_factory;
-  grpc_channel *channel =
-      grpc_channel_create(exec_ctx, target, args, GRPC_CLIENT_CHANNEL, NULL);
-  grpc_resolver *resolver = grpc_resolver_create(target, args);
-  if (resolver == NULL) {
-    GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, channel,
-                                "client_channel_factory_create_channel");
-    return NULL;
-  }
-  grpc_client_channel_finish_initialization(
-      exec_ctx, grpc_channel_get_channel_stack(channel), resolver, &f->base);
-  GRPC_RESOLVER_UNREF(exec_ctx, resolver, "create_channel");
+  // Add channel arg containing the server URI.
+  grpc_arg arg;
+  arg.type = GRPC_ARG_STRING;
+  arg.key = GRPC_ARG_SERVER_URI;
+  arg.value.string = (char *)target;
+  grpc_channel_args *new_args = grpc_channel_args_copy_and_add(args, &arg, 1);
+  grpc_channel *channel = grpc_channel_create(exec_ctx, target, new_args,
+                                              GRPC_CLIENT_CHANNEL, NULL);
+  grpc_channel_args_destroy(new_args);
   return channel;
 }
 
@@ -143,14 +138,6 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds,
     return grpc_lame_client_channel_create(
         target, GRPC_STATUS_INTERNAL, "Failed to create security connector.");
   }
-  grpc_arg connector_arg =
-      grpc_security_connector_to_arg(&security_connector->base);
-  grpc_channel_args *new_args = grpc_channel_args_copy_and_add(
-      new_args_from_connector != NULL ? new_args_from_connector : args,
-      &connector_arg, 1);
-  if (new_args_from_connector != NULL) {
-    grpc_channel_args_destroy(new_args_from_connector);
-  }
   // Create client channel factory.
   client_channel_factory *f = gpr_malloc(sizeof(*f));
   memset(f, 0, sizeof(*f));
@@ -159,13 +146,24 @@ grpc_channel *grpc_secure_channel_create(grpc_channel_credentials *creds,
   GRPC_SECURITY_CONNECTOR_REF(&security_connector->base,
                               "grpc_secure_channel_create");
   f->security_connector = security_connector;
+  // Add channel args containing the client channel factory and security
+  // connector.
+  grpc_arg new_args[2];
+  new_args[0] = grpc_client_channel_factory_create_channel_arg(&f->base);
+  new_args[1] = grpc_security_connector_to_arg(&security_connector->base);
+  grpc_channel_args *args_copy = grpc_channel_args_copy_and_add(
+      new_args_from_connector != NULL ? new_args_from_connector : args,
+      new_args, GPR_ARRAY_SIZE(new_args));
+  if (new_args_from_connector != NULL) {
+    grpc_channel_args_destroy(new_args_from_connector);
+  }
   // Create channel.
   grpc_channel *channel = client_channel_factory_create_channel(
-      &exec_ctx, &f->base, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, new_args);
+      &exec_ctx, &f->base, target, GRPC_CLIENT_CHANNEL_TYPE_REGULAR, args_copy);
   // Clean up.
   GRPC_SECURITY_CONNECTOR_UNREF(&f->security_connector->base,
                                 "secure_client_channel_factory_create_channel");
-  grpc_channel_args_destroy(new_args);
+  grpc_channel_args_destroy(args_copy);
   grpc_client_channel_factory_unref(&exec_ctx, &f->base);
   grpc_exec_ctx_finish(&exec_ctx);
   return channel; /* may be NULL */

src/core/ext/transport/chttp2/server/chttp2_server.c

@@ -53,13 +53,13 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-void grpc_chttp2_server_handshaker_factory_create_handshakers(
+void grpc_chttp2_server_handshaker_factory_add_handshakers(
     grpc_exec_ctx *exec_ctx,
     grpc_chttp2_server_handshaker_factory *handshaker_factory,
     grpc_handshake_manager *handshake_mgr) {
   if (handshaker_factory != NULL) {
-    handshaker_factory->vtable->create_handshakers(exec_ctx, handshaker_factory,
-                                                   handshake_mgr);
+    handshaker_factory->vtable->add_handshakers(exec_ctx, handshaker_factory,
+                                                handshake_mgr);
   }
 }
 
@@ -139,7 +139,7 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
     const char *error_str = grpc_error_string(error);
     gpr_log(GPR_ERROR, "Handshaking failed: %s", error_str);
     grpc_error_free_string(error_str);
-    if (error == GRPC_ERROR_NONE) {
+    if (error == GRPC_ERROR_NONE && args->endpoint != NULL) {
       // We were shut down after handshaking completed successfully, so
       // destroy the endpoint here.
       // TODO(ctiller): It is currently necessary to shutdown endpoints
@@ -153,19 +153,26 @@ static void on_handshake_done(grpc_exec_ctx *exec_ctx, void *arg,
       gpr_free(args->read_buffer);
     }
   } else {
-    grpc_transport *transport =
-        grpc_create_chttp2_transport(exec_ctx, args->args, args->endpoint, 0);
-    grpc_server_setup_transport(
-        exec_ctx, connection_state->server_state->server, transport,
-        connection_state->accepting_pollset, args->args);
-    grpc_chttp2_transport_start_reading(exec_ctx, transport, args->read_buffer);
-    grpc_channel_args_destroy(args->args);
+    // If the handshaking succeeded but there is no endpoint, then the
+    // handshaker may have handed off the connection to some external
+    // code, so we can just clean up here without creating a transport.
+    if (args->endpoint != NULL) {
+      grpc_transport *transport =
+          grpc_create_chttp2_transport(exec_ctx, args->args, args->endpoint, 0);
+      grpc_server_setup_transport(
+          exec_ctx, connection_state->server_state->server, transport,
+          connection_state->accepting_pollset, args->args);
+      grpc_chttp2_transport_start_reading(exec_ctx, transport,
+                                          args->read_buffer);
+      grpc_channel_args_destroy(args->args);
+    }
   }
   pending_handshake_manager_remove_locked(connection_state->server_state,
                                           connection_state->handshake_mgr);
   gpr_mu_unlock(&connection_state->server_state->mu);
   grpc_handshake_manager_destroy(exec_ctx, connection_state->handshake_mgr);
   grpc_tcp_server_unref(exec_ctx, connection_state->server_state->tcp_server);
+  gpr_free(connection_state->acceptor);
   gpr_free(connection_state);
 }
 
@@ -177,6 +184,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
   if (state->shutdown) {
     gpr_mu_unlock(&state->mu);
     grpc_endpoint_destroy(exec_ctx, tcp);
+    gpr_free(acceptor);
     return;
   }
   grpc_handshake_manager *handshake_mgr = grpc_handshake_manager_create();
@@ -189,7 +197,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
   connection_state->accepting_pollset = accepting_pollset;
   connection_state->acceptor = acceptor;
   connection_state->handshake_mgr = handshake_mgr;
-  grpc_chttp2_server_handshaker_factory_create_handshakers(
+  grpc_chttp2_server_handshaker_factory_add_handshakers(
       exec_ctx, state->handshaker_factory, connection_state->handshake_mgr);
   // TODO(roth): We should really get this timeout value from channel
   // args instead of hard-coding it.
@@ -273,7 +281,8 @@ grpc_error *grpc_chttp2_server_add_port(
   state = gpr_malloc(sizeof(*state));
   memset(state, 0, sizeof(*state));
   grpc_closure_init(&state->tcp_server_shutdown_complete,
-                    tcp_server_shutdown_complete, state);
+                    tcp_server_shutdown_complete, state,
+                    grpc_schedule_on_exec_ctx);
   err = grpc_tcp_server_create(exec_ctx, &state->tcp_server_shutdown_complete,
                                args, &tcp_server);
   if (err != GRPC_ERROR_NONE) {

src/core/ext/transport/chttp2/server/chttp2_server.h

@@ -45,7 +45,7 @@ typedef struct grpc_chttp2_server_handshaker_factory
     grpc_chttp2_server_handshaker_factory;
 
 typedef struct {
-  void (*create_handshakers)(
+  void (*add_handshakers)(
       grpc_exec_ctx *exec_ctx,
       grpc_chttp2_server_handshaker_factory *handshaker_factory,
       grpc_handshake_manager *handshake_mgr);
@@ -57,7 +57,7 @@ struct grpc_chttp2_server_handshaker_factory {
   const grpc_chttp2_server_handshaker_factory_vtable *vtable;
 };
 
-void grpc_chttp2_server_handshaker_factory_create_handshakers(
+void grpc_chttp2_server_handshaker_factory_add_handshakers(
     grpc_exec_ctx *exec_ctx,
     grpc_chttp2_server_handshaker_factory *handshaker_factory,
     grpc_handshake_manager *handshake_mgr);

src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c

@@ -54,12 +54,12 @@ typedef struct {
   grpc_server_security_connector *security_connector;
 } server_security_handshaker_factory;
 
-static void server_security_handshaker_factory_create_handshakers(
+static void server_security_handshaker_factory_add_handshakers(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_server_handshaker_factory *hf,
     grpc_handshake_manager *handshake_mgr) {
   server_security_handshaker_factory *handshaker_factory =
       (server_security_handshaker_factory *)hf;
-  grpc_server_security_connector_create_handshakers(
+  grpc_server_security_connector_add_handshakers(
       exec_ctx, handshaker_factory->security_connector, handshake_mgr);
 }
 
@@ -74,7 +74,7 @@ static void server_security_handshaker_factory_destroy(
 
 static const grpc_chttp2_server_handshaker_factory_vtable
     server_security_handshaker_factory_vtable = {
-        server_security_handshaker_factory_create_handshakers,
+        server_security_handshaker_factory_add_handshakers,
         server_security_handshaker_factory_destroy};
 
 int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,

src/core/ext/transport/chttp2/transport/chttp2_transport.c

@@ -73,20 +73,14 @@ static const grpc_transport_vtable vtable;
 static void write_action_begin_locked(grpc_exec_ctx *exec_ctx, void *t,
                                       grpc_error *error);
 static void write_action(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error);
-static void write_action_end(grpc_exec_ctx *exec_ctx, void *t,
-                             grpc_error *error);
 static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *t,
                                     grpc_error *error);
 
-static void read_action_begin(grpc_exec_ctx *exec_ctx, void *t,
-                              grpc_error *error);
 static void read_action_locked(grpc_exec_ctx *exec_ctx, void *t,
                                grpc_error *error);
 
 static void complete_fetch_locked(grpc_exec_ctx *exec_ctx, void *gs,
                                   grpc_error *error);
-static void complete_fetch(grpc_exec_ctx *exec_ctx, void *gs,
-                           grpc_error *error);
 /** Set a transport level setting, and push it to our peer */
 static void push_setting(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                          grpc_chttp2_setting_id id, uint32_t value);
@@ -112,12 +106,8 @@ static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx,
                                                 void *byte_stream,
                                                 grpc_error *error_ignored);
 
-static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *t,
-                             grpc_error *error);
 static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t,
                                     grpc_error *error);
-static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *t,
-                                  grpc_error *error);
 static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t,
                                          grpc_error *error);
 
@@ -166,8 +156,8 @@ static void destruct_transport(grpc_exec_ctx *exec_ctx,
      and maybe they hold resources that need to be freed */
   while (t->pings.next != &t->pings) {
     grpc_chttp2_outstanding_ping *ping = t->pings.next;
-    grpc_exec_ctx_sched(exec_ctx, ping->on_recv,
-                        GRPC_ERROR_CREATE("Transport closed"), NULL);
+    grpc_closure_sched(exec_ctx, ping->on_recv,
+                       GRPC_ERROR_CREATE("Transport closed"));
     ping->next->prev = ping->prev;
     ping->prev->next = ping->next;
     gpr_free(ping);
@@ -246,18 +236,15 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   grpc_slice_buffer_init(&t->outbuf);
   grpc_chttp2_hpack_compressor_init(&t->hpack_compressor);
 
-  grpc_closure_init(&t->write_action_begin_locked, write_action_begin_locked,
-                    t);
-  grpc_closure_init(&t->write_action, write_action, t);
-  grpc_closure_init(&t->write_action_end, write_action_end, t);
-  grpc_closure_init(&t->write_action_end_locked, write_action_end_locked, t);
-  grpc_closure_init(&t->read_action_begin, read_action_begin, t);
-  grpc_closure_init(&t->read_action_locked, read_action_locked, t);
-  grpc_closure_init(&t->benign_reclaimer, benign_reclaimer, t);
-  grpc_closure_init(&t->destructive_reclaimer, destructive_reclaimer, t);
-  grpc_closure_init(&t->benign_reclaimer_locked, benign_reclaimer_locked, t);
+  grpc_closure_init(&t->write_action, write_action, t,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&t->read_action_locked, read_action_locked, t,
+                    grpc_combiner_scheduler(t->combiner, false));
+  grpc_closure_init(&t->benign_reclaimer_locked, benign_reclaimer_locked, t,
+                    grpc_combiner_scheduler(t->combiner, false));
   grpc_closure_init(&t->destructive_reclaimer_locked,
-                    destructive_reclaimer_locked, t);
+                    destructive_reclaimer_locked, t,
+                    grpc_combiner_scheduler(t->combiner, false));
 
   grpc_chttp2_goaway_parser_init(&t->goaway_parser);
   grpc_chttp2_hpack_parser_init(&t->hpack_parser);
@@ -395,9 +382,10 @@ static void destroy_transport_locked(grpc_exec_ctx *exec_ctx, void *tp,
 
 static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {
   grpc_chttp2_transport *t = (grpc_chttp2_transport *)gt;
-  grpc_combiner_execute(exec_ctx, t->combiner,
-                        grpc_closure_create(destroy_transport_locked, t),
-                        GRPC_ERROR_NONE, false);
+  grpc_closure_sched(exec_ctx, grpc_closure_create(
+                                   destroy_transport_locked, t,
+                                   grpc_combiner_scheduler(t->combiner, false)),
+                     GRPC_ERROR_NONE);
 }
 
 static void close_transport_locked(grpc_exec_ctx *exec_ctx,
@@ -425,7 +413,6 @@ static void close_transport_locked(grpc_exec_ctx *exec_ctx,
     /* flush writable stream list to avoid dangling references */
     grpc_chttp2_stream *s;
     while (grpc_chttp2_list_pop_writable_stream(t, &s)) {
-      grpc_chttp2_leave_writing_lists(exec_ctx, t, s);
       GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:close");
     }
     end_all_the_calls(exec_ctx, t, GRPC_ERROR_REF(error));
@@ -472,8 +459,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   grpc_chttp2_data_parser_init(&s->data_parser);
   grpc_slice_buffer_init(&s->flow_controlled_buffer);
   s->deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
-  grpc_closure_init(&s->complete_fetch, complete_fetch, s);
-  grpc_closure_init(&s->complete_fetch_locked, complete_fetch_locked, s);
+  grpc_closure_init(&s->complete_fetch_locked, complete_fetch_locked, s,
+                    grpc_schedule_on_exec_ctx);
 
   GRPC_CHTTP2_REF_TRANSPORT(t, "stream");
 
@@ -521,10 +508,6 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp,
     }
   }
 
-  if (s->fail_pending_writes_on_writes_finished_error != NULL) {
-    GRPC_ERROR_UNREF(s->fail_pending_writes_on_writes_finished_error);
-  }
-
   GPR_ASSERT(s->send_initial_metadata_finished == NULL);
   GPR_ASSERT(s->fetching_send_message == NULL);
   GPR_ASSERT(s->send_trailing_metadata_finished == NULL);
@@ -552,9 +535,10 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   grpc_chttp2_stream *s = (grpc_chttp2_stream *)gs;
 
   s->destroy_stream_arg = and_free_memory;
-  grpc_closure_init(&s->destroy_stream, destroy_stream_locked, s);
-  grpc_combiner_execute(exec_ctx, t->combiner, &s->destroy_stream,
-                        GRPC_ERROR_NONE, false);
+  grpc_closure_sched(
+      exec_ctx, grpc_closure_init(&s->destroy_stream, destroy_stream_locked, s,
+                                  grpc_combiner_scheduler(t->combiner, false)),
+      GRPC_ERROR_NONE);
   GPR_TIMER_END("destroy_stream", 0);
 }
 
@@ -604,11 +588,13 @@ static void set_write_state(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
                                  write_state_name(t->write_state),
                                  write_state_name(st), reason));
   t->write_state = st;
-  if (st == GRPC_CHTTP2_WRITE_STATE_IDLE &&
-      t->close_transport_on_writes_finished != NULL) {
-    grpc_error *err = t->close_transport_on_writes_finished;
-    t->close_transport_on_writes_finished = NULL;
-    close_transport_locked(exec_ctx, t, err);
+  if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) {
+    grpc_closure_list_sched(exec_ctx, &t->run_after_write);
+    if (t->close_transport_on_writes_finished != NULL) {
+      grpc_error *err = t->close_transport_on_writes_finished;
+      t->close_transport_on_writes_finished = NULL;
+      close_transport_locked(exec_ctx, t, err);
+    }
   }
 }
 
@@ -621,9 +607,12 @@ void grpc_chttp2_initiate_write(grpc_exec_ctx *exec_ctx,
     case GRPC_CHTTP2_WRITE_STATE_IDLE:
       set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING, reason);
       GRPC_CHTTP2_REF_TRANSPORT(t, "writing");
-      grpc_combiner_execute_finally(exec_ctx, t->combiner,
-                                    &t->write_action_begin_locked,
-                                    GRPC_ERROR_NONE, covered_by_poller);
+      grpc_closure_sched(
+          exec_ctx,
+          grpc_closure_init(
+              &t->write_action_begin_locked, write_action_begin_locked, t,
+              grpc_combiner_finally_scheduler(t->combiner, covered_by_poller)),
+          GRPC_ERROR_NONE);
       break;
     case GRPC_CHTTP2_WRITE_STATE_WRITING:
       set_write_state(
@@ -665,7 +654,7 @@ static void write_action_begin_locked(grpc_exec_ctx *exec_ctx, void *gt,
   if (!t->closed && grpc_chttp2_begin_write(exec_ctx, t)) {
     set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING,
                     "begin writing");
-    grpc_exec_ctx_sched(exec_ctx, &t->write_action, GRPC_ERROR_NONE, NULL);
+    grpc_closure_sched(exec_ctx, &t->write_action, GRPC_ERROR_NONE);
   } else {
     set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_IDLE,
                     "begin writing nothing");
@@ -677,19 +666,13 @@ static void write_action_begin_locked(grpc_exec_ctx *exec_ctx, void *gt,
 static void write_action(grpc_exec_ctx *exec_ctx, void *gt, grpc_error *error) {
   grpc_chttp2_transport *t = gt;
   GPR_TIMER_BEGIN("write_action", 0);
-  grpc_endpoint_write(exec_ctx, t->ep, &t->outbuf, &t->write_action_end);
+  grpc_endpoint_write(
+      exec_ctx, t->ep, &t->outbuf,
+      grpc_closure_init(&t->write_action_end_locked, write_action_end_locked, t,
+                        grpc_combiner_scheduler(t->combiner, false)));
   GPR_TIMER_END("write_action", 0);
 }
 
-static void write_action_end(grpc_exec_ctx *exec_ctx, void *gt,
-                             grpc_error *error) {
-  grpc_chttp2_transport *t = gt;
-  GPR_TIMER_BEGIN("write_action_end", 0);
-  grpc_combiner_execute(exec_ctx, t->combiner, &t->write_action_end_locked,
-                        GRPC_ERROR_REF(error), false);
-  GPR_TIMER_END("write_action_end", 0);
-}
-
 static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *tp,
                                     grpc_error *error) {
   GPR_TIMER_BEGIN("terminate_writing_with_lock", 0);
@@ -719,18 +702,24 @@ static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *tp,
       set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING,
                       "continue writing [!covered]");
       GRPC_CHTTP2_REF_TRANSPORT(t, "writing");
-      grpc_combiner_execute_finally(exec_ctx, t->combiner,
-                                    &t->write_action_begin_locked,
-                                    GRPC_ERROR_NONE, false);
+      grpc_closure_run(
+          exec_ctx,
+          grpc_closure_init(
+              &t->write_action_begin_locked, write_action_begin_locked, t,
+              grpc_combiner_finally_scheduler(t->combiner, false)),
+          GRPC_ERROR_NONE);
       break;
     case GRPC_CHTTP2_WRITE_STATE_WRITING_WITH_MORE_AND_COVERED_BY_POLLER:
       GPR_TIMER_MARK("state=writing_stale_with_poller", 0);
       set_write_state(exec_ctx, t, GRPC_CHTTP2_WRITE_STATE_WRITING,
                       "continue writing [covered]");
       GRPC_CHTTP2_REF_TRANSPORT(t, "writing");
-      grpc_combiner_execute_finally(exec_ctx, t->combiner,
-                                    &t->write_action_begin_locked,
-                                    GRPC_ERROR_NONE, true);
+      grpc_closure_run(
+          exec_ctx,
+          grpc_closure_init(&t->write_action_begin_locked,
+                            write_action_begin_locked, t,
+                            grpc_combiner_finally_scheduler(t->combiner, true)),
+          GRPC_ERROR_NONE);
       break;
   }
 
@@ -825,7 +814,14 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx,
   }
 }
 
+/* Flag that this closure barrier wants stats to be updated before finishing */
 #define CLOSURE_BARRIER_STATS_BIT (1 << 0)
+/* Flag that this closure barrier may be covering a write in a pollset, and so
+   we should not complete this closure until we can prove that the write got
+   scheduled */
+#define CLOSURE_BARRIER_MAY_COVER_WRITE (1 << 1)
+/* First bit of the reference count, stored in the high order bits (with the low
+   bits being used for flags defined above) */
 #define CLOSURE_BARRIER_FIRST_REF_BIT (1 << 16)
 
 static grpc_closure *add_closure_barrier(grpc_closure *closure) {
@@ -852,6 +848,16 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
     return;
   }
   closure->next_data.scratch -= CLOSURE_BARRIER_FIRST_REF_BIT;
+  if (grpc_http_trace) {
+    const char *errstr = grpc_error_string(error);
+    gpr_log(GPR_DEBUG,
+            "complete_closure_step: %p refs=%d flags=0x%04x desc=%s err=%s",
+            closure,
+            (int)(closure->next_data.scratch / CLOSURE_BARRIER_FIRST_REF_BIT),
+            (int)(closure->next_data.scratch % CLOSURE_BARRIER_FIRST_REF_BIT),
+            desc, errstr);
+    grpc_error_free_string(errstr);
+  }
   if (error != GRPC_ERROR_NONE) {
     if (closure->error_data.error == GRPC_ERROR_NONE) {
       closure->error_data.error =
@@ -868,7 +874,13 @@ void grpc_chttp2_complete_closure_step(grpc_exec_ctx *exec_ctx,
       grpc_transport_move_stats(&s->stats, s->collecting_stats);
       s->collecting_stats = NULL;
     }
-    grpc_closure_run(exec_ctx, closure, closure->error_data.error);
+    if ((t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) ||
+        !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) {
+      grpc_closure_run(exec_ctx, closure, closure->error_data.error);
+    } else {
+      grpc_closure_list_append(&t->run_after_write, closure,
+                               closure->error_data.error);
+    }
   }
 }
 
@@ -945,15 +957,6 @@ static void complete_fetch_locked(grpc_exec_ctx *exec_ctx, void *gs,
   }
 }
 
-static void complete_fetch(grpc_exec_ctx *exec_ctx, void *gs,
-                           grpc_error *error) {
-  grpc_chttp2_stream *s = gs;
-  grpc_chttp2_transport *t = s->t;
-  grpc_combiner_execute(exec_ctx, t->combiner, &s->complete_fetch_locked,
-                        GRPC_ERROR_REF(error),
-                        s->complete_fetch_covered_by_poller);
-}
-
 static void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {}
 
 static void log_metadata(const grpc_metadata_batch *md_batch, uint32_t id,
@@ -989,7 +992,8 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
 
   grpc_closure *on_complete = op->on_complete;
   if (on_complete == NULL) {
-    on_complete = grpc_closure_create(do_nothing, NULL);
+    on_complete =
+        grpc_closure_create(do_nothing, NULL, grpc_schedule_on_exec_ctx);
   }
 
   /* use final_data as a barrier until enqueue time; the inital counter is
@@ -1013,6 +1017,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
 
   if (op->send_initial_metadata != NULL) {
     GPR_ASSERT(s->send_initial_metadata_finished == NULL);
+    on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->send_initial_metadata_finished = add_closure_barrier(on_complete);
     s->send_initial_metadata = op->send_initial_metadata;
     const size_t metadata_size =
@@ -1066,6 +1071,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
   }
 
   if (op->send_message != NULL) {
+    on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->fetching_send_message_finished = add_closure_barrier(op->on_complete);
     if (s->write_closed) {
       grpc_chttp2_complete_closure_step(
@@ -1103,6 +1109,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
 
   if (op->send_trailing_metadata != NULL) {
     GPR_ASSERT(s->send_trailing_metadata_finished == NULL);
+    on_complete->next_data.scratch |= CLOSURE_BARRIER_MAY_COVER_WRITE;
     s->send_trailing_metadata_finished = add_closure_barrier(on_complete);
     s->send_trailing_metadata = op->send_trailing_metadata;
     const size_t metadata_size =
@@ -1189,13 +1196,15 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
     gpr_free(str);
   }
 
-  grpc_closure_init(&op->transport_private.closure, perform_stream_op_locked,
-                    op);
   op->transport_private.args[0] = gt;
   op->transport_private.args[1] = gs;
   GRPC_CHTTP2_STREAM_REF(s, "perform_stream_op");
-  grpc_combiner_execute(exec_ctx, t->combiner, &op->transport_private.closure,
-                        GRPC_ERROR_NONE, op->covered_by_poller);
+  grpc_closure_sched(
+      exec_ctx,
+      grpc_closure_init(
+          &op->transport_private.closure, perform_stream_op_locked, op,
+          grpc_combiner_scheduler(t->combiner, op->covered_by_poller)),
+      GRPC_ERROR_NONE);
   GPR_TIMER_END("perform_stream_op", 0);
 }
 
@@ -1224,7 +1233,7 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   grpc_chttp2_outstanding_ping *ping;
   for (ping = t->pings.next; ping != &t->pings; ping = ping->next) {
     if (0 == memcmp(opaque_8bytes, ping->id, 8)) {
-      grpc_exec_ctx_sched(exec_ctx, ping->on_recv, GRPC_ERROR_NONE, NULL);
+      grpc_closure_sched(exec_ctx, ping->on_recv, GRPC_ERROR_NONE);
       ping->next->prev = ping->prev;
       ping->prev->next = ping->next;
       gpr_free(ping);
@@ -1298,11 +1307,12 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   char *msg = grpc_transport_op_string(op);
   gpr_free(msg);
   op->transport_private.args[0] = gt;
-  grpc_closure_init(&op->transport_private.closure, perform_transport_op_locked,
-                    op);
   GRPC_CHTTP2_REF_TRANSPORT(t, "transport_op");
-  grpc_combiner_execute(exec_ctx, t->combiner, &op->transport_private.closure,
-                        GRPC_ERROR_NONE, false);
+  grpc_closure_sched(
+      exec_ctx, grpc_closure_init(&op->transport_private.closure,
+                                  perform_transport_op_locked, op,
+                                  grpc_combiner_scheduler(t->combiner, false)),
+      GRPC_ERROR_NONE);
 }
 
 /*******************************************************************************
@@ -1406,7 +1416,6 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     }
   }
   if (grpc_chttp2_list_remove_writable_stream(t, s)) {
-    grpc_chttp2_leave_writing_lists(exec_ctx, t, s);
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:remove_stream");
   }
 
@@ -1537,41 +1546,9 @@ static grpc_error *removal_error(grpc_error *extra_error, grpc_chttp2_stream *s,
   return error;
 }
 
-void grpc_chttp2_leave_writing_lists(grpc_exec_ctx *exec_ctx,
-                                     grpc_chttp2_transport *t,
-                                     grpc_chttp2_stream *s) {
-  if (s->need_fail_pending_writes_on_writes_finished) {
-    grpc_error *error = s->fail_pending_writes_on_writes_finished_error;
-    s->fail_pending_writes_on_writes_finished_error = NULL;
-    s->need_fail_pending_writes_on_writes_finished = false;
-    grpc_chttp2_fail_pending_writes(exec_ctx, t, s, error);
-  }
-}
-
 void grpc_chttp2_fail_pending_writes(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_transport *t,
                                      grpc_chttp2_stream *s, grpc_error *error) {
-  if (s->need_fail_pending_writes_on_writes_finished ||
-      (t->write_state != GRPC_CHTTP2_WRITE_STATE_IDLE &&
-       (s->included[GRPC_CHTTP2_LIST_WRITABLE] ||
-        s->included[GRPC_CHTTP2_LIST_WRITING]))) {
-    /* If a write is in progress, and it involves this stream, wait for the
-     * write to complete before cancelling things out. If we don't do this, then
-     * our combiner lock might think that some operation on its queue might be
-     * covering a completion even though there is none, in which case we might
-     * offload to another thread, which isn't guarateed to exist */
-    if (error != GRPC_ERROR_NONE) {
-      if (s->fail_pending_writes_on_writes_finished_error == GRPC_ERROR_NONE) {
-        s->fail_pending_writes_on_writes_finished_error = GRPC_ERROR_CREATE(
-            "Post-poned fail writes due to in-progress write");
-      }
-      s->fail_pending_writes_on_writes_finished_error = grpc_error_add_child(
-          s->fail_pending_writes_on_writes_finished_error, error);
-    }
-    s->need_fail_pending_writes_on_writes_finished = true;
-    return; /* early out */
-  }
-
   error =
       removal_error(error, s, "Pending writes failed due to stream closure");
   s->send_initial_metadata = NULL;
@@ -1632,6 +1609,9 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx,
     if (s->id != 0) {
       remove_stream(exec_ctx, t, s->id,
                     removal_error(GRPC_ERROR_REF(error), s, "Stream removed"));
+    } else {
+      /* Purge streams waiting on concurrency still waiting for id assignment */
+      grpc_chttp2_list_remove_waiting_for_concurrency(t, s);
     }
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2");
   }
@@ -1808,19 +1788,6 @@ static void update_global_window(void *args, uint32_t id, void *stream) {
  * INPUT PROCESSING - PARSING
  */
 
-static void read_action_begin(grpc_exec_ctx *exec_ctx, void *tp,
-                              grpc_error *error) {
-  /* Control flow:
-     reading_action_locked ->
-       (parse_unlocked -> post_parse_locked)? ->
-       post_reading_action_locked */
-  GPR_TIMER_BEGIN("reading_action", 0);
-  grpc_chttp2_transport *t = tp;
-  grpc_combiner_execute(exec_ctx, t->combiner, &t->read_action_locked,
-                        GRPC_ERROR_REF(error), false);
-  GPR_TIMER_END("reading_action", 0);
-}
-
 static grpc_error *try_http_parsing(grpc_exec_ctx *exec_ctx,
                                     grpc_chttp2_transport *t) {
   grpc_http_parser parser;
@@ -1920,7 +1887,8 @@ static void read_action_locked(grpc_exec_ctx *exec_ctx, void *tp,
   grpc_slice_buffer_reset_and_unref(&t->read_buffer);
 
   if (keep_reading) {
-    grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer, &t->read_action_begin);
+    grpc_endpoint_read(exec_ctx, t->ep, &t->read_buffer,
+                       &t->read_action_locked);
     GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "keep_reading");
   } else {
     GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "reading_action");
@@ -2057,10 +2025,12 @@ static int incoming_byte_stream_next(grpc_exec_ctx *exec_ctx,
   bs->next_action.slice = slice;
   bs->next_action.max_size_hint = max_size_hint;
   bs->next_action.on_complete = on_complete;
-  grpc_closure_init(&bs->next_action.closure, incoming_byte_stream_next_locked,
-                    bs);
-  grpc_combiner_execute(exec_ctx, bs->transport->combiner,
-                        &bs->next_action.closure, GRPC_ERROR_NONE, false);
+  grpc_closure_sched(
+      exec_ctx,
+      grpc_closure_init(
+          &bs->next_action.closure, incoming_byte_stream_next_locked, bs,
+          grpc_combiner_scheduler(bs->transport->combiner, false)),
+      GRPC_ERROR_NONE);
   GPR_TIMER_END("incoming_byte_stream_next", 0);
   return 0;
 }
@@ -2082,10 +2052,12 @@ static void incoming_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
   GPR_TIMER_BEGIN("incoming_byte_stream_destroy", 0);
   grpc_chttp2_incoming_byte_stream *bs =
       (grpc_chttp2_incoming_byte_stream *)byte_stream;
-  grpc_closure_init(&bs->destroy_action, incoming_byte_stream_destroy_locked,
-                    bs);
-  grpc_combiner_execute(exec_ctx, bs->transport->combiner, &bs->destroy_action,
-                        GRPC_ERROR_NONE, false);
+  grpc_closure_sched(
+      exec_ctx,
+      grpc_closure_init(
+          &bs->destroy_action, incoming_byte_stream_destroy_locked, bs,
+          grpc_combiner_scheduler(bs->transport->combiner, false)),
+      GRPC_ERROR_NONE);
   GPR_TIMER_END("incoming_byte_stream_destroy", 0);
 }
 
@@ -2093,7 +2065,7 @@ static void incoming_byte_stream_publish_error(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs,
     grpc_error *error) {
   GPR_ASSERT(error != GRPC_ERROR_NONE);
-  grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_REF(error), NULL);
+  grpc_closure_sched(exec_ctx, bs->on_next, GRPC_ERROR_REF(error));
   bs->on_next = NULL;
   GRPC_ERROR_UNREF(bs->error);
   bs->error = error;
@@ -2110,7 +2082,7 @@ void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx,
     bs->remaining_bytes -= (uint32_t)GRPC_SLICE_LENGTH(slice);
     if (bs->on_next != NULL) {
       *bs->next = slice;
-      grpc_exec_ctx_sched(exec_ctx, bs->on_next, GRPC_ERROR_NONE, NULL);
+      grpc_closure_sched(exec_ctx, bs->on_next, GRPC_ERROR_NONE);
       bs->on_next = NULL;
     } else {
       grpc_slice_buffer_add(&bs->slices, slice);
@@ -2178,7 +2150,7 @@ static void post_benign_reclaimer(grpc_exec_ctx *exec_ctx,
     GRPC_CHTTP2_REF_TRANSPORT(t, "benign_reclaimer");
     grpc_resource_user_post_reclaimer(exec_ctx,
                                       grpc_endpoint_get_resource_user(t->ep),
-                                      false, &t->benign_reclaimer);
+                                      false, &t->benign_reclaimer_locked);
   }
 }
 
@@ -2189,24 +2161,10 @@ static void post_destructive_reclaimer(grpc_exec_ctx *exec_ctx,
     GRPC_CHTTP2_REF_TRANSPORT(t, "destructive_reclaimer");
     grpc_resource_user_post_reclaimer(exec_ctx,
                                       grpc_endpoint_get_resource_user(t->ep),
-                                      true, &t->destructive_reclaimer);
+                                      true, &t->destructive_reclaimer_locked);
   }
 }
 
-static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *arg,
-                             grpc_error *error) {
-  grpc_chttp2_transport *t = arg;
-  grpc_combiner_execute(exec_ctx, t->combiner, &t->benign_reclaimer_locked,
-                        GRPC_ERROR_REF(error), false);
-}
-
-static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *arg,
-                                  grpc_error *error) {
-  grpc_chttp2_transport *t = arg;
-  grpc_combiner_execute(exec_ctx, t->combiner, &t->destructive_reclaimer_locked,
-                        GRPC_ERROR_REF(error), false);
-}
-
 static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                     grpc_error *error) {
   grpc_chttp2_transport *t = arg;
@@ -2387,5 +2345,5 @@ void grpc_chttp2_transport_start_reading(grpc_exec_ctx *exec_ctx,
     grpc_slice_buffer_move_into(read_buffer, &t->read_buffer);
     gpr_free(read_buffer);
   }
-  read_action_begin(exec_ctx, t, GRPC_ERROR_NONE);
+  grpc_closure_sched(exec_ctx, &t->read_action_locked, GRPC_ERROR_NONE);
 }

src/core/ext/transport/chttp2/transport/hpack_parser.c

@@ -1634,10 +1634,11 @@ grpc_error *grpc_chttp2_header_parser_parse(grpc_exec_ctx *exec_ctx,
              however -- it might be that we receive a RST_STREAM following this
              and can avoid the extra write */
           GRPC_CHTTP2_STREAM_REF(s, "final_rst");
-          grpc_combiner_execute_finally(
-              exec_ctx, t->combiner,
-              grpc_closure_create(force_client_rst_stream, s), GRPC_ERROR_NONE,
-              false);
+          grpc_closure_sched(
+              exec_ctx, grpc_closure_create(force_client_rst_stream, s,
+                                            grpc_combiner_finally_scheduler(
+                                                t->combiner, false)),
+              GRPC_ERROR_NONE);
         }
         grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, false,
                                        GRPC_ERROR_NONE);

src/core/ext/transport/chttp2/transport/internal.h

@@ -212,10 +212,8 @@ struct grpc_chttp2_transport {
 
   grpc_closure write_action_begin_locked;
   grpc_closure write_action;
-  grpc_closure write_action_end;
   grpc_closure write_action_end_locked;
 
-  grpc_closure read_action_begin;
   grpc_closure read_action_locked;
 
   /** incoming read bytes */
@@ -327,16 +325,17 @@ struct grpc_chttp2_transport {
    */
   grpc_error *close_transport_on_writes_finished;
 
+  /* a list of closures to run after writes are finished */
+  grpc_closure_list run_after_write;
+
   /* buffer pool state */
   /** have we scheduled a benign cleanup? */
   bool benign_reclaimer_registered;
   /** have we scheduled a destructive cleanup? */
   bool destructive_reclaimer_registered;
   /** benign cleanup closure */
-  grpc_closure benign_reclaimer;
   grpc_closure benign_reclaimer_locked;
   /** destructive cleanup closure */
-  grpc_closure destructive_reclaimer;
   grpc_closure destructive_reclaimer_locked;
 };
 
@@ -409,9 +408,6 @@ struct grpc_chttp2_stream {
   grpc_error *read_closed_error;
   /** the error that resulted in this stream being write-closed */
   grpc_error *write_closed_error;
-  /** should any writes be cleared once this stream becomes non-writable */
-  bool need_fail_pending_writes_on_writes_finished;
-  grpc_error *fail_pending_writes_on_writes_finished_error;
 
   grpc_published_metadata_method published_metadata[2];
   bool final_metadata_requested;
@@ -496,6 +492,8 @@ void grpc_chttp2_list_add_waiting_for_concurrency(grpc_chttp2_transport *t,
                                                   grpc_chttp2_stream *s);
 int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t,
                                                  grpc_chttp2_stream **s);
+void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t,
+                                                     grpc_chttp2_stream *s);
 
 void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream *s);
@@ -692,9 +690,6 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx,
                                                        grpc_chttp2_transport *t,
                                                        grpc_chttp2_stream *s);
 
-void grpc_chttp2_leave_writing_lists(grpc_exec_ctx *exec_ctx,
-                                     grpc_chttp2_transport *t,
-                                     grpc_chttp2_stream *s);
 void grpc_chttp2_fail_pending_writes(grpc_exec_ctx *exec_ctx,
                                      grpc_chttp2_transport *t,
                                      grpc_chttp2_stream *s, grpc_error *error);

src/core/ext/transport/chttp2/transport/stream_lists.c

@@ -158,6 +158,11 @@ int grpc_chttp2_list_pop_waiting_for_concurrency(grpc_chttp2_transport *t,
   return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY);
 }
 
+void grpc_chttp2_list_remove_waiting_for_concurrency(grpc_chttp2_transport *t,
+                                                     grpc_chttp2_stream *s) {
+  stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY);
+}
+
 void grpc_chttp2_list_add_stalled_by_transport(grpc_chttp2_transport *t,
                                                grpc_chttp2_stream *s) {
   stream_list_add(t, s, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT);

src/core/ext/transport/chttp2/transport/writing.c

@@ -208,7 +208,6 @@ bool grpc_chttp2_begin_write(grpc_exec_ctx *exec_ctx,
         GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:already_writing");
       }
     } else {
-      grpc_chttp2_leave_writing_lists(exec_ctx, t, s);
       GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:no_write");
     }
   }
@@ -253,7 +252,6 @@ void grpc_chttp2_end_write(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
       grpc_chttp2_mark_stream_closed(exec_ctx, t, s, !t->is_client, 1,
                                      GRPC_ERROR_REF(error));
     }
-    grpc_chttp2_leave_writing_lists(exec_ctx, t, s);
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:end");
   }
   grpc_slice_buffer_reset_and_unref(&t->outbuf);

src/core/ext/transport/cronet/transport/cronet_transport.c

@@ -149,6 +149,9 @@ struct write_state {
 struct op_state {
   bool state_op_done[OP_NUM_OPS];
   bool state_callback_received[OP_NUM_OPS];
+  bool fail_state;
+  bool flush_read;
+  grpc_error *cancel_error;
   /* data structure for storing data coming from server */
   struct read_state rs;
   /* data structure for storing data going to the server */
@@ -248,6 +251,12 @@ static void free_read_buffer(stream_obj *s) {
   }
 }
 
+static grpc_error *make_error_with_desc(int error_code, const char *desc) {
+  grpc_error *error = GRPC_ERROR_CREATE(desc);
+  error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, error_code);
+  return error;
+}
+
 /*
   Add a new stream op to op storage.
 */
@@ -433,6 +442,18 @@ static void on_response_headers_received(
             grpc_mdstr_from_string(headers->headers[i].value)));
   }
   s->state.state_callback_received[OP_RECV_INITIAL_METADATA] = true;
+  if (!(s->state.state_op_done[OP_CANCEL_ERROR] ||
+        s->state.state_callback_received[OP_FAILED])) {
+    /* Do an extra read to trigger on_succeeded() callback in case connection
+     is closed */
+    GPR_ASSERT(s->state.rs.length_field_received == false);
+    s->state.rs.read_buffer = s->state.rs.grpc_header_bytes;
+    s->state.rs.received_bytes = 0;
+    s->state.rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES;
+    CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read(%p)", s->cbs);
+    cronet_bidirectional_stream_read(s->cbs, s->state.rs.read_buffer,
+                                     s->state.rs.remaining_bytes);
+  }
   gpr_mu_unlock(&s->mu);
   execute_from_storage(s);
 }
@@ -464,7 +485,11 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data,
              count);
   gpr_mu_lock(&s->mu);
   s->state.state_callback_received[OP_RECV_MESSAGE] = true;
-  if (count > 0) {
+  if (count > 0 && s->state.flush_read) {
+    CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read(%p)", s->cbs);
+    cronet_bidirectional_stream_read(s->cbs, s->state.rs.read_buffer, 4096);
+    gpr_mu_unlock(&s->mu);
+  } else if (count > 0) {
     s->state.rs.received_bytes += count;
     s->state.rs.remaining_bytes -= count;
     if (s->state.rs.remaining_bytes > 0) {
@@ -479,6 +504,10 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data,
       execute_from_storage(s);
     }
   } else {
+    if (s->state.flush_read) {
+      gpr_free(s->state.rs.read_buffer);
+      s->state.rs.read_buffer = NULL;
+    }
     s->state.rs.read_stream_closed = true;
     gpr_mu_unlock(&s->mu);
     execute_from_storage(s);
@@ -508,10 +537,27 @@ static void on_response_trailers_received(
             grpc_mdstr_from_string(trailers->headers[i].key),
             grpc_mdstr_from_string(trailers->headers[i].value)));
     s->state.rs.trailing_metadata_valid = true;
+    if (0 == strcmp(trailers->headers[i].key, "grpc-status") &&
+        0 != strcmp(trailers->headers[i].value, "0")) {
+      s->state.fail_state = true;
+    }
   }
   s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true;
-  gpr_mu_unlock(&s->mu);
-  execute_from_storage(s);
+  /* Send a EOS when server terminates the stream (testServerFinishesRequest) to
+   * trigger on_succeeded */
+  if (!s->state.state_op_done[OP_SEND_TRAILING_METADATA] &&
+      !(s->state.state_op_done[OP_CANCEL_ERROR] ||
+        s->state.state_callback_received[OP_FAILED])) {
+    CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, 0)", s->cbs);
+    s->state.state_callback_received[OP_SEND_MESSAGE] = false;
+    cronet_bidirectional_stream_write(s->cbs, "", 0, true);
+    s->state.state_op_done[OP_SEND_TRAILING_METADATA] = true;
+
+    gpr_mu_unlock(&s->mu);
+  } else {
+    gpr_mu_unlock(&s->mu);
+    execute_from_storage(s);
+  }
 }
 
 /*
@@ -632,9 +678,9 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op,
   /* When call is canceled, every op can be run, except under following
   conditions
   */
-  bool is_canceled_of_failed = stream_state->state_op_done[OP_CANCEL_ERROR] ||
+  bool is_canceled_or_failed = stream_state->state_op_done[OP_CANCEL_ERROR] ||
                                stream_state->state_callback_received[OP_FAILED];
-  if (is_canceled_of_failed) {
+  if (is_canceled_or_failed) {
     if (op_id == OP_SEND_INITIAL_METADATA) result = false;
     if (op_id == OP_SEND_MESSAGE) result = false;
     if (op_id == OP_SEND_TRAILING_METADATA) result = false;
@@ -778,16 +824,10 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
       op_can_be_run(stream_op, stream_state, &oas->state,
                     OP_SEND_INITIAL_METADATA)) {
     CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas);
-    /* This OP is the beginning. Reset various states */
-    memset(&s->header_array, 0, sizeof(s->header_array));
-    memset(&stream_state->rs, 0, sizeof(stream_state->rs));
-    memset(&stream_state->ws, 0, sizeof(stream_state->ws));
-    memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done));
-    memset(stream_state->state_callback_received, 0,
-           sizeof(stream_state->state_callback_received));
     /* Start new cronet stream. It is destroyed in on_succeeded, on_canceled,
      * on_failed */
     GPR_ASSERT(s->cbs == NULL);
+    GPR_ASSERT(!stream_state->state_op_done[OP_SEND_INITIAL_METADATA]);
     s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs,
                                                 &cronet_callbacks);
     CRONET_LOG(GPR_DEBUG, "%p = cronet_bidirectional_stream_create()", s->cbs);
@@ -808,15 +848,18 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_RECV_INITIAL_METADATA)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_RECV_INITIAL_METADATA", oas);
-    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
-        stream_state->state_callback_received[OP_FAILED]) {
-      grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
-                          GRPC_ERROR_CANCELLED, NULL);
+    if (stream_state->state_op_done[OP_CANCEL_ERROR]) {
+      grpc_closure_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
+                         GRPC_ERROR_CANCELLED);
+    } else if (stream_state->state_callback_received[OP_FAILED]) {
+      grpc_closure_sched(
+          exec_ctx, stream_op->recv_initial_metadata_ready,
+          make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."));
     } else {
       grpc_chttp2_incoming_metadata_buffer_publish(
           &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata);
-      grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
-                          GRPC_ERROR_NONE, NULL);
+      grpc_closure_sched(exec_ctx, stream_op->recv_initial_metadata_ready,
+                         GRPC_ERROR_NONE);
     }
     stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true;
     result = ACTION_TAKEN_NO_CALLBACK;
@@ -865,19 +908,27 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_RECV_MESSAGE)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_RECV_MESSAGE", oas);
-    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
-        stream_state->state_callback_received[OP_FAILED]) {
-      CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed.");
-      grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready,
-                          GRPC_ERROR_CANCELLED, NULL);
+    if (stream_state->state_op_done[OP_CANCEL_ERROR]) {
+      CRONET_LOG(GPR_DEBUG, "Stream is cancelled.");
+      grpc_closure_sched(exec_ctx, stream_op->recv_message_ready,
+                         GRPC_ERROR_CANCELLED);
+      stream_state->state_op_done[OP_RECV_MESSAGE] = true;
+      result = ACTION_TAKEN_NO_CALLBACK;
+    } else if (stream_state->state_callback_received[OP_FAILED]) {
+      CRONET_LOG(GPR_DEBUG, "Stream failed.");
+      grpc_closure_sched(
+          exec_ctx, stream_op->recv_message_ready,
+          make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."));
       stream_state->state_op_done[OP_RECV_MESSAGE] = true;
+      result = ACTION_TAKEN_NO_CALLBACK;
     } else if (stream_state->rs.read_stream_closed == true) {
       /* No more data will be received */
       CRONET_LOG(GPR_DEBUG, "read stream closed");
-      grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready,
-                          GRPC_ERROR_NONE, NULL);
+      grpc_closure_sched(exec_ctx, stream_op->recv_message_ready,
+                         GRPC_ERROR_NONE);
       stream_state->state_op_done[OP_RECV_MESSAGE] = true;
       oas->state.state_op_done[OP_RECV_MESSAGE] = true;
+      result = ACTION_TAKEN_NO_CALLBACK;
     } else if (stream_state->rs.length_field_received == false) {
       if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES &&
           stream_state->rs.remaining_bytes == 0) {
@@ -907,8 +958,8 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
                                         &stream_state->rs.read_slice_buffer, 0);
           *((grpc_byte_buffer **)stream_op->recv_message) =
               (grpc_byte_buffer *)&stream_state->rs.sbs;
-          grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready,
-                              GRPC_ERROR_NONE, NULL);
+          grpc_closure_sched(exec_ctx, stream_op->recv_message_ready,
+                             GRPC_ERROR_NONE);
           stream_state->state_op_done[OP_RECV_MESSAGE] = true;
           oas->state.state_op_done[OP_RECV_MESSAGE] = true;
           result = ACTION_TAKEN_NO_CALLBACK;
@@ -942,14 +993,19 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
                                     &stream_state->rs.read_slice_buffer, 0);
       *((grpc_byte_buffer **)stream_op->recv_message) =
           (grpc_byte_buffer *)&stream_state->rs.sbs;
-      grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready,
-                          GRPC_ERROR_NONE, NULL);
+      grpc_closure_sched(exec_ctx, stream_op->recv_message_ready,
+                         GRPC_ERROR_NONE);
       stream_state->state_op_done[OP_RECV_MESSAGE] = true;
       oas->state.state_op_done[OP_RECV_MESSAGE] = true;
-      /* Clear read state of the stream, so next read op (if it were to come)
-       * will work */
-      stream_state->rs.received_bytes = stream_state->rs.remaining_bytes =
-          stream_state->rs.length_field_received = 0;
+      /* Do an extra read to trigger on_succeeded() callback in case connection
+         is closed */
+      stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes;
+      stream_state->rs.received_bytes = 0;
+      stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES;
+      stream_state->rs.length_field_received = false;
+      CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read(%p)", s->cbs);
+      cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer,
+                                       stream_state->rs.remaining_bytes);
       result = ACTION_TAKEN_NO_CALLBACK;
     }
   } else if (stream_op->recv_trailing_metadata &&
@@ -986,23 +1042,30 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
     CRONET_LOG(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs);
     if (s->cbs) {
       cronet_bidirectional_stream_cancel(s->cbs);
+      result = ACTION_TAKEN_WITH_CALLBACK;
+    } else {
+      result = ACTION_TAKEN_NO_CALLBACK;
     }
     stream_state->state_op_done[OP_CANCEL_ERROR] = true;
-    result = ACTION_TAKEN_WITH_CALLBACK;
+    if (!stream_state->cancel_error) {
+      stream_state->cancel_error = GRPC_ERROR_REF(stream_op->cancel_error);
+    }
   } else if (stream_op->on_complete &&
              op_can_be_run(stream_op, stream_state, &oas->state,
                            OP_ON_COMPLETE)) {
     CRONET_LOG(GPR_DEBUG, "running: %p  OP_ON_COMPLETE", oas);
-    if (stream_state->state_op_done[OP_CANCEL_ERROR] ||
-        stream_state->state_callback_received[OP_FAILED]) {
-      grpc_exec_ctx_sched(exec_ctx, stream_op->on_complete,
-                          GRPC_ERROR_CANCELLED, NULL);
+    if (stream_state->state_op_done[OP_CANCEL_ERROR]) {
+      grpc_closure_sched(exec_ctx, stream_op->on_complete,
+                         GRPC_ERROR_REF(stream_state->cancel_error));
+    } else if (stream_state->state_callback_received[OP_FAILED]) {
+      grpc_closure_sched(
+          exec_ctx, stream_op->on_complete,
+          make_error_with_desc(GRPC_STATUS_UNAVAILABLE, "Unavailable."));
     } else {
       /* All actions in this stream_op are complete. Call the on_complete
        * callback
        */
-      grpc_exec_ctx_sched(exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE,
-                          NULL);
+      grpc_closure_sched(exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE);
     }
     oas->state.state_op_done[OP_ON_COMPLETE] = true;
     oas->done = true;
@@ -1017,6 +1080,15 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx,
       make a note */
     if (stream_op->recv_message)
       stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true;
+  } else if (stream_state->fail_state && !stream_state->flush_read) {
+    CRONET_LOG(GPR_DEBUG, "running: %p  flush read", oas);
+    if (stream_state->rs.read_buffer &&
+        stream_state->rs.read_buffer != stream_state->rs.grpc_header_bytes) {
+      gpr_free(stream_state->rs.read_buffer);
+      stream_state->rs.read_buffer = NULL;
+    }
+    stream_state->rs.read_buffer = gpr_malloc(4096);
+    stream_state->flush_read = true;
   } else {
     result = NO_ACTION_POSSIBLE;
   }
@@ -1042,6 +1114,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
   memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done));
   memset(s->state.state_callback_received, 0,
          sizeof(s->state.state_callback_received));
+  s->state.fail_state = s->state.flush_read = false;
+  s->state.cancel_error = NULL;
   gpr_mu_init(&s->mu);
   return 0;
 }
@@ -1088,7 +1162,10 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
 }
 
 static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
-                           grpc_stream *gs, void *and_free_memory) {}
+                           grpc_stream *gs, void *and_free_memory) {
+  stream_obj *s = (stream_obj *)gs;
+  GRPC_ERROR_UNREF(s->state.cancel_error);
+}
 
 static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {}
 

src/core/lib/channel/channel_stack.c

@@ -102,13 +102,11 @@ grpc_call_element *grpc_call_stack_element(grpc_call_stack *call_stack,
   return CALL_ELEMS_FROM_STACK(call_stack) + index;
 }
 
-void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
-                             grpc_iomgr_cb_func destroy, void *destroy_arg,
-                             const grpc_channel_filter **filters,
-                             size_t filter_count,
-                             const grpc_channel_args *channel_args,
-                             grpc_transport *optional_transport,
-                             const char *name, grpc_channel_stack *stack) {
+grpc_error *grpc_channel_stack_init(
+    grpc_exec_ctx *exec_ctx, int initial_refs, grpc_iomgr_cb_func destroy,
+    void *destroy_arg, const grpc_channel_filter **filters, size_t filter_count,
+    const grpc_channel_args *channel_args, grpc_transport *optional_transport,
+    const char *name, grpc_channel_stack *stack) {
   size_t call_size =
       ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(grpc_call_stack)) +
       ROUND_UP_TO_ALIGNMENT_SIZE(filter_count * sizeof(grpc_call_element));
@@ -126,6 +124,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
       ROUND_UP_TO_ALIGNMENT_SIZE(filter_count * sizeof(grpc_channel_element));
 
   /* init per-filter data */
+  grpc_error *first_error = GRPC_ERROR_NONE;
   for (i = 0; i < filter_count; i++) {
     args.channel_stack = stack;
     args.channel_args = channel_args;
@@ -134,7 +133,15 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
     args.is_last = i == (filter_count - 1);
     elems[i].filter = filters[i];
     elems[i].channel_data = user_data;
-    elems[i].filter->init_channel_elem(exec_ctx, &elems[i], &args);
+    grpc_error *error =
+        elems[i].filter->init_channel_elem(exec_ctx, &elems[i], &args);
+    if (error != GRPC_ERROR_NONE) {
+      if (first_error == GRPC_ERROR_NONE) {
+        first_error = error;
+      } else {
+        GRPC_ERROR_UNREF(error);
+      }
+    }
     user_data += ROUND_UP_TO_ALIGNMENT_SIZE(filters[i]->sizeof_channel_data);
     call_size += ROUND_UP_TO_ALIGNMENT_SIZE(filters[i]->sizeof_call_data);
   }
@@ -144,6 +151,7 @@ void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
              grpc_channel_stack_size(filters, filter_count));
 
   stack->call_stack_size = call_size;
+  return first_error;
 }
 
 void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx,
@@ -289,7 +297,8 @@ void grpc_call_element_send_cancel(grpc_exec_ctx *exec_ctx,
   grpc_transport_stream_op *op = gpr_malloc(sizeof(*op));
   memset(op, 0, sizeof(*op));
   op->cancel_error = GRPC_ERROR_CANCELLED;
-  op->on_complete = grpc_closure_create(destroy_op, op);
+  op->on_complete =
+      grpc_closure_create(destroy_op, op, grpc_schedule_on_exec_ctx);
   elem->filter->start_transport_stream_op(exec_ctx, elem, op);
 }
 
@@ -299,7 +308,8 @@ void grpc_call_element_send_cancel_with_message(grpc_exec_ctx *exec_ctx,
                                                 grpc_slice *optional_message) {
   grpc_transport_stream_op *op = gpr_malloc(sizeof(*op));
   memset(op, 0, sizeof(*op));
-  op->on_complete = grpc_closure_create(destroy_op, op);
+  op->on_complete =
+      grpc_closure_create(destroy_op, op, grpc_schedule_on_exec_ctx);
   grpc_transport_stream_op_add_cancellation_with_message(op, status,
                                                          optional_message);
   elem->filter->start_transport_stream_op(exec_ctx, elem, op);
@@ -311,7 +321,8 @@ void grpc_call_element_send_close_with_message(grpc_exec_ctx *exec_ctx,
                                                grpc_slice *optional_message) {
   grpc_transport_stream_op *op = gpr_malloc(sizeof(*op));
   memset(op, 0, sizeof(*op));
-  op->on_complete = grpc_closure_create(destroy_op, op);
+  op->on_complete =
+      grpc_closure_create(destroy_op, op, grpc_schedule_on_exec_ctx);
   grpc_transport_stream_op_add_close(op, status, optional_message);
   elem->filter->start_transport_stream_op(exec_ctx, elem, op);
 }

src/core/lib/channel/channel_stack.h

@@ -34,6 +34,13 @@
 #ifndef GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H
 #define GRPC_CORE_LIB_CHANNEL_CHANNEL_STACK_H
 
+//////////////////////////////////////////////////////////////////////////////
+// IMPORTANT NOTE:
+//
+// When you update this API, please make the corresponding changes to
+// the C++ API in src/cpp/common/channel_filter.{h,cc}
+//////////////////////////////////////////////////////////////////////////////
+
 /* A channel filter defines how operations on a channel are implemented.
    Channel filters are chained together to create full channels, and if those
    chains are linear, then channel stacks provide a mechanism to minimize
@@ -146,8 +153,9 @@ typedef struct {
      is_first, is_last designate this elements position in the stack, and are
      useful for asserting correct configuration by upper layer code.
      The filter does not need to do any chaining */
-  void (*init_channel_elem)(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem,
-                            grpc_channel_element_args *args);
+  grpc_error *(*init_channel_elem)(grpc_exec_ctx *exec_ctx,
+                                   grpc_channel_element *elem,
+                                   grpc_channel_element_args *args);
   /* Destroy per channel data.
      The filter does not need to do any chaining */
   void (*destroy_channel_elem)(grpc_exec_ctx *exec_ctx,
@@ -214,12 +222,11 @@ grpc_call_element *grpc_call_stack_element(grpc_call_stack *stack, size_t i);
 size_t grpc_channel_stack_size(const grpc_channel_filter **filters,
                                size_t filter_count);
 /* Initialize a channel stack given some filters */
-void grpc_channel_stack_init(grpc_exec_ctx *exec_ctx, int initial_refs,
-                             grpc_iomgr_cb_func destroy, void *destroy_arg,
-                             const grpc_channel_filter **filters,
-                             size_t filter_count, const grpc_channel_args *args,
-                             grpc_transport *optional_transport,
-                             const char *name, grpc_channel_stack *stack);
+grpc_error *grpc_channel_stack_init(
+    grpc_exec_ctx *exec_ctx, int initial_refs, grpc_iomgr_cb_func destroy,
+    void *destroy_arg, const grpc_channel_filter **filters, size_t filter_count,
+    const grpc_channel_args *args, grpc_transport *optional_transport,
+    const char *name, grpc_channel_stack *stack);
 /* Destroy a channel stack */
 void grpc_channel_stack_destroy(grpc_exec_ctx *exec_ctx,
                                 grpc_channel_stack *stack);

src/core/lib/channel/channel_stack_builder.c

@@ -227,11 +227,10 @@ void grpc_channel_stack_builder_destroy(grpc_channel_stack_builder *builder) {
   gpr_free(builder);
 }
 
-void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx,
-                                        grpc_channel_stack_builder *builder,
-                                        size_t prefix_bytes, int initial_refs,
-                                        grpc_iomgr_cb_func destroy,
-                                        void *destroy_arg) {
+grpc_error *grpc_channel_stack_builder_finish(
+    grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder,
+    size_t prefix_bytes, int initial_refs, grpc_iomgr_cb_func destroy,
+    void *destroy_arg, void **result) {
   // count the number of filters
   size_t num_filters = 0;
   for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) {
@@ -250,28 +249,35 @@ void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx,
   size_t channel_stack_size = grpc_channel_stack_size(filters, num_filters);
 
   // allocate memory, with prefix_bytes followed by channel_stack_size
-  char *result = gpr_malloc(prefix_bytes + channel_stack_size);
+  *result = gpr_malloc(prefix_bytes + channel_stack_size);
   // fetch a pointer to the channel stack
   grpc_channel_stack *channel_stack =
-      (grpc_channel_stack *)(result + prefix_bytes);
+      (grpc_channel_stack *)((char *)(*result) + prefix_bytes);
   // and initialize it
-  grpc_channel_stack_init(exec_ctx, initial_refs, destroy,
-                          destroy_arg == NULL ? result : destroy_arg, filters,
-                          num_filters, builder->args, builder->transport,
-                          builder->name, channel_stack);
-
-  // run post-initialization functions
-  i = 0;
-  for (filter_node *p = builder->begin.next; p != &builder->end; p = p->next) {
-    if (p->init != NULL) {
-      p->init(channel_stack, grpc_channel_stack_element(channel_stack, i),
-              p->init_arg);
+  grpc_error *error = grpc_channel_stack_init(
+      exec_ctx, initial_refs, destroy,
+      destroy_arg == NULL ? *result : destroy_arg, filters, num_filters,
+      builder->args, builder->transport, builder->name, channel_stack);
+
+  if (error != GRPC_ERROR_NONE) {
+    grpc_channel_stack_destroy(exec_ctx, channel_stack);
+    gpr_free(*result);
+    *result = NULL;
+  } else {
+    // run post-initialization functions
+    i = 0;
+    for (filter_node *p = builder->begin.next; p != &builder->end;
+         p = p->next) {
+      if (p->init != NULL) {
+        p->init(channel_stack, grpc_channel_stack_element(channel_stack, i),
+                p->init_arg);
+      }
+      i++;
     }
-    i++;
   }
 
   grpc_channel_stack_builder_destroy(builder);
   gpr_free((grpc_channel_filter **)filters);
 
-  return result;
+  return error;
 }

src/core/lib/channel/channel_stack_builder.h

@@ -146,16 +146,15 @@ bool grpc_channel_stack_builder_append_filter(
 void grpc_channel_stack_builder_iterator_destroy(
     grpc_channel_stack_builder_iterator *iterator);
 
-/// Destroy the builder, return the freshly minted channel stack
+/// Destroy the builder, return the freshly minted channel stack in \a result.
 /// Allocates \a prefix_bytes bytes before the channel stack
 /// Returns the base pointer of the allocated block
 /// \a initial_refs, \a destroy, \a destroy_arg are as per
 /// grpc_channel_stack_init
-void *grpc_channel_stack_builder_finish(grpc_exec_ctx *exec_ctx,
-                                        grpc_channel_stack_builder *builder,
-                                        size_t prefix_bytes, int initial_refs,
-                                        grpc_iomgr_cb_func destroy,
-                                        void *destroy_arg);
+grpc_error *grpc_channel_stack_builder_finish(
+    grpc_exec_ctx *exec_ctx, grpc_channel_stack_builder *builder,
+    size_t prefix_bytes, int initial_refs, grpc_iomgr_cb_func destroy,
+    void *destroy_arg, void **result);
 
 /// Destroy the builder without creating a channel stack
 void grpc_channel_stack_builder_destroy(grpc_channel_stack_builder *builder);

src/core/lib/channel/compress_filter.c

@@ -269,8 +269,10 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   /* initialize members */
   grpc_slice_buffer_init(&calld->slices);
   calld->has_compression_algorithm = 0;
-  grpc_closure_init(&calld->got_slice, got_slice, elem);
-  grpc_closure_init(&calld->send_done, send_done, elem);
+  grpc_closure_init(&calld->got_slice, got_slice, elem,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&calld->send_done, send_done, elem,
+                    grpc_schedule_on_exec_ctx);
 
   return GRPC_ERROR_NONE;
 }
@@ -285,9 +287,9 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 }
 
 /* Constructor for channel_data */
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
+static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_channel_element *elem,
+                                     grpc_channel_element_args *args) {
   channel_data *channeld = elem->channel_data;
 
   channeld->enabled_algorithms_bitset =
@@ -315,6 +317,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
   }
 
   GPR_ASSERT(!args->is_last);
+  return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel data */

src/core/lib/channel/connected_channel.c

@@ -114,12 +114,13 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 }
 
 /* Constructor for channel_data */
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
+static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_channel_element *elem,
+                                     grpc_channel_element_args *args) {
   channel_data *cd = (channel_data *)elem->channel_data;
   GPR_ASSERT(args->is_last);
   cd->transport = NULL;
+  return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel_data */

src/core/lib/channel/context.h

@@ -47,6 +47,9 @@ typedef enum {
   /// Value is a \a census_context.
   GRPC_CONTEXT_TRACING,
 
+  /// Reserved for traffic_class_context.
+  GRPC_CONTEXT_TRAFFIC,
+
   GRPC_CONTEXT_COUNT
 } grpc_context_index;
 

src/core/lib/channel/deadline_filter.c

@@ -123,7 +123,8 @@ static void on_complete(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
 static void inject_on_complete_cb(grpc_deadline_state* deadline_state,
                                   grpc_transport_stream_op* op) {
   deadline_state->next_on_complete = op->on_complete;
-  grpc_closure_init(&deadline_state->on_complete, on_complete, deadline_state);
+  grpc_closure_init(&deadline_state->on_complete, on_complete, deadline_state,
+                    grpc_schedule_on_exec_ctx);
   op->on_complete = &deadline_state->on_complete;
 }
 
@@ -172,8 +173,9 @@ void grpc_deadline_state_start(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
     struct start_timer_after_init_state* state = gpr_malloc(sizeof(*state));
     state->elem = elem;
     state->deadline = deadline;
-    grpc_closure_init(&state->closure, start_timer_after_init, state);
-    grpc_exec_ctx_sched(exec_ctx, &state->closure, GRPC_ERROR_NONE, NULL);
+    grpc_closure_init(&state->closure, start_timer_after_init, state,
+                      grpc_schedule_on_exec_ctx);
+    grpc_closure_sched(exec_ctx, &state->closure, GRPC_ERROR_NONE);
   }
 }
 
@@ -207,10 +209,11 @@ void grpc_deadline_state_client_start_transport_stream_op(
 //
 
 // Constructor for channel_data.  Used for both client and server filters.
-static void init_channel_elem(grpc_exec_ctx* exec_ctx,
-                              grpc_channel_element* elem,
-                              grpc_channel_element_args* args) {
+static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx,
+                                     grpc_channel_element* elem,
+                                     grpc_channel_element_args* args) {
   GPR_ASSERT(!args->is_last);
+  return GRPC_ERROR_NONE;
 }
 
 // Destructor for channel_data.  Used for both client and server filters.
@@ -289,7 +292,8 @@ static void server_start_transport_stream_op(grpc_exec_ctx* exec_ctx,
       calld->next_recv_initial_metadata_ready = op->recv_initial_metadata_ready;
       calld->recv_initial_metadata = op->recv_initial_metadata;
       grpc_closure_init(&calld->recv_initial_metadata_ready,
-                        recv_initial_metadata_ready, elem);
+                        recv_initial_metadata_ready, elem,
+                        grpc_schedule_on_exec_ctx);
       op->recv_initial_metadata_ready = &calld->recv_initial_metadata_ready;
     }
     // Make sure we know when the call is complete, so that we can cancel

src/core/lib/channel/handshaker.c

@@ -49,21 +49,21 @@ void grpc_handshaker_init(const grpc_handshaker_vtable* vtable,
   handshaker->vtable = vtable;
 }
 
-static void grpc_handshaker_destroy(grpc_exec_ctx* exec_ctx,
-                                    grpc_handshaker* handshaker) {
+void grpc_handshaker_destroy(grpc_exec_ctx* exec_ctx,
+                             grpc_handshaker* handshaker) {
   handshaker->vtable->destroy(exec_ctx, handshaker);
 }
 
-static void grpc_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
-                                     grpc_handshaker* handshaker) {
+void grpc_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
+                              grpc_handshaker* handshaker) {
   handshaker->vtable->shutdown(exec_ctx, handshaker);
 }
 
-static void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx,
-                                         grpc_handshaker* handshaker,
-                                         grpc_tcp_server_acceptor* acceptor,
-                                         grpc_closure* on_handshake_done,
-                                         grpc_handshaker_args* args) {
+void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx,
+                                  grpc_handshaker* handshaker,
+                                  grpc_tcp_server_acceptor* acceptor,
+                                  grpc_closure* on_handshake_done,
+                                  grpc_handshaker_args* args) {
   handshaker->vtable->do_handshake(exec_ctx, handshaker, acceptor,
                                    on_handshake_done, args);
 }
@@ -157,14 +157,15 @@ static bool call_next_handshaker_locked(grpc_exec_ctx* exec_ctx,
                                         grpc_handshake_manager* mgr,
                                         grpc_error* error) {
   GPR_ASSERT(mgr->index <= mgr->count);
-  // If we got an error or we've been shut down or we've finished the last
-  // handshaker, invoke the on_handshake_done callback.  Otherwise, call the
-  // next handshaker.
-  if (error != GRPC_ERROR_NONE || mgr->shutdown || mgr->index == mgr->count) {
+  // If we got an error or we've been shut down or we're exiting early or
+  // we've finished the last handshaker, invoke the on_handshake_done
+  // callback.  Otherwise, call the next handshaker.
+  if (error != GRPC_ERROR_NONE || mgr->shutdown || mgr->args.exit_early ||
+      mgr->index == mgr->count) {
     // Cancel deadline timer, since we're invoking the on_handshake_done
     // callback now.
     grpc_timer_cancel(exec_ctx, &mgr->deadline_timer);
-    grpc_exec_ctx_sched(exec_ctx, &mgr->on_handshake_done, error, NULL);
+    grpc_closure_sched(exec_ctx, &mgr->on_handshake_done, error);
     mgr->shutdown = true;
   } else {
     grpc_handshaker_do_handshake(exec_ctx, mgr->handshakers[mgr->index],
@@ -217,8 +218,10 @@ void grpc_handshake_manager_do_handshake(
   grpc_slice_buffer_init(mgr->args.read_buffer);
   // Initialize state needed for calling handshakers.
   mgr->acceptor = acceptor;
-  grpc_closure_init(&mgr->call_next_handshaker, call_next_handshaker, mgr);
-  grpc_closure_init(&mgr->on_handshake_done, on_handshake_done, &mgr->args);
+  grpc_closure_init(&mgr->call_next_handshaker, call_next_handshaker, mgr,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&mgr->on_handshake_done, on_handshake_done, &mgr->args,
+                    grpc_schedule_on_exec_ctx);
   // Start deadline timer, which owns a ref.
   gpr_ref(&mgr->refs);
   grpc_timer_init(exec_ctx, &mgr->deadline_timer,

src/core/lib/channel/handshaker.h

@@ -72,6 +72,9 @@ typedef struct {
   grpc_endpoint* endpoint;
   grpc_channel_args* args;
   grpc_slice_buffer* read_buffer;
+  // A handshaker may set this to true before invoking on_handshake_done
+  // to indicate that subsequent handshakers should be skipped.
+  bool exit_early;
   // User data passed through the handshake manager.  Not used by
   // individual handshakers.
   void* user_data;
@@ -105,6 +108,16 @@ struct grpc_handshaker {
 void grpc_handshaker_init(const grpc_handshaker_vtable* vtable,
                           grpc_handshaker* handshaker);
 
+void grpc_handshaker_destroy(grpc_exec_ctx* exec_ctx,
+                             grpc_handshaker* handshaker);
+void grpc_handshaker_shutdown(grpc_exec_ctx* exec_ctx,
+                              grpc_handshaker* handshaker);
+void grpc_handshaker_do_handshake(grpc_exec_ctx* exec_ctx,
+                                  grpc_handshaker* handshaker,
+                                  grpc_tcp_server_acceptor* acceptor,
+                                  grpc_closure* on_handshake_done,
+                                  grpc_handshaker_args* args);
+
 ///
 /// grpc_handshake_manager
 ///

src/core/lib/channel/http_client_filter.c

@@ -352,12 +352,17 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   calld->send_message_blocked = false;
   grpc_slice_buffer_init(&calld->slices);
   grpc_closure_init(&calld->hc_on_recv_initial_metadata,
-                    hc_on_recv_initial_metadata, elem);
+                    hc_on_recv_initial_metadata, elem,
+                    grpc_schedule_on_exec_ctx);
   grpc_closure_init(&calld->hc_on_recv_trailing_metadata,
-                    hc_on_recv_trailing_metadata, elem);
-  grpc_closure_init(&calld->hc_on_complete, hc_on_complete, elem);
-  grpc_closure_init(&calld->got_slice, got_slice, elem);
-  grpc_closure_init(&calld->send_done, send_done, elem);
+                    hc_on_recv_trailing_metadata, elem,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&calld->hc_on_complete, hc_on_complete, elem,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&calld->got_slice, got_slice, elem,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&calld->send_done, send_done, elem,
+                    grpc_schedule_on_exec_ctx);
   return GRPC_ERROR_NONE;
 }
 
@@ -457,9 +462,9 @@ static grpc_mdstr *user_agent_from_args(const grpc_channel_args *args,
 }
 
 /* Constructor for channel_data */
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
+static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_channel_element *elem,
+                                     grpc_channel_element_args *args) {
   channel_data *chand = elem->channel_data;
   GPR_ASSERT(!args->is_last);
   GPR_ASSERT(args->optional_transport != NULL);
@@ -470,6 +475,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx,
       GRPC_MDSTR_USER_AGENT,
       user_agent_from_args(args->channel_args,
                            args->optional_transport->vtable->name));
+  return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel data */

src/core/lib/channel/http_server_filter.c

@@ -334,9 +334,12 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
   call_data *calld = elem->call_data;
   /* initialize members */
   memset(calld, 0, sizeof(*calld));
-  grpc_closure_init(&calld->hs_on_recv, hs_on_recv, elem);
-  grpc_closure_init(&calld->hs_on_complete, hs_on_complete, elem);
-  grpc_closure_init(&calld->hs_recv_message_ready, hs_recv_message_ready, elem);
+  grpc_closure_init(&calld->hs_on_recv, hs_on_recv, elem,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&calld->hs_on_complete, hs_on_complete, elem,
+                    grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&calld->hs_recv_message_ready, hs_recv_message_ready, elem,
+                    grpc_schedule_on_exec_ctx);
   grpc_slice_buffer_init(&calld->read_slice_buffer);
   return GRPC_ERROR_NONE;
 }
@@ -350,10 +353,11 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem,
 }
 
 /* Constructor for channel_data */
-static void init_channel_elem(grpc_exec_ctx *exec_ctx,
-                              grpc_channel_element *elem,
-                              grpc_channel_element_args *args) {
+static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx,
+                                     grpc_channel_element *elem,
+                                     grpc_channel_element_args *args) {
   GPR_ASSERT(!args->is_last);
+  return GRPC_ERROR_NONE;
 }
 
 /* Destructor for channel data */

src/core/lib/channel/message_size_filter.c

@@ -124,7 +124,7 @@ static void recv_message_ready(grpc_exec_ctx* exec_ctx, void* user_data,
     gpr_free(message_string);
   }
   // Invoke the next callback.
-  grpc_exec_ctx_sched(exec_ctx, calld->next_recv_message_ready, error, NULL);
+  grpc_closure_sched(exec_ctx, calld->next_recv_message_ready, error);
 }
 
 // Start transport stream op.
@@ -160,7 +160,8 @@ static grpc_error* init_call_elem(grpc_exec_ctx* exec_ctx,
   channel_data* chand = elem->channel_data;
   call_data* calld = elem->call_data;
   calld->next_recv_message_ready = NULL;
-  grpc_closure_init(&calld->recv_message_ready, recv_message_ready, elem);
+  grpc_closure_init(&calld->recv_message_ready, recv_message_ready, elem,
+                    grpc_schedule_on_exec_ctx);
   // Get max sizes from channel data, then merge in per-method config values.
   // Note: Per-method config is only available on the client, so we
   // apply the max request size to the send limit and the max response
@@ -192,9 +193,9 @@ static void destroy_call_elem(grpc_exec_ctx* exec_ctx, grpc_call_element* elem,
                               void* ignored) {}
 
 // Constructor for channel_data.
-static void init_channel_elem(grpc_exec_ctx* exec_ctx,
-                              grpc_channel_element* elem,
-                              grpc_channel_element_args* args) {
+static grpc_error* init_channel_elem(grpc_exec_ctx* exec_ctx,
+                                     grpc_channel_element* elem,
+                                     grpc_channel_element_args* args) {
   GPR_ASSERT(!args->is_last);
   channel_data* chand = elem->channel_data;
   memset(chand, 0, sizeof(*chand));
@@ -231,6 +232,7 @@ static void init_channel_elem(grpc_exec_ctx* exec_ctx,
       grpc_service_config_destroy(service_config);
     }
   }
+  return GRPC_ERROR_NONE;
 }
 
 // Destructor for channel_data.

src/core/lib/http/httpcli.c

@@ -103,7 +103,7 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req,
                    grpc_error *error) {
   grpc_polling_entity_del_from_pollset_set(exec_ctx, req->pollent,
                                            req->context->pollset_set);
-  grpc_exec_ctx_sched(exec_ctx, req->on_done, error, NULL);
+  grpc_closure_sched(exec_ctx, req->on_done, error);
   grpc_http_parser_destroy(&req->parser);
   if (req->addresses != NULL) {
     grpc_resolved_addresses_destroy(req->addresses);
@@ -224,7 +224,8 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req,
     return;
   }
   addr = &req->addresses->addrs[req->next_address++];
-  grpc_closure_init(&req->connected, on_connected, req);
+  grpc_closure_init(&req->connected, on_connected, req,
+                    grpc_schedule_on_exec_ctx);
   grpc_arg arg;
   arg.key = GRPC_ARG_RESOURCE_QUOTA;
   arg.type = GRPC_ARG_POINTER;
@@ -266,8 +267,9 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx,
   req->pollent = pollent;
   req->overall_error = GRPC_ERROR_NONE;
   req->resource_quota = grpc_resource_quota_internal_ref(resource_quota);
-  grpc_closure_init(&req->on_read, on_read, req);
-  grpc_closure_init(&req->done_write, done_write, req);
+  grpc_closure_init(&req->on_read, on_read, req, grpc_schedule_on_exec_ctx);
+  grpc_closure_init(&req->done_write, done_write, req,
+                    grpc_schedule_on_exec_ctx);
   grpc_slice_buffer_init(&req->incoming);
   grpc_slice_buffer_init(&req->outgoing);
   grpc_iomgr_register_object(&req->iomgr_obj, name);
@@ -277,8 +279,11 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(pollent);
   grpc_polling_entity_add_to_pollset_set(exec_ctx, req->pollent,
                                          req->context->pollset_set);
-  grpc_resolve_address(exec_ctx, request->host, req->handshaker->default_port,
-                       grpc_closure_create(on_resolved, req), &req->addresses);
+  grpc_resolve_address(
+      exec_ctx, request->host, req->handshaker->default_port,
+      req->context->pollset_set,
+      grpc_closure_create(on_resolved, req, grpc_schedule_on_exec_ctx),
+      &req->addresses);
 }
 
 void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,

src/core/lib/http/httpcli_security_connector.c

@@ -60,9 +60,9 @@ static void httpcli_ssl_destroy(grpc_security_connector *sc) {
   gpr_free(sc);
 }
 
-static void httpcli_ssl_create_handshakers(
-    grpc_exec_ctx *exec_ctx, grpc_channel_security_connector *sc,
-    grpc_handshake_manager *handshake_mgr) {
+static void httpcli_ssl_add_handshakers(grpc_exec_ctx *exec_ctx,
+                                        grpc_channel_security_connector *sc,
+                                        grpc_handshake_manager *handshake_mgr) {
   grpc_httpcli_ssl_channel_security_connector *c =
       (grpc_httpcli_ssl_channel_security_connector *)sc;
   tsi_handshaker *handshaker = NULL;
@@ -74,8 +74,9 @@ static void httpcli_ssl_create_handshakers(
               tsi_result_to_string(result));
     }
   }
-  grpc_security_create_handshakers(exec_ctx, handshaker, &sc->base,
-                                   handshake_mgr);
+  grpc_handshake_manager_add(
+      handshake_mgr,
+      grpc_security_handshaker_create(exec_ctx, handshaker, &sc->base));
 }
 
 static void httpcli_ssl_check_peer(grpc_exec_ctx *exec_ctx,
@@ -95,7 +96,7 @@ static void httpcli_ssl_check_peer(grpc_exec_ctx *exec_ctx,
     error = GRPC_ERROR_CREATE(msg);
     gpr_free(msg);
   }
-  grpc_exec_ctx_sched(exec_ctx, on_peer_checked, error, NULL);
+  grpc_closure_sched(exec_ctx, on_peer_checked, error);
   tsi_peer_destruct(&peer);
 }
 
@@ -132,7 +133,7 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create(
     *sc = NULL;
     return GRPC_SECURITY_ERROR;
   }
-  c->base.create_handshakers = httpcli_ssl_create_handshakers;
+  c->base.add_handshakers = httpcli_ssl_add_handshakers;
   *sc = &c->base;
   return GRPC_SECURITY_OK;
 }
@@ -185,8 +186,8 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg,
   GPR_ASSERT(httpcli_ssl_channel_security_connector_create(
                  pem_root_certs, pem_root_certs_size, host, &sc) ==
              GRPC_SECURITY_OK);
-  grpc_channel_security_connector_create_handshakers(exec_ctx, sc,
-                                                     c->handshake_mgr);
+  grpc_channel_security_connector_add_handshakers(exec_ctx, sc,
+                                                  c->handshake_mgr);
   grpc_handshake_manager_do_handshake(
       exec_ctx, c->handshake_mgr, tcp, NULL /* channel_args */, deadline,
       NULL /* acceptor */, on_handshake_done, c /* user_data */);

src/core/lib/iomgr/closure.c

@@ -37,10 +37,13 @@
 
 #include "src/core/lib/profiling/timers.h"
 
-void grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb,
-                       void *cb_arg) {
+grpc_closure *grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb,
+                                void *cb_arg,
+                                grpc_closure_scheduler *scheduler) {
   closure->cb = cb;
   closure->cb_arg = cb_arg;
+  closure->scheduler = scheduler;
+  return closure;
 }
 
 void grpc_closure_list_init(grpc_closure_list *closure_list) {
@@ -105,11 +108,12 @@ static void closure_wrapper(grpc_exec_ctx *exec_ctx, void *arg,
   cb(exec_ctx, cb_arg, error);
 }
 
-grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg) {
+grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg,
+                                  grpc_closure_scheduler *scheduler) {
   wrapped_closure *wc = gpr_malloc(sizeof(*wc));
   wc->cb = cb;
   wc->cb_arg = cb_arg;
-  grpc_closure_init(&wc->wrapper, closure_wrapper, wc);
+  grpc_closure_init(&wc->wrapper, closure_wrapper, wc, scheduler);
   return &wc->wrapper;
 }
 
@@ -117,8 +121,30 @@ void grpc_closure_run(grpc_exec_ctx *exec_ctx, grpc_closure *c,
                       grpc_error *error) {
   GPR_TIMER_BEGIN("grpc_closure_run", 0);
   if (c != NULL) {
-    c->cb(exec_ctx, c->cb_arg, error);
+    c->scheduler->vtable->run(exec_ctx, c, error);
+  } else {
+    GRPC_ERROR_UNREF(error);
   }
-  GRPC_ERROR_UNREF(error);
   GPR_TIMER_END("grpc_closure_run", 0);
 }
+
+void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *c,
+                        grpc_error *error) {
+  GPR_TIMER_BEGIN("grpc_closure_sched", 0);
+  if (c != NULL) {
+    c->scheduler->vtable->sched(exec_ctx, c, error);
+  } else {
+    GRPC_ERROR_UNREF(error);
+  }
+  GPR_TIMER_END("grpc_closure_sched", 0);
+}
+
+void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) {
+  grpc_closure *c = list->head;
+  while (c != NULL) {
+    grpc_closure *next = c->next_data.next;
+    c->scheduler->vtable->sched(exec_ctx, c, c->error_data.error);
+    c = next;
+  }
+  list->head = list->tail = NULL;
+}

src/core/lib/iomgr/closure.h

@@ -59,6 +59,22 @@ typedef struct grpc_closure_list {
 typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_error *error);
 
+typedef struct grpc_closure_scheduler grpc_closure_scheduler;
+
+typedef struct grpc_closure_scheduler_vtable {
+  /* NOTE: for all these functions, closure->scheduler == the scheduler that was
+           used to find this vtable */
+  void (*run)(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+              grpc_error *error);
+  void (*sched)(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                grpc_error *error);
+} grpc_closure_scheduler_vtable;
+
+/** Abstract type that can schedule closures for execution */
+struct grpc_closure_scheduler {
+  const grpc_closure_scheduler_vtable *vtable;
+};
+
 /** A closure over a grpc_iomgr_cb_func. */
 struct grpc_closure {
   /** Once queued, next indicates the next queued closure; before then, scratch
@@ -75,6 +91,10 @@ struct grpc_closure {
   /** Arguments to be passed to "cb". */
   void *cb_arg;
 
+  /** Scheduler to schedule against: NULL to schedule against current execution
+      context */
+  grpc_closure_scheduler *scheduler;
+
   /** Once queued, the result of the closure. Before then: scratch space */
   union {
     grpc_error *error;
@@ -82,12 +102,14 @@ struct grpc_closure {
   } error_data;
 };
 
-/** Initializes \a closure with \a cb and \a cb_arg. */
-void grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb,
-                       void *cb_arg);
+/** Initializes \a closure with \a cb and \a cb_arg. Returns \a closure. */
+grpc_closure *grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb,
+                                void *cb_arg,
+                                grpc_closure_scheduler *scheduler);
 
 /* Create a heap allocated closure: try to avoid except for very rare events */
-grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg);
+grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg,
+                                  grpc_closure_scheduler *scheduler);
 
 #define GRPC_CLOSURE_LIST_INIT \
   { NULL, NULL }
@@ -115,4 +137,13 @@ bool grpc_closure_list_empty(grpc_closure_list list);
 void grpc_closure_run(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                       grpc_error *error);
 
+/** Schedule a closure to be run. Does not need to be run from a safe point. */
+void grpc_closure_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                        grpc_error *error);
+
+/** Schedule all closures in a list to be run. Does not need to be run from a
+ * safe point. */
+void grpc_closure_list_sched(grpc_exec_ctx *exec_ctx,
+                             grpc_closure_list *closure_list);
+
 #endif /* GRPC_CORE_LIB_IOMGR_CLOSURE_H */

src/core/lib/iomgr/combiner.c

@@ -56,6 +56,10 @@ int grpc_combiner_trace = 0;
 struct grpc_combiner {
   grpc_combiner *next_combiner_on_this_exec_ctx;
   grpc_workqueue *optional_workqueue;
+  grpc_closure_scheduler uncovered_scheduler;
+  grpc_closure_scheduler covered_scheduler;
+  grpc_closure_scheduler uncovered_finally_scheduler;
+  grpc_closure_scheduler covered_finally_scheduler;
   gpr_mpscq queue;
   // state is:
   // lower bit - zero if orphaned (STATE_UNORPHANED)
@@ -70,6 +74,26 @@ struct grpc_combiner {
   grpc_closure offload;
 };
 
+static void combiner_exec_uncovered(grpc_exec_ctx *exec_ctx,
+                                    grpc_closure *closure, grpc_error *error);
+static void combiner_exec_covered(grpc_exec_ctx *exec_ctx,
+                                  grpc_closure *closure, grpc_error *error);
+static void combiner_finally_exec_uncovered(grpc_exec_ctx *exec_ctx,
+                                            grpc_closure *closure,
+                                            grpc_error *error);
+static void combiner_finally_exec_covered(grpc_exec_ctx *exec_ctx,
+                                          grpc_closure *closure,
+                                          grpc_error *error);
+
+static const grpc_closure_scheduler_vtable scheduler_uncovered = {
+    combiner_exec_uncovered, combiner_exec_uncovered};
+static const grpc_closure_scheduler_vtable scheduler_covered = {
+    combiner_exec_covered, combiner_exec_covered};
+static const grpc_closure_scheduler_vtable finally_scheduler_uncovered = {
+    combiner_finally_exec_uncovered, combiner_finally_exec_uncovered};
+static const grpc_closure_scheduler_vtable finally_scheduler_covered = {
+    combiner_finally_exec_covered, combiner_finally_exec_covered};
+
 static void offload(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 
 typedef struct {
@@ -102,11 +126,16 @@ grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue) {
   lock->time_to_execute_final_list = false;
   lock->optional_workqueue = optional_workqueue;
   lock->final_list_covered_by_poller = false;
+  lock->uncovered_scheduler.vtable = &scheduler_uncovered;
+  lock->covered_scheduler.vtable = &scheduler_covered;
+  lock->uncovered_finally_scheduler.vtable = &finally_scheduler_uncovered;
+  lock->covered_finally_scheduler.vtable = &finally_scheduler_covered;
   gpr_atm_no_barrier_store(&lock->state, STATE_UNORPHANED);
   gpr_atm_no_barrier_store(&lock->elements_covered_by_poller, 0);
   gpr_mpscq_init(&lock->queue);
   grpc_closure_list_init(&lock->final_list);
-  grpc_closure_init(&lock->offload, offload, lock);
+  grpc_closure_init(&lock->offload, offload, lock,
+                    grpc_workqueue_scheduler(lock->optional_workqueue));
   GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p create", lock));
   return lock;
 }
@@ -148,9 +177,9 @@ static void push_first_on_exec_ctx(grpc_exec_ctx *exec_ctx,
   }
 }
 
-void grpc_combiner_execute(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
-                           grpc_closure *cl, grpc_error *error,
-                           bool covered_by_poller) {
+static void combiner_exec(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
+                          grpc_closure *cl, grpc_error *error,
+                          bool covered_by_poller) {
   GPR_TIMER_BEGIN("combiner.execute", 0);
   gpr_atm last = gpr_atm_full_fetch_add(&lock->state, STATE_ELEM_COUNT_LOW_BIT);
   GRPC_COMBINER_TRACE(gpr_log(
@@ -171,6 +200,24 @@ void grpc_combiner_execute(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
   GPR_TIMER_END("combiner.execute", 0);
 }
 
+#define COMBINER_FROM_CLOSURE_SCHEDULER(closure, scheduler_name) \
+  ((grpc_combiner *)(((char *)((closure)->scheduler)) -          \
+                     offsetof(grpc_combiner, scheduler_name)))
+
+static void combiner_exec_uncovered(grpc_exec_ctx *exec_ctx, grpc_closure *cl,
+                                    grpc_error *error) {
+  combiner_exec(exec_ctx,
+                COMBINER_FROM_CLOSURE_SCHEDULER(cl, uncovered_scheduler), cl,
+                error, false);
+}
+
+static void combiner_exec_covered(grpc_exec_ctx *exec_ctx, grpc_closure *cl,
+                                  grpc_error *error) {
+  combiner_exec(exec_ctx,
+                COMBINER_FROM_CLOSURE_SCHEDULER(cl, covered_scheduler), cl,
+                error, true);
+}
+
 static void move_next(grpc_exec_ctx *exec_ctx) {
   exec_ctx->active_combiner =
       exec_ctx->active_combiner->next_combiner_on_this_exec_ctx;
@@ -188,8 +235,7 @@ static void queue_offload(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
   move_next(exec_ctx);
   GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p queue_offload --> %p", lock,
                               lock->optional_workqueue));
-  grpc_workqueue_enqueue(exec_ctx, lock->optional_workqueue, &lock->offload,
-                         GRPC_ERROR_NONE);
+  grpc_closure_sched(exec_ctx, &lock->offload, GRPC_ERROR_NONE);
 }
 
 bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx) {
@@ -312,23 +358,22 @@ bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx) {
 }
 
 static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure,
-                            grpc_error *error) {
-  grpc_combiner_execute_finally(exec_ctx, exec_ctx->active_combiner, closure,
-                                GRPC_ERROR_REF(error), false);
-}
+                            grpc_error *error);
 
-void grpc_combiner_execute_finally(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
-                                   grpc_closure *closure, grpc_error *error,
-                                   bool covered_by_poller) {
+static void combiner_execute_finally(grpc_exec_ctx *exec_ctx,
+                                     grpc_combiner *lock, grpc_closure *closure,
+                                     grpc_error *error,
+                                     bool covered_by_poller) {
   GRPC_COMBINER_TRACE(gpr_log(
       GPR_DEBUG, "C:%p grpc_combiner_execute_finally c=%p; ac=%p; cov=%d", lock,
       closure, exec_ctx->active_combiner, covered_by_poller));
   GPR_TIMER_BEGIN("combiner.execute_finally", 0);
   if (exec_ctx->active_combiner != lock) {
     GPR_TIMER_MARK("slowpath", 0);
-    grpc_combiner_execute(exec_ctx, lock,
-                          grpc_closure_create(enqueue_finally, closure), error,
-                          false);
+    grpc_closure_sched(
+        exec_ctx, grpc_closure_create(enqueue_finally, closure,
+                                      grpc_combiner_scheduler(lock, false)),
+        error);
     GPR_TIMER_END("combiner.execute_finally", 0);
     return;
   }
@@ -342,3 +387,36 @@ void grpc_combiner_execute_finally(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
   grpc_closure_list_append(&lock->final_list, closure, error);
   GPR_TIMER_END("combiner.execute_finally", 0);
 }
+
+static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure,
+                            grpc_error *error) {
+  combiner_execute_finally(exec_ctx, exec_ctx->active_combiner, closure,
+                           GRPC_ERROR_REF(error), false);
+}
+
+static void combiner_finally_exec_uncovered(grpc_exec_ctx *exec_ctx,
+                                            grpc_closure *cl,
+                                            grpc_error *error) {
+  combiner_execute_finally(exec_ctx, COMBINER_FROM_CLOSURE_SCHEDULER(
+                                         cl, uncovered_finally_scheduler),
+                           cl, error, false);
+}
+
+static void combiner_finally_exec_covered(grpc_exec_ctx *exec_ctx,
+                                          grpc_closure *cl, grpc_error *error) {
+  combiner_execute_finally(
+      exec_ctx, COMBINER_FROM_CLOSURE_SCHEDULER(cl, covered_finally_scheduler),
+      cl, error, true);
+}
+
+grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *combiner,
+                                                bool covered_by_poller) {
+  return covered_by_poller ? &combiner->covered_scheduler
+                           : &combiner->uncovered_scheduler;
+}
+
+grpc_closure_scheduler *grpc_combiner_finally_scheduler(
+    grpc_combiner *combiner, bool covered_by_poller) {
+  return covered_by_poller ? &combiner->covered_finally_scheduler
+                           : &combiner->uncovered_finally_scheduler;
+}

src/core/lib/iomgr/combiner.h

@@ -50,14 +50,12 @@
 grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue);
 // Destroy the lock
 void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock);
-// Execute \a action within the lock.
-void grpc_combiner_execute(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
-                           grpc_closure *closure, grpc_error *error,
-                           bool covered_by_poller);
-// Execute \a action within the lock just prior to unlocking.
-void grpc_combiner_execute_finally(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
-                                   grpc_closure *closure, grpc_error *error,
-                                   bool covered_by_poller);
+// Fetch a scheduler to schedule closures against
+grpc_closure_scheduler *grpc_combiner_scheduler(grpc_combiner *lock,
+                                                bool covered_by_poller);
+// Scheduler to execute \a action within the lock just prior to unlocking.
+grpc_closure_scheduler *grpc_combiner_finally_scheduler(grpc_combiner *lock,
+                                                        bool covered_by_poller);
 
 bool grpc_combiner_continue_exec_ctx(grpc_exec_ctx *exec_ctx);
 

src/core/lib/iomgr/ev_poll_posix.c

@@ -397,7 +397,7 @@ static void close_fd_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   if (!fd->released) {
     close(fd->fd);
   }
-  grpc_exec_ctx_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE, NULL);
+  grpc_closure_sched(exec_ctx, fd->on_done_closure, GRPC_ERROR_NONE);
 }
 
 static int fd_wrapped_fd(grpc_fd *fd) {
@@ -457,16 +457,14 @@ static grpc_error *fd_shutdown_error(bool shutdown) {
 static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure **st, grpc_closure *closure) {
   if (fd->shutdown) {
-    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"),
-                        NULL);
+    grpc_closure_sched(exec_ctx, closure, GRPC_ERROR_CREATE("FD shutdown"));
   } else if (*st == CLOSURE_NOT_READY) {
     /* not ready ==> switch to a waiting state by setting the closure */
     *st = closure;
   } else if (*st == CLOSURE_READY) {
     /* already ready ==> queue the closure to run immediately */
     *st = CLOSURE_NOT_READY;
-    grpc_exec_ctx_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown),
-                        NULL);
+    grpc_closure_sched(exec_ctx, closure, fd_shutdown_error(fd->shutdown));
     maybe_wake_one_watcher_locked(fd);
   } else {
     /* upcallptr was set to a different closure.  This is an error! */
@@ -489,7 +487,7 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
     return 0;
   } else {
     /* waiting ==> queue closure */
-    grpc_exec_ctx_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown), NULL);
+    grpc_closure_sched(exec_ctx, *st, fd_shutdown_error(fd->shutdown));
     *st = CLOSURE_NOT_READY;
     return 1;
   }
@@ -852,7 +850,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
     GRPC_FD_UNREF(pollset->fds[i], "multipoller");
   }
   pollset->fd_count = 0;
-  grpc_exec_ctx_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE, NULL);
+  grpc_closure_sched(exec_ctx, pollset->shutdown_done, GRPC_ERROR_NONE);
 }
 
 static void work_combine_error(grpc_error **composite, grpc_error *error) {
@@ -901,7 +899,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (!pollset_has_workers(pollset) &&
       !grpc_closure_list_empty(pollset->idle_jobs)) {
     GPR_TIMER_MARK("pollset_work.idle_jobs", 0);
-    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+    grpc_closure_list_sched(exec_ctx, &pollset->idle_jobs);
     goto done;
   }
   /* If we're shutting down then we don't execute any extended work */
@@ -1081,7 +1079,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
        * TODO(dklempner): Can we refactor the shutdown logic to avoid this? */
       gpr_mu_lock(&pollset->mu);
     } else if (!grpc_closure_list_empty(pollset->idle_jobs)) {
-      grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+      grpc_closure_list_sched(exec_ctx, &pollset->idle_jobs);
       gpr_mu_unlock(&pollset->mu);
       grpc_exec_ctx_flush(exec_ctx);
       gpr_mu_lock(&pollset->mu);
@@ -1100,7 +1098,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   pollset->shutdown_done = closure;
   pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST);
   if (!pollset_has_workers(pollset)) {
-    grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
+    grpc_closure_list_sched(exec_ctx, &pollset->idle_jobs);
   }
   if (!pollset->called_shutdown && !pollset_has_workers(pollset)) {
     pollset->called_shutdown = 1;
@@ -1288,10 +1286,8 @@ static void workqueue_unref(grpc_exec_ctx *exec_ctx,
                             grpc_workqueue *workqueue) {}
 #endif
 
-static void workqueue_enqueue(grpc_exec_ctx *exec_ctx,
-                              grpc_workqueue *workqueue, grpc_closure *closure,
-                              grpc_error *error) {
-  grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
+static grpc_closure_scheduler *workqueue_scheduler(grpc_workqueue *workqueue) {
+  return grpc_schedule_on_exec_ctx;
 }
 
 /*******************************************************************************
@@ -1534,7 +1530,7 @@ static const grpc_event_engine_vtable vtable = {
 
     .workqueue_ref = workqueue_ref,
     .workqueue_unref = workqueue_unref,
-    .workqueue_enqueue = workqueue_enqueue,
+    .workqueue_scheduler = workqueue_scheduler,
 
     .shutdown_engine = shutdown_engine,
 };

src/core/lib/iomgr/ev_posix.c

@@ -275,9 +275,8 @@ void grpc_workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
 }
 #endif
 
-void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
-                            grpc_closure *closure, grpc_error *error) {
-  g_event_engine->workqueue_enqueue(exec_ctx, workqueue, closure, error);
+grpc_closure_scheduler *grpc_workqueue_scheduler(grpc_workqueue *workqueue) {
+  return g_event_engine->workqueue_scheduler(workqueue);
 }
 
 #endif  // GRPC_POSIX_SOCKET

src/core/lib/iomgr/ev_posix.h

@@ -106,8 +106,7 @@ typedef struct grpc_event_engine_vtable {
   grpc_workqueue *(*workqueue_ref)(grpc_workqueue *workqueue);
   void (*workqueue_unref)(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue);
 #endif
-  void (*workqueue_enqueue)(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
-                            grpc_closure *closure, grpc_error *error);
+  grpc_closure_scheduler *(*workqueue_scheduler)(grpc_workqueue *workqueue);
 } grpc_event_engine_vtable;
 
 void grpc_event_engine_init(void);

src/core/lib/iomgr/exec_ctx.c

@@ -57,7 +57,6 @@ bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored) {
   return true;
 }
 
-#ifndef GRPC_EXECUTION_CONTEXT_SANITIZER
 bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
   bool did_something = 0;
   GPR_TIMER_BEGIN("grpc_exec_ctx_flush", 0);
@@ -67,8 +66,10 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
       exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL;
       while (c != NULL) {
         grpc_closure *next = c->next_data.next;
+        grpc_error *error = c->error_data.error;
         did_something = true;
-        grpc_closure_run(exec_ctx, c, c->error_data.error);
+        c->cb(exec_ctx, c->cb_arg, error);
+        GRPC_ERROR_UNREF(error);
         c = next;
       }
     } else if (!grpc_combiner_continue_exec_ctx(exec_ctx)) {
@@ -76,30 +77,6 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) {
     }
   }
   GPR_ASSERT(exec_ctx->active_combiner == NULL);
-  if (exec_ctx->stealing_from_workqueue != NULL) {
-    if (grpc_exec_ctx_ready_to_finish(exec_ctx)) {
-      grpc_workqueue_enqueue(exec_ctx, exec_ctx->stealing_from_workqueue,
-                             exec_ctx->stolen_closure,
-                             exec_ctx->stolen_closure->error_data.error);
-      GRPC_WORKQUEUE_UNREF(exec_ctx, exec_ctx->stealing_from_workqueue,
-                           "exec_ctx_sched");
-      exec_ctx->stealing_from_workqueue = NULL;
-      exec_ctx->stolen_closure = NULL;
-    } else {
-      grpc_closure *c = exec_ctx->stolen_closure;
-      GRPC_WORKQUEUE_UNREF(exec_ctx, exec_ctx->stealing_from_workqueue,
-                           "exec_ctx_sched");
-      exec_ctx->stealing_from_workqueue = NULL;
-      exec_ctx->stolen_closure = NULL;
-      grpc_error *error = c->error_data.error;
-      GPR_TIMER_BEGIN("grpc_exec_ctx_flush.stolen_cb", 0);
-      c->cb(exec_ctx, c->cb_arg, error);
-      GRPC_ERROR_UNREF(error);
-      GPR_TIMER_END("grpc_exec_ctx_flush.stolen_cb", 0);
-      grpc_exec_ctx_flush(exec_ctx);
-      did_something = true;
-    }
-  }
   GPR_TIMER_END("grpc_exec_ctx_flush", 0);
   return did_something;
 }
@@ -109,104 +86,21 @@ void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {
   grpc_exec_ctx_flush(exec_ctx);
 }
 
-void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                         grpc_error *error,
-                         grpc_workqueue *offload_target_or_null) {
-  GPR_TIMER_BEGIN("grpc_exec_ctx_sched", 0);
-  if (offload_target_or_null == NULL) {
-    grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
-  } else if (exec_ctx->stealing_from_workqueue == NULL) {
-    exec_ctx->stealing_from_workqueue = offload_target_or_null;
-    closure->error_data.error = error;
-    exec_ctx->stolen_closure = closure;
-  } else if (exec_ctx->stealing_from_workqueue != offload_target_or_null) {
-    grpc_workqueue_enqueue(exec_ctx, offload_target_or_null, closure, error);
-    GRPC_WORKQUEUE_UNREF(exec_ctx, offload_target_or_null, "exec_ctx_sched");
-  } else { /* stealing_from_workqueue == offload_target_or_null */
-    grpc_workqueue_enqueue(exec_ctx, offload_target_or_null,
-                           exec_ctx->stolen_closure,
-                           exec_ctx->stolen_closure->error_data.error);
-    closure->error_data.error = error;
-    exec_ctx->stolen_closure = closure;
-    GRPC_WORKQUEUE_UNREF(exec_ctx, offload_target_or_null, "exec_ctx_sched");
-  }
-  GPR_TIMER_END("grpc_exec_ctx_sched", 0);
+static void exec_ctx_run(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                         grpc_error *error) {
+  closure->cb(exec_ctx, closure->cb_arg, error);
+  GRPC_ERROR_UNREF(error);
 }
 
-void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
-                                grpc_closure_list *list,
-                                grpc_workqueue *offload_target_or_null) {
-  grpc_closure_list_move(list, &exec_ctx->closure_list);
+static void exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
+                           grpc_error *error) {
+  grpc_closure_list_append(&exec_ctx->closure_list, closure, error);
 }
 
 void grpc_exec_ctx_global_init(void) {}
 void grpc_exec_ctx_global_shutdown(void) {}
-#else
-static gpr_mu g_mu;
-static gpr_cv g_cv;
-static int g_threads = 0;
-
-static void run_closure(void *arg) {
-  grpc_closure *closure = arg;
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  closure->cb(&exec_ctx, closure->cb_arg, (closure->final_data & 1) != 0);
-  grpc_exec_ctx_finish(&exec_ctx);
-  gpr_mu_lock(&g_mu);
-  if (--g_threads == 0) {
-    gpr_cv_signal(&g_cv);
-  }
-  gpr_mu_unlock(&g_mu);
-}
-
-static void start_closure(grpc_closure *closure) {
-  gpr_thd_id id;
-  gpr_mu_lock(&g_mu);
-  g_threads++;
-  gpr_mu_unlock(&g_mu);
-  gpr_thd_new(&id, run_closure, closure, NULL);
-}
-
-bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { return false; }
-
-void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) {}
 
-void grpc_exec_ctx_enqueue(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                           bool success,
-                           grpc_workqueue *offload_target_or_null) {
-  GPR_ASSERT(offload_target_or_null == NULL);
-  if (closure == NULL) return;
-  closure->final_data = success;
-  start_closure(closure);
-}
-
-void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
-                                grpc_closure_list *list,
-                                grpc_workqueue *offload_target_or_null) {
-  GPR_ASSERT(offload_target_or_null == NULL);
-  if (list == NULL) return;
-  grpc_closure *p = list->head;
-  while (p) {
-    grpc_closure *start = p;
-    p = grpc_closure_next(start);
-    start_closure(start);
-  }
-  grpc_closure_list r = GRPC_CLOSURE_LIST_INIT;
-  *list = r;
-}
-
-void grpc_exec_ctx_global_init(void) {
-  gpr_mu_init(&g_mu);
-  gpr_cv_init(&g_cv);
-}
-
-void grpc_exec_ctx_global_shutdown(void) {
-  gpr_mu_lock(&g_mu);
-  while (g_threads != 0) {
-    gpr_cv_wait(&g_cv, &g_mu, gpr_inf_future(GPR_CLOCK_REALTIME));
-  }
-  gpr_mu_unlock(&g_mu);
-
-  gpr_mu_destroy(&g_mu);
-  gpr_cv_destroy(&g_cv);
-}
-#endif
+static const grpc_closure_scheduler_vtable exec_ctx_scheduler_vtable = {
+    exec_ctx_run, exec_ctx_sched};
+static grpc_closure_scheduler exec_ctx_scheduler = {&exec_ctx_scheduler_vtable};
+grpc_closure_scheduler *grpc_schedule_on_exec_ctx = &exec_ctx_scheduler;

src/core/lib/iomgr/exec_ctx.h

@@ -66,17 +66,6 @@ typedef struct grpc_combiner grpc_combiner;
 #ifndef GRPC_EXECUTION_CONTEXT_SANITIZER
 struct grpc_exec_ctx {
   grpc_closure_list closure_list;
-  /** The workqueue we're stealing work from.
-      As items are queued to the execution context, we try to steal one
-      workqueue item and execute it inline (assuming the exec_ctx is not
-      finished) - doing so does not invalidate the workqueue's contract, and
-      provides a small latency win in cases where we get a hit */
-  grpc_workqueue *stealing_from_workqueue;
-  /** The workqueue item that was stolen from the workqueue above. When new
-      items are scheduled to be offloaded to that workqueue, we need to update
-      this like a 1-deep fifo to maintain the invariant that workqueue items
-      queued by one thread are started in order */
-  grpc_closure *stolen_closure;
   /** currently active combiner: updated only via combiner.c */
   grpc_combiner *active_combiner;
   /** last active combiner in the active combiner list */
@@ -89,10 +78,7 @@ struct grpc_exec_ctx {
 /* initializer for grpc_exec_ctx:
    prefer to use GRPC_EXEC_CTX_INIT whenever possible */
 #define GRPC_EXEC_CTX_INIT_WITH_FINISH_CHECK(finish_check, finish_check_arg) \
-  {                                                                          \
-    GRPC_CLOSURE_LIST_INIT, NULL, NULL, NULL, NULL, false, finish_check_arg, \
-        finish_check                                                         \
-  }
+  { GRPC_CLOSURE_LIST_INIT, NULL, NULL, false, finish_check_arg, finish_check }
 #else
 struct grpc_exec_ctx {
   bool cached_ready_to_finish;
@@ -108,6 +94,8 @@ struct grpc_exec_ctx {
 #define GRPC_EXEC_CTX_INIT \
   GRPC_EXEC_CTX_INIT_WITH_FINISH_CHECK(grpc_always_ready_to_finish, NULL)
 
+extern grpc_closure_scheduler *grpc_schedule_on_exec_ctx;
+
 /** Flush any work that has been enqueued onto this grpc_exec_ctx.
  *  Caller must guarantee that no interfering locks are held.
  *  Returns true if work was performed, false otherwise. */
@@ -115,14 +103,6 @@ bool grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx);
 /** Finish any pending work for a grpc_exec_ctx. Must be called before
  *  the instance is destroyed, or work may be lost. */
 void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx);
-/** Add a closure to be executed in the future.
-    If \a offload_target_or_null is NULL, the closure will be executed at the
-    next exec_ctx.{finish,flush} point.
-    If \a offload_target_or_null is non-NULL, the closure will be scheduled
-    against the workqueue, and a reference to the workqueue will be consumed. */
-void grpc_exec_ctx_sched(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
-                         grpc_error *error,
-                         grpc_workqueue *offload_target_or_null);
 /** Returns true if we'd like to leave this execution context as soon as
     possible: useful for deciding whether to do something more or not depending
     on outside context */
@@ -131,11 +111,6 @@ bool grpc_exec_ctx_ready_to_finish(grpc_exec_ctx *exec_ctx);
 bool grpc_never_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored);
 /** A finish check that is always ready to finish */
 bool grpc_always_ready_to_finish(grpc_exec_ctx *exec_ctx, void *arg_ignored);
-/** Add a list of closures to be executed at the next flush/finish point.
- *  Leaves \a list empty. */
-void grpc_exec_ctx_enqueue_list(grpc_exec_ctx *exec_ctx,
-                                grpc_closure_list *list,
-                                grpc_workqueue *offload_target_or_null);
 
 void grpc_exec_ctx_global_init(void);