Merge remote-tracking branch 'upstream/master' into grpclb_resolver_changes

BUILD

@@ -51,6 +51,7 @@ cc_library(
     "src/core/lib/support/backoff.h",
     "src/core/lib/support/block_annotate.h",
     "src/core/lib/support/env.h",
+    "src/core/lib/support/mpscq.h",
     "src/core/lib/support/murmur_hash.h",
     "src/core/lib/support/percent_encoding.h",
     "src/core/lib/support/stack_lockfree.h",
@@ -79,6 +80,7 @@ cc_library(
     "src/core/lib/support/log_linux.c",
     "src/core/lib/support/log_posix.c",
     "src/core/lib/support/log_windows.c",
+    "src/core/lib/support/mpscq.c",
     "src/core/lib/support/murmur_hash.c",
     "src/core/lib/support/percent_encoding.c",
     "src/core/lib/support/slice.c",
@@ -176,6 +178,7 @@ cc_library(
     "src/core/lib/http/httpcli.h",
     "src/core/lib/http/parser.h",
     "src/core/lib/iomgr/closure.h",
+    "src/core/lib/iomgr/combiner.h",
     "src/core/lib/iomgr/endpoint.h",
     "src/core/lib/iomgr/endpoint_pair.h",
     "src/core/lib/iomgr/error.h",
@@ -331,6 +334,7 @@ cc_library(
     "src/core/lib/http/httpcli.c",
     "src/core/lib/http/parser.c",
     "src/core/lib/iomgr/closure.c",
+    "src/core/lib/iomgr/combiner.c",
     "src/core/lib/iomgr/endpoint.c",
     "src/core/lib/iomgr/endpoint_pair_posix.c",
     "src/core/lib/iomgr/endpoint_pair_windows.c",
@@ -567,6 +571,7 @@ cc_library(
     "src/core/lib/http/httpcli.h",
     "src/core/lib/http/parser.h",
     "src/core/lib/iomgr/closure.h",
+    "src/core/lib/iomgr/combiner.h",
     "src/core/lib/iomgr/endpoint.h",
     "src/core/lib/iomgr/endpoint_pair.h",
     "src/core/lib/iomgr/error.h",
@@ -708,6 +713,7 @@ cc_library(
     "src/core/lib/http/httpcli.c",
     "src/core/lib/http/parser.c",
     "src/core/lib/iomgr/closure.c",
+    "src/core/lib/iomgr/combiner.c",
     "src/core/lib/iomgr/endpoint.c",
     "src/core/lib/iomgr/endpoint_pair_posix.c",
     "src/core/lib/iomgr/endpoint_pair_windows.c",
@@ -915,6 +921,7 @@ cc_library(
     "src/core/lib/http/httpcli.h",
     "src/core/lib/http/parser.h",
     "src/core/lib/iomgr/closure.h",
+    "src/core/lib/iomgr/combiner.h",
     "src/core/lib/iomgr/endpoint.h",
     "src/core/lib/iomgr/endpoint_pair.h",
     "src/core/lib/iomgr/error.h",
@@ -1047,6 +1054,7 @@ cc_library(
     "src/core/lib/http/httpcli.c",
     "src/core/lib/http/parser.c",
     "src/core/lib/iomgr/closure.c",
+    "src/core/lib/iomgr/combiner.c",
     "src/core/lib/iomgr/endpoint.c",
     "src/core/lib/iomgr/endpoint_pair_posix.c",
     "src/core/lib/iomgr/endpoint_pair_windows.c",
@@ -1258,6 +1266,7 @@ cc_library(
     "src/core/lib/http/httpcli.h",
     "src/core/lib/http/parser.h",
     "src/core/lib/iomgr/closure.h",
+    "src/core/lib/iomgr/combiner.h",
     "src/core/lib/iomgr/endpoint.h",
     "src/core/lib/iomgr/endpoint_pair.h",
     "src/core/lib/iomgr/error.h",
@@ -1370,6 +1379,7 @@ cc_library(
     "src/core/lib/http/httpcli.c",
     "src/core/lib/http/parser.c",
     "src/core/lib/iomgr/closure.c",
+    "src/core/lib/iomgr/combiner.c",
     "src/core/lib/iomgr/endpoint.c",
     "src/core/lib/iomgr/endpoint_pair_posix.c",
     "src/core/lib/iomgr/endpoint_pair_windows.c",
@@ -1662,6 +1672,7 @@ cc_library(
     "src/core/lib/http/httpcli.h",
     "src/core/lib/http/parser.h",
     "src/core/lib/iomgr/closure.h",
+    "src/core/lib/iomgr/combiner.h",
     "src/core/lib/iomgr/endpoint.h",
     "src/core/lib/iomgr/endpoint_pair.h",
     "src/core/lib/iomgr/error.h",
@@ -1769,6 +1780,7 @@ cc_library(
     "src/core/lib/http/httpcli.c",
     "src/core/lib/http/parser.c",
     "src/core/lib/iomgr/closure.c",
+    "src/core/lib/iomgr/combiner.c",
     "src/core/lib/iomgr/endpoint.c",
     "src/core/lib/iomgr/endpoint_pair_posix.c",
     "src/core/lib/iomgr/endpoint_pair_windows.c",
@@ -2044,6 +2056,7 @@ objc_library(
     "src/core/lib/support/log_linux.c",
     "src/core/lib/support/log_posix.c",
     "src/core/lib/support/log_windows.c",
+    "src/core/lib/support/mpscq.c",
     "src/core/lib/support/murmur_hash.c",
     "src/core/lib/support/percent_encoding.c",
     "src/core/lib/support/slice.c",
@@ -2115,6 +2128,7 @@ objc_library(
     "src/core/lib/support/backoff.h",
     "src/core/lib/support/block_annotate.h",
     "src/core/lib/support/env.h",
+    "src/core/lib/support/mpscq.h",
     "src/core/lib/support/murmur_hash.h",
     "src/core/lib/support/percent_encoding.h",
     "src/core/lib/support/stack_lockfree.h",
@@ -2153,6 +2167,7 @@ objc_library(
     "src/core/lib/http/httpcli.c",
     "src/core/lib/http/parser.c",
     "src/core/lib/iomgr/closure.c",
+    "src/core/lib/iomgr/combiner.c",
     "src/core/lib/iomgr/endpoint.c",
     "src/core/lib/iomgr/endpoint_pair_posix.c",
     "src/core/lib/iomgr/endpoint_pair_windows.c",
@@ -2368,6 +2383,7 @@ objc_library(
     "src/core/lib/http/httpcli.h",
     "src/core/lib/http/parser.h",
     "src/core/lib/iomgr/closure.h",
+    "src/core/lib/iomgr/combiner.h",
     "src/core/lib/iomgr/endpoint.h",
     "src/core/lib/iomgr/endpoint_pair.h",
     "src/core/lib/iomgr/error.h",

CMakeLists.txt

@@ -191,6 +191,7 @@ add_library(gpr
   src/core/lib/support/log_linux.c
   src/core/lib/support/log_posix.c
   src/core/lib/support/log_windows.c
+  src/core/lib/support/mpscq.c
   src/core/lib/support/murmur_hash.c
   src/core/lib/support/percent_encoding.c
   src/core/lib/support/slice.c
@@ -304,6 +305,7 @@ add_library(grpc
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
   src/core/lib/iomgr/closure.c
+  src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
   src/core/lib/iomgr/endpoint_pair_posix.c
   src/core/lib/iomgr/endpoint_pair_windows.c
@@ -558,6 +560,7 @@ add_library(grpc_cronet
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
   src/core/lib/iomgr/closure.c
+  src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
   src/core/lib/iomgr/endpoint_pair_posix.c
   src/core/lib/iomgr/endpoint_pair_windows.c
@@ -785,6 +788,7 @@ add_library(grpc_unsecure
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
   src/core/lib/iomgr/closure.c
+  src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
   src/core/lib/iomgr/endpoint_pair_posix.c
   src/core/lib/iomgr/endpoint_pair_windows.c
@@ -1038,6 +1042,7 @@ add_library(grpc++
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
   src/core/lib/iomgr/closure.c
+  src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
   src/core/lib/iomgr/endpoint_pair_posix.c
   src/core/lib/iomgr/endpoint_pair_windows.c
@@ -1389,6 +1394,7 @@ add_library(grpc++_unsecure
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
   src/core/lib/iomgr/closure.c
+  src/core/lib/iomgr/combiner.c
   src/core/lib/iomgr/endpoint.c
   src/core/lib/iomgr/endpoint_pair_posix.c
   src/core/lib/iomgr/endpoint_pair_windows.c

Makefile

@@ -915,6 +915,7 @@ chttp2_status_conversion_test: $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test
 chttp2_stream_map_test: $(BINDIR)/$(CONFIG)/chttp2_stream_map_test
 chttp2_varint_test: $(BINDIR)/$(CONFIG)/chttp2_varint_test
 client_fuzzer: $(BINDIR)/$(CONFIG)/client_fuzzer
+combiner_test: $(BINDIR)/$(CONFIG)/combiner_test
 compression_test: $(BINDIR)/$(CONFIG)/compression_test
 concurrent_connectivity_test: $(BINDIR)/$(CONFIG)/concurrent_connectivity_test
 dns_resolver_connectivity_test: $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test
@@ -940,6 +941,7 @@ gpr_env_test: $(BINDIR)/$(CONFIG)/gpr_env_test
 gpr_histogram_test: $(BINDIR)/$(CONFIG)/gpr_histogram_test
 gpr_host_port_test: $(BINDIR)/$(CONFIG)/gpr_host_port_test
 gpr_log_test: $(BINDIR)/$(CONFIG)/gpr_log_test
+gpr_mpscq_test: $(BINDIR)/$(CONFIG)/gpr_mpscq_test
 gpr_percent_encoding_test: $(BINDIR)/$(CONFIG)/gpr_percent_encoding_test
 gpr_slice_buffer_test: $(BINDIR)/$(CONFIG)/gpr_slice_buffer_test
 gpr_slice_test: $(BINDIR)/$(CONFIG)/gpr_slice_test
@@ -1240,6 +1242,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test \
   $(BINDIR)/$(CONFIG)/chttp2_stream_map_test \
   $(BINDIR)/$(CONFIG)/chttp2_varint_test \
+  $(BINDIR)/$(CONFIG)/combiner_test \
   $(BINDIR)/$(CONFIG)/compression_test \
   $(BINDIR)/$(CONFIG)/concurrent_connectivity_test \
   $(BINDIR)/$(CONFIG)/dns_resolver_connectivity_test \
@@ -1262,6 +1265,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/gpr_histogram_test \
   $(BINDIR)/$(CONFIG)/gpr_host_port_test \
   $(BINDIR)/$(CONFIG)/gpr_log_test \
+  $(BINDIR)/$(CONFIG)/gpr_mpscq_test \
   $(BINDIR)/$(CONFIG)/gpr_percent_encoding_test \
   $(BINDIR)/$(CONFIG)/gpr_slice_buffer_test \
   $(BINDIR)/$(CONFIG)/gpr_slice_test \
@@ -1557,6 +1561,8 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/chttp2_stream_map_test || ( echo test chttp2_stream_map_test failed ; exit 1 )
 	$(E) "[RUN]     Testing chttp2_varint_test"
 	$(Q) $(BINDIR)/$(CONFIG)/chttp2_varint_test || ( echo test chttp2_varint_test failed ; exit 1 )
+	$(E) "[RUN]     Testing combiner_test"
+	$(Q) $(BINDIR)/$(CONFIG)/combiner_test || ( echo test combiner_test failed ; exit 1 )
 	$(E) "[RUN]     Testing compression_test"
 	$(Q) $(BINDIR)/$(CONFIG)/compression_test || ( echo test compression_test failed ; exit 1 )
 	$(E) "[RUN]     Testing concurrent_connectivity_test"
@@ -1597,6 +1603,8 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/gpr_host_port_test || ( echo test gpr_host_port_test failed ; exit 1 )
 	$(E) "[RUN]     Testing gpr_log_test"
 	$(Q) $(BINDIR)/$(CONFIG)/gpr_log_test || ( echo test gpr_log_test failed ; exit 1 )
+	$(E) "[RUN]     Testing gpr_mpscq_test"
+	$(Q) $(BINDIR)/$(CONFIG)/gpr_mpscq_test || ( echo test gpr_mpscq_test failed ; exit 1 )
 	$(E) "[RUN]     Testing gpr_percent_encoding_test"
 	$(Q) $(BINDIR)/$(CONFIG)/gpr_percent_encoding_test || ( echo test gpr_percent_encoding_test failed ; exit 1 )
 	$(E) "[RUN]     Testing gpr_slice_buffer_test"
@@ -2378,6 +2386,7 @@ LIBGPR_SRC = \
     src/core/lib/support/log_linux.c \
     src/core/lib/support/log_posix.c \
     src/core/lib/support/log_windows.c \
+    src/core/lib/support/mpscq.c \
     src/core/lib/support/murmur_hash.c \
     src/core/lib/support/percent_encoding.c \
     src/core/lib/support/slice.c \
@@ -2525,6 +2534,7 @@ LIBGRPC_SRC = \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
     src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
     src/core/lib/iomgr/endpoint_pair_posix.c \
     src/core/lib/iomgr/endpoint_pair_windows.c \
@@ -2797,6 +2807,7 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
     src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
     src/core/lib/iomgr/endpoint_pair_posix.c \
     src/core/lib/iomgr/endpoint_pair_windows.c \
@@ -3058,6 +3069,7 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
     src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
     src/core/lib/iomgr/endpoint_pair_posix.c \
     src/core/lib/iomgr/endpoint_pair_windows.c \
@@ -3247,6 +3259,7 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
     src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
     src/core/lib/iomgr/endpoint_pair_posix.c \
     src/core/lib/iomgr/endpoint_pair_windows.c \
@@ -3583,6 +3596,7 @@ LIBGRPC++_SRC = \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
     src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
     src/core/lib/iomgr/endpoint_pair_posix.c \
     src/core/lib/iomgr/endpoint_pair_windows.c \
@@ -4209,6 +4223,7 @@ LIBGRPC++_UNSECURE_SRC = \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
     src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
     src/core/lib/iomgr/endpoint_pair_posix.c \
     src/core/lib/iomgr/endpoint_pair_windows.c \
@@ -7382,6 +7397,38 @@ endif
 endif
 
 
+COMBINER_TEST_SRC = \
+    test/core/iomgr/combiner_test.c \
+
+COMBINER_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(COMBINER_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/combiner_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/combiner_test: $(COMBINER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(COMBINER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/combiner_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/iomgr/combiner_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_combiner_test: $(COMBINER_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(COMBINER_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 COMPRESSION_TEST_SRC = \
     test/core/compression/compression_test.c \
 
@@ -8182,6 +8229,38 @@ endif
 endif
 
 
+GPR_MPSCQ_TEST_SRC = \
+    test/core/support/mpscq_test.c \
+
+GPR_MPSCQ_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GPR_MPSCQ_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/gpr_mpscq_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/gpr_mpscq_test: $(GPR_MPSCQ_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(GPR_MPSCQ_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/gpr_mpscq_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/support/mpscq_test.o:  $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_gpr_mpscq_test: $(GPR_MPSCQ_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(GPR_MPSCQ_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 GPR_PERCENT_ENCODING_TEST_SRC = \
     test/core/support/percent_encoding_test.c \
 

binding.gyp

@@ -515,6 +515,7 @@
         'src/core/lib/support/log_linux.c',
         'src/core/lib/support/log_posix.c',
         'src/core/lib/support/log_windows.c',
+        'src/core/lib/support/mpscq.c',
         'src/core/lib/support/murmur_hash.c',
         'src/core/lib/support/percent_encoding.c',
         'src/core/lib/support/slice.c',
@@ -579,6 +580,7 @@
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
         'src/core/lib/iomgr/closure.c',
+        'src/core/lib/iomgr/combiner.c',
         'src/core/lib/iomgr/endpoint.c',
         'src/core/lib/iomgr/endpoint_pair_posix.c',
         'src/core/lib/iomgr/endpoint_pair_windows.c',

build.yaml

@@ -82,6 +82,7 @@ filegroups:
   - src/core/lib/support/backoff.h
   - src/core/lib/support/block_annotate.h
   - src/core/lib/support/env.h
+  - src/core/lib/support/mpscq.h
   - src/core/lib/support/murmur_hash.h
   - src/core/lib/support/percent_encoding.h
   - src/core/lib/support/stack_lockfree.h
@@ -111,6 +112,7 @@ filegroups:
   - src/core/lib/support/log_linux.c
   - src/core/lib/support/log_posix.c
   - src/core/lib/support/log_windows.c
+  - src/core/lib/support/mpscq.c
   - src/core/lib/support/murmur_hash.c
   - src/core/lib/support/percent_encoding.c
   - src/core/lib/support/slice.c
@@ -178,6 +180,7 @@ filegroups:
   - src/core/lib/http/httpcli.h
   - src/core/lib/http/parser.h
   - src/core/lib/iomgr/closure.h
+  - src/core/lib/iomgr/combiner.h
   - src/core/lib/iomgr/endpoint.h
   - src/core/lib/iomgr/endpoint_pair.h
   - src/core/lib/iomgr/error.h
@@ -258,6 +261,7 @@ filegroups:
   - src/core/lib/http/httpcli.c
   - src/core/lib/http/parser.c
   - src/core/lib/iomgr/closure.c
+  - src/core/lib/iomgr/combiner.c
   - src/core/lib/iomgr/endpoint.c
   - src/core/lib/iomgr/endpoint_pair_posix.c
   - src/core/lib/iomgr/endpoint_pair_windows.c
@@ -1405,6 +1409,17 @@ targets:
   - test/core/end2end/fuzzers/client_fuzzer_corpus
   dict: test/core/end2end/fuzzers/hpack.dictionary
   maxlen: 2048
+- name: combiner_test
+  cpu_cost: 30
+  build: test
+  language: c
+  src:
+  - test/core/iomgr/combiner_test.c
+  deps:
+  - grpc_test_util
+  - grpc
+  - gpr_test_util
+  - gpr
 - name: compression_test
   build: test
   language: c
@@ -1662,6 +1677,15 @@ targets:
   deps:
   - gpr_test_util
   - gpr
+- name: gpr_mpscq_test
+  cpu_cost: 30
+  build: test
+  language: c
+  src:
+  - test/core/support/mpscq_test.c
+  deps:
+  - gpr_test_util
+  - gpr
 - name: gpr_percent_encoding_test
   build: test
   language: c

config.m4

@@ -56,6 +56,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/support/log_linux.c \
     src/core/lib/support/log_posix.c \
     src/core/lib/support/log_windows.c \
+    src/core/lib/support/mpscq.c \
     src/core/lib/support/murmur_hash.c \
     src/core/lib/support/percent_encoding.c \
     src/core/lib/support/slice.c \
@@ -98,6 +99,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
     src/core/lib/iomgr/closure.c \
+    src/core/lib/iomgr/combiner.c \
     src/core/lib/iomgr/endpoint.c \
     src/core/lib/iomgr/endpoint_pair_posix.c \
     src/core/lib/iomgr/endpoint_pair_windows.c \

examples/cpp/helloworld/greeter_async_client.cc

@@ -36,6 +36,7 @@
 #include <string>
 
 #include <grpc++/grpc++.h>
+#include <grpc/support/log.h>
 
 #include "helloworld.grpc.pb.h"
 

examples/cpp/helloworld/greeter_async_client2.cc

@@ -36,6 +36,7 @@
 #include <string>
 
 #include <grpc++/grpc++.h>
+#include <grpc/support/log.h>
 #include <thread>
 
 #include "helloworld.grpc.pb.h"

examples/cpp/helloworld/greeter_async_server.cc

@@ -37,6 +37,7 @@
 #include <thread>
 
 #include <grpc++/grpc++.h>
+#include <grpc/support/log.h>
 
 #include "helloworld.grpc.pb.h"
 

gRPC-Core.podspec

@@ -193,6 +193,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/support/backoff.h',
                       'src/core/lib/support/block_annotate.h',
                       'src/core/lib/support/env.h',
+                      'src/core/lib/support/mpscq.h',
                       'src/core/lib/support/murmur_hash.h',
                       'src/core/lib/support/percent_encoding.h',
                       'src/core/lib/support/stack_lockfree.h',
@@ -221,6 +222,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/support/log_linux.c',
                       'src/core/lib/support/log_posix.c',
                       'src/core/lib/support/log_windows.c',
+                      'src/core/lib/support/mpscq.c',
                       'src/core/lib/support/murmur_hash.c',
                       'src/core/lib/support/percent_encoding.c',
                       'src/core/lib/support/slice.c',
@@ -263,6 +265,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/http/httpcli.h',
                       'src/core/lib/http/parser.h',
                       'src/core/lib/iomgr/closure.h',
+                      'src/core/lib/iomgr/combiner.h',
                       'src/core/lib/iomgr/endpoint.h',
                       'src/core/lib/iomgr/endpoint_pair.h',
                       'src/core/lib/iomgr/error.h',
@@ -422,6 +425,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/http/httpcli.c',
                       'src/core/lib/http/parser.c',
                       'src/core/lib/iomgr/closure.c',
+                      'src/core/lib/iomgr/combiner.c',
                       'src/core/lib/iomgr/endpoint.c',
                       'src/core/lib/iomgr/endpoint_pair_posix.c',
                       'src/core/lib/iomgr/endpoint_pair_windows.c',
@@ -601,6 +605,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/support/backoff.h',
                               'src/core/lib/support/block_annotate.h',
                               'src/core/lib/support/env.h',
+                              'src/core/lib/support/mpscq.h',
                               'src/core/lib/support/murmur_hash.h',
                               'src/core/lib/support/percent_encoding.h',
                               'src/core/lib/support/stack_lockfree.h',
@@ -625,6 +630,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/http/httpcli.h',
                               'src/core/lib/http/parser.h',
                               'src/core/lib/iomgr/closure.h',
+                              'src/core/lib/iomgr/combiner.h',
                               'src/core/lib/iomgr/endpoint.h',
                               'src/core/lib/iomgr/endpoint_pair.h',
                               'src/core/lib/iomgr/error.h',

grpc.gemspec

@@ -87,6 +87,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/support/backoff.h )
   s.files += %w( src/core/lib/support/block_annotate.h )
   s.files += %w( src/core/lib/support/env.h )
+  s.files += %w( src/core/lib/support/mpscq.h )
   s.files += %w( src/core/lib/support/murmur_hash.h )
   s.files += %w( src/core/lib/support/percent_encoding.h )
   s.files += %w( src/core/lib/support/stack_lockfree.h )
@@ -115,6 +116,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/support/log_linux.c )
   s.files += %w( src/core/lib/support/log_posix.c )
   s.files += %w( src/core/lib/support/log_windows.c )
+  s.files += %w( src/core/lib/support/mpscq.c )
   s.files += %w( src/core/lib/support/murmur_hash.c )
   s.files += %w( src/core/lib/support/percent_encoding.c )
   s.files += %w( src/core/lib/support/slice.c )
@@ -183,6 +185,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/http/httpcli.h )
   s.files += %w( src/core/lib/http/parser.h )
   s.files += %w( src/core/lib/iomgr/closure.h )
+  s.files += %w( src/core/lib/iomgr/combiner.h )
   s.files += %w( src/core/lib/iomgr/endpoint.h )
   s.files += %w( src/core/lib/iomgr/endpoint_pair.h )
   s.files += %w( src/core/lib/iomgr/error.h )
@@ -342,6 +345,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/http/httpcli.c )
   s.files += %w( src/core/lib/http/parser.c )
   s.files += %w( src/core/lib/iomgr/closure.c )
+  s.files += %w( src/core/lib/iomgr/combiner.c )
   s.files += %w( src/core/lib/iomgr/endpoint.c )
   s.files += %w( src/core/lib/iomgr/endpoint_pair_posix.c )
   s.files += %w( src/core/lib/iomgr/endpoint_pair_windows.c )

include/grpc/impl/codegen/atm.h

@@ -75,6 +75,9 @@
    int gpr_atm_no_barrier_cas(gpr_atm *p, gpr_atm o, gpr_atm n);
    int gpr_atm_acq_cas(gpr_atm *p, gpr_atm o, gpr_atm n);
    int gpr_atm_rel_cas(gpr_atm *p, gpr_atm o, gpr_atm n);
+
+   // Atomically, set *p=n and return the old value of *p
+   gpr_atm gpr_atm_full_xchg(gpr_atm *p, gpr_atm n);
 */
 
 #include <grpc/impl/codegen/port_platform.h>

include/grpc/impl/codegen/atm_gcc_atomic.h

@@ -69,4 +69,6 @@ static __inline int gpr_atm_rel_cas(gpr_atm *p, gpr_atm o, gpr_atm n) {
                                      __ATOMIC_RELAXED);
 }
 
+#define gpr_atm_full_xchg(p, n) __atomic_exchange_n((p), (n), __ATOMIC_ACQ_REL)
+
 #endif /* GRPC_IMPL_CODEGEN_ATM_GCC_ATOMIC_H */

include/grpc/impl/codegen/atm_gcc_sync.h

@@ -84,4 +84,12 @@ static __inline void gpr_atm_no_barrier_store(gpr_atm *p, gpr_atm value) {
 #define gpr_atm_acq_cas(p, o, n) (__sync_bool_compare_and_swap((p), (o), (n)))
 #define gpr_atm_rel_cas(p, o, n) gpr_atm_acq_cas((p), (o), (n))
 
+static __inline gpr_atm gpr_atm_full_xchg(gpr_atm *p, gpr_atm n) {
+  gpr_atm cur;
+  do {
+    cur = gpr_atm_acq_load(p);
+  } while (!gpr_atm_rel_cas(p, cur, n));
+  return cur;
+}
+
 #endif /* GRPC_IMPL_CODEGEN_ATM_GCC_SYNC_H */

include/grpc/impl/codegen/atm_windows.h

@@ -122,4 +122,8 @@ static __inline gpr_atm gpr_atm_full_fetch_add(gpr_atm *p, gpr_atm delta) {
   return old;
 }
 
+static __inline gpr_atm gpr_atm_full_xchg(gpr_atm *p, gpr_atm n) {
+  return (gpr_atm)InterlockedExchangePointer((PVOID *)p, (PVOID)n);
+}
+
 #endif /* GRPC_IMPL_CODEGEN_ATM_WINDOWS_H */

package.xml

@@ -94,6 +94,7 @@
     <file baseinstalldir="/" name="src/core/lib/support/backoff.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/block_annotate.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/env.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/support/mpscq.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/murmur_hash.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/percent_encoding.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/stack_lockfree.h" role="src" />
@@ -122,6 +123,7 @@
     <file baseinstalldir="/" name="src/core/lib/support/log_linux.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/log_posix.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/log_windows.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/support/mpscq.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/murmur_hash.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/percent_encoding.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/support/slice.c" role="src" />
@@ -190,6 +192,7 @@
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/closure.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/combiner.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint_pair.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/error.h" role="src" />
@@ -349,6 +352,7 @@
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/closure.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/combiner.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint_pair_posix.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/endpoint_pair_windows.c" role="src" />

src/core/ext/client_config/client_channel.c

@@ -387,7 +387,7 @@ typedef struct client_channel_call_data {
   grpc_connected_subchannel *connected_subchannel;
   grpc_polling_entity *pollent;
 
-  grpc_transport_stream_op *waiting_ops;
+  grpc_transport_stream_op **waiting_ops;
   size_t waiting_ops_count;
   size_t waiting_ops_capacity;
 
@@ -406,7 +406,7 @@ static void add_waiting_locked(call_data *calld, grpc_transport_stream_op *op) {
         gpr_realloc(calld->waiting_ops,
                     calld->waiting_ops_capacity * sizeof(*calld->waiting_ops));
   }
-  calld->waiting_ops[calld->waiting_ops_count++] = *op;
+  calld->waiting_ops[calld->waiting_ops_count++] = op;
   GPR_TIMER_END("add_waiting_locked", 0);
 }
 
@@ -415,14 +415,14 @@ static void fail_locked(grpc_exec_ctx *exec_ctx, call_data *calld,
   size_t i;
   for (i = 0; i < calld->waiting_ops_count; i++) {
     grpc_transport_stream_op_finish_with_failure(
-        exec_ctx, &calld->waiting_ops[i], GRPC_ERROR_REF(error));
+        exec_ctx, calld->waiting_ops[i], GRPC_ERROR_REF(error));
   }
   calld->waiting_ops_count = 0;
   GRPC_ERROR_UNREF(error);
 }
 
 typedef struct {
-  grpc_transport_stream_op *ops;
+  grpc_transport_stream_op **ops;
   size_t nops;
   grpc_subchannel_call *call;
 } retry_ops_args;
@@ -431,7 +431,7 @@ static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
   retry_ops_args *a = args;
   size_t i;
   for (i = 0; i < a->nops; i++) {
-    grpc_subchannel_call_process_op(exec_ctx, a->call, &a->ops[i]);
+    grpc_subchannel_call_process_op(exec_ctx, a->call, a->ops[i]);
   }
   GRPC_SUBCHANNEL_CALL_UNREF(exec_ctx, a->call, "retry_ops");
   gpr_free(a->ops);
@@ -439,6 +439,10 @@ static void retry_ops(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
 }
 
 static void retry_waiting_locked(grpc_exec_ctx *exec_ctx, call_data *calld) {
+  if (calld->waiting_ops_count == 0) {
+    return;
+  }
+
   retry_ops_args *a = gpr_malloc(sizeof(*a));
   a->ops = calld->waiting_ops;
   a->nops = calld->waiting_ops_count;

src/core/ext/client_config/subchannel.c

@@ -504,14 +504,13 @@ static void connected_subchannel_state_op(grpc_exec_ctx *exec_ctx,
                                           grpc_pollset_set *interested_parties,
                                           grpc_connectivity_state *state,
                                           grpc_closure *closure) {
-  grpc_transport_op op;
+  grpc_transport_op *op = grpc_make_transport_op(NULL);
   grpc_channel_element *elem;
-  memset(&op, 0, sizeof(op));
-  op.connectivity_state = state;
-  op.on_connectivity_state_change = closure;
-  op.bind_pollset_set = interested_parties;
+  op->connectivity_state = state;
+  op->on_connectivity_state_change = closure;
+  op->bind_pollset_set = interested_parties;
   elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(con), 0);
-  elem->filter->start_transport_op(exec_ctx, elem, &op);
+  elem->filter->start_transport_op(exec_ctx, elem, op);
 }
 
 void grpc_connected_subchannel_notify_on_state_change(
@@ -525,12 +524,11 @@ void grpc_connected_subchannel_notify_on_state_change(
 void grpc_connected_subchannel_ping(grpc_exec_ctx *exec_ctx,
                                     grpc_connected_subchannel *con,
                                     grpc_closure *closure) {
-  grpc_transport_op op;
+  grpc_transport_op *op = grpc_make_transport_op(NULL);
   grpc_channel_element *elem;
-  memset(&op, 0, sizeof(op));
-  op.send_ping = closure;
+  op->send_ping = closure;
   elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CONNECTION(con), 0);
-  elem->filter->start_transport_op(exec_ctx, elem, &op);
+  elem->filter->start_transport_op(exec_ctx, elem, op);
 }
 
 static void publish_transport_locked(grpc_exec_ctx *exec_ctx,

src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c

@@ -74,7 +74,7 @@ const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3] = {
 
 const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3] = {
     PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, grpc_lb_v1_InitialLoadBalanceResponse, load_balancer_delegate, load_balancer_delegate, 0),
-    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v1_Duration_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v1_Duration_fields),
     PB_LAST_FIELD
 };
 

src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h

@@ -146,7 +146,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse {
 #define grpc_lb_v1_Server_load_balance_token_tag 3
 #define grpc_lb_v1_Server_drop_request_tag       4
 #define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 1
-#define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 3
+#define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 2
 #define grpc_lb_v1_LoadBalanceRequest_initial_request_tag 1
 #define grpc_lb_v1_LoadBalanceRequest_client_stats_tag 2
 #define grpc_lb_v1_ServerList_servers_tag        1

src/core/ext/transport/chttp2/transport/internal.h

@@ -48,6 +48,7 @@
 #include "src/core/ext/transport/chttp2/transport/hpack_parser.h"
 #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h"
 #include "src/core/ext/transport/chttp2/transport/stream_map.h"
+#include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/transport_impl.h"
@@ -161,9 +162,20 @@ struct grpc_chttp2_incoming_byte_stream {
   grpc_chttp2_transport *transport;
   grpc_chttp2_stream *stream;
   int is_tail;
+
+  gpr_mu slice_mu;  // protects slices, on_next
   gpr_slice_buffer slices;
   grpc_closure *on_next;
   gpr_slice *next;
+
+  struct {
+    grpc_closure closure;
+    gpr_slice *slice;
+    size_t max_size_hint;
+    grpc_closure *on_complete;
+  } next_action;
+  grpc_closure destroy_action;
+  grpc_closure finished_action;
 };
 
 typedef struct {
@@ -296,23 +308,11 @@ struct grpc_chttp2_transport_parsing {
   int64_t outgoing_window;
 };
 
-typedef void (*grpc_chttp2_locked_action)(grpc_exec_ctx *ctx,
-                                          grpc_chttp2_transport *t,
-                                          grpc_chttp2_stream *s, void *arg);
-
-typedef struct grpc_chttp2_executor_action_header {
-  grpc_chttp2_stream *stream;
-  grpc_chttp2_locked_action action;
-  struct grpc_chttp2_executor_action_header *next;
-  void *arg;
-} grpc_chttp2_executor_action_header;
-
 typedef enum {
+  /** no writing activity allowed */
+  GRPC_CHTTP2_WRITES_CORKED,
   /** no writing activity */
   GRPC_CHTTP2_WRITING_INACTIVE,
-  /** write has been requested, but not scheduled yet */
-  GRPC_CHTTP2_WRITE_REQUESTED_WITH_POLLER,
-  GRPC_CHTTP2_WRITE_REQUESTED_NO_POLLER,
   /** write has been requested and scheduled against the workqueue */
   GRPC_CHTTP2_WRITE_SCHEDULED,
   /** write has been initiated after being reaped from the workqueue */
@@ -333,7 +333,7 @@ struct grpc_chttp2_transport {
   gpr_refcount shutdown_ep_refs;
 
   struct {
-    gpr_mu mu;
+    grpc_combiner *combiner;
 
     /** is a thread currently in the global lock */
     bool global_active;
@@ -341,9 +341,8 @@ struct grpc_chttp2_transport {
     bool parsing_active;
     /** write execution state of the transport */
     grpc_chttp2_write_state write_state;
-
-    grpc_chttp2_executor_action_header *pending_actions_head;
-    grpc_chttp2_executor_action_header *pending_actions_tail;
+    /** has a check_read_ops been scheduled */
+    bool check_read_ops_scheduled;
   } executor;
 
   /** is the transport destroying itself? */
@@ -380,10 +379,16 @@ struct grpc_chttp2_transport {
   grpc_closure writing_action;
   /** closure to start reading from the endpoint */
   grpc_closure reading_action;
+  grpc_closure reading_action_locked;
+  grpc_closure post_parse_locked;
   /** closure to actually do parsing */
   grpc_closure parsing_action;
   /** closure to initiate writing */
   grpc_closure initiate_writing;
+  /** closure to finish writing */
+  grpc_closure terminate_writing;
+  /** closure to flush read state up the stack */
+  grpc_closure initiate_read_flush_locked;
 
   /** incoming read bytes */
   gpr_slice_buffer read_buffer;
@@ -527,11 +532,16 @@ struct grpc_chttp2_stream_parsing {
 };
 
 struct grpc_chttp2_stream {
+  grpc_chttp2_transport *t;
   grpc_stream_refcount *refcount;
   grpc_chttp2_stream_global global;
   grpc_chttp2_stream_writing writing;
   grpc_chttp2_stream_parsing parsing;
 
+  grpc_closure init_stream;
+  grpc_closure destroy_stream;
+  void *destroy_stream_arg;
+
   grpc_chttp2_stream_link links[STREAM_LIST_COUNT];
   uint8_t included[STREAM_LIST_COUNT];
 };
@@ -626,7 +636,7 @@ int grpc_chttp2_list_pop_waiting_for_concurrency(
     grpc_chttp2_stream_global **stream_global);
 
 void grpc_chttp2_list_add_check_read_ops(
-    grpc_chttp2_transport_global *transport_global,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global);
 bool grpc_chttp2_list_remove_check_read_ops(
     grpc_chttp2_transport_global *transport_global,
@@ -706,12 +716,6 @@ void grpc_chttp2_complete_closure_step(
     grpc_chttp2_stream_global *stream_global, grpc_closure **pclosure,
     grpc_error *error);
 
-void grpc_chttp2_run_with_global_lock(grpc_exec_ctx *exec_ctx,
-                                      grpc_chttp2_transport *transport,
-                                      grpc_chttp2_stream *optional_stream,
-                                      grpc_chttp2_locked_action action,
-                                      void *arg, size_t sizeof_arg);
-
 #define GRPC_CHTTP2_CLIENT_CONNECT_STRING "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
 #define GRPC_CHTTP2_CLIENT_CONNECT_STRLEN \
   (sizeof(GRPC_CHTTP2_CLIENT_CONNECT_STRING) - 1)

src/core/ext/transport/chttp2/transport/parsing.c

@@ -177,7 +177,8 @@ void grpc_chttp2_publish_reads(
       stream_global->seen_error = true;
       stream_global->exceeded_metadata_size =
           stream_parsing->exceeded_metadata_size;
-      grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
+      grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global,
+                                          stream_global);
     }
 
     /* flush stats to global stream state */
@@ -203,7 +204,8 @@ void grpc_chttp2_publish_reads(
       stream_global->incoming_frames.tail->is_tail = 0;
     }
     if (stream_parsing->data_parser.incoming_frames.head != NULL) {
-      grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
+      grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global,
+                                          stream_global);
     }
     grpc_chttp2_incoming_frame_queue_merge(
         &stream_global->incoming_frames,
@@ -219,7 +221,8 @@ void grpc_chttp2_publish_reads(
       GPR_SWAP(grpc_chttp2_incoming_metadata_buffer,
                stream_parsing->metadata_buffer[0],
                stream_global->received_initial_metadata);
-      grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
+      grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global,
+                                          stream_global);
     }
     if (!stream_global->published_trailing_metadata &&
         stream_parsing->got_metadata_on_parse[1]) {
@@ -228,7 +231,8 @@ void grpc_chttp2_publish_reads(
       GPR_SWAP(grpc_chttp2_incoming_metadata_buffer,
                stream_parsing->metadata_buffer[1],
                stream_global->received_trailing_metadata);
-      grpc_chttp2_list_add_check_read_ops(transport_global, stream_global);
+      grpc_chttp2_list_add_check_read_ops(exec_ctx, transport_global,
+                                          stream_global);
     }
 
     if (stream_parsing->forced_close_error != GRPC_ERROR_NONE) {

src/core/ext/transport/chttp2/transport/stream_lists.c

@@ -298,8 +298,15 @@ int grpc_chttp2_list_pop_waiting_for_concurrency(
 }
 
 void grpc_chttp2_list_add_check_read_ops(
-    grpc_chttp2_transport_global *transport_global,
+    grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global) {
+  grpc_chttp2_transport *t = TRANSPORT_FROM_GLOBAL(transport_global);
+  if (!t->executor.check_read_ops_scheduled) {
+    grpc_combiner_execute_finally(exec_ctx, t->executor.combiner,
+                                  &t->initiate_read_flush_locked,
+                                  GRPC_ERROR_NONE, false);
+    t->executor.check_read_ops_scheduled = true;
+  }
   stream_list_add(TRANSPORT_FROM_GLOBAL(transport_global),
                   STREAM_FROM_GLOBAL(stream_global),
                   GRPC_CHTTP2_LIST_CHECK_READ_OPS);

src/core/ext/transport/chttp2/transport/writing.c

@@ -55,15 +55,6 @@ int grpc_chttp2_unlocking_check_writes(
       transport_global->settings[GRPC_ACKED_SETTINGS]
                                 [GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE];
 
-  /* simple writes are queued to qbuf, and flushed here */
-  gpr_slice_buffer_swap(&transport_global->qbuf, &transport_writing->outbuf);
-  GPR_ASSERT(transport_global->qbuf.count == 0);
-
-  grpc_chttp2_hpack_compressor_set_max_table_size(
-      &transport_writing->hpack_compressor,
-      transport_global->settings[GRPC_PEER_SETTINGS]
-                                [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]);
-
   if (transport_global->dirtied_local_settings &&
       !transport_global->sent_local_settings) {
     gpr_slice_buffer_add(
@@ -77,6 +68,16 @@ int grpc_chttp2_unlocking_check_writes(
     transport_global->sent_local_settings = 1;
   }
 
+  /* simple writes are queued to qbuf, and flushed here */
+  gpr_slice_buffer_move_into(&transport_global->qbuf,
+                             &transport_writing->outbuf);
+  GPR_ASSERT(transport_global->qbuf.count == 0);
+
+  grpc_chttp2_hpack_compressor_set_max_table_size(
+      &transport_writing->hpack_compressor,
+      transport_global->settings[GRPC_PEER_SETTINGS]
+                                [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]);
+
   GRPC_CHTTP2_FLOW_MOVE_TRANSPORT("write", transport_writing, outgoing_window,
                                   transport_global, outgoing_window);
   if (transport_writing->outgoing_window > 0) {
@@ -344,6 +345,7 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx,
 void grpc_chttp2_cleanup_writing(
     grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_transport_writing *transport_writing) {
+  GPR_TIMER_BEGIN("grpc_chttp2_cleanup_writing", 0);
   grpc_chttp2_stream_writing *stream_writing;
   grpc_chttp2_stream_global *stream_global;
 
@@ -382,4 +384,5 @@ void grpc_chttp2_cleanup_writing(
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, stream_global, "chttp2_writing");
   }
   gpr_slice_buffer_reset_and_unref(&transport_writing->outbuf);
+  GPR_TIMER_END("grpc_chttp2_cleanup_writing", 0);
 }

src/core/lib/channel/channel_stack.c

@@ -32,6 +32,7 @@
  */
 
 #include "src/core/lib/channel/channel_stack.h"
+#include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 
 #include <stdlib.h>
@@ -270,21 +271,27 @@ grpc_call_stack *grpc_call_stack_from_top_element(grpc_call_element *elem) {
       sizeof(grpc_call_stack)));
 }
 
+static void destroy_op(grpc_exec_ctx *exec_ctx, void *op, grpc_error *error) {
+  gpr_free(op);
+}
+
 void grpc_call_element_send_cancel(grpc_exec_ctx *exec_ctx,
                                    grpc_call_element *cur_elem) {
-  grpc_transport_stream_op op;
-  memset(&op, 0, sizeof(op));
-  op.cancel_error = GRPC_ERROR_CANCELLED;
-  grpc_call_next_op(exec_ctx, cur_elem, &op);
+  grpc_transport_stream_op *op = gpr_malloc(sizeof(*op));
+  memset(op, 0, sizeof(*op));
+  op->cancel_error = GRPC_ERROR_CANCELLED;
+  op->on_complete = grpc_closure_create(destroy_op, op);
+  grpc_call_next_op(exec_ctx, cur_elem, op);
 }
 
 void grpc_call_element_send_cancel_with_message(grpc_exec_ctx *exec_ctx,
                                                 grpc_call_element *cur_elem,
                                                 grpc_status_code status,
                                                 gpr_slice *optional_message) {
-  grpc_transport_stream_op op;
-  memset(&op, 0, sizeof(op));
-  grpc_transport_stream_op_add_cancellation_with_message(&op, status,
+  grpc_transport_stream_op *op = gpr_malloc(sizeof(*op));
+  memset(op, 0, sizeof(*op));
+  op->on_complete = grpc_closure_create(destroy_op, op);
+  grpc_transport_stream_op_add_cancellation_with_message(op, status,
                                                          optional_message);
-  grpc_call_next_op(exec_ctx, cur_elem, &op);
+  grpc_call_next_op(exec_ctx, cur_elem, op);
 }

src/core/lib/channel/compress_filter.c

@@ -60,7 +60,7 @@ typedef struct call_data {
   /** If true, contents of \a compression_algorithm are authoritative */
   int has_compression_algorithm;
 
-  grpc_transport_stream_op send_op;
+  grpc_transport_stream_op *send_op;
   uint32_t send_length;
   uint32_t send_flags;
   gpr_slice incoming_slice;
@@ -199,11 +199,11 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx,
 
   grpc_slice_buffer_stream_init(&calld->replacement_stream, &calld->slices,
                                 calld->send_flags);
-  calld->send_op.send_message = &calld->replacement_stream.base;
-  calld->post_send = calld->send_op.on_complete;
-  calld->send_op.on_complete = &calld->send_done;
+  calld->send_op->send_message = &calld->replacement_stream.base;
+  calld->post_send = calld->send_op->on_complete;
+  calld->send_op->on_complete = &calld->send_done;
 
-  grpc_call_next_op(exec_ctx, elem, &calld->send_op);
+  grpc_call_next_op(exec_ctx, elem, calld->send_op);
 }
 
 static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
@@ -220,7 +220,7 @@ static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) {
 static void continue_send_message(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem) {
   call_data *calld = elem->call_data;
-  while (grpc_byte_stream_next(exec_ctx, calld->send_op.send_message,
+  while (grpc_byte_stream_next(exec_ctx, calld->send_op->send_message,
                                &calld->incoming_slice, ~(size_t)0,
                                &calld->got_slice)) {
     gpr_slice_buffer_add(&calld->slices, calld->incoming_slice);
@@ -243,7 +243,7 @@ static void compress_start_transport_stream_op(grpc_exec_ctx *exec_ctx,
   }
   if (op->send_message != NULL && !skip_compression(elem) &&
       0 == (op->send_message->flags & GRPC_WRITE_NO_COMPRESS)) {
-    calld->send_op = *op;
+    calld->send_op = op;
     calld->send_length = op->send_message->length;
     calld->send_flags = op->send_message->flags;
     continue_send_message(exec_ctx, elem);

src/core/lib/iomgr/closure.c

@@ -41,6 +41,10 @@ void grpc_closure_init(grpc_closure *closure, grpc_iomgr_cb_func cb,
   closure->cb_arg = cb_arg;
 }
 
+void grpc_closure_list_init(grpc_closure_list *closure_list) {
+  closure_list->head = closure_list->tail = NULL;
+}
+
 void grpc_closure_list_append(grpc_closure_list *closure_list,
                               grpc_closure *closure, grpc_error *error) {
   if (closure == NULL) {

src/core/lib/iomgr/closure.h

@@ -37,6 +37,7 @@
 #include <grpc/support/port_platform.h>
 #include <stdbool.h>
 #include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/support/mpscq.h"
 
 struct grpc_closure;
 typedef struct grpc_closure grpc_closure;
@@ -60,6 +61,14 @@ typedef void (*grpc_iomgr_cb_func)(grpc_exec_ctx *exec_ctx, void *arg,
 
 /** A closure over a grpc_iomgr_cb_func. */
 struct grpc_closure {
+  /** Once queued, next indicates the next queued closure; before then, scratch
+   *  space */
+  union {
+    grpc_closure *next;
+    gpr_mpscq_node atm_next;
+    uintptr_t scratch;
+  } next_data;
+
   /** Bound callback. */
   grpc_iomgr_cb_func cb;
 
@@ -68,13 +77,6 @@ struct grpc_closure {
 
   /** Once queued, the result of the closure. Before then: scratch space */
   grpc_error *error;
-
-  /** Once queued, next indicates the next queued closure; before then, scratch
-   *  space */
-  union {
-    grpc_closure *next;
-    uintptr_t scratch;
-  } next_data;
 };
 
 /** Initializes \a closure with \a cb and \a cb_arg. */
@@ -87,6 +89,8 @@ grpc_closure *grpc_closure_create(grpc_iomgr_cb_func cb, void *cb_arg);
 #define GRPC_CLOSURE_LIST_INIT \
   { NULL, NULL }
 
+void grpc_closure_list_init(grpc_closure_list *list);
+
 /** add \a closure to the end of \a list
     and set \a closure's result to \a error */
 void grpc_closure_list_append(grpc_closure_list *list, grpc_closure *closure,

src/core/lib/iomgr/combiner.c

@@ -0,0 +1,293 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/combiner.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+#include "src/core/lib/iomgr/workqueue.h"
+#include "src/core/lib/profiling/timers.h"
+
+int grpc_combiner_trace = 0;
+
+#define GRPC_COMBINER_TRACE(fn) \
+  do {                          \
+    if (grpc_combiner_trace) {  \
+      fn;                       \
+    }                           \
+  } while (0)
+
+struct grpc_combiner {
+  grpc_workqueue *optional_workqueue;
+  gpr_mpscq queue;
+  // state is:
+  // lower bit - zero if orphaned
+  // other bits - number of items queued on the lock
+  gpr_atm state;
+  bool take_async_break_before_final_list;
+  grpc_closure_list final_list;
+  grpc_closure continue_finishing;
+};
+
+grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue) {
+  grpc_combiner *lock = gpr_malloc(sizeof(*lock));
+  lock->optional_workqueue = optional_workqueue;
+  gpr_atm_no_barrier_store(&lock->state, 1);
+  gpr_mpscq_init(&lock->queue);
+  lock->take_async_break_before_final_list = false;
+  grpc_closure_list_init(&lock->final_list);
+  GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p create", lock));
+  return lock;
+}
+
+static void really_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG, "C:%p really_destroy", lock));
+  GPR_ASSERT(gpr_atm_no_barrier_load(&lock->state) == 0);
+  gpr_mpscq_destroy(&lock->queue);
+  GRPC_WORKQUEUE_UNREF(exec_ctx, lock->optional_workqueue, "combiner");
+  gpr_free(lock);
+}
+
+void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  gpr_atm old_state = gpr_atm_full_fetch_add(&lock->state, -1);
+  GRPC_COMBINER_TRACE(gpr_log(
+      GPR_DEBUG, "C:%p really_destroy old_state=%" PRIdPTR, lock, old_state));
+  if (old_state == 1) {
+    really_destroy(exec_ctx, lock);
+  }
+}
+
+static bool maybe_finish_one(grpc_exec_ctx *exec_ctx, grpc_combiner *lock);
+static void finish(grpc_exec_ctx *exec_ctx, grpc_combiner *lock);
+
+static void continue_finishing_mainline(grpc_exec_ctx *exec_ctx, void *arg,
+                                        grpc_error *error) {
+  GPR_TIMER_BEGIN("combiner.continue_executing_mainline", 0);
+  grpc_combiner *lock = arg;
+  GRPC_COMBINER_TRACE(
+      gpr_log(GPR_DEBUG, "C:%p continue_finishing_mainline", lock));
+  GPR_ASSERT(exec_ctx->active_combiner == NULL);
+  exec_ctx->active_combiner = lock;
+  if (maybe_finish_one(exec_ctx, lock)) finish(exec_ctx, lock);
+  GPR_ASSERT(exec_ctx->active_combiner == lock);
+  exec_ctx->active_combiner = NULL;
+  GPR_TIMER_END("combiner.continue_executing_mainline", 0);
+}
+
+static void execute_final(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  GPR_TIMER_BEGIN("combiner.execute_final", 0);
+  grpc_closure *c = lock->final_list.head;
+  GPR_ASSERT(c != NULL);
+  grpc_closure_list_init(&lock->final_list);
+  lock->take_async_break_before_final_list = false;
+  int loops = 0;
+  while (c != NULL) {
+    GRPC_COMBINER_TRACE(
+        gpr_log(GPR_DEBUG, "C:%p execute_final[%d] c=%p", lock, loops, c));
+    grpc_closure *next = c->next_data.next;
+    grpc_error *error = c->error;
+    c->cb(exec_ctx, c->cb_arg, error);
+    GRPC_ERROR_UNREF(error);
+    c = next;
+    loops++;
+  }
+  GPR_TIMER_END("combiner.execute_final", 0);
+}
+
+static void continue_executing_final(grpc_exec_ctx *exec_ctx, void *arg,
+                                     grpc_error *error) {
+  GPR_TIMER_BEGIN("combiner.continue_executing_final", 0);
+  grpc_combiner *lock = arg;
+  GRPC_COMBINER_TRACE(
+      gpr_log(GPR_DEBUG, "C:%p continue_executing_final", lock));
+  GPR_ASSERT(exec_ctx->active_combiner == NULL);
+  exec_ctx->active_combiner = lock;
+  // quick peek to see if new things have turned up on the queue: if so, go back
+  // to executing them before the final list
+  if ((gpr_atm_acq_load(&lock->state) >> 1) > 1) {
+    if (maybe_finish_one(exec_ctx, lock)) finish(exec_ctx, lock);
+  } else {
+    execute_final(exec_ctx, lock);
+    finish(exec_ctx, lock);
+  }
+  GPR_ASSERT(exec_ctx->active_combiner == lock);
+  exec_ctx->active_combiner = NULL;
+  GPR_TIMER_END("combiner.continue_executing_final", 0);
+}
+
+static bool start_execute_final(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  GPR_TIMER_BEGIN("combiner.start_execute_final", 0);
+  GPR_ASSERT(exec_ctx->active_combiner == lock);
+  GRPC_COMBINER_TRACE(
+      gpr_log(GPR_DEBUG,
+              "C:%p start_execute_final take_async_break_before_final_list=%d",
+              lock, lock->take_async_break_before_final_list));
+  if (lock->take_async_break_before_final_list) {
+    grpc_closure_init(&lock->continue_finishing, continue_executing_final,
+                      lock);
+    grpc_exec_ctx_sched(exec_ctx, &lock->continue_finishing, GRPC_ERROR_NONE,
+                        GRPC_WORKQUEUE_REF(lock->optional_workqueue, "sched"));
+    GPR_TIMER_END("combiner.start_execute_final", 0);
+    return false;
+  } else {
+    execute_final(exec_ctx, lock);
+    GPR_TIMER_END("combiner.start_execute_final", 0);
+    return true;
+  }
+}
+
+static bool maybe_finish_one(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  GPR_TIMER_BEGIN("combiner.maybe_finish_one", 0);
+  gpr_mpscq_node *n = gpr_mpscq_pop(&lock->queue);
+  GRPC_COMBINER_TRACE(
+      gpr_log(GPR_DEBUG, "C:%p maybe_finish_one n=%p", lock, n));
+  GPR_ASSERT(exec_ctx->active_combiner == lock);
+  if (n == NULL) {
+    // Queue is in an transiently inconsistent state: a new item is being queued
+    // but is not visible to this thread yet.
+    // Use this as a cue that we should go off and do something else for a while
+    // (and come back later)
+    grpc_closure_init(&lock->continue_finishing, continue_finishing_mainline,
+                      lock);
+    grpc_exec_ctx_sched(exec_ctx, &lock->continue_finishing, GRPC_ERROR_NONE,
+                        GRPC_WORKQUEUE_REF(lock->optional_workqueue, "sched"));
+    GPR_TIMER_END("combiner.maybe_finish_one", 0);
+    return false;
+  }
+  grpc_closure *cl = (grpc_closure *)n;
+  grpc_error *error = cl->error;
+  cl->cb(exec_ctx, cl->cb_arg, error);
+  GRPC_ERROR_UNREF(error);
+  GPR_TIMER_END("combiner.maybe_finish_one", 0);
+  return true;
+}
+
+static void finish(grpc_exec_ctx *exec_ctx, grpc_combiner *lock) {
+  bool (*executor)(grpc_exec_ctx * exec_ctx, grpc_combiner * lock);
+  GPR_TIMER_BEGIN("combiner.finish", 0);
+  int loops = 0;
+  do {
+    executor = maybe_finish_one;
+    gpr_atm old_state = gpr_atm_full_fetch_add(&lock->state, -2);
+    GRPC_COMBINER_TRACE(gpr_log(GPR_DEBUG,
+                                "C:%p finish[%d] old_state=%" PRIdPTR, lock,
+                                loops, old_state));
+    switch (old_state) {
+      default:
+        // we have multiple queued work items: just continue executing them
+        break;
+      case 5:  // we're down to one queued item: if it's the final list we
+      case 4:  // should do that
+        if (!grpc_closure_list_empty(lock->final_list)) {
+          executor = start_execute_final;
+        }
+        break;
+      case 3:  // had one count, one unorphaned --> unlocked unorphaned
+        GPR_TIMER_END("combiner.finish", 0);
+        return;
+      case 2:  // and one count, one orphaned --> unlocked and orphaned
+        really_destroy(exec_ctx, lock);
+        GPR_TIMER_END("combiner.finish", 0);
+        return;
+      case 1:
+      case 0:
+        // these values are illegal - representing an already unlocked or
+        // deleted lock
+        GPR_UNREACHABLE_CODE(return );
+    }
+    loops++;
+  } while (executor(exec_ctx, lock));
+  GPR_TIMER_END("combiner.finish", 0);
+}
+
+void grpc_combiner_execute(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
+                           grpc_closure *cl, grpc_error *error) {
+  GRPC_COMBINER_TRACE(
+      gpr_log(GPR_DEBUG, "C:%p grpc_combiner_execute c=%p", lock, cl));
+  GPR_TIMER_BEGIN("combiner.execute", 0);
+  gpr_atm last = gpr_atm_full_fetch_add(&lock->state, 2);
+  GPR_ASSERT(last & 1);  // ensure lock has not been destroyed
+  if (last == 1) {
+    exec_ctx->active_combiner = lock;
+    GPR_TIMER_BEGIN("combiner.execute_first_cb", 0);
+    cl->cb(exec_ctx, cl->cb_arg, error);
+    GPR_TIMER_END("combiner.execute_first_cb", 0);
+    GRPC_ERROR_UNREF(error);
+    finish(exec_ctx, lock);
+    GPR_ASSERT(exec_ctx->active_combiner == lock);
+    exec_ctx->active_combiner = NULL;
+  } else {
+    cl->error = error;
+    gpr_mpscq_push(&lock->queue, &cl->next_data.atm_next);
+  }
+  GPR_TIMER_END("combiner.execute", 0);
+}
+
+static void enqueue_finally(grpc_exec_ctx *exec_ctx, void *closure,
+                            grpc_error *error) {
+  grpc_combiner_execute_finally(exec_ctx, exec_ctx->active_combiner, closure,
+                                GRPC_ERROR_REF(error), false);
+}
+
+void grpc_combiner_execute_finally(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
+                                   grpc_closure *closure, grpc_error *error,
+                                   bool force_async_break) {
+  GRPC_COMBINER_TRACE(gpr_log(
+      GPR_DEBUG,
+      "C:%p grpc_combiner_execute_finally c=%p force_async_break=%d; ac=%p",
+      lock, closure, force_async_break, exec_ctx->active_combiner));
+  GPR_TIMER_BEGIN("combiner.execute_finally", 0);
+  if (exec_ctx->active_combiner != lock) {
+    GPR_TIMER_MARK("slowpath", 0);
+    grpc_combiner_execute(exec_ctx, lock,
+                          grpc_closure_create(enqueue_finally, closure), error);
+    GPR_TIMER_END("combiner.execute_finally", 0);
+    return;
+  }
+
+  if (force_async_break) {
+    lock->take_async_break_before_final_list = true;
+  }
+  if (grpc_closure_list_empty(lock->final_list)) {
+    gpr_atm_full_fetch_add(&lock->state, 2);
+  }
+  grpc_closure_list_append(&lock->final_list, closure, error);
+  GPR_TIMER_END("combiner.execute_finally", 0);
+}
+
+void grpc_combiner_force_async_finally(grpc_combiner *lock) {
+  lock->take_async_break_before_final_list = true;
+}

src/core/lib/iomgr/combiner.h

@@ -0,0 +1,71 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_COMBINER_H
+#define GRPC_CORE_LIB_IOMGR_COMBINER_H
+
+#include <stddef.h>
+
+#include <grpc/support/atm.h>
+#include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/support/mpscq.h"
+
+// Provides serialized access to some resource.
+// Each action queued on a combiner is executed serially in a borrowed thread.
+// The actual thread executing actions may change over time (but there will only
+// every be one at a time).
+
+// Initialize the lock, with an optional workqueue to shift load to when
+// necessary
+grpc_combiner *grpc_combiner_create(grpc_workqueue *optional_workqueue);
+// Destroy the lock
+void grpc_combiner_destroy(grpc_exec_ctx *exec_ctx, grpc_combiner *lock);
+// Execute \a action within the lock.
+void grpc_combiner_execute(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
+                           grpc_closure *closure, grpc_error *error);
+// Execute \a action within the lock just prior to unlocking.
+// if \a hint_async_break is true, the combiner tries to hand execution to
+// another thread before finishing the primary queue of combined closures and
+// executing the finally list.
+// Deprecation warning: \a hint_async_break will be removed in a future version
+// Takes a very slow and round-about path if not called from a
+// grpc_combiner_execute closure.
+void grpc_combiner_execute_finally(grpc_exec_ctx *exec_ctx, grpc_combiner *lock,
+                                   grpc_closure *closure, grpc_error *error,
+                                   bool hint_async_break);
+// Deprecated: force the finally list execution onto another thread
+void grpc_combiner_force_async_finally(grpc_combiner *lock);
+
+extern int grpc_combiner_trace;
+
+#endif /* GRPC_CORE_LIB_IOMGR_COMBINER_H */

src/core/lib/iomgr/error.c

@@ -332,7 +332,7 @@ grpc_error *grpc_error_add_child(grpc_error *src, grpc_error *child) {
   return new;
 }
 
-static const char *no_error_string = "null";
+static const char *no_error_string = "\"No Error\"";
 static const char *oom_error_string = "\"Out of memory\"";
 static const char *cancelled_error_string = "\"Cancelled\"";
 

src/core/lib/iomgr/ev_epoll_linux.c

@@ -1531,6 +1531,8 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
 static void pollset_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
                            grpc_fd *fd) {
+  GPR_TIMER_BEGIN("pollset_add_fd", 0);
+
   grpc_error *error = GRPC_ERROR_NONE;
 
   gpr_mu_lock(&pollset->mu);
@@ -1643,6 +1645,8 @@ retry:
   gpr_mu_unlock(&pollset->mu);
 
   GRPC_LOG_IF_ERROR("pollset_add_fd", error);
+
+  GPR_TIMER_END("pollset_add_fd", 0);
 }
 
 /*******************************************************************************

src/core/lib/iomgr/exec_ctx.h

@@ -40,8 +40,8 @@
 
 /** A workqueue represents a list of work to be executed asynchronously.
     Forward declared here to avoid a circular dependency with workqueue.h. */
-struct grpc_workqueue;
 typedef struct grpc_workqueue grpc_workqueue;
+typedef struct grpc_combiner grpc_combiner;
 
 #ifndef GRPC_EXECUTION_CONTEXT_SANITIZER
 /** Execution context.
@@ -66,13 +66,15 @@ typedef struct grpc_workqueue grpc_workqueue;
  */
 struct grpc_exec_ctx {
   grpc_closure_list closure_list;
+  /** currently active combiner: updated only via combiner.c */
+  grpc_combiner *active_combiner;
   bool cached_ready_to_finish;
   void *check_ready_to_finish_arg;
   bool (*check_ready_to_finish)(grpc_exec_ctx *exec_ctx, void *arg);
 };
 
 #define GRPC_EXEC_CTX_INIT_WITH_FINISH_CHECK(finish_check, finish_check_arg) \
-  { GRPC_CLOSURE_LIST_INIT, false, finish_check_arg, finish_check }
+  { GRPC_CLOSURE_LIST_INIT, NULL, false, finish_check_arg, finish_check }
 #else
 struct grpc_exec_ctx {
   bool cached_ready_to_finish;

src/core/lib/iomgr/tcp_posix.c

@@ -379,10 +379,19 @@ static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
   }
 
   if (!tcp_flush(tcp, &error)) {
+    if (grpc_tcp_trace) {
+      gpr_log(GPR_DEBUG, "write: delayed");
+    }
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
     cb = tcp->write_cb;
     tcp->write_cb = NULL;
+    if (grpc_tcp_trace) {
+      const char *str = grpc_error_string(error);
+      gpr_log(GPR_DEBUG, "write: %s", str);
+      grpc_error_free_string(str);
+    }
+
     GPR_TIMER_BEGIN("tcp_handle_write.cb", 0);
     cb->cb(exec_ctx, cb->cb_arg, error);
     GPR_TIMER_END("tcp_handle_write.cb", 0);
@@ -425,8 +434,16 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   if (!tcp_flush(tcp, &error)) {
     TCP_REF(tcp, "write");
     tcp->write_cb = cb;
+    if (grpc_tcp_trace) {
+      gpr_log(GPR_DEBUG, "write: delayed");
+    }
     grpc_fd_notify_on_write(exec_ctx, tcp->em_fd, &tcp->write_closure);
   } else {
+    if (grpc_tcp_trace) {
+      const char *str = grpc_error_string(error);
+      gpr_log(GPR_DEBUG, "write: %s", str);
+      grpc_error_free_string(str);
+    }
     grpc_exec_ctx_sched(exec_ctx, cb, error, NULL);
   }
 

src/core/lib/iomgr/workqueue.h

@@ -50,10 +50,6 @@
 
 /* grpc_workqueue is forward declared in exec_ctx.h */
 
-/* Deprecated: do not use.
-   This has *already* been removed in a future commit. */
-void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue);
-
 /* Reference counting functions. Use the macro's always
    (GRPC_WORKQUEUE_{REF,UNREF}).
 

src/core/lib/iomgr/workqueue_posix.c

@@ -44,6 +44,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/profiling/timers.h"
 
 static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error);
 
@@ -52,8 +53,7 @@ grpc_error *grpc_workqueue_create(grpc_exec_ctx *exec_ctx,
   char name[32];
   *workqueue = gpr_malloc(sizeof(grpc_workqueue));
   gpr_ref_init(&(*workqueue)->refs, 1);
-  gpr_mu_init(&(*workqueue)->mu);
-  (*workqueue)->closure_list.head = (*workqueue)->closure_list.tail = NULL;
+  gpr_atm_no_barrier_store(&(*workqueue)->state, 1);
   grpc_error *err = grpc_wakeup_fd_init(&(*workqueue)->wakeup_fd);
   if (err != GRPC_ERROR_NONE) {
     gpr_free(*workqueue);
@@ -62,6 +62,7 @@ grpc_error *grpc_workqueue_create(grpc_exec_ctx *exec_ctx,
   sprintf(name, "workqueue:%p", (void *)(*workqueue));
   (*workqueue)->wakeup_read_fd = grpc_fd_create(
       GRPC_WAKEUP_FD_GET_READ_FD(&(*workqueue)->wakeup_fd), name);
+  gpr_mpscq_init(&(*workqueue)->queue);
   grpc_closure_init(&(*workqueue)->read_closure, on_readable, *workqueue);
   grpc_fd_notify_on_read(exec_ctx, (*workqueue)->wakeup_read_fd,
                          &(*workqueue)->read_closure);
@@ -70,57 +71,79 @@ grpc_error *grpc_workqueue_create(grpc_exec_ctx *exec_ctx,
 
 static void workqueue_destroy(grpc_exec_ctx *exec_ctx,
                               grpc_workqueue *workqueue) {
-  grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
   grpc_fd_shutdown(exec_ctx, workqueue->wakeup_read_fd);
 }
 
+static void workqueue_orphan(grpc_exec_ctx *exec_ctx,
+                             grpc_workqueue *workqueue) {
+  if (gpr_atm_full_fetch_add(&workqueue->state, -1) == 1) {
+    workqueue_destroy(exec_ctx, workqueue);
+  }
+}
+
 #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG
 void grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file, int line,
                         const char *reason) {
+  if (workqueue == NULL) return;
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "WORKQUEUE:%p   ref %d -> %d %s",
           workqueue, (int)workqueue->refs.count, (int)workqueue->refs.count + 1,
           reason);
+  gpr_ref(&workqueue->refs);
+}
 #else
 void grpc_workqueue_ref(grpc_workqueue *workqueue) {
-#endif
+  if (workqueue == NULL) return;
   gpr_ref(&workqueue->refs);
 }
+#endif
 
 #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG
 void grpc_workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
                           const char *file, int line, const char *reason) {
+  if (workqueue == NULL) return;
   gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "WORKQUEUE:%p unref %d -> %d %s",
           workqueue, (int)workqueue->refs.count, (int)workqueue->refs.count - 1,
           reason);
+  if (gpr_unref(&workqueue->refs)) {
+    workqueue_orphan(exec_ctx, workqueue);
+  }
+}
 #else
 void grpc_workqueue_unref(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
-#endif
+  if (workqueue == NULL) return;
   if (gpr_unref(&workqueue->refs)) {
-    workqueue_destroy(exec_ctx, workqueue);
+    workqueue_orphan(exec_ctx, workqueue);
   }
 }
+#endif
+
+static void drain(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
+  abort();
+}
 
-void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
-  gpr_mu_lock(&workqueue->mu);
-  grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
-  gpr_mu_unlock(&workqueue->mu);
+static void wakeup(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {
+  GPR_TIMER_MARK("workqueue.wakeup", 0);
+  grpc_error *err = grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
+  if (!GRPC_LOG_IF_ERROR("wakeupfd_wakeup", err)) {
+    drain(exec_ctx, workqueue);
+  }
 }
 
 static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  GPR_TIMER_BEGIN("workqueue.on_readable", 0);
+
   grpc_workqueue *workqueue = arg;
 
   if (error != GRPC_ERROR_NONE) {
-    gpr_mu_destroy(&workqueue->mu);
     /* HACK: let wakeup_fd code know that we stole the fd */
     workqueue->wakeup_fd.read_fd = 0;
     grpc_wakeup_fd_destroy(&workqueue->wakeup_fd);
     grpc_fd_orphan(exec_ctx, workqueue->wakeup_read_fd, NULL, NULL, "destroy");
+    GPR_ASSERT(gpr_atm_no_barrier_load(&workqueue->state) == 0);
     gpr_free(workqueue);
   } else {
-    gpr_mu_lock(&workqueue->mu);
-    grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
     error = grpc_wakeup_fd_consume_wakeup(&workqueue->wakeup_fd);
-    gpr_mu_unlock(&workqueue->mu);
+    gpr_mpscq_node *n = gpr_mpscq_pop(&workqueue->queue);
     if (error == GRPC_ERROR_NONE) {
       grpc_fd_notify_on_read(exec_ctx, workqueue->wakeup_read_fd,
                              &workqueue->read_closure);
@@ -128,24 +151,46 @@ static void on_readable(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
       /* recurse to get error handling */
       on_readable(exec_ctx, arg, error);
     }
+    if (n == NULL) {
+      /* try again - queue in an inconsistant state */
+      wakeup(exec_ctx, workqueue);
+    } else {
+      switch (gpr_atm_full_fetch_add(&workqueue->state, -2)) {
+        case 3:  // had one count, one unorphaned --> done, unorphaned
+          break;
+        case 2:  // had one count, one orphaned --> done, orphaned
+          workqueue_destroy(exec_ctx, workqueue);
+          break;
+        case 1:
+        case 0:
+          // these values are illegal - representing an already done or
+          // deleted workqueue
+          GPR_UNREACHABLE_CODE(break);
+        default:
+          // schedule a wakeup since there's more to do
+          wakeup(exec_ctx, workqueue);
+      }
+      grpc_closure *cl = (grpc_closure *)n;
+      grpc_error *clerr = cl->error;
+      cl->cb(exec_ctx, cl->cb_arg, clerr);
+      GRPC_ERROR_UNREF(clerr);
+    }
   }
+
+  GPR_TIMER_END("workqueue.on_readable", 0);
 }
 
 void grpc_workqueue_enqueue(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue,
                             grpc_closure *closure, grpc_error *error) {
-  grpc_error *push_error = GRPC_ERROR_NONE;
-  gpr_mu_lock(&workqueue->mu);
-  if (grpc_closure_list_empty(workqueue->closure_list)) {
-    push_error = grpc_wakeup_fd_wakeup(&workqueue->wakeup_fd);
-  }
-  grpc_closure_list_append(&workqueue->closure_list, closure, error);
-  if (push_error != GRPC_ERROR_NONE) {
-    const char *msg = grpc_error_string(push_error);
-    gpr_log(GPR_ERROR, "Failed to push to workqueue: %s", msg);
-    grpc_error_free_string(msg);
-    grpc_exec_ctx_enqueue_list(exec_ctx, &workqueue->closure_list, NULL);
+  GPR_TIMER_BEGIN("workqueue.enqueue", 0);
+  gpr_atm last = gpr_atm_full_fetch_add(&workqueue->state, 2);
+  GPR_ASSERT(last & 1);
+  closure->error = error;
+  gpr_mpscq_push(&workqueue->queue, &closure->next_data.atm_next);
+  if (last == 1) {
+    wakeup(exec_ctx, workqueue);
   }
-  gpr_mu_unlock(&workqueue->mu);
+  GPR_TIMER_END("workqueue.enqueue", 0);
 }
 
 #endif /* GPR_POSIX_SOCKET */

src/core/lib/iomgr/workqueue_posix.h

@@ -35,14 +35,17 @@
 #define GRPC_CORE_LIB_IOMGR_WORKQUEUE_POSIX_H
 
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
+#include "src/core/lib/support/mpscq.h"
 
 struct grpc_fd;
 
 struct grpc_workqueue {
   gpr_refcount refs;
-
-  gpr_mu mu;
-  grpc_closure_list closure_list;
+  gpr_mpscq queue;
+  // state is:
+  // lower bit - zero if orphaned
+  // other bits - number of items enqueued
+  gpr_atm state;
 
   grpc_wakeup_fd wakeup_fd;
   struct grpc_fd *wakeup_read_fd;

src/core/lib/iomgr/workqueue_windows.c

@@ -42,8 +42,6 @@
 // context, which is at least correct, if not performant or in the spirit of
 // workqueues.
 
-void grpc_workqueue_flush(grpc_exec_ctx *exec_ctx, grpc_workqueue *workqueue) {}
-
 #ifdef GRPC_WORKQUEUE_REFCOUNT_DEBUG
 void grpc_workqueue_ref(grpc_workqueue *workqueue, const char *file, int line,
                         const char *reason) {}

src/core/lib/security/transport/client_auth_filter.c

@@ -40,6 +40,7 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/security/transport/security_connector.h"
@@ -218,6 +219,8 @@ static void on_host_checked(grpc_exec_ctx *exec_ctx, void *user_data,
 static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
                                     grpc_call_element *elem,
                                     grpc_transport_stream_op *op) {
+  GPR_TIMER_BEGIN("auth_start_transport_op", 0);
+
   /* grab pointers to our data from the call element */
   call_data *calld = elem->call_data;
   channel_data *chand = elem->channel_data;
@@ -258,12 +261,14 @@ static void auth_start_transport_op(grpc_exec_ctx *exec_ctx,
       grpc_channel_security_connector_check_call_host(
           exec_ctx, chand->security_connector, call_host, chand->auth_context,
           on_host_checked, elem);
+      GPR_TIMER_END("auth_start_transport_op", 0);
       return; /* early exit */
     }
   }
 
   /* pass control down the stack */
   grpc_call_next_op(exec_ctx, elem, op);
+  GPR_TIMER_END("auth_start_transport_op", 0);
 }
 
 /* Constructor for call_data */

src/core/lib/security/transport/secure_endpoint.c

@@ -38,6 +38,7 @@
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/sync.h>
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/security/transport/tsi_error.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
@@ -248,6 +249,8 @@ static void flush_write_staging_buffer(secure_endpoint *ep, uint8_t **cur,
 
 static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
                            gpr_slice_buffer *slices, grpc_closure *cb) {
+  GPR_TIMER_BEGIN("secure_endpoint.endpoint_write", 0);
+
   unsigned i;
   tsi_result result = TSI_OK;
   secure_endpoint *ep = (secure_endpoint *)secure_ep;
@@ -323,10 +326,12 @@ static void endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep,
         exec_ctx, cb,
         grpc_set_tsi_error_result(GRPC_ERROR_CREATE("Wrap failed"), result),
         NULL);
+    GPR_TIMER_END("secure_endpoint.endpoint_write", 0);
     return;
   }
 
   grpc_endpoint_write(exec_ctx, ep->wrapped_ep, &ep->output_buffer, cb);
+  GPR_TIMER_END("secure_endpoint.endpoint_write", 0);
 }
 
 static void endpoint_shutdown(grpc_exec_ctx *exec_ctx,

src/core/lib/security/transport/server_auth_filter.c

@@ -48,7 +48,7 @@ typedef struct call_data {
      up-call on transport_op, and remember to call our on_done_recv member after
      handling it. */
   grpc_closure auth_on_recv;
-  grpc_transport_stream_op transport_op;
+  grpc_transport_stream_op *transport_op;
   grpc_metadata_array md;
   const grpc_metadata *consumed_md;
   size_t num_consumed_md;
@@ -106,6 +106,10 @@ static grpc_mdelem *remove_consumed_md(void *user_data, grpc_mdelem *md) {
   return md;
 }
 
+static void destroy_op(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  gpr_free(arg);
+}
+
 /* called from application code */
 static void on_md_processing_done(
     void *user_data, const grpc_metadata *consumed_md, size_t num_consumed_md,
@@ -131,21 +135,22 @@ static void on_md_processing_done(
     grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv, GRPC_ERROR_NONE, NULL);
   } else {
     gpr_slice message;
-    grpc_transport_stream_op close_op;
-    memset(&close_op, 0, sizeof(close_op));
+    grpc_transport_stream_op *close_op = gpr_malloc(sizeof(*close_op));
+    memset(close_op, 0, sizeof(*close_op));
     grpc_metadata_array_destroy(&calld->md);
     error_details = error_details != NULL
                         ? error_details
                         : "Authentication metadata processing failed.";
     message = gpr_slice_from_copied_string(error_details);
-    calld->transport_op.send_initial_metadata = NULL;
-    if (calld->transport_op.send_message != NULL) {
-      grpc_byte_stream_destroy(&exec_ctx, calld->transport_op.send_message);
-      calld->transport_op.send_message = NULL;
+    calld->transport_op->send_initial_metadata = NULL;
+    if (calld->transport_op->send_message != NULL) {
+      grpc_byte_stream_destroy(&exec_ctx, calld->transport_op->send_message);
+      calld->transport_op->send_message = NULL;
     }
-    calld->transport_op.send_trailing_metadata = NULL;
-    grpc_transport_stream_op_add_close(&close_op, status, &message);
-    grpc_call_next_op(&exec_ctx, elem, &close_op);
+    calld->transport_op->send_trailing_metadata = NULL;
+    close_op->on_complete = grpc_closure_create(destroy_op, close_op);
+    grpc_transport_stream_op_add_close(close_op, status, &message);
+    grpc_call_next_op(&exec_ctx, elem, close_op);
     grpc_exec_ctx_sched(&exec_ctx, calld->on_done_recv,
                         grpc_error_set_int(GRPC_ERROR_CREATE(error_details),
                                            GRPC_ERROR_INT_GRPC_STATUS, status),
@@ -182,7 +187,7 @@ static void set_recv_ops_md_callbacks(grpc_call_element *elem,
     calld->recv_initial_metadata = op->recv_initial_metadata;
     calld->on_done_recv = op->recv_initial_metadata_ready;
     op->recv_initial_metadata_ready = &calld->auth_on_recv;
-    calld->transport_op = *op;
+    calld->transport_op = op;
   }
 }
 

src/core/lib/support/mpscq.c

@@ -0,0 +1,83 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/support/mpscq.h"
+
+#include <grpc/support/log.h>
+
+void gpr_mpscq_init(gpr_mpscq *q) {
+  gpr_atm_no_barrier_store(&q->head, (gpr_atm)&q->stub);
+  q->tail = &q->stub;
+  gpr_atm_no_barrier_store(&q->stub.next, (gpr_atm)NULL);
+}
+
+void gpr_mpscq_destroy(gpr_mpscq *q) {
+  GPR_ASSERT(gpr_atm_no_barrier_load(&q->head) == (gpr_atm)&q->stub);
+  GPR_ASSERT(q->tail == &q->stub);
+}
+
+void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n) {
+  gpr_atm_no_barrier_store(&n->next, (gpr_atm)NULL);
+  gpr_mpscq_node *prev =
+      (gpr_mpscq_node *)gpr_atm_full_xchg(&q->head, (gpr_atm)n);
+  gpr_atm_rel_store(&prev->next, (gpr_atm)n);
+}
+
+gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q) {
+  gpr_mpscq_node *tail = q->tail;
+  gpr_mpscq_node *next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next);
+  if (tail == &q->stub) {
+    // indicates the list is actually (ephemerally) empty
+    if (next == NULL) return NULL;
+    q->tail = next;
+    tail = next;
+    next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next);
+  }
+  if (next != NULL) {
+    q->tail = next;
+    return tail;
+  }
+  gpr_mpscq_node *head = (gpr_mpscq_node *)gpr_atm_acq_load(&q->head);
+  if (tail != head) {
+    // indicates a retry is in order: we're still adding
+    return NULL;
+  }
+  gpr_mpscq_push(q, &q->stub);
+  next = (gpr_mpscq_node *)gpr_atm_acq_load(&tail->next);
+  if (next != NULL) {
+    q->tail = next;
+    return tail;
+  }
+  // indicates a retry is in order: we're still adding
+  return NULL;
+}

src/core/lib/support/mpscq.h

@@ -0,0 +1,65 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SUPPORT_MPSCQ_H
+#define GRPC_CORE_LIB_SUPPORT_MPSCQ_H
+
+#include <grpc/support/atm.h>
+#include <stddef.h>
+
+// Multiple-producer single-consumer lock free queue, based upon the
+// implementation from Dmitry Vyukov here:
+// http://www.1024cores.net/home/lock-free-algorithms/queues/intrusive-mpsc-node-based-queue
+
+// List node (include this in a data structure at the top, and add application
+// fields after it - to simulate inheritance)
+typedef struct gpr_mpscq_node { gpr_atm next; } gpr_mpscq_node;
+
+// Actual queue type
+typedef struct gpr_mpscq {
+  gpr_atm head;
+  // make sure head & tail don't share a cacheline
+  char padding[GPR_CACHELINE_SIZE];
+  gpr_mpscq_node *tail;
+  gpr_mpscq_node stub;
+} gpr_mpscq;
+
+void gpr_mpscq_init(gpr_mpscq *q);
+void gpr_mpscq_destroy(gpr_mpscq *q);
+// Push a node
+void gpr_mpscq_push(gpr_mpscq *q, gpr_mpscq_node *n);
+// Pop a node (returns NULL if no node is ready - which doesn't indicate that
+// the queue is empty!!)
+gpr_mpscq_node *gpr_mpscq_pop(gpr_mpscq *q);
+
+#endif /* GRPC_CORE_LIB_SUPPORT_MPSCQ_H */

src/core/lib/surface/call.c

@@ -109,6 +109,10 @@ typedef struct batch_control {
   uint8_t recv_message;
   uint8_t recv_final_op;
   uint8_t is_notify_tag_closure;
+
+  /* TODO(ctiller): now that this is inlined, figure out how much of the above
+                    state can be eliminated */
+  grpc_transport_stream_op op;
 } batch_control;
 
 struct grpc_call {
@@ -778,6 +782,7 @@ typedef struct termination_closure {
   grpc_error *error;
   grpc_closure *op_closure;
   enum { TC_CANCEL, TC_CLOSE } type;
+  grpc_transport_stream_op op;
 } termination_closure;
 
 static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp,
@@ -797,26 +802,24 @@ static void done_termination(grpc_exec_ctx *exec_ctx, void *tcp,
 }
 
 static void send_cancel(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
-  grpc_transport_stream_op op;
   termination_closure *tc = tcp;
-  memset(&op, 0, sizeof(op));
-  op.cancel_error = tc->error;
+  memset(&tc->op, 0, sizeof(tc->op));
+  tc->op.cancel_error = tc->error;
   /* reuse closure to catch completion */
   grpc_closure_init(&tc->closure, done_termination, tc);
-  op.on_complete = &tc->closure;
-  execute_op(exec_ctx, tc->call, &op);
+  tc->op.on_complete = &tc->closure;
+  execute_op(exec_ctx, tc->call, &tc->op);
 }
 
 static void send_close(grpc_exec_ctx *exec_ctx, void *tcp, grpc_error *error) {
-  grpc_transport_stream_op op;
   termination_closure *tc = tcp;
-  memset(&op, 0, sizeof(op));
-  op.close_error = tc->error;
+  memset(&tc->op, 0, sizeof(tc->op));
+  tc->op.close_error = tc->error;
   /* reuse closure to catch completion */
   grpc_closure_init(&tc->closure, done_termination, tc);
-  tc->op_closure = op.on_complete;
-  op.on_complete = &tc->closure;
-  execute_op(exec_ctx, tc->call, &op);
+  tc->op_closure = tc->op.on_complete;
+  tc->op.on_complete = &tc->closure;
+  execute_op(exec_ctx, tc->call, &tc->op);
 }
 
 static grpc_call_error terminate_with_status(grpc_exec_ctx *exec_ctx,
@@ -1370,7 +1373,6 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
                                         grpc_call *call, const grpc_op *ops,
                                         size_t nops, void *notify_tag,
                                         int is_notify_tag_closure) {
-  grpc_transport_stream_op stream_op;
   size_t i;
   const grpc_op *op;
   batch_control *bctl;
@@ -1384,8 +1386,6 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
 
   GRPC_CALL_LOG_BATCH(GPR_INFO, call, ops, nops, notify_tag);
 
-  memset(&stream_op, 0, sizeof(stream_op));
-
   /* TODO(ctiller): this feels like it could be made lock-free */
   gpr_mu_lock(&call->mu);
   bctl = allocate_batch_control(call);
@@ -1394,6 +1394,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
   bctl->notify_tag = notify_tag;
   bctl->is_notify_tag_closure = (uint8_t)(is_notify_tag_closure != 0);
 
+  grpc_transport_stream_op *stream_op = &bctl->op;
+  memset(stream_op, 0, sizeof(*stream_op));
+
   if (nops == 0) {
     GRPC_CALL_INTERNAL_REF(call, "completion");
     bctl->error = GRPC_ERROR_NONE;
@@ -1471,9 +1474,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         }
         /* TODO(ctiller): just make these the same variable? */
         call->metadata_batch[0][0].deadline = call->send_deadline;
-        stream_op.send_initial_metadata =
+        stream_op->send_initial_metadata =
             &call->metadata_batch[0 /* is_receiving */][0 /* is_trailing */];
-        stream_op.send_initial_metadata_flags = op->flags;
+        stream_op->send_initial_metadata_flags = op->flags;
         break;
       case GRPC_OP_SEND_MESSAGE:
         if (!are_write_flags_valid(op->flags)) {
@@ -1493,7 +1496,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         grpc_slice_buffer_stream_init(
             &call->sending_stream,
             &op->data.send_message->data.raw.slice_buffer, op->flags);
-        stream_op.send_message = &call->sending_stream.base;
+        stream_op->send_message = &call->sending_stream.base;
         break;
       case GRPC_OP_SEND_CLOSE_FROM_CLIENT:
         /* Flag validation: currently allow no flags */
@@ -1511,7 +1514,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         }
         bctl->send_final_op = 1;
         call->sent_final_op = 1;
-        stream_op.send_trailing_metadata =
+        stream_op->send_trailing_metadata =
             &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */];
         break;
       case GRPC_OP_SEND_STATUS_FROM_SERVER:
@@ -1558,7 +1561,7 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
           error = GRPC_CALL_ERROR_INVALID_METADATA;
           goto done_with_error;
         }
-        stream_op.send_trailing_metadata =
+        stream_op->send_trailing_metadata =
             &call->metadata_batch[0 /* is_receiving */][1 /* is_trailing */];
         break;
       case GRPC_OP_RECV_INITIAL_METADATA:
@@ -1576,9 +1579,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         grpc_closure_init(&call->receiving_initial_metadata_ready,
                           receiving_initial_metadata_ready, bctl);
         bctl->recv_initial_metadata = 1;
-        stream_op.recv_initial_metadata =
+        stream_op->recv_initial_metadata =
             &call->metadata_batch[1 /* is_receiving */][0 /* is_trailing */];
-        stream_op.recv_initial_metadata_ready =
+        stream_op->recv_initial_metadata_ready =
             &call->receiving_initial_metadata_ready;
         num_completion_callbacks_needed++;
         break;
@@ -1595,10 +1598,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         call->receiving_message = 1;
         bctl->recv_message = 1;
         call->receiving_buffer = op->data.recv_message;
-        stream_op.recv_message = &call->receiving_stream;
+        stream_op->recv_message = &call->receiving_stream;
         grpc_closure_init(&call->receiving_stream_ready, receiving_stream_ready,
                           bctl);
-        stream_op.recv_message_ready = &call->receiving_stream_ready;
+        stream_op->recv_message_ready = &call->receiving_stream_ready;
         num_completion_callbacks_needed++;
         break;
       case GRPC_OP_RECV_STATUS_ON_CLIENT:
@@ -1624,9 +1627,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         call->final_op.client.status_details_capacity =
             op->data.recv_status_on_client.status_details_capacity;
         bctl->recv_final_op = 1;
-        stream_op.recv_trailing_metadata =
+        stream_op->recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
-        stream_op.collect_stats =
+        stream_op->collect_stats =
             &call->final_info.stats.transport_stream_stats;
         break;
       case GRPC_OP_RECV_CLOSE_ON_SERVER:
@@ -1647,9 +1650,9 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
         call->final_op.server.cancelled =
             op->data.recv_close_on_server.cancelled;
         bctl->recv_final_op = 1;
-        stream_op.recv_trailing_metadata =
+        stream_op->recv_trailing_metadata =
             &call->metadata_batch[1 /* is_receiving */][1 /* is_trailing */];
-        stream_op.collect_stats =
+        stream_op->collect_stats =
             &call->final_info.stats.transport_stream_stats;
         break;
     }
@@ -1661,12 +1664,12 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
   }
   gpr_ref_init(&bctl->steps_to_complete, num_completion_callbacks_needed);
 
-  stream_op.context = call->context;
+  stream_op->context = call->context;
   grpc_closure_init(&bctl->finish_batch, finish_batch, bctl);
-  stream_op.on_complete = &bctl->finish_batch;
+  stream_op->on_complete = &bctl->finish_batch;
   gpr_mu_unlock(&call->mu);
 
-  execute_op(exec_ctx, call, &stream_op);
+  execute_op(exec_ctx, call, stream_op);
 
 done:
   GPR_TIMER_END("grpc_call_start_batch", 0);

src/core/lib/surface/channel.c

@@ -334,14 +334,13 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, void *arg,
 }
 
 void grpc_channel_destroy(grpc_channel *channel) {
-  grpc_transport_op op;
+  grpc_transport_op *op = grpc_make_transport_op(NULL);
   grpc_channel_element *elem;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   GRPC_API_TRACE("grpc_channel_destroy(channel=%p)", 1, (channel));
-  memset(&op, 0, sizeof(op));
-  op.disconnect_with_error = GRPC_ERROR_CREATE("Channel Destroyed");
+  op->disconnect_with_error = GRPC_ERROR_CREATE("Channel Destroyed");
   elem = grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0);
-  elem->filter->start_transport_op(&exec_ctx, elem, &op);
+  elem->filter->start_transport_op(&exec_ctx, elem, op);
 
   GRPC_CHANNEL_INTERNAL_UNREF(&exec_ctx, channel, "channel");
 

src/core/lib/surface/channel_ping.c

@@ -61,19 +61,20 @@ static void ping_done(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 
 void grpc_channel_ping(grpc_channel *channel, grpc_completion_queue *cq,
                        void *tag, void *reserved) {
-  grpc_transport_op op;
+  GRPC_API_TRACE("grpc_channel_ping(channel=%p, cq=%p, tag=%p, reserved=%p)", 4,
+                 (channel, cq, tag, reserved));
+  grpc_transport_op *op = grpc_make_transport_op(NULL);
   ping_result *pr = gpr_malloc(sizeof(*pr));
   grpc_channel_element *top_elem =
       grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0);
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   GPR_ASSERT(reserved == NULL);
-  memset(&op, 0, sizeof(op));
   pr->tag = tag;
   pr->cq = cq;
   grpc_closure_init(&pr->closure, ping_done, pr);
-  op.send_ping = &pr->closure;
-  op.bind_pollset = grpc_cq_pollset(cq);
+  op->send_ping = &pr->closure;
+  op->bind_pollset = grpc_cq_pollset(cq);
   grpc_cq_begin_op(cq, tag);
-  top_elem->filter->start_transport_op(&exec_ctx, top_elem, &op);
+  top_elem->filter->start_transport_op(&exec_ctx, top_elem, op);
   grpc_exec_ctx_finish(&exec_ctx);
 }

src/core/lib/surface/init.c

@@ -47,6 +47,7 @@
 #include "src/core/lib/channel/http_server_filter.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/profiling/timers.h"
@@ -165,6 +166,7 @@ void grpc_init(void) {
     grpc_register_tracer("http1", &grpc_http1_trace);
     grpc_register_tracer("compression", &grpc_compression_trace);
     grpc_register_tracer("queue_pluck", &grpc_cq_pluck_trace);
+    grpc_register_tracer("combiner", &grpc_combiner_trace);
     // Default pluck trace to 1
     grpc_cq_pluck_trace = 1;
     grpc_register_tracer("queue_timeout", &grpc_cq_event_timeout_trace);

src/core/lib/surface/lame_client.c

@@ -97,14 +97,14 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx,
     grpc_exec_ctx_sched(exec_ctx, op->on_connectivity_state_change,
                         GRPC_ERROR_NONE, NULL);
   }
-  if (op->on_consumed != NULL) {
-    grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
-  }
   if (op->send_ping != NULL) {
     grpc_exec_ctx_sched(exec_ctx, op->send_ping,
                         GRPC_ERROR_CREATE("lame client channel"), NULL);
   }
   GRPC_ERROR_UNREF(op->disconnect_with_error);
+  if (op->on_consumed != NULL) {
+    grpc_exec_ctx_sched(exec_ctx, op->on_consumed, GRPC_ERROR_NONE, NULL);
+  }
 }
 
 static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,

src/core/lib/surface/server.c

@@ -273,23 +273,20 @@ static void shutdown_cleanup(grpc_exec_ctx *exec_ctx, void *arg,
 }
 
 static void send_shutdown(grpc_exec_ctx *exec_ctx, grpc_channel *channel,
-                          bool send_goaway, grpc_error *send_disconnect) {
-  grpc_transport_op op;
-  struct shutdown_cleanup_args *sc;
+                          int send_goaway, grpc_error *send_disconnect) {
+  struct shutdown_cleanup_args *sc = gpr_malloc(sizeof(*sc));
+  grpc_closure_init(&sc->closure, shutdown_cleanup, sc);
+  grpc_transport_op *op = grpc_make_transport_op(&sc->closure);
   grpc_channel_element *elem;
 
-  memset(&op, 0, sizeof(op));
-  op.send_goaway = send_goaway;
-  sc = gpr_malloc(sizeof(*sc));
+  op->send_goaway = send_goaway;
   sc->slice = gpr_slice_from_copied_string("Server shutdown");
-  op.goaway_message = &sc->slice;
-  op.goaway_status = GRPC_STATUS_OK;
-  op.disconnect_with_error = send_disconnect;
-  grpc_closure_init(&sc->closure, shutdown_cleanup, sc);
-  op.on_consumed = &sc->closure;
+  op->goaway_message = &sc->slice;
+  op->goaway_status = GRPC_STATUS_OK;
+  op->disconnect_with_error = send_disconnect;
 
   elem = grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0);
-  elem->filter->start_transport_op(exec_ctx, elem, &op);
+  elem->filter->start_transport_op(exec_ctx, elem, op);
 }
 
 static void channel_broadcaster_shutdown(grpc_exec_ctx *exec_ctx,
@@ -432,7 +429,8 @@ static void finish_destroy_channel(grpc_exec_ctx *exec_ctx, void *cd,
   server_unref(exec_ctx, server);
 }
 
-static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand) {
+static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand,
+                            grpc_error *error) {
   if (is_channel_orphaned(chand)) return;
   GPR_ASSERT(chand->server != NULL);
   orphan_channel(chand);
@@ -441,14 +439,20 @@ static void destroy_channel(grpc_exec_ctx *exec_ctx, channel_data *chand) {
   chand->finish_destroy_channel_closure.cb = finish_destroy_channel;
   chand->finish_destroy_channel_closure.cb_arg = chand;
 
-  grpc_transport_op op;
-  memset(&op, 0, sizeof(op));
-  op.set_accept_stream = true;
-  op.on_consumed = &chand->finish_destroy_channel_closure;
+  grpc_transport_op *op =
+      grpc_make_transport_op(&chand->finish_destroy_channel_closure);
+  op->set_accept_stream = true;
   grpc_channel_next_op(exec_ctx,
                        grpc_channel_stack_element(
                            grpc_channel_get_channel_stack(chand->channel), 0),
-                       &op);
+                       op);
+
+  if (error != GRPC_ERROR_NONE) {
+    const char *msg = grpc_error_string(error);
+    gpr_log(GPR_INFO, "Disconnected client: %s", msg);
+    grpc_error_free_string(msg);
+  }
+  GRPC_ERROR_UNREF(error);
 }
 
 static void cpstr(char **dest, size_t *capacity, grpc_mdstr *value) {
@@ -845,17 +849,16 @@ static void channel_connectivity_changed(grpc_exec_ctx *exec_ctx, void *cd,
   channel_data *chand = cd;
   grpc_server *server = chand->server;
   if (chand->connectivity_state != GRPC_CHANNEL_SHUTDOWN) {
-    grpc_transport_op op;
-    memset(&op, 0, sizeof(op));
-    op.on_connectivity_state_change = &chand->channel_connectivity_changed,
-    op.connectivity_state = &chand->connectivity_state;
+    grpc_transport_op *op = grpc_make_transport_op(NULL);
+    op->on_connectivity_state_change = &chand->channel_connectivity_changed,
+    op->connectivity_state = &chand->connectivity_state;
     grpc_channel_next_op(exec_ctx,
                          grpc_channel_stack_element(
                              grpc_channel_get_channel_stack(chand->channel), 0),
-                         &op);
+                         op);
   } else {
     gpr_mu_lock(&server->mu_global);
-    destroy_channel(exec_ctx, chand);
+    destroy_channel(exec_ctx, chand, GRPC_ERROR_REF(error));
     gpr_mu_unlock(&server->mu_global);
     GRPC_CHANNEL_INTERNAL_UNREF(exec_ctx, chand->channel, "connectivity");
   }
@@ -1119,7 +1122,7 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   size_t slots;
   uint32_t probes;
   uint32_t max_probes = 0;
-  grpc_transport_op op;
+  grpc_transport_op *op = NULL;
 
   channel =
       grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
@@ -1179,16 +1182,16 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   gpr_mu_unlock(&s->mu_global);
 
   GRPC_CHANNEL_INTERNAL_REF(channel, "connectivity");
-  memset(&op, 0, sizeof(op));
-  op.set_accept_stream = true;
-  op.set_accept_stream_fn = accept_stream;
-  op.set_accept_stream_user_data = chand;
-  op.on_connectivity_state_change = &chand->channel_connectivity_changed;
-  op.connectivity_state = &chand->connectivity_state;
+  op = grpc_make_transport_op(NULL);
+  op->set_accept_stream = true;
+  op->set_accept_stream_fn = accept_stream;
+  op->set_accept_stream_user_data = chand;
+  op->on_connectivity_state_change = &chand->channel_connectivity_changed;
+  op->connectivity_state = &chand->connectivity_state;
   if (gpr_atm_acq_load(&s->shutdown_flag) != 0) {
-    op.disconnect_with_error = GRPC_ERROR_CREATE("Server shutdown");
+    op->disconnect_with_error = GRPC_ERROR_CREATE("Server shutdown");
   }
-  grpc_transport_perform_op(exec_ctx, transport, &op);
+  grpc_transport_perform_op(exec_ctx, transport, op);
 }
 
 void done_published_shutdown(grpc_exec_ctx *exec_ctx, void *done_arg,

src/core/lib/transport/transport.c

@@ -32,10 +32,14 @@
  */
 
 #include "src/core/lib/transport/transport.h"
+
+#include <string.h>
+
 #include <grpc/support/alloc.h>
 #include <grpc/support/atm.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
+
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/transport/transport_impl.h"
 
@@ -247,3 +251,26 @@ void grpc_transport_stream_op_add_close(grpc_transport_stream_op *op,
   error = grpc_error_set_int(error, GRPC_ERROR_INT_GRPC_STATUS, status);
   add_error(op, &op->close_error, error);
 }
+
+typedef struct {
+  grpc_closure outer_on_complete;
+  grpc_closure *inner_on_complete;
+  grpc_transport_op op;
+} made_transport_op;
+
+static void destroy_made_transport_op(grpc_exec_ctx *exec_ctx, void *arg,
+                                      grpc_error *error) {
+  made_transport_op *op = arg;
+  grpc_exec_ctx_sched(exec_ctx, op->inner_on_complete, GRPC_ERROR_REF(error),
+                      NULL);
+  gpr_free(op);
+}
+
+grpc_transport_op *grpc_make_transport_op(grpc_closure *on_complete) {
+  made_transport_op *op = gpr_malloc(sizeof(*op));
+  grpc_closure_init(&op->outer_on_complete, destroy_made_transport_op, op);
+  op->inner_on_complete = on_complete;
+  memset(&op->op, 0, sizeof(op->op));
+  op->op.on_consumed = &op->outer_on_complete;
+  return &op->op;
+}

src/core/lib/transport/transport.h

@@ -100,6 +100,11 @@ void grpc_transport_move_one_way_stats(grpc_transport_one_way_stats *from,
 void grpc_transport_move_stats(grpc_transport_stream_stats *from,
                                grpc_transport_stream_stats *to);
 
+typedef struct {
+  grpc_closure closure;
+  void *args[2];
+} grpc_transport_private_op_data;
+
 /* Transport stream op: a set of operations to perform on a transport
    against a single stream */
 typedef struct grpc_transport_stream_op {
@@ -149,6 +154,12 @@ typedef struct grpc_transport_stream_op {
 
   /* Indexes correspond to grpc_context_index enum values */
   grpc_call_context_element *context;
+
+  /***************************************************************************
+   * remaining fields are initialized and used at the discretion of the
+   * transport implementation */
+
+  grpc_transport_private_op_data transport_private;
 } grpc_transport_stream_op;
 
 /** Transport op: a set of operations to perform on a transport as a whole */
@@ -182,6 +193,12 @@ typedef struct grpc_transport_op {
   grpc_pollset_set *bind_pollset_set;
   /** send a ping, call this back if not NULL */
   grpc_closure *send_ping;
+
+  /***************************************************************************
+   * remaining fields are initialized and used at the discretion of the
+   * transport implementation */
+
+  grpc_transport_private_op_data transport_private;
 } grpc_transport_op;
 
 /* Returns the amount of memory required to store a grpc_stream for this
@@ -273,6 +290,10 @@ void grpc_transport_destroy(grpc_exec_ctx *exec_ctx, grpc_transport *transport);
 char *grpc_transport_get_peer(grpc_exec_ctx *exec_ctx,
                               grpc_transport *transport);
 
+/* Allocate a grpc_transport_op, and preconfigure the on_consumed closure to
+   \a on_consumed and then delete the returned transport op */
+grpc_transport_op *grpc_make_transport_op(grpc_closure *on_consumed);
+
 #ifdef __cplusplus
 }
 #endif

src/proto/grpc/lb/v1/load_balancer.proto

@@ -101,7 +101,7 @@ message InitialLoadBalanceResponse {
   // This interval defines how often the client should send the client stats
   // to the load balancer. Stats should only be reported when the duration is
   // positive.
-  Duration client_stats_report_interval = 3;
+  Duration client_stats_report_interval = 2;
 }
 
 message ServerList {

src/python/grpcio/grpc_core_dependencies.py

@@ -50,6 +50,7 @@ CORE_SOURCE_FILES = [
   'src/core/lib/support/log_linux.c',
   'src/core/lib/support/log_posix.c',
   'src/core/lib/support/log_windows.c',
+  'src/core/lib/support/mpscq.c',
   'src/core/lib/support/murmur_hash.c',
   'src/core/lib/support/percent_encoding.c',
   'src/core/lib/support/slice.c',
@@ -92,6 +93,7 @@ CORE_SOURCE_FILES = [
   'src/core/lib/http/httpcli.c',
   'src/core/lib/http/parser.c',
   'src/core/lib/iomgr/closure.c',
+  'src/core/lib/iomgr/combiner.c',
   'src/core/lib/iomgr/endpoint.c',
   'src/core/lib/iomgr/endpoint_pair_posix.c',
   'src/core/lib/iomgr/endpoint_pair_windows.c',

test/core/iomgr/combiner_test.c

@@ -0,0 +1,164 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/combiner.h"
+
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/thd.h>
+#include <grpc/support/useful.h>
+
+#include "test/core/util/test_config.h"
+
+static void test_no_op(void) {
+  gpr_log(GPR_DEBUG, "test_no_op");
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_combiner_destroy(&exec_ctx, grpc_combiner_create(NULL));
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void set_bool_to_true(grpc_exec_ctx *exec_ctx, void *value,
+                             grpc_error *error) {
+  *(bool *)value = true;
+}
+
+static void test_execute_one(void) {
+  gpr_log(GPR_DEBUG, "test_execute_one");
+
+  grpc_combiner *lock = grpc_combiner_create(NULL);
+  bool done = false;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_combiner_execute(&exec_ctx, lock,
+                        grpc_closure_create(set_bool_to_true, &done),
+                        GRPC_ERROR_NONE);
+  grpc_exec_ctx_flush(&exec_ctx);
+  GPR_ASSERT(done);
+  grpc_combiner_destroy(&exec_ctx, lock);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+typedef struct {
+  size_t ctr;
+  grpc_combiner *lock;
+} thd_args;
+
+typedef struct {
+  size_t *ctr;
+  size_t value;
+} ex_args;
+
+static void check_one(grpc_exec_ctx *exec_ctx, void *a, grpc_error *error) {
+  ex_args *args = a;
+  GPR_ASSERT(*args->ctr == args->value - 1);
+  *args->ctr = args->value;
+  gpr_free(a);
+}
+
+static void execute_many_loop(void *a) {
+  thd_args *args = a;
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  size_t n = 1;
+  for (size_t i = 0; i < 10; i++) {
+    for (size_t j = 0; j < 10000; j++) {
+      ex_args *c = gpr_malloc(sizeof(*c));
+      c->ctr = &args->ctr;
+      c->value = n++;
+      grpc_combiner_execute(&exec_ctx, args->lock,
+                            grpc_closure_create(check_one, c), GRPC_ERROR_NONE);
+      grpc_exec_ctx_flush(&exec_ctx);
+    }
+    // sleep for a little bit, to test a combiner draining and another thread
+    // picking it up
+    gpr_sleep_until(GRPC_TIMEOUT_MILLIS_TO_DEADLINE(100));
+  }
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void test_execute_many(void) {
+  gpr_log(GPR_DEBUG, "test_execute_many");
+
+  grpc_combiner *lock = grpc_combiner_create(NULL);
+  gpr_thd_id thds[100];
+  thd_args ta[GPR_ARRAY_SIZE(thds)];
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
+    gpr_thd_options options = gpr_thd_options_default();
+    gpr_thd_options_set_joinable(&options);
+    ta[i].ctr = 0;
+    ta[i].lock = lock;
+    GPR_ASSERT(gpr_thd_new(&thds[i], execute_many_loop, &ta[i], &options));
+  }
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
+    gpr_thd_join(thds[i]);
+  }
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_combiner_destroy(&exec_ctx, lock);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static bool got_in_finally = false;
+
+static void in_finally(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  got_in_finally = true;
+}
+
+static void add_finally(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
+  grpc_combiner_execute_finally(exec_ctx, arg,
+                                grpc_closure_create(in_finally, NULL),
+                                GRPC_ERROR_NONE, false);
+}
+
+static void test_execute_finally(void) {
+  gpr_log(GPR_DEBUG, "test_execute_finally");
+
+  grpc_combiner *lock = grpc_combiner_create(NULL);
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_combiner_execute(&exec_ctx, lock, grpc_closure_create(add_finally, lock),
+                        GRPC_ERROR_NONE);
+  grpc_exec_ctx_flush(&exec_ctx);
+  GPR_ASSERT(got_in_finally);
+  grpc_combiner_destroy(&exec_ctx, lock);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+int main(int argc, char **argv) {
+  grpc_test_init(argc, argv);
+  grpc_init();
+  test_no_op();
+  test_execute_one();
+  test_execute_finally();
+  test_execute_many();
+  grpc_shutdown();
+
+  return 0;
+}

test/core/support/mpscq_test.c

@@ -0,0 +1,206 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/support/mpscq.h"
+
+#include <stdlib.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/thd.h>
+#include <grpc/support/useful.h>
+#include "test/core/util/test_config.h"
+
+typedef struct test_node {
+  gpr_mpscq_node node;
+  size_t i;
+  size_t *ctr;
+} test_node;
+
+static test_node *new_node(size_t i, size_t *ctr) {
+  test_node *n = gpr_malloc(sizeof(test_node));
+  n->i = i;
+  n->ctr = ctr;
+  return n;
+}
+
+static void test_serial(void) {
+  gpr_log(GPR_DEBUG, "test_serial");
+  gpr_mpscq q;
+  gpr_mpscq_init(&q);
+  for (size_t i = 0; i < 10000000; i++) {
+    gpr_mpscq_push(&q, &new_node(i, NULL)->node);
+  }
+  for (size_t i = 0; i < 10000000; i++) {
+    test_node *n = (test_node *)gpr_mpscq_pop(&q);
+    GPR_ASSERT(n);
+    GPR_ASSERT(n->i == i);
+    gpr_free(n);
+  }
+}
+
+typedef struct {
+  size_t ctr;
+  gpr_mpscq *q;
+  gpr_event *start;
+} thd_args;
+
+#define THREAD_ITERATIONS 100000
+
+static void test_thread(void *args) {
+  thd_args *a = args;
+  gpr_event_wait(a->start, gpr_inf_future(GPR_CLOCK_REALTIME));
+  for (size_t i = 1; i <= THREAD_ITERATIONS; i++) {
+    gpr_mpscq_push(a->q, &new_node(i, &a->ctr)->node);
+  }
+}
+
+static void test_mt(void) {
+  gpr_log(GPR_DEBUG, "test_mt");
+  gpr_event start;
+  gpr_event_init(&start);
+  gpr_thd_id thds[100];
+  thd_args ta[GPR_ARRAY_SIZE(thds)];
+  gpr_mpscq q;
+  gpr_mpscq_init(&q);
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
+    gpr_thd_options options = gpr_thd_options_default();
+    gpr_thd_options_set_joinable(&options);
+    ta[i].ctr = 0;
+    ta[i].q = &q;
+    ta[i].start = &start;
+    GPR_ASSERT(gpr_thd_new(&thds[i], test_thread, &ta[i], &options));
+  }
+  size_t num_done = 0;
+  size_t spins = 0;
+  gpr_event_set(&start, (void *)1);
+  while (num_done != GPR_ARRAY_SIZE(thds)) {
+    gpr_mpscq_node *n;
+    while ((n = gpr_mpscq_pop(&q)) == NULL) {
+      spins++;
+    }
+    test_node *tn = (test_node *)n;
+    GPR_ASSERT(*tn->ctr == tn->i - 1);
+    *tn->ctr = tn->i;
+    if (tn->i == THREAD_ITERATIONS) num_done++;
+    gpr_free(tn);
+  }
+  gpr_log(GPR_DEBUG, "spins: %" PRIdPTR, spins);
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
+    gpr_thd_join(thds[i]);
+  }
+  gpr_mpscq_destroy(&q);
+}
+
+typedef struct {
+  thd_args *ta;
+  size_t num_thds;
+  gpr_mu mu;
+  size_t num_done;
+  size_t spins;
+  gpr_mpscq *q;
+  gpr_event *start;
+} pull_args;
+
+static void pull_thread(void *arg) {
+  pull_args *pa = arg;
+  gpr_event_wait(pa->start, gpr_inf_future(GPR_CLOCK_REALTIME));
+
+  for (;;) {
+    gpr_mu_lock(&pa->mu);
+    if (pa->num_done == pa->num_thds) {
+      gpr_mu_unlock(&pa->mu);
+      return;
+    }
+    gpr_mpscq_node *n;
+    while ((n = gpr_mpscq_pop(pa->q)) == NULL) {
+      pa->spins++;
+    }
+    test_node *tn = (test_node *)n;
+    GPR_ASSERT(*tn->ctr == tn->i - 1);
+    *tn->ctr = tn->i;
+    if (tn->i == THREAD_ITERATIONS) pa->num_done++;
+    gpr_free(tn);
+    gpr_mu_unlock(&pa->mu);
+  }
+}
+
+static void test_mt_multipop(void) {
+  gpr_log(GPR_DEBUG, "test_mt_multipop");
+  gpr_event start;
+  gpr_event_init(&start);
+  gpr_thd_id thds[100];
+  gpr_thd_id pull_thds[100];
+  thd_args ta[GPR_ARRAY_SIZE(thds)];
+  gpr_mpscq q;
+  gpr_mpscq_init(&q);
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
+    gpr_thd_options options = gpr_thd_options_default();
+    gpr_thd_options_set_joinable(&options);
+    ta[i].ctr = 0;
+    ta[i].q = &q;
+    ta[i].start = &start;
+    GPR_ASSERT(gpr_thd_new(&thds[i], test_thread, &ta[i], &options));
+  }
+  pull_args pa;
+  pa.ta = ta;
+  pa.num_thds = GPR_ARRAY_SIZE(thds);
+  pa.spins = 0;
+  pa.num_done = 0;
+  pa.q = &q;
+  pa.start = &start;
+  gpr_mu_init(&pa.mu);
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(pull_thds); i++) {
+    gpr_thd_options options = gpr_thd_options_default();
+    gpr_thd_options_set_joinable(&options);
+    GPR_ASSERT(gpr_thd_new(&pull_thds[i], pull_thread, &pa, &options));
+  }
+  gpr_event_set(&start, (void *)1);
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(pull_thds); i++) {
+    gpr_thd_join(pull_thds[i]);
+  }
+  gpr_log(GPR_DEBUG, "spins: %" PRIdPTR, pa.spins);
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(thds); i++) {
+    gpr_thd_join(thds[i]);
+  }
+  gpr_mpscq_destroy(&q);
+}
+
+int main(int argc, char **argv) {
+  grpc_test_init(argc, argv);
+  test_serial();
+  test_mt();
+  test_mt_multipop();
+  return 0;
+}

test/core/surface/lame_client_test.c

@@ -49,32 +49,31 @@ static void *tag(intptr_t x) { return (void *)x; }
 
 void verify_connectivity(grpc_exec_ctx *exec_ctx, void *arg,
                          grpc_error *error) {
-  grpc_transport_op *op = arg;
-  GPR_ASSERT(GRPC_CHANNEL_SHUTDOWN == *op->connectivity_state);
+  grpc_connectivity_state *state = arg;
+  GPR_ASSERT(GRPC_CHANNEL_SHUTDOWN == *state);
   GPR_ASSERT(error == GRPC_ERROR_NONE);
 }
 
 void do_nothing(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {}
 
 void test_transport_op(grpc_channel *channel) {
-  grpc_transport_op op;
+  grpc_transport_op *op;
   grpc_channel_element *elem;
   grpc_connectivity_state state = GRPC_CHANNEL_IDLE;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
 
-  memset(&op, 0, sizeof(op));
-  grpc_closure_init(&transport_op_cb, verify_connectivity, &op);
+  grpc_closure_init(&transport_op_cb, verify_connectivity, &state);
 
-  op.on_connectivity_state_change = &transport_op_cb;
-  op.connectivity_state = &state;
+  op = grpc_make_transport_op(NULL);
+  op->on_connectivity_state_change = &transport_op_cb;
+  op->connectivity_state = &state;
   elem = grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0);
-  elem->filter->start_transport_op(&exec_ctx, elem, &op);
+  elem->filter->start_transport_op(&exec_ctx, elem, op);
   grpc_exec_ctx_finish(&exec_ctx);
 
-  memset(&op, 0, sizeof(op));
   grpc_closure_init(&transport_op_cb, do_nothing, NULL);
-  op.on_consumed = &transport_op_cb;
-  elem->filter->start_transport_op(&exec_ctx, elem, &op);
+  op = grpc_make_transport_op(&transport_op_cb);
+  elem->filter->start_transport_op(&exec_ctx, elem, op);
   grpc_exec_ctx_finish(&exec_ctx);
 }
 

tools/doxygen/Doxyfile.c++.internal

@@ -886,6 +886,7 @@ src/core/lib/http/format_request.h \
 src/core/lib/http/httpcli.h \
 src/core/lib/http/parser.h \
 src/core/lib/iomgr/closure.h \
+src/core/lib/iomgr/combiner.h \
 src/core/lib/iomgr/endpoint.h \
 src/core/lib/iomgr/endpoint_pair.h \
 src/core/lib/iomgr/error.h \
@@ -998,6 +999,7 @@ src/core/lib/http/format_request.c \
 src/core/lib/http/httpcli.c \
 src/core/lib/http/parser.c \
 src/core/lib/iomgr/closure.c \
+src/core/lib/iomgr/combiner.c \
 src/core/lib/iomgr/endpoint.c \
 src/core/lib/iomgr/endpoint_pair_posix.c \
 src/core/lib/iomgr/endpoint_pair_windows.c \

tools/doxygen/Doxyfile.core.internal

@@ -802,6 +802,7 @@ src/core/lib/http/format_request.h \
 src/core/lib/http/httpcli.h \
 src/core/lib/http/parser.h \
 src/core/lib/iomgr/closure.h \
+src/core/lib/iomgr/combiner.h \
 src/core/lib/iomgr/endpoint.h \
 src/core/lib/iomgr/endpoint_pair.h \
 src/core/lib/iomgr/error.h \
@@ -961,6 +962,7 @@ src/core/lib/http/format_request.c \
 src/core/lib/http/httpcli.c \
 src/core/lib/http/parser.c \
 src/core/lib/iomgr/closure.c \
+src/core/lib/iomgr/combiner.c \
 src/core/lib/iomgr/endpoint.c \
 src/core/lib/iomgr/endpoint_pair_posix.c \
 src/core/lib/iomgr/endpoint_pair_windows.c \
@@ -1178,6 +1180,7 @@ src/core/lib/profiling/timers.h \
 src/core/lib/support/backoff.h \
 src/core/lib/support/block_annotate.h \
 src/core/lib/support/env.h \
+src/core/lib/support/mpscq.h \
 src/core/lib/support/murmur_hash.h \
 src/core/lib/support/percent_encoding.h \
 src/core/lib/support/stack_lockfree.h \
@@ -1206,6 +1209,7 @@ src/core/lib/support/log_android.c \
 src/core/lib/support/log_linux.c \
 src/core/lib/support/log_posix.c \
 src/core/lib/support/log_windows.c \
+src/core/lib/support/mpscq.c \
 src/core/lib/support/murmur_hash.c \
 src/core/lib/support/percent_encoding.c \
 src/core/lib/support/slice.c \

tools/run_tests/performance/scenario_config.py

@@ -38,6 +38,7 @@ BENCHMARK_SECONDS=30
 SMOKETEST='smoketest'
 SCALABLE='scalable'
 SWEEP='sweep'
+DEFAULT_CATEGORIES=[SCALABLE, SMOKETEST]
 
 SECURE_SECARGS = {'use_test_ca': True,
                   'server_host_override': 'foo.test.google.fr'}
@@ -94,6 +95,13 @@ def remove_nonproto_fields(scenario):
   return scenario
 
 
+def geometric_progression(start, stop, step):
+  n = start
+  while n < stop:
+    yield int(round(n))
+    n *= step
+
+
 def _ping_pong_scenario(name, rpc_type,
                         client_type, server_type,
                         secure=True,
@@ -104,8 +112,9 @@ def _ping_pong_scenario(name, rpc_type,
                         server_core_limit=0,
                         async_server_threads=0,
                         warmup_seconds=WARMUP_SECONDS,
-                        categories=[],
-                        channels=None):
+                        categories=DEFAULT_CATEGORIES,
+                        channels=None,
+                        outstanding=None):
   """Creates a basic ping pong scenario."""
   scenario = {
     'name': name,
@@ -142,8 +151,9 @@ def _ping_pong_scenario(name, rpc_type,
     scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD
 
   if unconstrained_client:
+    outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client]
     wide = channels if channels is not None else WIDE
-    deep = int(math.ceil(1.0 * OUTSTANDING_REQUESTS[unconstrained_client] / wide))
+    deep = int(math.ceil(1.0 * outstanding_calls / wide))
 
     scenario['num_clients'] = 0  # use as many client as available.
     scenario['client_config']['outstanding_rpcs_per_channel'] = deep
@@ -245,13 +255,17 @@ class CXXLanguage:
             secure=secure,
             categories=[SCALABLE])
 
-        for channels in [1, 3, 10, 31, 100, 316, 1000]:
-          yield _ping_pong_scenario(
-              'cpp_protobuf_%s_unary_qps_unconstrained_%s_%d_channels' % (synchronicity, secstr, channels),
-              rpc_type='UNARY',
-              client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-              unconstrained_client=synchronicity, secure=secure,
-              categories=[SWEEP], channels=channels)
+        for channels in geometric_progression(1, 500, math.sqrt(10)):
+          for outstanding in geometric_progression(1, 20000, math.sqrt(10)):
+              if synchronicity == 'sync' and outstanding > 1200: continue
+              if outstanding < channels: continue
+              yield _ping_pong_scenario(
+                  'cpp_protobuf_%s_unary_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, secstr, channels, outstanding),
+                  rpc_type='UNARY',
+                  client_type='%s_CLIENT' % synchronicity.upper(),
+                  server_type='%s_SERVER' % synchronicity.upper(),
+                  unconstrained_client=synchronicity, secure=secure,
+                  categories=[SWEEP], channels=channels, outstanding=outstanding)
 
   def __str__(self):
     return 'c++'

tools/run_tests/run_performance_tests.py

@@ -312,8 +312,8 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
   for language in languages:
     for scenario_json in language.scenarios():
       if re.search(args.regex, scenario_json['name']):
-        categories = scenario_json.get('CATEGORIES', [])
-        if category in categories or (category == 'all' and categories != ['sweep']):
+        categories = scenario_json.get('CATEGORIES', ['scalable', 'smoketest'])
+        if category in categories or category == 'all':
           workers = workers_by_lang[str(language)][:]
           # 'SERVER_LANGUAGE' is an indicator for this script to pick
           # a server in different language.

tools/run_tests/sources_and_headers.json

@@ -252,6 +252,22 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util", 
+      "grpc", 
+      "grpc_test_util"
+    ], 
+    "headers": [], 
+    "language": "c", 
+    "name": "combiner_test", 
+    "src": [
+      "test/core/iomgr/combiner_test.c"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
@@ -624,6 +640,20 @@
     "third_party": false, 
     "type": "target"
   }, 
+  {
+    "deps": [
+      "gpr", 
+      "gpr_test_util"
+    ], 
+    "headers": [], 
+    "language": "c", 
+    "name": "gpr_mpscq_test", 
+    "src": [
+      "test/core/support/mpscq_test.c"
+    ], 
+    "third_party": false, 
+    "type": "target"
+  }, 
   {
     "deps": [
       "gpr", 
@@ -5735,6 +5765,7 @@
       "src/core/lib/support/backoff.h", 
       "src/core/lib/support/block_annotate.h", 
       "src/core/lib/support/env.h", 
+      "src/core/lib/support/mpscq.h", 
       "src/core/lib/support/murmur_hash.h", 
       "src/core/lib/support/percent_encoding.h", 
       "src/core/lib/support/stack_lockfree.h", 
@@ -5799,6 +5830,8 @@
       "src/core/lib/support/log_linux.c", 
       "src/core/lib/support/log_posix.c", 
       "src/core/lib/support/log_windows.c", 
+      "src/core/lib/support/mpscq.c", 
+      "src/core/lib/support/mpscq.h", 
       "src/core/lib/support/murmur_hash.c", 
       "src/core/lib/support/murmur_hash.h", 
       "src/core/lib/support/percent_encoding.c", 
@@ -5899,6 +5932,7 @@
       "src/core/lib/http/httpcli.h", 
       "src/core/lib/http/parser.h", 
       "src/core/lib/iomgr/closure.h", 
+      "src/core/lib/iomgr/combiner.h", 
       "src/core/lib/iomgr/endpoint.h", 
       "src/core/lib/iomgr/endpoint_pair.h", 
       "src/core/lib/iomgr/error.h", 
@@ -6005,6 +6039,8 @@
       "src/core/lib/http/parser.h", 
       "src/core/lib/iomgr/closure.c", 
       "src/core/lib/iomgr/closure.h", 
+      "src/core/lib/iomgr/combiner.c", 
+      "src/core/lib/iomgr/combiner.h", 
       "src/core/lib/iomgr/endpoint.c", 
       "src/core/lib/iomgr/endpoint.h", 
       "src/core/lib/iomgr/endpoint_pair.h", 

vsprojects/buildtests_c.sln

@@ -175,6 +175,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "chttp2_varint_test", "vcxpr
 		{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}
 	EndProjectSection
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "combiner_test", "vcxproj\test\combiner_test\combiner_test.vcxproj", "{C237D1E4-8825-80BA-1FC3-5E147E53E96E}"
+	ProjectSection(myProperties) = preProject
+        	lib = "False"
+	EndProjectSection
+	ProjectSection(ProjectDependencies) = postProject
+		{17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B}
+		{29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9}
+		{EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037}
+		{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}
+	EndProjectSection
+EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "compression_test", "vcxproj\test\compression_test\compression_test.vcxproj", "{5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}"
 	ProjectSection(myProperties) = preProject
         	lib = "False"
@@ -382,6 +393,15 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "gpr_log_test", "vcxproj\tes
 		{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}
 	EndProjectSection
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "gpr_mpscq_test", "vcxproj\test\gpr_mpscq_test\gpr_mpscq_test.vcxproj", "{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}"
+	ProjectSection(myProperties) = preProject
+        	lib = "False"
+	EndProjectSection
+	ProjectSection(ProjectDependencies) = postProject
+		{EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037}
+		{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}
+	EndProjectSection
+EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "gpr_percent_encoding_test", "vcxproj\test\gpr_percent_encoding_test\gpr_percent_encoding_test.vcxproj", "{8313AE17-FCFA-8110-95C7-7AF2F814D188}"
 	ProjectSection(myProperties) = preProject
         	lib = "False"
@@ -1720,6 +1740,22 @@ Global
 		{6B29F634-1277-74B8-47F6-78756190BA7B}.Release-DLL|Win32.Build.0 = Release|Win32
 		{6B29F634-1277-74B8-47F6-78756190BA7B}.Release-DLL|x64.ActiveCfg = Release|x64
 		{6B29F634-1277-74B8-47F6-78756190BA7B}.Release-DLL|x64.Build.0 = Release|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug|Win32.ActiveCfg = Debug|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug|x64.ActiveCfg = Debug|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release|Win32.ActiveCfg = Release|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release|x64.ActiveCfg = Release|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug|Win32.Build.0 = Debug|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug|x64.Build.0 = Debug|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release|Win32.Build.0 = Release|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release|x64.Build.0 = Release|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug-DLL|Win32.ActiveCfg = Debug|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug-DLL|Win32.Build.0 = Debug|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug-DLL|x64.ActiveCfg = Debug|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Debug-DLL|x64.Build.0 = Debug|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release-DLL|Win32.ActiveCfg = Release|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release-DLL|Win32.Build.0 = Release|Win32
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release-DLL|x64.ActiveCfg = Release|x64
+		{C237D1E4-8825-80BA-1FC3-5E147E53E96E}.Release-DLL|x64.Build.0 = Release|x64
 		{5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Debug|Win32.ActiveCfg = Debug|Win32
 		{5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Debug|x64.ActiveCfg = Debug|x64
 		{5AFE7D17-A4A7-D68E-4491-CBC852F9D2A0}.Release|Win32.ActiveCfg = Release|Win32
@@ -2072,6 +2108,22 @@ Global
 		{38797EE3-62CC-3CBF-18D5-009ED6DD0BEC}.Release-DLL|Win32.Build.0 = Release|Win32
 		{38797EE3-62CC-3CBF-18D5-009ED6DD0BEC}.Release-DLL|x64.ActiveCfg = Release|x64
 		{38797EE3-62CC-3CBF-18D5-009ED6DD0BEC}.Release-DLL|x64.Build.0 = Release|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug|Win32.ActiveCfg = Debug|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug|x64.ActiveCfg = Debug|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release|Win32.ActiveCfg = Release|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release|x64.ActiveCfg = Release|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug|Win32.Build.0 = Debug|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug|x64.Build.0 = Debug|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release|Win32.Build.0 = Release|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release|x64.Build.0 = Release|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug-DLL|Win32.ActiveCfg = Debug|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug-DLL|Win32.Build.0 = Debug|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug-DLL|x64.ActiveCfg = Debug|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Debug-DLL|x64.Build.0 = Debug|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release-DLL|Win32.ActiveCfg = Release|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release-DLL|Win32.Build.0 = Release|Win32
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release-DLL|x64.ActiveCfg = Release|x64
+		{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}.Release-DLL|x64.Build.0 = Release|x64
 		{8313AE17-FCFA-8110-95C7-7AF2F814D188}.Debug|Win32.ActiveCfg = Debug|Win32
 		{8313AE17-FCFA-8110-95C7-7AF2F814D188}.Debug|x64.ActiveCfg = Debug|x64
 		{8313AE17-FCFA-8110-95C7-7AF2F814D188}.Release|Win32.ActiveCfg = Release|Win32

vsprojects/vcxproj/gpr/gpr.vcxproj

@@ -192,6 +192,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\backoff.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\block_annotate.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\env.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\mpscq.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\murmur_hash.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\percent_encoding.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\stack_lockfree.h" />
@@ -242,6 +243,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\log_windows.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\mpscq.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\murmur_hash.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\percent_encoding.c">

vsprojects/vcxproj/gpr/gpr.vcxproj.filters

@@ -61,6 +61,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\log_windows.c">
       <Filter>src\core\lib\support</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\mpscq.c">
+      <Filter>src\core\lib\support</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\support\murmur_hash.c">
       <Filter>src\core\lib\support</Filter>
     </ClCompile>
@@ -272,6 +275,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\env.h">
       <Filter>src\core\lib\support</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\mpscq.h">
+      <Filter>src\core\lib\support</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\support\murmur_hash.h">
       <Filter>src\core\lib\support</Filter>
     </ClInclude>

vsprojects/vcxproj/grpc++/grpc++.vcxproj

@@ -386,6 +386,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\httpcli.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\parser.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\error.h" />
@@ -548,6 +549,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair_posix.c">

vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters

@@ -145,6 +145,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
@@ -752,6 +755,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>

vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj

@@ -382,6 +382,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\httpcli.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\parser.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\error.h" />
@@ -534,6 +535,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair_posix.c">

vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters

@@ -130,6 +130,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
@@ -725,6 +728,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>

vsprojects/vcxproj/grpc/grpc.vcxproj

@@ -311,6 +311,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\httpcli.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\parser.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\error.h" />
@@ -488,6 +489,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair_posix.c">

vsprojects/vcxproj/grpc/grpc.vcxproj.filters

@@ -49,6 +49,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
@@ -701,6 +704,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>

vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj

@@ -202,6 +202,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\httpcli.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\parser.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\error.h" />
@@ -332,6 +333,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair_posix.c">

vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters

@@ -97,6 +97,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
@@ -482,6 +485,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>

vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj

@@ -301,6 +301,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\httpcli.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\http\parser.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\error.h" />
@@ -456,6 +457,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint_pair_posix.c">

vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters

@@ -52,6 +52,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.c">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.c">
       <Filter>src\core\lib\iomgr</Filter>
     </ClCompile>
@@ -611,6 +614,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\closure.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\combiner.h">
+      <Filter>src\core\lib\iomgr</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\iomgr\endpoint.h">
       <Filter>src\core\lib\iomgr</Filter>
     </ClInclude>

vsprojects/vcxproj/test/combiner_test/combiner_test.vcxproj

@@ -0,0 +1,199 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\1.0.204.1.props')" />
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{C237D1E4-8825-80BA-1FC3-5E147E53E96E}</ProjectGuid>
+    <IgnoreWarnIntDirInTempDetected>true</IgnoreWarnIntDirInTempDetected>
+    <IntDir>$(SolutionDir)IntDir\$(MSBuildProjectName)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '10.0'" Label="Configuration">
+    <PlatformToolset>v100</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '11.0'" Label="Configuration">
+    <PlatformToolset>v110</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '12.0'" Label="Configuration">
+    <PlatformToolset>v120</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '14.0'" Label="Configuration">
+    <PlatformToolset>v140</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="$(SolutionDir)\..\vsprojects\global.props" />
+    <Import Project="$(SolutionDir)\..\vsprojects\openssl.props" />
+    <Import Project="$(SolutionDir)\..\vsprojects\winsock.props" />
+    <Import Project="$(SolutionDir)\..\vsprojects\zlib.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <TargetName>combiner_test</TargetName>
+    <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib>
+    <Configuration-grpc_dependencies_zlib>Debug</Configuration-grpc_dependencies_zlib>
+    <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl>
+    <Configuration-grpc_dependencies_openssl>Debug</Configuration-grpc_dependencies_openssl>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <TargetName>combiner_test</TargetName>
+    <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib>
+    <Configuration-grpc_dependencies_zlib>Release</Configuration-grpc_dependencies_zlib>
+    <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl>
+    <Configuration-grpc_dependencies_openssl>Release</Configuration-grpc_dependencies_openssl>
+  </PropertyGroup>
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <ClCompile Include="$(SolutionDir)\..\test\core\iomgr\combiner_test.c">
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\grpc_test_util\grpc_test_util.vcxproj">
+      <Project>{17BCAFC0-5FDC-4C94-AEB9-95F3E220614B}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\grpc\grpc.vcxproj">
+      <Project>{29D16885-7228-4C31-81ED-5F9187C7F2A9}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr_test_util\gpr_test_util.vcxproj">
+      <Project>{EAB0A629-17A9-44DB-B5FF-E91A721FE037}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr\gpr.vcxproj">
+      <Project>{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" />
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" />
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" />
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" />
+  </ImportGroup>
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" />
+  </Target>
+</Project>
+

vsprojects/vcxproj/test/combiner_test/combiner_test.vcxproj.filters

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <ClCompile Include="$(SolutionDir)\..\test\core\iomgr\combiner_test.c">
+      <Filter>test\core\iomgr</Filter>
+    </ClCompile>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Filter Include="test">
+      <UniqueIdentifier>{82bca2af-d499-b405-fd05-4d345372496c}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="test\core">
+      <UniqueIdentifier>{c32d8e20-b719-532d-ba23-bd9d523fac15}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="test\core\iomgr">
+      <UniqueIdentifier>{b4fa8ca1-e6c7-dec5-6d62-8a62396825c6}</UniqueIdentifier>
+    </Filter>
+  </ItemGroup>
+</Project>
+

vsprojects/vcxproj/test/gpr_mpscq_test/gpr_mpscq_test.vcxproj

@@ -0,0 +1,193 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\1.0.204.1.props')" />
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{B3D7760B-8BEA-2EF6-F1D4-9F9020E166D6}</ProjectGuid>
+    <IgnoreWarnIntDirInTempDetected>true</IgnoreWarnIntDirInTempDetected>
+    <IntDir>$(SolutionDir)IntDir\$(MSBuildProjectName)\</IntDir>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '10.0'" Label="Configuration">
+    <PlatformToolset>v100</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '11.0'" Label="Configuration">
+    <PlatformToolset>v110</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '12.0'" Label="Configuration">
+    <PlatformToolset>v120</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(VisualStudioVersion)' == '14.0'" Label="Configuration">
+    <PlatformToolset>v140</PlatformToolset>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+    <Import Project="$(SolutionDir)\..\vsprojects\global.props" />
+    <Import Project="$(SolutionDir)\..\vsprojects\openssl.props" />
+    <Import Project="$(SolutionDir)\..\vsprojects\winsock.props" />
+    <Import Project="$(SolutionDir)\..\vsprojects\zlib.props" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)'=='Debug'">
+    <TargetName>gpr_mpscq_test</TargetName>
+    <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib>
+    <Configuration-grpc_dependencies_zlib>Debug</Configuration-grpc_dependencies_zlib>
+    <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl>
+    <Configuration-grpc_dependencies_openssl>Debug</Configuration-grpc_dependencies_openssl>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <TargetName>gpr_mpscq_test</TargetName>
+    <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib>
+    <Configuration-grpc_dependencies_zlib>Release</Configuration-grpc_dependencies_zlib>
+    <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl>
+    <Configuration-grpc_dependencies_openssl>Release</Configuration-grpc_dependencies_openssl>
+  </PropertyGroup>
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+
+    <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <PrecompiledHeader>NotUsing</PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>MaxSpeed</Optimization>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <SDLCheck>true</SDLCheck>
+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+      <TreatWarningAsError>true</TreatWarningAsError>
+      <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat>
+      <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation>
+      <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+
+  <ItemGroup>
+    <ClCompile Include="$(SolutionDir)\..\test\core\support\mpscq_test.c">
+    </ClCompile>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr_test_util\gpr_test_util.vcxproj">
+      <Project>{EAB0A629-17A9-44DB-B5FF-E91A721FE037}</Project>
+    </ProjectReference>
+    <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr\gpr.vcxproj">
+      <Project>{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" />
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" />
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" />
+  <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" />
+  </ImportGroup>
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" />
+    <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" />
+  </Target>
+</Project>
+

vsprojects/vcxproj/test/gpr_mpscq_test/gpr_mpscq_test.vcxproj.filters

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <ClCompile Include="$(SolutionDir)\..\test\core\support\mpscq_test.c">
+      <Filter>test\core\support</Filter>
+    </ClCompile>
+  </ItemGroup>
+
+  <ItemGroup>
+    <Filter Include="test">
+      <UniqueIdentifier>{e82fb80c-10ba-2959-55d6-8653715f1e4f}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="test\core">
+      <UniqueIdentifier>{32120233-25e6-f3e4-f828-c6408d47ec04}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="test\core\support">
+      <UniqueIdentifier>{aa3a22bc-229a-c00a-dd4a-924c818c6a49}</UniqueIdentifier>
+    </Filter>
+  </ItemGroup>
+</Project>
+