Merge remote-tracking branch 'upstream/master' into lb_policy_name_channel_arg

BUILD

@@ -204,6 +204,7 @@ cc_library(
     "src/core/lib/iomgr/pollset_windows.h",
     "src/core/lib/iomgr/port.h",
     "src/core/lib/iomgr/resolve_address.h",
+    "src/core/lib/iomgr/resource_quota.h",
     "src/core/lib/iomgr/sockaddr.h",
     "src/core/lib/iomgr/sockaddr_posix.h",
     "src/core/lib/iomgr/sockaddr_utils.h",
@@ -212,6 +213,7 @@ cc_library(
     "src/core/lib/iomgr/socket_utils_posix.h",
     "src/core/lib/iomgr/socket_windows.h",
     "src/core/lib/iomgr/tcp_client.h",
+    "src/core/lib/iomgr/tcp_client_posix.h",
     "src/core/lib/iomgr/tcp_posix.h",
     "src/core/lib/iomgr/tcp_server.h",
     "src/core/lib/iomgr/tcp_uv.h",
@@ -375,6 +377,7 @@ cc_library(
     "src/core/lib/iomgr/resolve_address_posix.c",
     "src/core/lib/iomgr/resolve_address_uv.c",
     "src/core/lib/iomgr/resolve_address_windows.c",
+    "src/core/lib/iomgr/resource_quota.c",
     "src/core/lib/iomgr/sockaddr_utils.c",
     "src/core/lib/iomgr/socket_utils_common_posix.c",
     "src/core/lib/iomgr/socket_utils_linux.c",
@@ -625,6 +628,7 @@ cc_library(
     "src/core/lib/iomgr/pollset_windows.h",
     "src/core/lib/iomgr/port.h",
     "src/core/lib/iomgr/resolve_address.h",
+    "src/core/lib/iomgr/resource_quota.h",
     "src/core/lib/iomgr/sockaddr.h",
     "src/core/lib/iomgr/sockaddr_posix.h",
     "src/core/lib/iomgr/sockaddr_utils.h",
@@ -633,6 +637,7 @@ cc_library(
     "src/core/lib/iomgr/socket_utils_posix.h",
     "src/core/lib/iomgr/socket_windows.h",
     "src/core/lib/iomgr/tcp_client.h",
+    "src/core/lib/iomgr/tcp_client_posix.h",
     "src/core/lib/iomgr/tcp_posix.h",
     "src/core/lib/iomgr/tcp_server.h",
     "src/core/lib/iomgr/tcp_uv.h",
@@ -781,6 +786,7 @@ cc_library(
     "src/core/lib/iomgr/resolve_address_posix.c",
     "src/core/lib/iomgr/resolve_address_uv.c",
     "src/core/lib/iomgr/resolve_address_windows.c",
+    "src/core/lib/iomgr/resource_quota.c",
     "src/core/lib/iomgr/sockaddr_utils.c",
     "src/core/lib/iomgr/socket_utils_common_posix.c",
     "src/core/lib/iomgr/socket_utils_linux.c",
@@ -1001,6 +1007,7 @@ cc_library(
     "src/core/lib/iomgr/pollset_windows.h",
     "src/core/lib/iomgr/port.h",
     "src/core/lib/iomgr/resolve_address.h",
+    "src/core/lib/iomgr/resource_quota.h",
     "src/core/lib/iomgr/sockaddr.h",
     "src/core/lib/iomgr/sockaddr_posix.h",
     "src/core/lib/iomgr/sockaddr_utils.h",
@@ -1009,6 +1016,7 @@ cc_library(
     "src/core/lib/iomgr/socket_utils_posix.h",
     "src/core/lib/iomgr/socket_windows.h",
     "src/core/lib/iomgr/tcp_client.h",
+    "src/core/lib/iomgr/tcp_client_posix.h",
     "src/core/lib/iomgr/tcp_posix.h",
     "src/core/lib/iomgr/tcp_server.h",
     "src/core/lib/iomgr/tcp_uv.h",
@@ -1149,6 +1157,7 @@ cc_library(
     "src/core/lib/iomgr/resolve_address_posix.c",
     "src/core/lib/iomgr/resolve_address_uv.c",
     "src/core/lib/iomgr/resolve_address_windows.c",
+    "src/core/lib/iomgr/resource_quota.c",
     "src/core/lib/iomgr/sockaddr_utils.c",
     "src/core/lib/iomgr/socket_utils_common_posix.c",
     "src/core/lib/iomgr/socket_utils_linux.c",
@@ -1333,6 +1342,7 @@ cc_library(
     "src/cpp/common/channel_filter.h",
     "src/cpp/server/dynamic_thread_pool.h",
     "src/cpp/server/thread_pool_interface.h",
+    "src/cpp/thread_manager/thread_manager.h",
     "src/cpp/client/insecure_credentials.cc",
     "src/cpp/client/secure_credentials.cc",
     "src/cpp/common/auth_property_iterator.cc",
@@ -1352,6 +1362,7 @@ cc_library(
     "src/cpp/common/channel_filter.cc",
     "src/cpp/common/completion_queue_cc.cc",
     "src/cpp/common/core_codegen.cc",
+    "src/cpp/common/resource_quota_cc.cc",
     "src/cpp/common/rpc_method.cc",
     "src/cpp/server/async_generic_service.cc",
     "src/cpp/server/create_default_thread_pool.cc",
@@ -1361,6 +1372,7 @@ cc_library(
     "src/cpp/server/server_context.cc",
     "src/cpp/server/server_credentials.cc",
     "src/cpp/server/server_posix.cc",
+    "src/cpp/thread_manager/thread_manager.cc",
     "src/cpp/util/byte_buffer_cc.cc",
     "src/cpp/util/slice_cc.cc",
     "src/cpp/util/status.cc",
@@ -1396,6 +1408,7 @@ cc_library(
     "include/grpc++/impl/thd.h",
     "include/grpc++/impl/thd_cxx11.h",
     "include/grpc++/impl/thd_no_cxx11.h",
+    "include/grpc++/resource_quota.h",
     "include/grpc++/security/auth_context.h",
     "include/grpc++/security/auth_metadata_processor.h",
     "include/grpc++/security/credentials.h",
@@ -1485,6 +1498,7 @@ cc_library(
     "src/cpp/common/channel_filter.h",
     "src/cpp/server/dynamic_thread_pool.h",
     "src/cpp/server/thread_pool_interface.h",
+    "src/cpp/thread_manager/thread_manager.h",
     "src/cpp/client/cronet_credentials.cc",
     "src/cpp/client/insecure_credentials.cc",
     "src/cpp/common/insecure_create_auth_context.cc",
@@ -1500,6 +1514,7 @@ cc_library(
     "src/cpp/common/channel_filter.cc",
     "src/cpp/common/completion_queue_cc.cc",
     "src/cpp/common/core_codegen.cc",
+    "src/cpp/common/resource_quota_cc.cc",
     "src/cpp/common/rpc_method.cc",
     "src/cpp/server/async_generic_service.cc",
     "src/cpp/server/create_default_thread_pool.cc",
@@ -1509,6 +1524,7 @@ cc_library(
     "src/cpp/server/server_context.cc",
     "src/cpp/server/server_credentials.cc",
     "src/cpp/server/server_posix.cc",
+    "src/cpp/thread_manager/thread_manager.cc",
     "src/cpp/util/byte_buffer_cc.cc",
     "src/cpp/util/slice_cc.cc",
     "src/cpp/util/status.cc",
@@ -1544,6 +1560,7 @@ cc_library(
     "include/grpc++/impl/thd.h",
     "include/grpc++/impl/thd_cxx11.h",
     "include/grpc++/impl/thd_no_cxx11.h",
+    "include/grpc++/resource_quota.h",
     "include/grpc++/security/auth_context.h",
     "include/grpc++/security/auth_metadata_processor.h",
     "include/grpc++/security/credentials.h",
@@ -1708,6 +1725,7 @@ cc_library(
     "src/cpp/common/channel_filter.h",
     "src/cpp/server/dynamic_thread_pool.h",
     "src/cpp/server/thread_pool_interface.h",
+    "src/cpp/thread_manager/thread_manager.h",
     "src/cpp/client/insecure_credentials.cc",
     "src/cpp/common/insecure_create_auth_context.cc",
     "src/cpp/server/insecure_server_credentials.cc",
@@ -1722,6 +1740,7 @@ cc_library(
     "src/cpp/common/channel_filter.cc",
     "src/cpp/common/completion_queue_cc.cc",
     "src/cpp/common/core_codegen.cc",
+    "src/cpp/common/resource_quota_cc.cc",
     "src/cpp/common/rpc_method.cc",
     "src/cpp/server/async_generic_service.cc",
     "src/cpp/server/create_default_thread_pool.cc",
@@ -1731,6 +1750,7 @@ cc_library(
     "src/cpp/server/server_context.cc",
     "src/cpp/server/server_credentials.cc",
     "src/cpp/server/server_posix.cc",
+    "src/cpp/thread_manager/thread_manager.cc",
     "src/cpp/util/byte_buffer_cc.cc",
     "src/cpp/util/slice_cc.cc",
     "src/cpp/util/status.cc",
@@ -1766,6 +1786,7 @@ cc_library(
     "include/grpc++/impl/thd.h",
     "include/grpc++/impl/thd_cxx11.h",
     "include/grpc++/impl/thd_no_cxx11.h",
+    "include/grpc++/resource_quota.h",
     "include/grpc++/security/auth_context.h",
     "include/grpc++/security/auth_metadata_processor.h",
     "include/grpc++/security/credentials.h",
@@ -2073,6 +2094,7 @@ objc_library(
     "src/core/lib/iomgr/resolve_address_posix.c",
     "src/core/lib/iomgr/resolve_address_uv.c",
     "src/core/lib/iomgr/resolve_address_windows.c",
+    "src/core/lib/iomgr/resource_quota.c",
     "src/core/lib/iomgr/sockaddr_utils.c",
     "src/core/lib/iomgr/socket_utils_common_posix.c",
     "src/core/lib/iomgr/socket_utils_linux.c",
@@ -2302,6 +2324,7 @@ objc_library(
     "src/core/lib/iomgr/pollset_windows.h",
     "src/core/lib/iomgr/port.h",
     "src/core/lib/iomgr/resolve_address.h",
+    "src/core/lib/iomgr/resource_quota.h",
     "src/core/lib/iomgr/sockaddr.h",
     "src/core/lib/iomgr/sockaddr_posix.h",
     "src/core/lib/iomgr/sockaddr_utils.h",
@@ -2310,6 +2333,7 @@ objc_library(
     "src/core/lib/iomgr/socket_utils_posix.h",
     "src/core/lib/iomgr/socket_windows.h",
     "src/core/lib/iomgr/tcp_client.h",
+    "src/core/lib/iomgr/tcp_client_posix.h",
     "src/core/lib/iomgr/tcp_posix.h",
     "src/core/lib/iomgr/tcp_server.h",
     "src/core/lib/iomgr/tcp_uv.h",

CMakeLists.txt

@@ -334,6 +334,7 @@ add_library(grpc
   src/core/lib/iomgr/resolve_address_posix.c
   src/core/lib/iomgr/resolve_address_uv.c
   src/core/lib/iomgr/resolve_address_windows.c
+  src/core/lib/iomgr/resource_quota.c
   src/core/lib/iomgr/sockaddr_utils.c
   src/core/lib/iomgr/socket_utils_common_posix.c
   src/core/lib/iomgr/socket_utils_linux.c
@@ -605,6 +606,7 @@ add_library(grpc_cronet
   src/core/lib/iomgr/resolve_address_posix.c
   src/core/lib/iomgr/resolve_address_uv.c
   src/core/lib/iomgr/resolve_address_windows.c
+  src/core/lib/iomgr/resource_quota.c
   src/core/lib/iomgr/sockaddr_utils.c
   src/core/lib/iomgr/socket_utils_common_posix.c
   src/core/lib/iomgr/socket_utils_linux.c
@@ -848,6 +850,7 @@ add_library(grpc_unsecure
   src/core/lib/iomgr/resolve_address_posix.c
   src/core/lib/iomgr/resolve_address_uv.c
   src/core/lib/iomgr/resolve_address_windows.c
+  src/core/lib/iomgr/resource_quota.c
   src/core/lib/iomgr/sockaddr_utils.c
   src/core/lib/iomgr/socket_utils_common_posix.c
   src/core/lib/iomgr/socket_utils_linux.c
@@ -1061,6 +1064,7 @@ add_library(grpc++
   src/cpp/common/channel_filter.cc
   src/cpp/common/completion_queue_cc.cc
   src/cpp/common/core_codegen.cc
+  src/cpp/common/resource_quota_cc.cc
   src/cpp/common/rpc_method.cc
   src/cpp/server/async_generic_service.cc
   src/cpp/server/create_default_thread_pool.cc
@@ -1070,6 +1074,7 @@ add_library(grpc++
   src/cpp/server/server_context.cc
   src/cpp/server/server_credentials.cc
   src/cpp/server/server_posix.cc
+  src/cpp/thread_manager/thread_manager.cc
   src/cpp/util/byte_buffer_cc.cc
   src/cpp/util/slice_cc.cc
   src/cpp/util/status.cc
@@ -1122,6 +1127,7 @@ foreach(_hdr
   include/grpc++/impl/thd.h
   include/grpc++/impl/thd_cxx11.h
   include/grpc++/impl/thd_no_cxx11.h
+  include/grpc++/resource_quota.h
   include/grpc++/security/auth_context.h
   include/grpc++/security/auth_metadata_processor.h
   include/grpc++/security/credentials.h
@@ -1224,6 +1230,7 @@ add_library(grpc++_cronet
   src/cpp/common/channel_filter.cc
   src/cpp/common/completion_queue_cc.cc
   src/cpp/common/core_codegen.cc
+  src/cpp/common/resource_quota_cc.cc
   src/cpp/common/rpc_method.cc
   src/cpp/server/async_generic_service.cc
   src/cpp/server/create_default_thread_pool.cc
@@ -1233,6 +1240,7 @@ add_library(grpc++_cronet
   src/cpp/server/server_context.cc
   src/cpp/server/server_credentials.cc
   src/cpp/server/server_posix.cc
+  src/cpp/thread_manager/thread_manager.cc
   src/cpp/util/byte_buffer_cc.cc
   src/cpp/util/slice_cc.cc
   src/cpp/util/status.cc
@@ -1285,6 +1293,7 @@ foreach(_hdr
   include/grpc++/impl/thd.h
   include/grpc++/impl/thd_cxx11.h
   include/grpc++/impl/thd_no_cxx11.h
+  include/grpc++/resource_quota.h
   include/grpc++/security/auth_context.h
   include/grpc++/security/auth_metadata_processor.h
   include/grpc++/security/credentials.h
@@ -1478,6 +1487,7 @@ add_library(grpc++_unsecure
   src/cpp/common/channel_filter.cc
   src/cpp/common/completion_queue_cc.cc
   src/cpp/common/core_codegen.cc
+  src/cpp/common/resource_quota_cc.cc
   src/cpp/common/rpc_method.cc
   src/cpp/server/async_generic_service.cc
   src/cpp/server/create_default_thread_pool.cc
@@ -1487,6 +1497,7 @@ add_library(grpc++_unsecure
   src/cpp/server/server_context.cc
   src/cpp/server/server_credentials.cc
   src/cpp/server/server_posix.cc
+  src/cpp/thread_manager/thread_manager.cc
   src/cpp/util/byte_buffer_cc.cc
   src/cpp/util/slice_cc.cc
   src/cpp/util/status.cc
@@ -1539,6 +1550,7 @@ foreach(_hdr
   include/grpc++/impl/thd.h
   include/grpc++/impl/thd_cxx11.h
   include/grpc++/impl/thd_no_cxx11.h
+  include/grpc++/resource_quota.h
   include/grpc++/security/auth_context.h
   include/grpc++/security/auth_metadata_processor.h
   include/grpc++/security/credentials.h

Makefile

@@ -1008,6 +1008,7 @@ no_server_test: $(BINDIR)/$(CONFIG)/no_server_test
 percent_decode_fuzzer: $(BINDIR)/$(CONFIG)/percent_decode_fuzzer
 percent_encode_fuzzer: $(BINDIR)/$(CONFIG)/percent_encode_fuzzer
 resolve_address_test: $(BINDIR)/$(CONFIG)/resolve_address_test
+resource_quota_test: $(BINDIR)/$(CONFIG)/resource_quota_test
 secure_channel_create_test: $(BINDIR)/$(CONFIG)/secure_channel_create_test
 secure_endpoint_test: $(BINDIR)/$(CONFIG)/secure_endpoint_test
 sequential_connectivity_test: $(BINDIR)/$(CONFIG)/sequential_connectivity_test
@@ -1086,6 +1087,7 @@ shutdown_test: $(BINDIR)/$(CONFIG)/shutdown_test
 status_test: $(BINDIR)/$(CONFIG)/status_test
 streaming_throughput_test: $(BINDIR)/$(CONFIG)/streaming_throughput_test
 stress_test: $(BINDIR)/$(CONFIG)/stress_test
+thread_manager_test: $(BINDIR)/$(CONFIG)/thread_manager_test
 thread_stress_test: $(BINDIR)/$(CONFIG)/thread_stress_test
 public_headers_must_be_c89: $(BINDIR)/$(CONFIG)/public_headers_must_be_c89
 boringssl_aes_test: $(BINDIR)/$(CONFIG)/boringssl_aes_test
@@ -1332,6 +1334,7 @@ buildtests_c: privatelibs_c \
   $(BINDIR)/$(CONFIG)/murmur_hash_test \
   $(BINDIR)/$(CONFIG)/no_server_test \
   $(BINDIR)/$(CONFIG)/resolve_address_test \
+  $(BINDIR)/$(CONFIG)/resource_quota_test \
   $(BINDIR)/$(CONFIG)/secure_channel_create_test \
   $(BINDIR)/$(CONFIG)/secure_endpoint_test \
   $(BINDIR)/$(CONFIG)/sequential_connectivity_test \
@@ -1463,6 +1466,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/status_test \
   $(BINDIR)/$(CONFIG)/streaming_throughput_test \
   $(BINDIR)/$(CONFIG)/stress_test \
+  $(BINDIR)/$(CONFIG)/thread_manager_test \
   $(BINDIR)/$(CONFIG)/thread_stress_test \
   $(BINDIR)/$(CONFIG)/boringssl_aes_test \
   $(BINDIR)/$(CONFIG)/boringssl_asn1_test \
@@ -1552,6 +1556,7 @@ buildtests_cxx: privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/status_test \
   $(BINDIR)/$(CONFIG)/streaming_throughput_test \
   $(BINDIR)/$(CONFIG)/stress_test \
+  $(BINDIR)/$(CONFIG)/thread_manager_test \
   $(BINDIR)/$(CONFIG)/thread_stress_test \
 
 endif
@@ -1720,6 +1725,8 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/no_server_test || ( echo test no_server_test failed ; exit 1 )
 	$(E) "[RUN]     Testing resolve_address_test"
 	$(Q) $(BINDIR)/$(CONFIG)/resolve_address_test || ( echo test resolve_address_test failed ; exit 1 )
+	$(E) "[RUN]     Testing resource_quota_test"
+	$(Q) $(BINDIR)/$(CONFIG)/resource_quota_test || ( echo test resource_quota_test failed ; exit 1 )
 	$(E) "[RUN]     Testing secure_channel_create_test"
 	$(Q) $(BINDIR)/$(CONFIG)/secure_channel_create_test || ( echo test secure_channel_create_test failed ; exit 1 )
 	$(E) "[RUN]     Testing secure_endpoint_test"
@@ -1866,6 +1873,8 @@ test_cxx: buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/status_test || ( echo test status_test failed ; exit 1 )
 	$(E) "[RUN]     Testing streaming_throughput_test"
 	$(Q) $(BINDIR)/$(CONFIG)/streaming_throughput_test || ( echo test streaming_throughput_test failed ; exit 1 )
+	$(E) "[RUN]     Testing thread_manager_test"
+	$(Q) $(BINDIR)/$(CONFIG)/thread_manager_test || ( echo test thread_manager_test failed ; exit 1 )
 	$(E) "[RUN]     Testing thread_stress_test"
 	$(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 )
 
@@ -2624,6 +2633,7 @@ LIBGRPC_SRC = \
     src/core/lib/iomgr/resolve_address_posix.c \
     src/core/lib/iomgr/resolve_address_uv.c \
     src/core/lib/iomgr/resolve_address_windows.c \
+    src/core/lib/iomgr/resource_quota.c \
     src/core/lib/iomgr/sockaddr_utils.c \
     src/core/lib/iomgr/socket_utils_common_posix.c \
     src/core/lib/iomgr/socket_utils_linux.c \
@@ -2913,6 +2923,7 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/iomgr/resolve_address_posix.c \
     src/core/lib/iomgr/resolve_address_uv.c \
     src/core/lib/iomgr/resolve_address_windows.c \
+    src/core/lib/iomgr/resource_quota.c \
     src/core/lib/iomgr/sockaddr_utils.c \
     src/core/lib/iomgr/socket_utils_common_posix.c \
     src/core/lib/iomgr/socket_utils_linux.c \
@@ -3193,6 +3204,7 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/lib/iomgr/resolve_address_posix.c \
     src/core/lib/iomgr/resolve_address_uv.c \
     src/core/lib/iomgr/resolve_address_windows.c \
+    src/core/lib/iomgr/resource_quota.c \
     src/core/lib/iomgr/sockaddr_utils.c \
     src/core/lib/iomgr/socket_utils_common_posix.c \
     src/core/lib/iomgr/socket_utils_linux.c \
@@ -3402,6 +3414,7 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/iomgr/resolve_address_posix.c \
     src/core/lib/iomgr/resolve_address_uv.c \
     src/core/lib/iomgr/resolve_address_windows.c \
+    src/core/lib/iomgr/resource_quota.c \
     src/core/lib/iomgr/sockaddr_utils.c \
     src/core/lib/iomgr/socket_utils_common_posix.c \
     src/core/lib/iomgr/socket_utils_linux.c \
@@ -3698,6 +3711,7 @@ LIBGRPC++_SRC = \
     src/cpp/common/channel_filter.cc \
     src/cpp/common/completion_queue_cc.cc \
     src/cpp/common/core_codegen.cc \
+    src/cpp/common/resource_quota_cc.cc \
     src/cpp/common/rpc_method.cc \
     src/cpp/server/async_generic_service.cc \
     src/cpp/server/create_default_thread_pool.cc \
@@ -3707,6 +3721,7 @@ LIBGRPC++_SRC = \
     src/cpp/server/server_context.cc \
     src/cpp/server/server_credentials.cc \
     src/cpp/server/server_posix.cc \
+    src/cpp/thread_manager/thread_manager.cc \
     src/cpp/util/byte_buffer_cc.cc \
     src/cpp/util/slice_cc.cc \
     src/cpp/util/status.cc \
@@ -3742,6 +3757,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc++/impl/thd.h \
     include/grpc++/impl/thd_cxx11.h \
     include/grpc++/impl/thd_no_cxx11.h \
+    include/grpc++/resource_quota.h \
     include/grpc++/security/auth_context.h \
     include/grpc++/security/auth_metadata_processor.h \
     include/grpc++/security/credentials.h \
@@ -3890,6 +3906,7 @@ LIBGRPC++_CRONET_SRC = \
     src/cpp/common/channel_filter.cc \
     src/cpp/common/completion_queue_cc.cc \
     src/cpp/common/core_codegen.cc \
+    src/cpp/common/resource_quota_cc.cc \
     src/cpp/common/rpc_method.cc \
     src/cpp/server/async_generic_service.cc \
     src/cpp/server/create_default_thread_pool.cc \
@@ -3899,6 +3916,7 @@ LIBGRPC++_CRONET_SRC = \
     src/cpp/server/server_context.cc \
     src/cpp/server/server_credentials.cc \
     src/cpp/server/server_posix.cc \
+    src/cpp/thread_manager/thread_manager.cc \
     src/cpp/util/byte_buffer_cc.cc \
     src/cpp/util/slice_cc.cc \
     src/cpp/util/status.cc \
@@ -3934,6 +3952,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc++/impl/thd.h \
     include/grpc++/impl/thd_cxx11.h \
     include/grpc++/impl/thd_no_cxx11.h \
+    include/grpc++/resource_quota.h \
     include/grpc++/security/auth_context.h \
     include/grpc++/security/auth_metadata_processor.h \
     include/grpc++/security/credentials.h \
@@ -4469,6 +4488,7 @@ LIBGRPC++_UNSECURE_SRC = \
     src/cpp/common/channel_filter.cc \
     src/cpp/common/completion_queue_cc.cc \
     src/cpp/common/core_codegen.cc \
+    src/cpp/common/resource_quota_cc.cc \
     src/cpp/common/rpc_method.cc \
     src/cpp/server/async_generic_service.cc \
     src/cpp/server/create_default_thread_pool.cc \
@@ -4478,6 +4498,7 @@ LIBGRPC++_UNSECURE_SRC = \
     src/cpp/server/server_context.cc \
     src/cpp/server/server_credentials.cc \
     src/cpp/server/server_posix.cc \
+    src/cpp/thread_manager/thread_manager.cc \
     src/cpp/util/byte_buffer_cc.cc \
     src/cpp/util/slice_cc.cc \
     src/cpp/util/status.cc \
@@ -4513,6 +4534,7 @@ PUBLIC_HEADERS_CXX += \
     include/grpc++/impl/thd.h \
     include/grpc++/impl/thd_cxx11.h \
     include/grpc++/impl/thd_no_cxx11.h \
+    include/grpc++/resource_quota.h \
     include/grpc++/security/auth_context.h \
     include/grpc++/security/auth_metadata_processor.h \
     include/grpc++/security/credentials.h \
@@ -6947,6 +6969,7 @@ LIBEND2END_TESTS_SRC = \
     test/core/end2end/tests/registered_call.c \
     test/core/end2end/tests/request_with_flags.c \
     test/core/end2end/tests/request_with_payload.c \
+    test/core/end2end/tests/resource_quota_server.c \
     test/core/end2end/tests/server_finishes_request.c \
     test/core/end2end/tests/shutdown_finishes_calls.c \
     test/core/end2end/tests/shutdown_finishes_tags.c \
@@ -7029,6 +7052,7 @@ LIBEND2END_NOSEC_TESTS_SRC = \
     test/core/end2end/tests/registered_call.c \
     test/core/end2end/tests/request_with_flags.c \
     test/core/end2end/tests/request_with_payload.c \
+    test/core/end2end/tests/resource_quota_server.c \
     test/core/end2end/tests/server_finishes_request.c \
     test/core/end2end/tests/shutdown_finishes_calls.c \
     test/core/end2end/tests/shutdown_finishes_tags.c \
@@ -10425,6 +10449,38 @@ endif
 endif
 
 
+RESOURCE_QUOTA_TEST_SRC = \
+    test/core/iomgr/resource_quota_test.c \
+
+RESOURCE_QUOTA_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(RESOURCE_QUOTA_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/resource_quota_test: openssl_dep_error
+
+else
+
+
+
+$(BINDIR)/$(CONFIG)/resource_quota_test: $(RESOURCE_QUOTA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LD) $(LDFLAGS) $(RESOURCE_QUOTA_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/resource_quota_test
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/core/iomgr/resource_quota_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
+
+deps_resource_quota_test: $(RESOURCE_QUOTA_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(RESOURCE_QUOTA_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 SECURE_CHANNEL_CREATE_TEST_SRC = \
     test/core/surface/secure_channel_create_test.c \
 
@@ -13522,6 +13578,49 @@ $(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_test.o: $(GENDIR)/src/proto/grpc/tes
 $(OBJDIR)/$(CONFIG)/test/cpp/util/metrics_server.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc
 
 
+THREAD_MANAGER_TEST_SRC = \
+    test/cpp/thread_manager/thread_manager_test.cc \
+
+THREAD_MANAGER_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(THREAD_MANAGER_TEST_SRC))))
+ifeq ($(NO_SECURE),true)
+
+# You can't build secure targets if you don't have OpenSSL.
+
+$(BINDIR)/$(CONFIG)/thread_manager_test: openssl_dep_error
+
+else
+
+
+
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
+
+$(BINDIR)/$(CONFIG)/thread_manager_test: protobuf_dep_error
+
+else
+
+$(BINDIR)/$(CONFIG)/thread_manager_test: $(PROTOBUF_DEP) $(THREAD_MANAGER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+	$(E) "[LD]      Linking $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) $(LDXX) $(LDFLAGS) $(THREAD_MANAGER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/thread_manager_test
+
+endif
+
+endif
+
+$(OBJDIR)/$(CONFIG)/test/cpp/thread_manager/thread_manager_test.o:  $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
+
+deps_thread_manager_test: $(THREAD_MANAGER_TEST_OBJS:.o=.dep)
+
+ifneq ($(NO_SECURE),true)
+ifneq ($(NO_DEPS),true)
+-include $(THREAD_MANAGER_TEST_OBJS:.o=.dep)
+endif
+endif
+
+
 THREAD_STRESS_TEST_SRC = \
     test/cpp/end2end/thread_stress_test.cc \
 

binding.gyp

@@ -612,6 +612,7 @@
         'src/core/lib/iomgr/resolve_address_posix.c',
         'src/core/lib/iomgr/resolve_address_uv.c',
         'src/core/lib/iomgr/resolve_address_windows.c',
+        'src/core/lib/iomgr/resource_quota.c',
         'src/core/lib/iomgr/sockaddr_utils.c',
         'src/core/lib/iomgr/socket_utils_common_posix.c',
         'src/core/lib/iomgr/socket_utils_linux.c',

build.yaml

@@ -208,6 +208,7 @@ filegroups:
   - src/core/lib/iomgr/pollset_windows.h
   - src/core/lib/iomgr/port.h
   - src/core/lib/iomgr/resolve_address.h
+  - src/core/lib/iomgr/resource_quota.h
   - src/core/lib/iomgr/sockaddr.h
   - src/core/lib/iomgr/sockaddr_posix.h
   - src/core/lib/iomgr/sockaddr_utils.h
@@ -216,6 +217,7 @@ filegroups:
   - src/core/lib/iomgr/socket_utils_posix.h
   - src/core/lib/iomgr/socket_windows.h
   - src/core/lib/iomgr/tcp_client.h
+  - src/core/lib/iomgr/tcp_client_posix.h
   - src/core/lib/iomgr/tcp_posix.h
   - src/core/lib/iomgr/tcp_server.h
   - src/core/lib/iomgr/tcp_uv.h
@@ -303,6 +305,7 @@ filegroups:
   - src/core/lib/iomgr/resolve_address_posix.c
   - src/core/lib/iomgr/resolve_address_uv.c
   - src/core/lib/iomgr/resolve_address_windows.c
+  - src/core/lib/iomgr/resource_quota.c
   - src/core/lib/iomgr/sockaddr_utils.c
   - src/core/lib/iomgr/socket_utils_common_posix.c
   - src/core/lib/iomgr/socket_utils_linux.c
@@ -713,6 +716,7 @@ filegroups:
   - include/grpc++/impl/thd.h
   - include/grpc++/impl/thd_cxx11.h
   - include/grpc++/impl/thd_no_cxx11.h
+  - include/grpc++/resource_quota.h
   - include/grpc++/security/auth_context.h
   - include/grpc++/security/auth_metadata_processor.h
   - include/grpc++/security/credentials.h
@@ -738,6 +742,7 @@ filegroups:
   - src/cpp/common/channel_filter.h
   - src/cpp/server/dynamic_thread_pool.h
   - src/cpp/server/thread_pool_interface.h
+  - src/cpp/thread_manager/thread_manager.h
   src:
   - src/cpp/client/channel_cc.cc
   - src/cpp/client/client_context.cc
@@ -750,6 +755,7 @@ filegroups:
   - src/cpp/common/channel_filter.cc
   - src/cpp/common/completion_queue_cc.cc
   - src/cpp/common/core_codegen.cc
+  - src/cpp/common/resource_quota_cc.cc
   - src/cpp/common/rpc_method.cc
   - src/cpp/server/async_generic_service.cc
   - src/cpp/server/create_default_thread_pool.cc
@@ -759,6 +765,7 @@ filegroups:
   - src/cpp/server/server_context.cc
   - src/cpp/server/server_credentials.cc
   - src/cpp/server/server_posix.cc
+  - src/cpp/thread_manager/thread_manager.cc
   - src/cpp/util/byte_buffer_cc.cc
   - src/cpp/util/slice_cc.cc
   - src/cpp/util/status.cc
@@ -2427,6 +2434,17 @@ targets:
   - gpr
   exclude_iomgrs:
   - uv
+- name: resource_quota_test
+  cpu_cost: 30
+  build: test
+  language: c
+  src:
+  - test/core/iomgr/resource_quota_test.c
+  deps:
+  - grpc_test_util
+  - grpc
+  - gpr_test_util
+  - gpr
 - name: secure_channel_create_test
   build: test
   language: c
@@ -3487,6 +3505,16 @@ targets:
   - gpr_test_util
   - gpr
   - grpc++_test_config
+- name: thread_manager_test
+  build: test
+  language: c++
+  src:
+  - test/cpp/thread_manager/thread_manager_test.cc
+  deps:
+  - grpc++
+  - grpc
+  - gpr
+  - grpc++_test_config
 - name: thread_stress_test
   gtest: true
   cpu_cost: 100

config.m4

@@ -128,6 +128,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/iomgr/resolve_address_posix.c \
     src/core/lib/iomgr/resolve_address_uv.c \
     src/core/lib/iomgr/resolve_address_windows.c \
+    src/core/lib/iomgr/resource_quota.c \
     src/core/lib/iomgr/sockaddr_utils.c \
     src/core/lib/iomgr/socket_utils_common_posix.c \
     src/core/lib/iomgr/socket_utils_linux.c \

gRPC-Core.podspec

@@ -35,7 +35,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-Core'
-  version = '1.0.0'
+  version = '1.0.1'
   s.version  = version
   s.summary  = 'Core cross-platform gRPC library, written in C'
   s.homepage = 'http://www.grpc.io'
@@ -186,7 +186,7 @@ Pod::Spec.new do |s|
     ss.header_mappings_dir = '.'
     ss.libraries = 'z'
     ss.dependency "#{s.name}/Interface", version
-    ss.dependency 'BoringSSL', '~> 6.0'
+    ss.dependency 'BoringSSL', '~> 7.0'
 
     # To save you from scrolling, this is the last part of the podspec.
     ss.source_files = 'src/core/lib/profiling/timers.h',
@@ -291,6 +291,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/iomgr/pollset_windows.h',
                       'src/core/lib/iomgr/port.h',
                       'src/core/lib/iomgr/resolve_address.h',
+                      'src/core/lib/iomgr/resource_quota.h',
                       'src/core/lib/iomgr/sockaddr.h',
                       'src/core/lib/iomgr/sockaddr_posix.h',
                       'src/core/lib/iomgr/sockaddr_utils.h',
@@ -299,6 +300,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/iomgr/socket_utils_posix.h',
                       'src/core/lib/iomgr/socket_windows.h',
                       'src/core/lib/iomgr/tcp_client.h',
+                      'src/core/lib/iomgr/tcp_client_posix.h',
                       'src/core/lib/iomgr/tcp_posix.h',
                       'src/core/lib/iomgr/tcp_server.h',
                       'src/core/lib/iomgr/tcp_uv.h',
@@ -466,6 +468,7 @@ Pod::Spec.new do |s|
                       'src/core/lib/iomgr/resolve_address_posix.c',
                       'src/core/lib/iomgr/resolve_address_uv.c',
                       'src/core/lib/iomgr/resolve_address_windows.c',
+                      'src/core/lib/iomgr/resource_quota.c',
                       'src/core/lib/iomgr/sockaddr_utils.c',
                       'src/core/lib/iomgr/socket_utils_common_posix.c',
                       'src/core/lib/iomgr/socket_utils_linux.c',
@@ -684,6 +687,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/iomgr/pollset_windows.h',
                               'src/core/lib/iomgr/port.h',
                               'src/core/lib/iomgr/resolve_address.h',
+                              'src/core/lib/iomgr/resource_quota.h',
                               'src/core/lib/iomgr/sockaddr.h',
                               'src/core/lib/iomgr/sockaddr_posix.h',
                               'src/core/lib/iomgr/sockaddr_utils.h',
@@ -692,6 +696,7 @@ Pod::Spec.new do |s|
                               'src/core/lib/iomgr/socket_utils_posix.h',
                               'src/core/lib/iomgr/socket_windows.h',
                               'src/core/lib/iomgr/tcp_client.h',
+                              'src/core/lib/iomgr/tcp_client_posix.h',
                               'src/core/lib/iomgr/tcp_posix.h',
                               'src/core/lib/iomgr/tcp_server.h',
                               'src/core/lib/iomgr/tcp_uv.h',

gRPC-ProtoRPC.podspec

@@ -30,7 +30,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-ProtoRPC'
-  version = '1.0.0'
+  version = '1.0.1'
   s.version  = version
   s.summary  = 'RPC library for Protocol Buffers, based on gRPC'
   s.homepage = 'http://www.grpc.io'

gRPC-RxLibrary.podspec

@@ -30,7 +30,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC-RxLibrary'
-  version = '1.0.0'
+  version = '1.0.1'
   s.version  = version
   s.summary  = 'Reactive Extensions library for iOS/OSX.'
   s.homepage = 'http://www.grpc.io'

gRPC.podspec

@@ -30,7 +30,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'gRPC'
-  version = '1.0.0'
+  version = '1.0.1'
   s.version  = version
   s.summary  = 'gRPC client library for iOS/OSX'
   s.homepage = 'http://www.grpc.io'

grpc.def

@@ -94,6 +94,11 @@ EXPORTS
     grpc_header_nonbin_value_is_legal
     grpc_is_binary_header
     grpc_call_error_to_string
+    grpc_resource_quota_create
+    grpc_resource_quota_ref
+    grpc_resource_quota_unref
+    grpc_resource_quota_resize
+    grpc_resource_quota_arg_vtable
     grpc_insecure_channel_create_from_fd
     grpc_server_add_insecure_channel_from_fd
     grpc_use_signal

grpc.gemspec

@@ -211,6 +211,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/iomgr/pollset_windows.h )
   s.files += %w( src/core/lib/iomgr/port.h )
   s.files += %w( src/core/lib/iomgr/resolve_address.h )
+  s.files += %w( src/core/lib/iomgr/resource_quota.h )
   s.files += %w( src/core/lib/iomgr/sockaddr.h )
   s.files += %w( src/core/lib/iomgr/sockaddr_posix.h )
   s.files += %w( src/core/lib/iomgr/sockaddr_utils.h )
@@ -219,6 +220,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/iomgr/socket_utils_posix.h )
   s.files += %w( src/core/lib/iomgr/socket_windows.h )
   s.files += %w( src/core/lib/iomgr/tcp_client.h )
+  s.files += %w( src/core/lib/iomgr/tcp_client_posix.h )
   s.files += %w( src/core/lib/iomgr/tcp_posix.h )
   s.files += %w( src/core/lib/iomgr/tcp_server.h )
   s.files += %w( src/core/lib/iomgr/tcp_uv.h )
@@ -386,6 +388,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/iomgr/resolve_address_posix.c )
   s.files += %w( src/core/lib/iomgr/resolve_address_uv.c )
   s.files += %w( src/core/lib/iomgr/resolve_address_windows.c )
+  s.files += %w( src/core/lib/iomgr/resource_quota.c )
   s.files += %w( src/core/lib/iomgr/sockaddr_utils.c )
   s.files += %w( src/core/lib/iomgr/socket_utils_common_posix.c )
   s.files += %w( src/core/lib/iomgr/socket_utils_linux.c )

include/grpc++/impl/codegen/server_interface.h

@@ -126,12 +126,6 @@ class ServerInterface : public CallHook {
   /// \return true on a successful shutdown.
   virtual bool Start(ServerCompletionQueue** cqs, size_t num_cqs) = 0;
 
-  /// Process one or more incoming calls.
-  virtual void RunRpc() = 0;
-
-  /// Schedule \a RunRpc to run in the threadpool.
-  virtual void ScheduleCallback() = 0;
-
   virtual void ShutdownInternal(gpr_timespec deadline) = 0;
 
   virtual int max_receive_message_size() const = 0;

include/grpc++/resource_quota.h

@@ -0,0 +1,70 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_RESOURCE_QUOTA_H
+#define GRPCXX_RESOURCE_QUOTA_H
+
+struct grpc_resource_quota;
+
+#include <grpc++/impl/codegen/config.h>
+
+namespace grpc {
+
+/// ResourceQuota represents a bound on memory usage by the gRPC library.
+/// A ResourceQuota can be attached to a server (via ServerBuilder), or a client
+/// channel (via ChannelArguments). gRPC will attempt to keep memory used by
+/// all attached entities below the ResourceQuota bound.
+class ResourceQuota GRPC_FINAL {
+ public:
+  explicit ResourceQuota(const grpc::string& name);
+  ResourceQuota();
+  ~ResourceQuota();
+
+  /// Resize this ResourceQuota to a new size. If new_size is smaller than the
+  /// current size of the pool, memory usage will be monotonically decreased
+  /// until it falls under new_size. No time bound is given for this to occur
+  /// however.
+  ResourceQuota& Resize(size_t new_size);
+
+  grpc_resource_quota* c_resource_quota() const { return impl_; }
+
+ private:
+  ResourceQuota(const ResourceQuota& rhs);
+  ResourceQuota& operator=(const ResourceQuota& rhs);
+
+  grpc_resource_quota* const impl_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPCXX_RESOURCE_QUOTA_H

include/grpc++/server.h

@@ -105,18 +105,41 @@ class Server GRPC_FINAL : public ServerInterface, private GrpcLibraryCodegen {
   class AsyncRequest;
   class ShutdownRequest;
 
+  /// SyncRequestThreadManager is an implementation of ThreadManager. This class
+  /// is responsible for polling for incoming RPCs and calling the RPC handlers.
+  /// This is only used in case of a Sync server (i.e a server exposing a sync
+  /// interface)
+  class SyncRequestThreadManager;
+
   class UnimplementedAsyncRequestContext;
   class UnimplementedAsyncRequest;
   class UnimplementedAsyncResponse;
 
   /// Server constructors. To be used by \a ServerBuilder only.
   ///
-  /// \param thread_pool The threadpool instance to use for call processing.
-  /// \param thread_pool_owned Does the server own the \a thread_pool instance?
-  /// \param max_receive_message_size Maximum message length that the channel
-  /// can receive.
-  Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
-         int max_receive_message_size, ChannelArguments* args);
+  /// \param max_message_size Maximum message length that the channel can
+  /// receive.
+  ///
+  /// \param args The channel args
+  ///
+  /// \param sync_server_cqs The completion queues to use if the server is a
+  /// synchronous server (or a hybrid server). The server polls for new RPCs on
+  /// these queues
+  ///
+  /// \param min_pollers The minimum number of polling threads per server
+  /// completion queue (in param sync_server_cqs) to use for listening to
+  /// incoming requests (used only in case of sync server)
+  ///
+  /// \param max_pollers The maximum number of polling threads per server
+  /// completion queue (in param sync_server_cqs) to use for listening to
+  /// incoming requests (used only in case of sync server)
+  ///
+  /// \param sync_cq_timeout_msec The timeout to use when calling AsyncNext() on
+  /// server completion queues passed via sync_server_cqs param.
+  Server(int max_message_size, ChannelArguments* args,
+         std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
+             sync_server_cqs,
+         int min_pollers, int max_pollers, int sync_cq_timeout_msec);
 
   /// Register a service. This call does not take ownership of the service.
   /// The service must exist for the lifetime of the Server instance.
@@ -151,12 +174,6 @@ class Server GRPC_FINAL : public ServerInterface, private GrpcLibraryCodegen {
   /// \return true on a successful shutdown.
   bool Start(ServerCompletionQueue** cqs, size_t num_cqs) GRPC_OVERRIDE;
 
-  /// Process one or more incoming calls.
-  void RunRpc() GRPC_OVERRIDE;
-
-  /// Schedule \a RunRpc to run in the threadpool.
-  void ScheduleCallback() GRPC_OVERRIDE;
-
   void PerformOpsOnCall(CallOpSetInterface* ops, Call* call) GRPC_OVERRIDE;
 
   void ShutdownInternal(gpr_timespec deadline) GRPC_OVERRIDE;
@@ -171,34 +188,31 @@ class Server GRPC_FINAL : public ServerInterface, private GrpcLibraryCodegen {
 
   const int max_receive_message_size_;
 
-  // Completion queue.
-  CompletionQueue cq_;
+  /// The following completion queues are ONLY used in case of Sync API i.e if
+  /// the server has any services with sync methods. The server uses these
+  /// completion queues to poll for new RPCs
+  std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
+      sync_server_cqs_;
+
+  /// List of ThreadManager instances (one for each cq in the sync_server_cqs)
+  std::vector<std::unique_ptr<SyncRequestThreadManager>> sync_req_mgrs_;
 
   // Sever status
   grpc::mutex mu_;
   bool started_;
   bool shutdown_;
-  bool shutdown_notified_;
-  // The number of threads which are running callbacks.
-  int num_running_cb_;
-  grpc::condition_variable callback_cv_;
+  bool shutdown_notified_;  // Was notify called on the shutdown_cv_
 
   grpc::condition_variable shutdown_cv_;
 
   std::shared_ptr<GlobalCallbacks> global_callbacks_;
 
-  std::list<SyncRequest>* sync_methods_;
   std::vector<grpc::string> services_;
-  std::unique_ptr<RpcServiceMethod> unknown_method_;
   bool has_generic_service_;
 
-  // Pointer to the c grpc server.
+  // Pointer to the wrapped grpc_server.
   grpc_server* server_;
 
-  ThreadPoolInterface* thread_pool_;
-  // Whether the thread pool is created and owned by the server.
-  bool thread_pool_owned_;
-
   std::unique_ptr<ServerInitializer> server_initializer_;
 };
 

include/grpc++/server_builder.h

@@ -34,6 +34,7 @@
 #ifndef GRPCXX_SERVER_BUILDER_H
 #define GRPCXX_SERVER_BUILDER_H
 
+#include <climits>
 #include <map>
 #include <memory>
 #include <vector>
@@ -42,10 +43,15 @@
 #include <grpc++/impl/server_builder_plugin.h>
 #include <grpc++/support/config.h>
 #include <grpc/compression.h>
+#include <grpc/support/cpu.h>
+#include <grpc/support/useful.h>
+
+struct grpc_resource_quota;
 
 namespace grpc {
 
 class AsyncGenericService;
+class ResourceQuota;
 class CompletionQueue;
 class RpcService;
 class Server;
@@ -61,6 +67,9 @@ class ServerBuilderPluginTest;
 class ServerBuilder {
  public:
   ServerBuilder();
+  ~ServerBuilder();
+
+  enum SyncServerOption { NUM_CQS, MIN_POLLERS, MAX_POLLERS, CQ_TIMEOUT_MSEC };
 
   /// Register a service. This call does not take ownership of the service.
   /// The service must exist for the lifetime of the \a Server instance returned
@@ -113,8 +122,14 @@ class ServerBuilder {
   ServerBuilder& SetDefaultCompressionAlgorithm(
       grpc_compression_algorithm algorithm);
 
+  /// Set the attached buffer pool for this server
+  ServerBuilder& SetResourceQuota(const ResourceQuota& resource_quota);
+
   ServerBuilder& SetOption(std::unique_ptr<ServerBuilderOption> option);
 
+  /// Only useful if this is a Synchronous server.
+  ServerBuilder& SetSyncServerOption(SyncServerOption option, int value);
+
   /// Tries to bind \a server to the given \a addr.
   ///
   /// It can be invoked multiple times.
@@ -170,6 +185,28 @@ class ServerBuilder {
     int* selected_port;
   };
 
+  struct SyncServerSettings {
+    SyncServerSettings()
+        : num_cqs(GPR_MAX(gpr_cpu_num_cores(), 4)),
+          min_pollers(1),
+          max_pollers(INT_MAX),
+          cq_timeout_msec(1000) {}
+
+    // Number of server completion queues to create to listen to incoming RPCs.
+    int num_cqs;
+
+    // Minimum number of threads per completion queue that should be listening
+    // to incoming RPCs.
+    int min_pollers;
+
+    // Maximum number of threads per completion queue that can be listening to
+    // incoming RPCs.
+    int max_pollers;
+
+    // The timeout for server completion queue's AsyncNext call.
+    int cq_timeout_msec;
+  };
+
   typedef std::unique_ptr<grpc::string> HostString;
   struct NamedService {
     explicit NamedService(Service* s) : service(s) {}
@@ -184,9 +221,15 @@ class ServerBuilder {
   std::vector<std::unique_ptr<ServerBuilderOption>> options_;
   std::vector<std::unique_ptr<NamedService>> services_;
   std::vector<Port> ports_;
+
+  SyncServerSettings sync_server_settings_;
+
+  // List of completion queues added via AddCompletionQueue() method
   std::vector<ServerCompletionQueue*> cqs_;
+
   std::shared_ptr<ServerCredentials> creds_;
   std::vector<std::unique_ptr<ServerBuilderPlugin>> plugins_;
+  grpc_resource_quota* resource_quota_;
   AsyncGenericService* generic_service_;
   struct {
     bool is_set;

include/grpc++/support/channel_arguments.h

@@ -46,6 +46,8 @@ namespace testing {
 class ChannelArgumentsTest;
 }  // namespace testing
 
+class ResourceQuota;
+
 /// Options for channel creation. The user can use generic setters to pass
 /// key value pairs down to c channel creation code. For grpc related options,
 /// concrete setters are provided.
@@ -80,6 +82,9 @@ class ChannelArguments {
   /// The given string will be sent at the front of the user agent string.
   void SetUserAgentPrefix(const grpc::string& user_agent_prefix);
 
+  /// The given buffer pool will be attached to the constructed channel
+  void SetResourceQuota(const ResourceQuota& resource_quota);
+
   // Set LB policy name.
   // Note that if the name resolver returns only balancer addresses, the
   // grpclb LB policy will be used, regardless of what is specified here.
@@ -93,6 +98,9 @@ class ChannelArguments {
   /// Set a pointer argument \a value under \a key. Owership is not transferred.
   void SetPointer(const grpc::string& key, void* value);
 
+  void SetPointerWithVtable(const grpc::string& key, void* value,
+                            const grpc_arg_pointer_vtable* vtable);
+
   /// Set a textual argument \a value under \a key.
   void SetString(const grpc::string& key, const grpc::string& value);
 

include/grpc/grpc.h

@@ -401,6 +401,23 @@ GRPCAPI int grpc_is_binary_header(const char *key, size_t length);
 /** Convert grpc_call_error values to a string */
 GRPCAPI const char *grpc_call_error_to_string(grpc_call_error error);
 
+/** Create a buffer pool */
+GRPCAPI grpc_resource_quota *grpc_resource_quota_create(const char *trace_name);
+
+/** Add a reference to a buffer pool */
+GRPCAPI void grpc_resource_quota_ref(grpc_resource_quota *resource_quota);
+
+/** Drop a reference to a buffer pool */
+GRPCAPI void grpc_resource_quota_unref(grpc_resource_quota *resource_quota);
+
+/** Update the size of a buffer pool */
+GRPCAPI void grpc_resource_quota_resize(grpc_resource_quota *resource_quota,
+                                        size_t new_size);
+
+/** Fetch a vtable for a grpc_channel_arg that points to a grpc_resource_quota
+ */
+GRPCAPI const grpc_arg_pointer_vtable *grpc_resource_quota_arg_vtable(void);
+
 #ifdef __cplusplus
 }
 #endif

include/grpc/impl/codegen/grpc_types.h

@@ -201,6 +201,9 @@ typedef struct {
 #define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size"
 /** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */
 #define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport"
+/** If non-zero, a pointer to a buffer pool (use grpc_resource_quota_arg_vtable
+   to fetch an appropriate pointer arg vtable */
+#define GRPC_ARG_RESOURCE_QUOTA "grpc.resource_quota"
 /** Service config data, to be passed to subchannels.
     Not intended for external use. */
 #define GRPC_ARG_SERVICE_CONFIG "grpc.service_config"
@@ -467,6 +470,8 @@ typedef struct grpc_op {
   } data;
 } grpc_op;
 
+typedef struct grpc_resource_quota grpc_resource_quota;
+
 #ifdef __cplusplus
 }
 #endif

package.json

@@ -25,24 +25,28 @@
     "coverage": "./node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha src/node/test",
     "install": "./node_modules/.bin/node-pre-gyp install --fallback-to-build"
   },
-  "bundledDependencies": ["node-pre-gyp"],
+  "bundledDependencies": [
+    "node-pre-gyp"
+  ],
   "dependencies": {
     "arguejs": "^0.2.3",
-    "lodash": "^3.9.3",
+    "lodash": "^4.15.0",
     "nan": "^2.0.0",
-    "protobufjs": "^4.0.0"
+    "node-pre-gyp": "^0.6.0",
+    "protobufjs": "^5.0.0"
   },
   "devDependencies": {
-    "async": "^1.5.0",
+    "async": "^2.0.1",
+    "body-parser": "^1.15.2",
+    "express": "^4.14.0",
     "google-auth-library": "^0.9.2",
     "google-protobuf": "^3.0.0",
-    "istanbul": "^0.3.21",
+    "istanbul": "^0.4.4",
     "jsdoc": "^3.3.2",
     "jshint": "^2.5.0",
     "minimist": "^1.1.0",
-    "mocha": "^2.3.4",
-    "mocha-jenkins-reporter": "^0.1.9",
-    "mustache": "^2.0.0",
+    "mocha": "^3.0.2",
+    "mocha-jenkins-reporter": "^0.2.3",
     "poisson-process": "^0.2.1"
   },
   "engines": {
@@ -50,11 +54,10 @@
   },
   "binary": {
     "module_name": "grpc_node",
-    "module_path": "./build/Release/",
+    "module_path": "src/node/extension_binary",
     "host": "https://storage.googleapis.com/",
     "remote_path": "grpc-precompiled-binaries/node/{name}/v{version}",
-    "package_name": "{node_abi}-{platform}-{arch}.tar.gz",
-    "module_path": "src/node/extension_binary"
+    "package_name": "{node_abi}-{platform}-{arch}.tar.gz"
   },
   "files": [
     "LICENSE",
@@ -75,7 +78,7 @@
   ],
   "main": "src/node/index.js",
   "license": "BSD-3-Clause",
-  "jshintConfig" : {
+  "jshintConfig": {
     "bitwise": true,
     "curly": true,
     "eqeqeq": true,

package.xml

@@ -218,6 +218,7 @@
     <file baseinstalldir="/" name="src/core/lib/iomgr/pollset_windows.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/port.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/resolve_address.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/resource_quota.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/sockaddr.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/sockaddr_posix.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/sockaddr_utils.h" role="src" />
@@ -226,6 +227,7 @@
     <file baseinstalldir="/" name="src/core/lib/iomgr/socket_utils_posix.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/socket_windows.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/tcp_client.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/tcp_client_posix.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/tcp_posix.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/tcp_server.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/tcp_uv.h" role="src" />
@@ -393,6 +395,7 @@
     <file baseinstalldir="/" name="src/core/lib/iomgr/resolve_address_posix.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/resolve_address_uv.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/resolve_address_windows.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/iomgr/resource_quota.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/sockaddr_utils.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/socket_utils_common_posix.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/iomgr/socket_utils_linux.c" role="src" />

src/compiler/python_generator.cc

@@ -35,6 +35,8 @@
 #include <cassert>
 #include <cctype>
 #include <cstring>
+#include <fstream>
+#include <iostream>
 #include <map>
 #include <memory>
 #include <ostream>
@@ -66,66 +68,11 @@ using std::vector;
 
 namespace grpc_python_generator {
 
-GeneratorConfiguration::GeneratorConfiguration()
-    : grpc_package_root("grpc"), beta_package_root("grpc.beta") {}
-
-PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config)
-    : config_(config) {}
-
-PythonGrpcGenerator::~PythonGrpcGenerator() {}
-
-bool PythonGrpcGenerator::Generate(const FileDescriptor* file,
-                                   const grpc::string& parameter,
-                                   GeneratorContext* context,
-                                   grpc::string* error) const {
-  // Get output file name.
-  grpc::string file_name;
-  static const int proto_suffix_length = strlen(".proto");
-  if (file->name().size() > static_cast<size_t>(proto_suffix_length) &&
-      file->name().find_last_of(".proto") == file->name().size() - 1) {
-    file_name =
-        file->name().substr(0, file->name().size() - proto_suffix_length) +
-        "_pb2.py";
-  } else {
-    *error = "Invalid proto file name. Proto file must end with .proto";
-    return false;
-  }
-
-  std::unique_ptr<ZeroCopyOutputStream> output(
-      context->OpenForInsert(file_name, "module_scope"));
-  CodedOutputStream coded_out(output.get());
-  bool success = false;
-  grpc::string code = "";
-  tie(success, code) = grpc_python_generator::GetServices(file, config_);
-  if (success) {
-    coded_out.WriteRaw(code.data(), code.size());
-    return true;
-  } else {
-    return false;
-  }
-}
-
 namespace {
-//////////////////////////////////
-// BEGIN FORMATTING BOILERPLATE //
-//////////////////////////////////
-
-// Converts an initializer list of the form { key0, value0, key1, value1, ... }
-// into a map of key* to value*. Is merely a readability helper for later code.
-map<grpc::string, grpc::string> ListToDict(
-    const initializer_list<grpc::string>& values) {
-  assert(values.size() % 2 == 0);
-  map<grpc::string, grpc::string> value_map;
-  auto value_iter = values.begin();
-  for (unsigned i = 0; i < values.size() / 2; ++i) {
-    grpc::string key = *value_iter;
-    ++value_iter;
-    grpc::string value = *value_iter;
-    value_map[key] = value;
-    ++value_iter;
-  }
-  return value_map;
-}
+
+typedef vector<const Descriptor*> DescriptorVector;
+typedef map<grpc::string, grpc::string> StringMap;
+typedef vector<grpc::string> StringVector;
 
 // Provides RAII indentation handling. Use as:
 // {
@@ -146,10 +93,6 @@ class IndentScope {
   Printer* printer_;
 };
 
-////////////////////////////////
-// END FORMATTING BOILERPLATE //
-////////////////////////////////
-
 // TODO(https://github.com/google/protobuf/issues/888):
 // Export `ModuleName` from protobuf's
 // `src/google/protobuf/compiler/python/python_generator.cc` file.
@@ -173,11 +116,61 @@ grpc::string ModuleAlias(const grpc::string& filename) {
   return module_name;
 }
 
-bool GetModuleAndMessagePath(const Descriptor* type,
-                             const ServiceDescriptor* service,
-                             grpc::string* out) {
+// Tucks all generator state in an anonymous namespace away from
+// PythonGrpcGenerator and the header file, mostly to encourage future changes
+// to not require updates to the grpcio-tools C++ code part. Assumes that it is
+// only ever used from a single thread.
+struct PrivateGenerator {
+  const GeneratorConfiguration& config;
+  const FileDescriptor* file;
+
+  bool generate_in_pb2_grpc;
+
+  Printer* out;
+
+  PrivateGenerator(const GeneratorConfiguration& config,
+                   const FileDescriptor* file);
+
+  std::pair<bool, grpc::string> GetGrpcServices();
+
+ private:
+  bool PrintPreamble();
+  bool PrintBetaPreamble();
+  bool PrintGAServices();
+  bool PrintBetaServices();
+
+  bool PrintAddServicerToServer(
+      const grpc::string& package_qualified_service_name,
+      const ServiceDescriptor* service);
+  bool PrintServicer(const ServiceDescriptor* service);
+  bool PrintStub(const grpc::string& package_qualified_service_name,
+                 const ServiceDescriptor* service);
+
+  bool PrintBetaServicer(const ServiceDescriptor* service);
+  bool PrintBetaServerFactory(
+      const grpc::string& package_qualified_service_name,
+      const ServiceDescriptor* service);
+  bool PrintBetaStub(const ServiceDescriptor* service);
+  bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
+                            const ServiceDescriptor* service);
+
+  // Get all comments (leading, leading_detached, trailing) and print them as a
+  // docstring. Any leading space of a line will be removed, but the line
+  // wrapping will not be changed.
+  template <typename DescriptorType>
+  void PrintAllComments(const DescriptorType* descriptor);
+
+  bool GetModuleAndMessagePath(const Descriptor* type, grpc::string* out);
+};
+
+PrivateGenerator::PrivateGenerator(const GeneratorConfiguration& config,
+                                   const FileDescriptor* file)
+    : config(config), file(file) {}
+
+bool PrivateGenerator::GetModuleAndMessagePath(const Descriptor* type,
+                                               grpc::string* out) {
   const Descriptor* path_elem_type = type;
-  vector<const Descriptor*> message_path;
+  DescriptorVector message_path;
   do {
     message_path.push_back(path_elem_type);
     path_elem_type = path_elem_type->containing_type();
@@ -188,12 +181,16 @@ bool GetModuleAndMessagePath(const Descriptor* type,
         file_name.find_last_of(".proto") == file_name.size() - 1)) {
     return false;
   }
-  grpc::string service_file_name = service->file()->name();
-  grpc::string module =
-      service_file_name == file_name ? "" : ModuleAlias(file_name) + ".";
+  grpc::string generator_file_name = file->name();
+  grpc::string module;
+  if (generator_file_name != file_name || generate_in_pb2_grpc) {
+    module = ModuleAlias(file_name) + ".";
+  } else {
+    module = "";
+  }
   grpc::string message_type;
-  for (auto path_iter = message_path.rbegin(); path_iter != message_path.rend();
-       ++path_iter) {
+  for (DescriptorVector::reverse_iterator path_iter = message_path.rbegin();
+       path_iter != message_path.rend(); ++path_iter) {
     message_type += (*path_iter)->name() + ".";
   }
   // no pop_back prior to C++11
@@ -202,33 +199,31 @@ bool GetModuleAndMessagePath(const Descriptor* type,
   return true;
 }
 
-// Get all comments (leading, leading_detached, trailing) and print them as a
-// docstring. Any leading space of a line will be removed, but the line wrapping
-// will not be changed.
 template <typename DescriptorType>
-static void PrintAllComments(const DescriptorType* desc, Printer* printer) {
-  std::vector<grpc::string> comments;
-  grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_LEADING_DETACHED,
-                             &comments);
-  grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_LEADING,
+void PrivateGenerator::PrintAllComments(const DescriptorType* descriptor) {
+  StringVector comments;
+  grpc_generator::GetComment(
+      descriptor, grpc_generator::COMMENTTYPE_LEADING_DETACHED, &comments);
+  grpc_generator::GetComment(descriptor, grpc_generator::COMMENTTYPE_LEADING,
                              &comments);
-  grpc_generator::GetComment(desc, grpc_generator::COMMENTTYPE_TRAILING,
+  grpc_generator::GetComment(descriptor, grpc_generator::COMMENTTYPE_TRAILING,
                              &comments);
   if (comments.empty()) {
     return;
   }
-  printer->Print("\"\"\"");
-  for (auto it = comments.begin(); it != comments.end(); ++it) {
+  out->Print("\"\"\"");
+  for (StringVector::iterator it = comments.begin(); it != comments.end();
+       ++it) {
     size_t start_pos = it->find_first_not_of(' ');
     if (start_pos != grpc::string::npos) {
-      printer->Print(it->c_str() + start_pos);
+      out->Print(it->c_str() + start_pos);
     }
-    printer->Print("\n");
+    out->Print("\n");
   }
-  printer->Print("\"\"\"\n");
+  out->Print("\"\"\"\n");
 }
 
-bool PrintBetaServicer(const ServiceDescriptor* service, Printer* out) {
+bool PrivateGenerator::PrintBetaServicer(const ServiceDescriptor* service) {
   out->Print("\n\n");
   out->Print("class Beta$Service$Servicer(object):\n", "Service",
              service->name());
@@ -241,16 +236,16 @@ bool PrintBetaServicer(const ServiceDescriptor* service, Printer* out) {
         "generated\n"
         "only to ease transition from grpcio<0.15.0 to "
         "grpcio>=0.15.0.\"\"\"\n");
-    PrintAllComments(service, out);
+    PrintAllComments(service);
     for (int i = 0; i < service->method_count(); ++i) {
-      auto meth = service->method(i);
+      const MethodDescriptor* method = service->method(i);
       grpc::string arg_name =
-          meth->client_streaming() ? "request_iterator" : "request";
+          method->client_streaming() ? "request_iterator" : "request";
       out->Print("def $Method$(self, $ArgName$, context):\n", "Method",
-                 meth->name(), "ArgName", arg_name);
+                 method->name(), "ArgName", arg_name);
       {
         IndentScope raii_method_indent(out);
-        PrintAllComments(meth, out);
+        PrintAllComments(method);
         out->Print("context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)\n");
       }
     }
@@ -258,7 +253,7 @@ bool PrintBetaServicer(const ServiceDescriptor* service, Printer* out) {
   return true;
 }
 
-bool PrintBetaStub(const ServiceDescriptor* service, Printer* out) {
+bool PrivateGenerator::PrintBetaStub(const ServiceDescriptor* service) {
   out->Print("\n\n");
   out->Print("class Beta$Service$Stub(object):\n", "Service", service->name());
   {
@@ -270,30 +265,33 @@ bool PrintBetaStub(const ServiceDescriptor* service, Printer* out) {
         "generated\n"
         "only to ease transition from grpcio<0.15.0 to "
         "grpcio>=0.15.0.\"\"\"\n");
-    PrintAllComments(service, out);
+    PrintAllComments(service);
     for (int i = 0; i < service->method_count(); ++i) {
-      const MethodDescriptor* meth = service->method(i);
+      const MethodDescriptor* method = service->method(i);
       grpc::string arg_name =
-          meth->client_streaming() ? "request_iterator" : "request";
-      auto methdict = ListToDict({"Method", meth->name(), "ArgName", arg_name});
-      out->Print(methdict,
+          method->client_streaming() ? "request_iterator" : "request";
+      StringMap method_dict;
+      method_dict["Method"] = method->name();
+      method_dict["ArgName"] = arg_name;
+      out->Print(method_dict,
                  "def $Method$(self, $ArgName$, timeout, metadata=None, "
                  "with_call=False, protocol_options=None):\n");
       {
         IndentScope raii_method_indent(out);
-        PrintAllComments(meth, out);
+        PrintAllComments(method);
         out->Print("raise NotImplementedError()\n");
       }
-      if (!meth->server_streaming()) {
-        out->Print(methdict, "$Method$.future = None\n");
+      if (!method->server_streaming()) {
+        out->Print(method_dict, "$Method$.future = None\n");
       }
     }
   }
   return true;
 }
 
-bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
-                            const ServiceDescriptor* service, Printer* out) {
+bool PrivateGenerator::PrintBetaServerFactory(
+    const grpc::string& package_qualified_service_name,
+    const ServiceDescriptor* service) {
   out->Print("\n\n");
   out->Print(
       "def beta_create_$Service$_server(servicer, pool=None, "
@@ -307,9 +305,9 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
         "file not marked beta) for all further purposes. This function was\n"
         "generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"
         "\"\"\"\n");
-    map<grpc::string, grpc::string> method_implementation_constructors;
-    map<grpc::string, grpc::string> input_message_modules_and_classes;
-    map<grpc::string, grpc::string> output_message_modules_and_classes;
+    StringMap method_implementation_constructors;
+    StringMap input_message_modules_and_classes;
+    StringMap output_message_modules_and_classes;
     for (int i = 0; i < service->method_count(); ++i) {
       const MethodDescriptor* method = service->method(i);
       const grpc::string method_implementation_constructor =
@@ -317,12 +315,12 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
           grpc::string(method->server_streaming() ? "stream_" : "unary_") +
           "inline";
       grpc::string input_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->input_type(), service,
+      if (!GetModuleAndMessagePath(method->input_type(),
                                    &input_message_module_and_class)) {
         return false;
       }
       grpc::string output_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->output_type(), service,
+      if (!GetModuleAndMessagePath(method->output_type(),
                                    &output_message_module_and_class)) {
         return false;
       }
@@ -334,7 +332,7 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
           make_pair(method->name(), output_message_module_and_class));
     }
     out->Print("request_deserializers = {\n");
-    for (auto name_and_input_module_class_pair =
+    for (StringMap::iterator name_and_input_module_class_pair =
              input_message_modules_and_classes.begin();
          name_and_input_module_class_pair !=
          input_message_modules_and_classes.end();
@@ -349,7 +347,7 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
     }
     out->Print("}\n");
     out->Print("response_serializers = {\n");
-    for (auto name_and_output_module_class_pair =
+    for (StringMap::iterator name_and_output_module_class_pair =
              output_message_modules_and_classes.begin();
          name_and_output_module_class_pair !=
          output_message_modules_and_classes.end();
@@ -365,7 +363,7 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
     }
     out->Print("}\n");
     out->Print("method_implementations = {\n");
-    for (auto name_and_implementation_constructor =
+    for (StringMap::iterator name_and_implementation_constructor =
              method_implementation_constructors.begin();
          name_and_implementation_constructor !=
          method_implementation_constructors.end();
@@ -395,11 +393,11 @@ bool PrintBetaServerFactory(const grpc::string& package_qualified_service_name,
   return true;
 }
 
-bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
-                          const ServiceDescriptor* service, Printer* out) {
-  map<grpc::string, grpc::string> dict = ListToDict({
-      "Service", service->name(),
-  });
+bool PrivateGenerator::PrintBetaStubFactory(
+    const grpc::string& package_qualified_service_name,
+    const ServiceDescriptor* service) {
+  StringMap dict;
+  dict["Service"] = service->name();
   out->Print("\n\n");
   out->Print(dict,
              "def beta_create_$Service$_stub(channel, host=None,"
@@ -412,21 +410,21 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
         "file not marked beta) for all further purposes. This function was\n"
         "generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0"
         "\"\"\"\n");
-    map<grpc::string, grpc::string> method_cardinalities;
-    map<grpc::string, grpc::string> input_message_modules_and_classes;
-    map<grpc::string, grpc::string> output_message_modules_and_classes;
+    StringMap method_cardinalities;
+    StringMap input_message_modules_and_classes;
+    StringMap output_message_modules_and_classes;
     for (int i = 0; i < service->method_count(); ++i) {
       const MethodDescriptor* method = service->method(i);
       const grpc::string method_cardinality =
           grpc::string(method->client_streaming() ? "STREAM" : "UNARY") + "_" +
           grpc::string(method->server_streaming() ? "STREAM" : "UNARY");
       grpc::string input_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->input_type(), service,
+      if (!GetModuleAndMessagePath(method->input_type(),
                                    &input_message_module_and_class)) {
         return false;
       }
       grpc::string output_message_module_and_class;
-      if (!GetModuleAndMessagePath(method->output_type(), service,
+      if (!GetModuleAndMessagePath(method->output_type(),
                                    &output_message_module_and_class)) {
         return false;
       }
@@ -438,7 +436,7 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
           make_pair(method->name(), output_message_module_and_class));
     }
     out->Print("request_serializers = {\n");
-    for (auto name_and_input_module_class_pair =
+    for (StringMap::iterator name_and_input_module_class_pair =
              input_message_modules_and_classes.begin();
          name_and_input_module_class_pair !=
          input_message_modules_and_classes.end();
@@ -453,7 +451,7 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
     }
     out->Print("}\n");
     out->Print("response_deserializers = {\n");
-    for (auto name_and_output_module_class_pair =
+    for (StringMap::iterator name_and_output_module_class_pair =
              output_message_modules_and_classes.begin();
          name_and_output_module_class_pair !=
          output_message_modules_and_classes.end();
@@ -469,7 +467,8 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
     }
     out->Print("}\n");
     out->Print("cardinalities = {\n");
-    for (auto name_and_cardinality = method_cardinalities.begin();
+    for (StringMap::iterator name_and_cardinality =
+             method_cardinalities.begin();
          name_and_cardinality != method_cardinalities.end();
          name_and_cardinality++) {
       IndentScope raii_descriptions_indent(out);
@@ -493,13 +492,14 @@ bool PrintBetaStubFactory(const grpc::string& package_qualified_service_name,
   return true;
 }
 
-bool PrintStub(const grpc::string& package_qualified_service_name,
-               const ServiceDescriptor* service, Printer* out) {
+bool PrivateGenerator::PrintStub(
+    const grpc::string& package_qualified_service_name,
+    const ServiceDescriptor* service) {
   out->Print("\n\n");
   out->Print("class $Service$Stub(object):\n", "Service", service->name());
   {
     IndentScope raii_class_indent(out);
-    PrintAllComments(service, out);
+    PrintAllComments(service);
     out->Print("\n");
     out->Print("def __init__(self, channel):\n");
     {
@@ -513,17 +513,17 @@ bool PrintStub(const grpc::string& package_qualified_service_name,
       }
       out->Print("\"\"\"\n");
       for (int i = 0; i < service->method_count(); ++i) {
-        auto method = service->method(i);
-        auto multi_callable_constructor =
+        const MethodDescriptor* method = service->method(i);
+        grpc::string multi_callable_constructor =
             grpc::string(method->client_streaming() ? "stream" : "unary") +
             "_" + grpc::string(method->server_streaming() ? "stream" : "unary");
         grpc::string request_module_and_class;
-        if (!GetModuleAndMessagePath(method->input_type(), service,
+        if (!GetModuleAndMessagePath(method->input_type(),
                                      &request_module_and_class)) {
           return false;
         }
         grpc::string response_module_and_class;
-        if (!GetModuleAndMessagePath(method->output_type(), service,
+        if (!GetModuleAndMessagePath(method->output_type(),
                                      &response_module_and_class)) {
           return false;
         }
@@ -550,14 +550,14 @@ bool PrintStub(const grpc::string& package_qualified_service_name,
   return true;
 }
 
-bool PrintServicer(const ServiceDescriptor* service, Printer* out) {
+bool PrivateGenerator::PrintServicer(const ServiceDescriptor* service) {
   out->Print("\n\n");
   out->Print("class $Service$Servicer(object):\n", "Service", service->name());
   {
     IndentScope raii_class_indent(out);
-    PrintAllComments(service, out);
+    PrintAllComments(service);
     for (int i = 0; i < service->method_count(); ++i) {
-      auto method = service->method(i);
+      const MethodDescriptor* method = service->method(i);
       grpc::string arg_name =
           method->client_streaming() ? "request_iterator" : "request";
       out->Print("\n");
@@ -565,7 +565,7 @@ bool PrintServicer(const ServiceDescriptor* service, Printer* out) {
                  method->name(), "ArgName", arg_name);
       {
         IndentScope raii_method_indent(out);
-        PrintAllComments(method, out);
+        PrintAllComments(method);
         out->Print("context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n");
         out->Print("context.set_details('Method not implemented!')\n");
         out->Print("raise NotImplementedError('Method not implemented!')\n");
@@ -575,9 +575,9 @@ bool PrintServicer(const ServiceDescriptor* service, Printer* out) {
   return true;
 }
 
-bool PrintAddServicerToServer(
+bool PrivateGenerator::PrintAddServicerToServer(
     const grpc::string& package_qualified_service_name,
-    const ServiceDescriptor* service, Printer* out) {
+    const ServiceDescriptor* service) {
   out->Print("\n\n");
   out->Print("def add_$Service$Servicer_to_server(servicer, server):\n",
              "Service", service->name());
@@ -588,19 +588,19 @@ bool PrintAddServicerToServer(
       IndentScope raii_dict_first_indent(out);
       IndentScope raii_dict_second_indent(out);
       for (int i = 0; i < service->method_count(); ++i) {
-        auto method = service->method(i);
-        auto method_handler_constructor =
+        const MethodDescriptor* method = service->method(i);
+        grpc::string method_handler_constructor =
             grpc::string(method->client_streaming() ? "stream" : "unary") +
             "_" +
             grpc::string(method->server_streaming() ? "stream" : "unary") +
             "_rpc_method_handler";
         grpc::string request_module_and_class;
-        if (!GetModuleAndMessagePath(method->input_type(), service,
+        if (!GetModuleAndMessagePath(method->input_type(),
                                      &request_module_and_class)) {
           return false;
         }
         grpc::string response_module_and_class;
-        if (!GetModuleAndMessagePath(method->output_type(), service,
+        if (!GetModuleAndMessagePath(method->output_type(),
                                      &response_module_and_class)) {
           return false;
         }
@@ -635,53 +635,173 @@ bool PrintAddServicerToServer(
   return true;
 }
 
-bool PrintPreamble(const FileDescriptor* file,
-                   const GeneratorConfiguration& config, Printer* out) {
-  out->Print("import $Package$\n", "Package", config.grpc_package_root);
+bool PrivateGenerator::PrintBetaPreamble() {
   out->Print("from $Package$ import implementations as beta_implementations\n",
              "Package", config.beta_package_root);
   out->Print("from $Package$ import interfaces as beta_interfaces\n", "Package",
              config.beta_package_root);
+  return true;
+}
+
+bool PrivateGenerator::PrintPreamble() {
+  out->Print("import $Package$\n", "Package", config.grpc_package_root);
   out->Print("from grpc.framework.common import cardinality\n");
   out->Print(
       "from grpc.framework.interfaces.face import utilities as "
       "face_utilities\n");
+  if (generate_in_pb2_grpc) {
+    out->Print("\n");
+    for (int i = 0; i < file->service_count(); ++i) {
+      const ServiceDescriptor* service = file->service(i);
+      for (int j = 0; j < service->method_count(); ++j) {
+        const MethodDescriptor* method = service->method(j);
+        const Descriptor* types[2] = {method->input_type(),
+                                      method->output_type()};
+        for (int k = 0; k < 2; ++k) {
+          const Descriptor* type = types[k];
+          grpc::string type_file_name = type->file()->name();
+          grpc::string module_name = ModuleName(type_file_name);
+          grpc::string module_alias = ModuleAlias(type_file_name);
+          out->Print("import $ModuleName$ as $ModuleAlias$\n", "ModuleName",
+                     module_name, "ModuleAlias", module_alias);
+        }
+      }
+    }
+  }
   return true;
 }
 
-}  // namespace
+bool PrivateGenerator::PrintGAServices() {
+  grpc::string package = file->package();
+  if (!package.empty()) {
+    package = package.append(".");
+  }
+  for (int i = 0; i < file->service_count(); ++i) {
+    const ServiceDescriptor* service = file->service(i);
+    grpc::string package_qualified_service_name = package + service->name();
+    if (!(PrintStub(package_qualified_service_name, service) &&
+          PrintServicer(service) &&
+          PrintAddServicerToServer(package_qualified_service_name, service))) {
+      return false;
+    }
+  }
+  return true;
+}
 
-pair<bool, grpc::string> GetServices(const FileDescriptor* file,
-                                     const GeneratorConfiguration& config) {
+bool PrivateGenerator::PrintBetaServices() {
+  grpc::string package = file->package();
+  if (!package.empty()) {
+    package = package.append(".");
+  }
+  for (int i = 0; i < file->service_count(); ++i) {
+    const ServiceDescriptor* service = file->service(i);
+    grpc::string package_qualified_service_name = package + service->name();
+    if (!(PrintBetaServicer(service) && PrintBetaStub(service) &&
+          PrintBetaServerFactory(package_qualified_service_name, service) &&
+          PrintBetaStubFactory(package_qualified_service_name, service))) {
+      return false;
+    }
+  }
+  return true;
+}
+
+pair<bool, grpc::string> PrivateGenerator::GetGrpcServices() {
   grpc::string output;
   {
     // Scope the output stream so it closes and finalizes output to the string.
     StringOutputStream output_stream(&output);
-    Printer out(&output_stream, '$');
-    if (!PrintPreamble(file, config, &out)) {
-      return make_pair(false, "");
-    }
-    auto package = file->package();
-    if (!package.empty()) {
-      package = package.append(".");
-    }
-    for (int i = 0; i < file->service_count(); ++i) {
-      auto service = file->service(i);
-      auto package_qualified_service_name = package + service->name();
-      if (!(PrintStub(package_qualified_service_name, service, &out) &&
-            PrintServicer(service, &out) &&
-            PrintAddServicerToServer(package_qualified_service_name, service,
-                                     &out) &&
-            PrintBetaServicer(service, &out) && PrintBetaStub(service, &out) &&
-            PrintBetaServerFactory(package_qualified_service_name, service,
-                                   &out) &&
-            PrintBetaStubFactory(package_qualified_service_name, service,
-                                 &out))) {
+    Printer out_printer(&output_stream, '$');
+    out = &out_printer;
+
+    if (generate_in_pb2_grpc) {
+      if (!PrintPreamble()) {
+        return make_pair(false, "");
+      }
+      if (!PrintGAServices()) {
         return make_pair(false, "");
       }
+    } else {
+      out->Print("try:\n");
+      {
+        IndentScope raii_dict_try_indent(out);
+        out->Print(
+            "# THESE ELEMENTS WILL BE DEPRECATED.\n"
+            "# Please use the generated *_pb2_grpc.py files instead.\n");
+        if (!PrintPreamble()) {
+          return make_pair(false, "");
+        }
+        if (!PrintBetaPreamble()) {
+          return make_pair(false, "");
+        }
+        if (!PrintGAServices()) {
+          return make_pair(false, "");
+        }
+        if (!PrintBetaServices()) {
+          return make_pair(false, "");
+        }
+      }
+      out->Print("except ImportError:\n");
+      {
+        IndentScope raii_dict_except_indent(out);
+        out->Print("pass");
+      }
     }
   }
   return make_pair(true, std::move(output));
 }
 
+}  // namespace
+
+GeneratorConfiguration::GeneratorConfiguration()
+    : grpc_package_root("grpc"), beta_package_root("grpc.beta") {}
+
+PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config)
+    : config_(config) {}
+
+PythonGrpcGenerator::~PythonGrpcGenerator() {}
+
+bool PythonGrpcGenerator::Generate(const FileDescriptor* file,
+                                   const grpc::string& parameter,
+                                   GeneratorContext* context,
+                                   grpc::string* error) const {
+  // Get output file name.
+  grpc::string pb2_file_name;
+  grpc::string pb2_grpc_file_name;
+  static const int proto_suffix_length = strlen(".proto");
+  if (file->name().size() > static_cast<size_t>(proto_suffix_length) &&
+      file->name().find_last_of(".proto") == file->name().size() - 1) {
+    grpc::string base =
+        file->name().substr(0, file->name().size() - proto_suffix_length);
+    pb2_file_name = base + "_pb2.py";
+    pb2_grpc_file_name = base + "_pb2_grpc.py";
+  } else {
+    *error = "Invalid proto file name. Proto file must end with .proto";
+    return false;
+  }
+
+  PrivateGenerator generator(config_, file);
+
+  std::unique_ptr<ZeroCopyOutputStream> pb2_output(
+      context->OpenForAppend(pb2_file_name));
+  std::unique_ptr<ZeroCopyOutputStream> grpc_output(
+      context->Open(pb2_grpc_file_name));
+  CodedOutputStream pb2_coded_out(pb2_output.get());
+  CodedOutputStream grpc_coded_out(grpc_output.get());
+  bool success = false;
+  grpc::string pb2_code;
+  grpc::string grpc_code;
+  generator.generate_in_pb2_grpc = false;
+  tie(success, pb2_code) = generator.GetGrpcServices();
+  if (success) {
+    generator.generate_in_pb2_grpc = true;
+    tie(success, grpc_code) = generator.GetGrpcServices();
+    if (success) {
+      pb2_coded_out.WriteRaw(pb2_code.data(), pb2_code.size());
+      grpc_coded_out.WriteRaw(grpc_code.data(), grpc_code.size());
+      return true;
+    }
+  }
+  return false;
+}
+
 }  // namespace grpc_python_generator

src/compiler/python_generator.h

@@ -62,10 +62,6 @@ class PythonGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
   GeneratorConfiguration config_;
 };
 
-std::pair<bool, grpc::string> GetServices(
-    const grpc::protobuf::FileDescriptor* file,
-    const GeneratorConfiguration& config);
-
 }  // namespace grpc_python_generator
 
 #endif  // GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H

src/compiler/ruby_generator.cc

@@ -119,6 +119,43 @@ void PrintService(const ServiceDescriptor *service, const grpc::string &package,
 
 }  // namespace
 
+// The following functions are copied directly from the source for the protoc
+// ruby generator
+// to ensure compatibility (with the exception of int and string type changes).
+// See
+// https://github.com/google/protobuf/blob/master/src/google/protobuf/compiler/ruby/ruby_generator.cc#L250
+// TODO: keep up to date with protoc code generation, though this behavior isn't
+// expected to change
+bool IsLower(char ch) { return ch >= 'a' && ch <= 'z'; }
+
+char ToUpper(char ch) { return IsLower(ch) ? (ch - 'a' + 'A') : ch; }
+
+// Package names in protobuf are snake_case by convention, but Ruby module
+// names must be PascalCased.
+//
+//   foo_bar_baz -> FooBarBaz
+grpc::string PackageToModule(const grpc::string &name) {
+  bool next_upper = true;
+  grpc::string result;
+  result.reserve(name.size());
+
+  for (grpc::string::size_type i = 0; i < name.size(); i++) {
+    if (name[i] == '_') {
+      next_upper = true;
+    } else {
+      if (next_upper) {
+        result.push_back(ToUpper(name[i]));
+      } else {
+        result.push_back(name[i]);
+      }
+      next_upper = false;
+    }
+  }
+
+  return result;
+}
+// end copying of protoc generator for ruby code
+
 grpc::string GetServices(const FileDescriptor *file) {
   grpc::string output;
   {
@@ -162,7 +199,7 @@ grpc::string GetServices(const FileDescriptor *file) {
     std::vector<grpc::string> modules = Split(file->package(), '.');
     for (size_t i = 0; i < modules.size(); ++i) {
       std::map<grpc::string, grpc::string> module_vars = ListToDict({
-          "module.name", CapitalizeFirst(modules[i]),
+          "module.name", PackageToModule(modules[i]),
       });
       out.Print(module_vars, "module $module.name$\n");
       out.Indent();

src/core/ext/client_channel/client_channel.c

@@ -1022,6 +1022,10 @@ static void cc_destroy_call_elem(grpc_exec_ctx *exec_ctx,
   GPR_ASSERT(calld->creation_phase == GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING);
   gpr_mu_destroy(&calld->mu);
   GPR_ASSERT(calld->waiting_ops_count == 0);
+  if (calld->connected_subchannel != NULL) {
+    GRPC_CONNECTED_SUBCHANNEL_UNREF(exec_ctx, calld->connected_subchannel,
+                                    "picked");
+  }
   gpr_free(calld->waiting_ops);
   gpr_free(and_free_memory);
 }

src/core/ext/client_channel/subchannel.c

@@ -183,9 +183,10 @@ static void connection_destroy(grpc_exec_ctx *exec_ctx, void *arg,
   gpr_free(c);
 }
 
-void grpc_connected_subchannel_ref(
+grpc_connected_subchannel *grpc_connected_subchannel_ref(
     grpc_connected_subchannel *c GRPC_SUBCHANNEL_REF_EXTRA_ARGS) {
   GRPC_CHANNEL_STACK_REF(CHANNEL_STACK_FROM_CONNECTION(c), REF_REASON);
+  return c;
 }
 
 void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx,

src/core/ext/client_channel/subchannel.h

@@ -97,7 +97,7 @@ grpc_subchannel *grpc_subchannel_weak_ref(
 void grpc_subchannel_weak_unref(grpc_exec_ctx *exec_ctx,
                                 grpc_subchannel *channel
                                     GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
-void grpc_connected_subchannel_ref(
+grpc_connected_subchannel *grpc_connected_subchannel_ref(
     grpc_connected_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS);
 void grpc_connected_subchannel_unref(grpc_exec_ctx *exec_ctx,
                                      grpc_connected_subchannel *channel

src/core/ext/lb_policy/pick_first/pick_first.c

@@ -209,7 +209,7 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   /* Check atomically for a selected channel */
   grpc_connected_subchannel *selected = GET_SELECTED(p);
   if (selected != NULL) {
-    *target = selected;
+    *target = GRPC_CONNECTED_SUBCHANNEL_REF(selected, "picked");
     return 1;
   }
 
@@ -218,7 +218,7 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   selected = GET_SELECTED(p);
   if (selected) {
     gpr_mu_unlock(&p->mu);
-    *target = selected;
+    *target = GRPC_CONNECTED_SUBCHANNEL_REF(selected, "picked");
     return 1;
   } else {
     if (!p->started_picking) {
@@ -310,7 +310,7 @@ static void pf_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         /* update any calls that were waiting for a pick */
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
-          *pp->target = selected;
+          *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(selected, "picked");
           grpc_exec_ctx_sched(exec_ctx, pp->on_complete, GRPC_ERROR_NONE, NULL);
           gpr_free(pp);
         }

src/core/ext/lb_policy/round_robin/round_robin.c

@@ -397,7 +397,9 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   gpr_mu_lock(&p->mu);
   if ((selected = peek_next_connected_locked(p))) {
     /* readily available, report right away */
-    *target = grpc_subchannel_get_connected_subchannel(selected->subchannel);
+    *target = GRPC_CONNECTED_SUBCHANNEL_REF(
+        grpc_subchannel_get_connected_subchannel(selected->subchannel),
+        "picked");
 
     if (user_data != NULL) {
       *user_data = selected->user_data;
@@ -463,8 +465,9 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg,
         while ((pp = p->pending_picks)) {
           p->pending_picks = pp->next;
 
-          *pp->target =
-              grpc_subchannel_get_connected_subchannel(selected->subchannel);
+          *pp->target = GRPC_CONNECTED_SUBCHANNEL_REF(
+              grpc_subchannel_get_connected_subchannel(selected->subchannel),
+              "picked");
           if (pp->user_data != NULL) {
             *pp->user_data = selected->user_data;
           }
@@ -578,7 +581,9 @@ static void rr_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
   gpr_mu_lock(&p->mu);
   if ((selected = peek_next_connected_locked(p))) {
     gpr_mu_unlock(&p->mu);
-    target = grpc_subchannel_get_connected_subchannel(selected->subchannel);
+    target = GRPC_CONNECTED_SUBCHANNEL_REF(
+        grpc_subchannel_get_connected_subchannel(selected->subchannel),
+        "picked");
     grpc_connected_subchannel_ping(exec_ctx, target, closure);
   } else {
     gpr_mu_unlock(&p->mu);

src/core/ext/transport/chttp2/client/insecure/channel_create.c

@@ -154,7 +154,8 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con,
   c->tcp = NULL;
   grpc_closure_init(&c->connected, connected, c);
   grpc_tcp_client_connect(exec_ctx, &c->connected, &c->tcp,
-                          args->interested_parties, args->addr, args->deadline);
+                          args->interested_parties, args->channel_args,
+                          args->addr, args->deadline);
 }
 
 static const grpc_connector_vtable connector_vtable = {

src/core/ext/transport/chttp2/client/insecure/channel_create_posix.c

@@ -44,6 +44,7 @@
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
+#include "src/core/lib/iomgr/tcp_client_posix.h"
 #include "src/core/lib/iomgr/tcp_posix.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
@@ -65,9 +66,8 @@ grpc_channel *grpc_insecure_channel_create_from_fd(
   int flags = fcntl(fd, F_GETFL, 0);
   GPR_ASSERT(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == 0);
 
-  grpc_endpoint *client =
-      grpc_tcp_create(grpc_fd_create(fd, "client"),
-                      GRPC_TCP_DEFAULT_READ_SLICE_SIZE, "fd-client");
+  grpc_endpoint *client = grpc_tcp_client_create_from_fd(
+      &exec_ctx, grpc_fd_create(fd, "client"), args, "fd-client");
 
   grpc_transport *transport =
       grpc_create_chttp2_transport(&exec_ctx, final_args, client, 1);

src/core/ext/transport/chttp2/client/secure/secure_channel_create.c

@@ -212,9 +212,9 @@ static void connector_connect(grpc_exec_ctx *exec_ctx, grpc_connector *con,
   GPR_ASSERT(c->connecting_endpoint == NULL);
   gpr_mu_unlock(&c->mu);
   grpc_closure_init(&c->connected_closure, connected, c);
-  grpc_tcp_client_connect(exec_ctx, &c->connected_closure,
-                          &c->newly_connecting_endpoint,
-                          args->interested_parties, args->addr, args->deadline);
+  grpc_tcp_client_connect(
+      exec_ctx, &c->connected_closure, &c->newly_connecting_endpoint,
+      args->interested_parties, args->channel_args, args->addr, args->deadline);
 }
 
 static const grpc_connector_vtable connector_vtable = {

src/core/ext/transport/chttp2/server/insecure/server_chttp2.c

@@ -139,8 +139,8 @@ int grpc_server_add_insecure_http2_port(grpc_server *server, const char *addr) {
     goto error;
   }
 
-  err =
-      grpc_tcp_server_create(NULL, grpc_server_get_channel_args(server), &tcp);
+  err = grpc_tcp_server_create(&exec_ctx, NULL,
+                               grpc_server_get_channel_args(server), &tcp);
   if (err != GRPC_ERROR_NONE) {
     goto error;
   }

src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c

@@ -57,8 +57,12 @@ void grpc_server_add_insecure_channel_from_fd(grpc_server *server,
   char *name;
   gpr_asprintf(&name, "fd:%d", fd);
 
-  grpc_endpoint *server_endpoint = grpc_tcp_create(
-      grpc_fd_create(fd, name), GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name);
+  grpc_resource_quota *resource_quota = grpc_resource_quota_from_channel_args(
+      grpc_server_get_channel_args(server));
+  grpc_endpoint *server_endpoint =
+      grpc_tcp_create(grpc_fd_create(fd, name), resource_quota,
+                      GRPC_TCP_DEFAULT_READ_SLICE_SIZE, name);
+  grpc_resource_quota_internal_unref(&exec_ctx, resource_quota);
 
   gpr_free(name);
 

src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c

@@ -271,7 +271,8 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
   memset(server_state, 0, sizeof(*server_state));
   grpc_closure_init(&server_state->tcp_server_shutdown_complete,
                     tcp_server_shutdown_complete, server_state);
-  err = grpc_tcp_server_create(&server_state->tcp_server_shutdown_complete,
+  err = grpc_tcp_server_create(&exec_ctx,
+                               &server_state->tcp_server_shutdown_complete,
                                grpc_server_get_channel_args(server), &tcp);
   if (err != GRPC_ERROR_NONE) {
     goto error;

src/core/ext/transport/chttp2/transport/chttp2_transport.c

@@ -114,6 +114,20 @@ static void fail_pending_writes(grpc_exec_ctx *exec_ctx,
                                 grpc_chttp2_transport *t, grpc_chttp2_stream *s,
                                 grpc_error *error);
 
+static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *t,
+                             grpc_error *error);
+static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t,
+                                    grpc_error *error);
+static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *t,
+                                  grpc_error *error);
+static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t,
+                                         grpc_error *error);
+
+static void post_benign_reclaimer(grpc_exec_ctx *exec_ctx,
+                                  grpc_chttp2_transport *t);
+static void post_destructive_reclaimer(grpc_exec_ctx *exec_ctx,
+                                       grpc_chttp2_transport *t);
+
 static void close_transport_locked(grpc_exec_ctx *exec_ctx,
                                    grpc_chttp2_transport *t, grpc_error *error);
 static void end_all_the_calls(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
@@ -241,6 +255,11 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   grpc_closure_init(&t->write_action_end_locked, write_action_end_locked, t);
   grpc_closure_init(&t->read_action_begin, read_action_begin, t);
   grpc_closure_init(&t->read_action_locked, read_action_locked, t);
+  grpc_closure_init(&t->benign_reclaimer, benign_reclaimer, t);
+  grpc_closure_init(&t->destructive_reclaimer, destructive_reclaimer, t);
+  grpc_closure_init(&t->benign_reclaimer_locked, benign_reclaimer_locked, t);
+  grpc_closure_init(&t->destructive_reclaimer_locked,
+                    destructive_reclaimer_locked, t);
 
   grpc_chttp2_goaway_parser_init(&t->goaway_parser);
   grpc_chttp2_hpack_parser_init(&t->hpack_parser);
@@ -362,6 +381,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   }
 
   grpc_chttp2_initiate_write(exec_ctx, t, false, "init");
+  post_benign_reclaimer(exec_ctx, t);
 }
 
 static void destroy_transport_locked(grpc_exec_ctx *exec_ctx, void *tp,
@@ -467,6 +487,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt,
                    [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
     *t->accepting_stream = s;
     grpc_chttp2_stream_map_add(&t->stream_map, s->id, s);
+    post_destructive_reclaimer(exec_ctx, t);
   }
 
   GPR_TIMER_END("init_stream", 0);
@@ -675,6 +696,13 @@ static void write_action_end_locked(grpc_exec_ctx *exec_ctx, void *tp,
     close_transport_locked(exec_ctx, t, GRPC_ERROR_REF(error));
   }
 
+  if (t->sent_goaway_state == GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED) {
+    t->sent_goaway_state = GRPC_CHTTP2_GOAWAY_SENT;
+    if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) {
+      close_transport_locked(exec_ctx, t, GRPC_ERROR_CREATE("goaway sent"));
+    }
+  }
+
   grpc_chttp2_end_write(exec_ctx, t, GRPC_ERROR_REF(error));
 
   switch (t->write_state) {
@@ -780,6 +808,7 @@ static void maybe_start_some_streams(grpc_exec_ctx *exec_ctx,
                    [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
     s->max_recv_bytes = GPR_MAX(stream_incoming_window, s->max_recv_bytes);
     grpc_chttp2_stream_map_add(&t->stream_map, s->id, s);
+    post_destructive_reclaimer(exec_ctx, t);
     grpc_chttp2_become_writable(exec_ctx, t, s, true, "new_stream");
   }
   /* cancel out streams that will never be started */
@@ -993,9 +1022,14 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op,
       }
       if (!s->write_closed) {
         if (t->is_client) {
-          GPR_ASSERT(s->id == 0);
-          grpc_chttp2_list_add_waiting_for_concurrency(t, s);
-          maybe_start_some_streams(exec_ctx, t);
+          if (!t->closed) {
+            GPR_ASSERT(s->id == 0);
+            grpc_chttp2_list_add_waiting_for_concurrency(t, s);
+            maybe_start_some_streams(exec_ctx, t);
+          } else {
+            grpc_chttp2_cancel_stream(exec_ctx, t, s,
+                                      GRPC_ERROR_CREATE("Transport closed"));
+          }
         } else {
           GPR_ASSERT(s->id != 0);
           grpc_chttp2_become_writable(exec_ctx, t, s, true,
@@ -1185,6 +1219,14 @@ void grpc_chttp2_ack_ping(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
   gpr_free(msg);
 }
 
+static void send_goaway(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
+                        grpc_chttp2_error_code error, gpr_slice data) {
+  t->sent_goaway_state = GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED;
+  grpc_chttp2_goaway_append(t->last_new_stream_id, (uint32_t)error, data,
+                            &t->qbuf);
+  grpc_chttp2_initiate_write(exec_ctx, t, false, "goaway_sent");
+}
+
 static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
                                         void *stream_op,
                                         grpc_error *error_ignored) {
@@ -1199,15 +1241,9 @@ static void perform_transport_op_locked(grpc_exec_ctx *exec_ctx,
   }
 
   if (op->send_goaway) {
-    t->sent_goaway = 1;
-    grpc_chttp2_goaway_append(
-        t->last_new_stream_id,
-        (uint32_t)grpc_chttp2_grpc_status_to_http2_error(op->goaway_status),
-        gpr_slice_ref(*op->goaway_message), &t->qbuf);
-    close_transport = grpc_chttp2_stream_map_size(&t->stream_map) == 0
-                          ? GRPC_ERROR_CREATE("GOAWAY sent")
-                          : GRPC_ERROR_NONE;
-    grpc_chttp2_initiate_write(exec_ctx, t, false, "goaway_sent");
+    send_goaway(exec_ctx, t,
+                grpc_chttp2_grpc_status_to_http2_error(op->goaway_status),
+                gpr_slice_ref(*op->goaway_message));
   }
 
   if (op->set_accept_stream) {
@@ -1341,10 +1377,14 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t,
     s->data_parser.parsing_frame = NULL;
   }
 
-  if (grpc_chttp2_stream_map_size(&t->stream_map) == 0 && t->sent_goaway) {
-    close_transport_locked(
-        exec_ctx, t, GRPC_ERROR_CREATE_REFERENCING(
-                         "Last stream closed after sending GOAWAY", &error, 1));
+  if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) {
+    post_benign_reclaimer(exec_ctx, t);
+    if (t->sent_goaway_state == GRPC_CHTTP2_GOAWAY_SENT) {
+      close_transport_locked(
+          exec_ctx, t,
+          GRPC_ERROR_CREATE_REFERENCING(
+              "Last stream closed after sending GOAWAY", &error, 1));
+    }
   }
   if (grpc_chttp2_list_remove_writable_stream(t, s)) {
     GRPC_CHTTP2_STREAM_UNREF(exec_ctx, s, "chttp2_writing:remove_stream");
@@ -2071,6 +2111,103 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create(
   return incoming_byte_stream;
 }
 
+/*******************************************************************************
+ * RESOURCE QUOTAS
+ */
+
+static void post_benign_reclaimer(grpc_exec_ctx *exec_ctx,
+                                  grpc_chttp2_transport *t) {
+  if (!t->benign_reclaimer_registered) {
+    t->benign_reclaimer_registered = true;
+    GRPC_CHTTP2_REF_TRANSPORT(t, "benign_reclaimer");
+    grpc_resource_user_post_reclaimer(exec_ctx,
+                                      grpc_endpoint_get_resource_user(t->ep),
+                                      false, &t->benign_reclaimer);
+  }
+}
+
+static void post_destructive_reclaimer(grpc_exec_ctx *exec_ctx,
+                                       grpc_chttp2_transport *t) {
+  if (!t->destructive_reclaimer_registered) {
+    t->destructive_reclaimer_registered = true;
+    GRPC_CHTTP2_REF_TRANSPORT(t, "destructive_reclaimer");
+    grpc_resource_user_post_reclaimer(exec_ctx,
+                                      grpc_endpoint_get_resource_user(t->ep),
+                                      true, &t->destructive_reclaimer);
+  }
+}
+
+static void benign_reclaimer(grpc_exec_ctx *exec_ctx, void *arg,
+                             grpc_error *error) {
+  grpc_chttp2_transport *t = arg;
+  grpc_combiner_execute(exec_ctx, t->combiner, &t->benign_reclaimer_locked,
+                        GRPC_ERROR_REF(error), false);
+}
+
+static void destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *arg,
+                                  grpc_error *error) {
+  grpc_chttp2_transport *t = arg;
+  grpc_combiner_execute(exec_ctx, t->combiner, &t->destructive_reclaimer_locked,
+                        GRPC_ERROR_REF(error), false);
+}
+
+static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                                    grpc_error *error) {
+  grpc_chttp2_transport *t = arg;
+  if (error == GRPC_ERROR_NONE &&
+      grpc_chttp2_stream_map_size(&t->stream_map) == 0) {
+    /* Channel with no active streams: send a goaway to try and make it
+     * disconnect cleanly */
+    if (grpc_resource_quota_trace) {
+      gpr_log(GPR_DEBUG, "HTTP2: %s - send goaway to free memory",
+              t->peer_string);
+    }
+    send_goaway(exec_ctx, t, GRPC_CHTTP2_ENHANCE_YOUR_CALM,
+                gpr_slice_from_static_string("Buffers full"));
+  } else if (error == GRPC_ERROR_NONE && grpc_resource_quota_trace) {
+    gpr_log(GPR_DEBUG,
+            "HTTP2: %s - skip benign reclamation, there are still %" PRIdPTR
+            " streams",
+            t->peer_string, grpc_chttp2_stream_map_size(&t->stream_map));
+  }
+  t->benign_reclaimer_registered = false;
+  if (error != GRPC_ERROR_CANCELLED) {
+    grpc_resource_user_finish_reclamation(
+        exec_ctx, grpc_endpoint_get_resource_user(t->ep));
+  }
+  GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "benign_reclaimer");
+}
+
+static void destructive_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *arg,
+                                         grpc_error *error) {
+  grpc_chttp2_transport *t = arg;
+  size_t n = grpc_chttp2_stream_map_size(&t->stream_map);
+  t->destructive_reclaimer_registered = false;
+  if (error == GRPC_ERROR_NONE && n > 0) {
+    grpc_chttp2_stream *s = grpc_chttp2_stream_map_rand(&t->stream_map);
+    if (grpc_resource_quota_trace) {
+      gpr_log(GPR_DEBUG, "HTTP2: %s - abandon stream id %d", t->peer_string,
+              s->id);
+    }
+    grpc_chttp2_cancel_stream(
+        exec_ctx, t, s, grpc_error_set_int(GRPC_ERROR_CREATE("Buffers full"),
+                                           GRPC_ERROR_INT_HTTP2_ERROR,
+                                           GRPC_CHTTP2_ENHANCE_YOUR_CALM));
+    if (n > 1) {
+      /* Since we cancel one stream per destructive reclamation, if
+         there are more streams left, we can immediately post a new
+         reclaimer in case the resource quota needs to free more
+         memory */
+      post_destructive_reclaimer(exec_ctx, t);
+    }
+  }
+  if (error != GRPC_ERROR_CANCELLED) {
+    grpc_resource_user_finish_reclamation(
+        exec_ctx, grpc_endpoint_get_resource_user(t->ep));
+  }
+  GRPC_CHTTP2_UNREF_TRANSPORT(exec_ctx, t, "destructive_reclaimer");
+}
+
 /*******************************************************************************
  * TRACING
  */

src/core/ext/transport/chttp2/transport/internal.h

@@ -138,6 +138,12 @@ typedef enum {
   GRPC_NUM_SETTING_SETS
 } grpc_chttp2_setting_set;
 
+typedef enum {
+  GRPC_CHTTP2_NO_GOAWAY_SEND,
+  GRPC_CHTTP2_GOAWAY_SEND_SCHEDULED,
+  GRPC_CHTTP2_GOAWAY_SENT,
+} grpc_chttp2_sent_goaway_state;
+
 /* Outstanding ping request data */
 typedef struct grpc_chttp2_outstanding_ping {
   uint8_t id[8];
@@ -249,7 +255,7 @@ struct grpc_chttp2_transport {
   /** have we seen a goaway */
   uint8_t seen_goaway;
   /** have we sent a goaway */
-  uint8_t sent_goaway;
+  grpc_chttp2_sent_goaway_state sent_goaway_state;
 
   /** are the local settings dirty and need to be sent? */
   uint8_t dirtied_local_settings;
@@ -320,6 +326,18 @@ struct grpc_chttp2_transport {
   /* if non-NULL, close the transport with this error when writes are finished
    */
   grpc_error *close_transport_on_writes_finished;
+
+  /* buffer pool state */
+  /** have we scheduled a benign cleanup? */
+  bool benign_reclaimer_registered;
+  /** have we scheduled a destructive cleanup? */
+  bool destructive_reclaimer_registered;
+  /** benign cleanup closure */
+  grpc_closure benign_reclaimer;
+  grpc_closure benign_reclaimer_locked;
+  /** destructive cleanup closure */
+  grpc_closure destructive_reclaimer;
+  grpc_closure destructive_reclaimer_locked;
 };
 
 typedef enum {

src/core/ext/transport/chttp2/transport/stream_map.c

@@ -151,6 +151,17 @@ size_t grpc_chttp2_stream_map_size(grpc_chttp2_stream_map *map) {
   return map->count - map->free;
 }
 
+void *grpc_chttp2_stream_map_rand(grpc_chttp2_stream_map *map) {
+  if (map->count == map->free) {
+    return NULL;
+  }
+  if (map->free != 0) {
+    map->count = compact(map->keys, map->values, map->count);
+    map->free = 0;
+  }
+  return map->values[((size_t)rand()) % map->count];
+}
+
 void grpc_chttp2_stream_map_for_each(grpc_chttp2_stream_map *map,
                                      void (*f)(void *user_data, uint32_t key,
                                                void *value),

src/core/ext/transport/chttp2/transport/stream_map.h

@@ -68,6 +68,9 @@ void *grpc_chttp2_stream_map_delete(grpc_chttp2_stream_map *map, uint32_t key);
 /* Return an existing key, or NULL if it does not exist */
 void *grpc_chttp2_stream_map_find(grpc_chttp2_stream_map *map, uint32_t key);
 
+/* Return a random entry */
+void *grpc_chttp2_stream_map_rand(grpc_chttp2_stream_map *map);
+
 /* How many (populated) entries are in the stream map? */
 size_t grpc_chttp2_stream_map_size(grpc_chttp2_stream_map *map);
 

src/core/lib/http/httpcli.c

@@ -70,6 +70,7 @@ typedef struct {
   grpc_closure done_write;
   grpc_closure connected;
   grpc_error *overall_error;
+  grpc_resource_quota *resource_quota;
 } internal_request;
 
 static grpc_httpcli_get_override g_get_override = NULL;
@@ -117,6 +118,7 @@ static void finish(grpc_exec_ctx *exec_ctx, internal_request *req,
   gpr_slice_buffer_destroy(&req->incoming);
   gpr_slice_buffer_destroy(&req->outgoing);
   GRPC_ERROR_UNREF(req->overall_error);
+  grpc_resource_quota_internal_unref(exec_ctx, req->resource_quota);
   gpr_free(req);
 }
 
@@ -223,8 +225,15 @@ static void next_address(grpc_exec_ctx *exec_ctx, internal_request *req,
   }
   addr = &req->addresses->addrs[req->next_address++];
   grpc_closure_init(&req->connected, on_connected, req);
+  grpc_arg arg;
+  arg.key = GRPC_ARG_RESOURCE_QUOTA;
+  arg.type = GRPC_ARG_POINTER;
+  arg.value.pointer.p = req->resource_quota;
+  arg.value.pointer.vtable = grpc_resource_quota_arg_vtable();
+  grpc_channel_args args = {1, &arg};
   grpc_tcp_client_connect(exec_ctx, &req->connected, &req->ep,
-                          req->context->pollset_set, addr, req->deadline);
+                          req->context->pollset_set, &args, addr,
+                          req->deadline);
 }
 
 static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
@@ -240,6 +249,7 @@ static void on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
 static void internal_request_begin(grpc_exec_ctx *exec_ctx,
                                    grpc_httpcli_context *context,
                                    grpc_polling_entity *pollent,
+                                   grpc_resource_quota *resource_quota,
                                    const grpc_httpcli_request *request,
                                    gpr_timespec deadline, grpc_closure *on_done,
                                    grpc_httpcli_response *response,
@@ -255,6 +265,7 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx,
   req->context = context;
   req->pollent = pollent;
   req->overall_error = GRPC_ERROR_NONE;
+  req->resource_quota = grpc_resource_quota_internal_ref(resource_quota);
   grpc_closure_init(&req->on_read, on_read, req);
   grpc_closure_init(&req->done_write, done_write, req);
   gpr_slice_buffer_init(&req->incoming);
@@ -272,6 +283,7 @@ static void internal_request_begin(grpc_exec_ctx *exec_ctx,
 
 void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
                       grpc_polling_entity *pollent,
+                      grpc_resource_quota *resource_quota,
                       const grpc_httpcli_request *request,
                       gpr_timespec deadline, grpc_closure *on_done,
                       grpc_httpcli_response *response) {
@@ -281,14 +293,15 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
     return;
   }
   gpr_asprintf(&name, "HTTP:GET:%s:%s", request->host, request->http.path);
-  internal_request_begin(exec_ctx, context, pollent, request, deadline, on_done,
-                         response, name,
+  internal_request_begin(exec_ctx, context, pollent, resource_quota, request,
+                         deadline, on_done, response, name,
                          grpc_httpcli_format_get_request(request));
   gpr_free(name);
 }
 
 void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
                        grpc_polling_entity *pollent,
+                       grpc_resource_quota *resource_quota,
                        const grpc_httpcli_request *request,
                        const char *body_bytes, size_t body_size,
                        gpr_timespec deadline, grpc_closure *on_done,
@@ -301,7 +314,8 @@ void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
   }
   gpr_asprintf(&name, "HTTP:POST:%s:%s", request->host, request->http.path);
   internal_request_begin(
-      exec_ctx, context, pollent, request, deadline, on_done, response, name,
+      exec_ctx, context, pollent, resource_quota, request, deadline, on_done,
+      response, name,
       grpc_httpcli_format_post_request(request, body_bytes, body_size));
   gpr_free(name);
 }

src/core/lib/http/httpcli.h

@@ -96,6 +96,7 @@ void grpc_httpcli_context_destroy(grpc_httpcli_context *context);
    'on_response' is a callback to report results to */
 void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
                       grpc_polling_entity *pollent,
+                      grpc_resource_quota *resource_quota,
                       const grpc_httpcli_request *request,
                       gpr_timespec deadline, grpc_closure *on_complete,
                       grpc_httpcli_response *response);
@@ -116,6 +117,7 @@ void grpc_httpcli_get(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
    Does not support ?var1=val1&var2=val2 in the path. */
 void grpc_httpcli_post(grpc_exec_ctx *exec_ctx, grpc_httpcli_context *context,
                        grpc_polling_entity *pollent,
+                       grpc_resource_quota *resource_quota,
                        const grpc_httpcli_request *request,
                        const char *body_bytes, size_t body_size,
                        gpr_timespec deadline, grpc_closure *on_complete,

src/core/lib/iomgr/endpoint.c

@@ -69,3 +69,7 @@ char* grpc_endpoint_get_peer(grpc_endpoint* ep) {
 grpc_workqueue* grpc_endpoint_get_workqueue(grpc_endpoint* ep) {
   return ep->vtable->get_workqueue(ep);
 }
+
+grpc_resource_user* grpc_endpoint_get_resource_user(grpc_endpoint* ep) {
+  return ep->vtable->get_resource_user(ep);
+}

src/core/lib/iomgr/endpoint.h

@@ -39,6 +39,7 @@
 #include <grpc/support/time.h>
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/iomgr/pollset_set.h"
+#include "src/core/lib/iomgr/resource_quota.h"
 
 /* An endpoint caps a streaming channel between two communicating processes.
    Examples may be: a tcp socket, <stdin+stdout>, or some shared memory. */
@@ -58,6 +59,7 @@ struct grpc_endpoint_vtable {
                              grpc_pollset_set *pollset);
   void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);
   void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);
+  grpc_resource_user *(*get_resource_user)(grpc_endpoint *ep);
   char *(*get_peer)(grpc_endpoint *ep);
 };
 
@@ -100,6 +102,8 @@ void grpc_endpoint_add_to_pollset_set(grpc_exec_ctx *exec_ctx,
                                       grpc_endpoint *ep,
                                       grpc_pollset_set *pollset_set);
 
+grpc_resource_user *grpc_endpoint_get_resource_user(grpc_endpoint *endpoint);
+
 struct grpc_endpoint {
   const grpc_endpoint_vtable *vtable;
 };

src/core/lib/iomgr/endpoint_pair.h

@@ -41,7 +41,8 @@ typedef struct {
   grpc_endpoint *server;
 } grpc_endpoint_pair;
 
-grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name,
-                                                   size_t read_slice_size);
+grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(
+    const char *name, grpc_resource_quota *resource_quota,
+    size_t read_slice_size);
 
 #endif /* GRPC_CORE_LIB_IOMGR_ENDPOINT_PAIR_H */

src/core/lib/iomgr/endpoint_pair_posix.c

@@ -62,20 +62,21 @@ static void create_sockets(int sv[2]) {
   GPR_ASSERT(grpc_set_socket_no_sigpipe_if_possible(sv[1]) == GRPC_ERROR_NONE);
 }
 
-grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name,
-                                                   size_t read_slice_size) {
+grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(
+    const char *name, grpc_resource_quota *resource_quota,
+    size_t read_slice_size) {
   int sv[2];
   grpc_endpoint_pair p;
   char *final_name;
   create_sockets(sv);
 
   gpr_asprintf(&final_name, "%s:client", name);
-  p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name), read_slice_size,
-                             "socketpair-server");
+  p.client = grpc_tcp_create(grpc_fd_create(sv[1], final_name), resource_quota,
+                             read_slice_size, "socketpair-server");
   gpr_free(final_name);
   gpr_asprintf(&final_name, "%s:server", name);
-  p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name), read_slice_size,
-                             "socketpair-client");
+  p.server = grpc_tcp_create(grpc_fd_create(sv[0], final_name), resource_quota,
+                             read_slice_size, "socketpair-client");
   gpr_free(final_name);
   return p;
 }

src/core/lib/iomgr/endpoint_pair_uv.c

@@ -41,8 +41,9 @@
 
 #include "src/core/lib/iomgr/endpoint_pair.h"
 
-grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name,
-                                                   size_t read_slice_size) {
+grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(
+    const char *name, grpc_resource_quota *resource_quota,
+    size_t read_slice_size) {
   grpc_endpoint_pair endpoint_pair;
   // TODO(mlumish): implement this properly under libuv
   GPR_ASSERT(false &&

src/core/lib/iomgr/endpoint_pair_windows.c

@@ -82,15 +82,16 @@ static void create_sockets(SOCKET sv[2]) {
   sv[0] = svr_sock;
 }
 
-grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(const char *name,
-                                                   size_t read_slice_size) {
+grpc_endpoint_pair grpc_iomgr_create_endpoint_pair(
+    const char *name, grpc_resource_quota *resource_quota,
+    size_t read_slice_size) {
   SOCKET sv[2];
   grpc_endpoint_pair p;
   create_sockets(sv);
   p.client = grpc_tcp_create(grpc_winsocket_create(sv[1], "endpoint:client"),
-                             "endpoint:server");
+                             resource_quota, "endpoint:server");
   p.server = grpc_tcp_create(grpc_winsocket_create(sv[0], "endpoint:server"),
-                             "endpoint:client");
+                             resource_quota, "endpoint:client");
   return p;
 }
 

src/core/lib/iomgr/ev_epoll_linux.c

@@ -1711,6 +1711,12 @@ retry:
             "pollset_add_fd: Raced creating new polling island. pi_new: %p "
             "(fd: %d, pollset: %p)",
             (void *)pi_new, fd->fd, (void *)pollset);
+
+        /* No need to lock 'pi_new' here since this is a new polling island and
+         * no one has a reference to it yet */
+        polling_island_remove_all_fds_locked(pi_new, true, &error);
+
+        /* Ref and unref so that the polling island gets deleted during unref */
         PI_ADD_REF(pi_new, "dance_of_destruction");
         PI_UNREF(exec_ctx, pi_new, "dance_of_destruction");
         goto retry;

src/core/lib/iomgr/port.h

@@ -90,7 +90,6 @@
 #define GRPC_POSIX_SOCKETUTILS
 #endif
 #elif defined(GPR_APPLE)
-#define GRPC_HAVE_IP_PKTINFO 1
 #define GRPC_HAVE_SO_NOSIGPIPE 1
 #define GRPC_HAVE_UNIX_SOCKET 1
 #define GRPC_MSG_IOVLEN_TYPE int
@@ -102,7 +101,6 @@
 #define GRPC_TIMER_USE_GENERIC 1
 #elif defined(GPR_FREEBSD)
 #define GRPC_HAVE_IPV6_RECVPKTINFO 1
-#define GRPC_HAVE_IP_PKTINFO 1
 #define GRPC_HAVE_SO_NOSIGPIPE 1
 #define GRPC_HAVE_UNIX_SOCKET 1
 #define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1

src/core/lib/iomgr/resource_quota.c

@@ -0,0 +1,717 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/iomgr/resource_quota.h"
+
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/iomgr/combiner.h"
+
+int grpc_resource_quota_trace = 0;
+
+struct grpc_resource_quota {
+  /* refcount */
+  gpr_refcount refs;
+
+  /* Master combiner lock: all activity on a quota executes under this combiner
+   * (so no mutex is needed for this data structure)
+   */
+  grpc_combiner *combiner;
+  /* Size of the resource quota */
+  int64_t size;
+  /* Amount of free memory in the resource quota */
+  int64_t free_pool;
+
+  /* Has rq_step been scheduled to occur? */
+  bool step_scheduled;
+  /* Are we currently reclaiming memory */
+  bool reclaiming;
+  /* Closure around rq_step */
+  grpc_closure rq_step_closure;
+  /* Closure around rq_reclamation_done */
+  grpc_closure rq_reclamation_done_closure;
+
+  /* Roots of all resource user lists */
+  grpc_resource_user *roots[GRPC_RULIST_COUNT];
+
+  char *name;
+};
+
+/*******************************************************************************
+ * list management
+ */
+
+static void rulist_add_head(grpc_resource_user *resource_user,
+                            grpc_rulist list) {
+  grpc_resource_quota *resource_quota = resource_user->resource_quota;
+  grpc_resource_user **root = &resource_quota->roots[list];
+  if (*root == NULL) {
+    *root = resource_user;
+    resource_user->links[list].next = resource_user->links[list].prev =
+        resource_user;
+  } else {
+    resource_user->links[list].next = *root;
+    resource_user->links[list].prev = (*root)->links[list].prev;
+    resource_user->links[list].next->links[list].prev =
+        resource_user->links[list].prev->links[list].next = resource_user;
+    *root = resource_user;
+  }
+}
+
+static void rulist_add_tail(grpc_resource_user *resource_user,
+                            grpc_rulist list) {
+  grpc_resource_quota *resource_quota = resource_user->resource_quota;
+  grpc_resource_user **root = &resource_quota->roots[list];
+  if (*root == NULL) {
+    *root = resource_user;
+    resource_user->links[list].next = resource_user->links[list].prev =
+        resource_user;
+  } else {
+    resource_user->links[list].next = (*root)->links[list].next;
+    resource_user->links[list].prev = *root;
+    resource_user->links[list].next->links[list].prev =
+        resource_user->links[list].prev->links[list].next = resource_user;
+  }
+}
+
+static bool rulist_empty(grpc_resource_quota *resource_quota,
+                         grpc_rulist list) {
+  return resource_quota->roots[list] == NULL;
+}
+
+static grpc_resource_user *rulist_pop_head(grpc_resource_quota *resource_quota,
+                                           grpc_rulist list) {
+  grpc_resource_user **root = &resource_quota->roots[list];
+  grpc_resource_user *resource_user = *root;
+  if (resource_user == NULL) {
+    return NULL;
+  }
+  if (resource_user->links[list].next == resource_user) {
+    *root = NULL;
+  } else {
+    resource_user->links[list].next->links[list].prev =
+        resource_user->links[list].prev;
+    resource_user->links[list].prev->links[list].next =
+        resource_user->links[list].next;
+    *root = resource_user->links[list].next;
+  }
+  resource_user->links[list].next = resource_user->links[list].prev = NULL;
+  return resource_user;
+}
+
+static void rulist_remove(grpc_resource_user *resource_user, grpc_rulist list) {
+  if (resource_user->links[list].next == NULL) return;
+  grpc_resource_quota *resource_quota = resource_user->resource_quota;
+  if (resource_quota->roots[list] == resource_user) {
+    resource_quota->roots[list] = resource_user->links[list].next;
+    if (resource_quota->roots[list] == resource_user) {
+      resource_quota->roots[list] = NULL;
+    }
+  }
+  resource_user->links[list].next->links[list].prev =
+      resource_user->links[list].prev;
+  resource_user->links[list].prev->links[list].next =
+      resource_user->links[list].next;
+}
+
+/*******************************************************************************
+ * resource quota state machine
+ */
+
+static bool rq_alloc(grpc_exec_ctx *exec_ctx,
+                     grpc_resource_quota *resource_quota);
+static bool rq_reclaim_from_per_user_free_pool(
+    grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota);
+static bool rq_reclaim(grpc_exec_ctx *exec_ctx,
+                       grpc_resource_quota *resource_quota, bool destructive);
+
+static void rq_step(grpc_exec_ctx *exec_ctx, void *rq, grpc_error *error) {
+  grpc_resource_quota *resource_quota = rq;
+  resource_quota->step_scheduled = false;
+  do {
+    if (rq_alloc(exec_ctx, resource_quota)) goto done;
+  } while (rq_reclaim_from_per_user_free_pool(exec_ctx, resource_quota));
+
+  if (!rq_reclaim(exec_ctx, resource_quota, false)) {
+    rq_reclaim(exec_ctx, resource_quota, true);
+  }
+
+done:
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
+}
+
+static void rq_step_sched(grpc_exec_ctx *exec_ctx,
+                          grpc_resource_quota *resource_quota) {
+  if (resource_quota->step_scheduled) return;
+  resource_quota->step_scheduled = true;
+  grpc_resource_quota_internal_ref(resource_quota);
+  grpc_combiner_execute_finally(exec_ctx, resource_quota->combiner,
+                                &resource_quota->rq_step_closure,
+                                GRPC_ERROR_NONE, false);
+}
+
+/* returns true if all allocations are completed */
+static bool rq_alloc(grpc_exec_ctx *exec_ctx,
+                     grpc_resource_quota *resource_quota) {
+  grpc_resource_user *resource_user;
+  while ((resource_user = rulist_pop_head(resource_quota,
+                                          GRPC_RULIST_AWAITING_ALLOCATION))) {
+    gpr_mu_lock(&resource_user->mu);
+    if (resource_user->free_pool < 0 &&
+        -resource_user->free_pool <= resource_quota->free_pool) {
+      int64_t amt = -resource_user->free_pool;
+      resource_user->free_pool = 0;
+      resource_quota->free_pool -= amt;
+      if (grpc_resource_quota_trace) {
+        gpr_log(GPR_DEBUG, "RQ %s %s: grant alloc %" PRId64
+                           " bytes; rq_free_pool -> %" PRId64,
+                resource_quota->name, resource_user->name, amt,
+                resource_quota->free_pool);
+      }
+    } else if (grpc_resource_quota_trace && resource_user->free_pool >= 0) {
+      gpr_log(GPR_DEBUG, "RQ %s %s: discard already satisfied alloc request",
+              resource_quota->name, resource_user->name);
+    }
+    if (resource_user->free_pool >= 0) {
+      resource_user->allocating = false;
+      grpc_exec_ctx_enqueue_list(exec_ctx, &resource_user->on_allocated, NULL);
+      gpr_mu_unlock(&resource_user->mu);
+    } else {
+      rulist_add_head(resource_user, GRPC_RULIST_AWAITING_ALLOCATION);
+      gpr_mu_unlock(&resource_user->mu);
+      return false;
+    }
+  }
+  return true;
+}
+
+/* returns true if any memory could be reclaimed from buffers */
+static bool rq_reclaim_from_per_user_free_pool(
+    grpc_exec_ctx *exec_ctx, grpc_resource_quota *resource_quota) {
+  grpc_resource_user *resource_user;
+  while ((resource_user = rulist_pop_head(resource_quota,
+                                          GRPC_RULIST_NON_EMPTY_FREE_POOL))) {
+    gpr_mu_lock(&resource_user->mu);
+    if (resource_user->free_pool > 0) {
+      int64_t amt = resource_user->free_pool;
+      resource_user->free_pool = 0;
+      resource_quota->free_pool += amt;
+      if (grpc_resource_quota_trace) {
+        gpr_log(GPR_DEBUG, "RQ %s %s: reclaim_from_per_user_free_pool %" PRId64
+                           " bytes; rq_free_pool -> %" PRId64,
+                resource_quota->name, resource_user->name, amt,
+                resource_quota->free_pool);
+      }
+      gpr_mu_unlock(&resource_user->mu);
+      return true;
+    } else {
+      gpr_mu_unlock(&resource_user->mu);
+    }
+  }
+  return false;
+}
+
+/* returns true if reclamation is proceeding */
+static bool rq_reclaim(grpc_exec_ctx *exec_ctx,
+                       grpc_resource_quota *resource_quota, bool destructive) {
+  if (resource_quota->reclaiming) return true;
+  grpc_rulist list = destructive ? GRPC_RULIST_RECLAIMER_DESTRUCTIVE
+                                 : GRPC_RULIST_RECLAIMER_BENIGN;
+  grpc_resource_user *resource_user = rulist_pop_head(resource_quota, list);
+  if (resource_user == NULL) return false;
+  if (grpc_resource_quota_trace) {
+    gpr_log(GPR_DEBUG, "RQ %s %s: initiate %s reclamation",
+            resource_quota->name, resource_user->name,
+            destructive ? "destructive" : "benign");
+  }
+  resource_quota->reclaiming = true;
+  grpc_resource_quota_internal_ref(resource_quota);
+  grpc_closure *c = resource_user->reclaimers[destructive];
+  resource_user->reclaimers[destructive] = NULL;
+  grpc_closure_run(exec_ctx, c, GRPC_ERROR_NONE);
+  return true;
+}
+
+/*******************************************************************************
+ * ru_slice: a slice implementation that is backed by a grpc_resource_user
+ */
+
+typedef struct {
+  gpr_slice_refcount base;
+  gpr_refcount refs;
+  grpc_resource_user *resource_user;
+  size_t size;
+} ru_slice_refcount;
+
+static void ru_slice_ref(void *p) {
+  ru_slice_refcount *rc = p;
+  gpr_ref(&rc->refs);
+}
+
+static void ru_slice_unref(void *p) {
+  ru_slice_refcount *rc = p;
+  if (gpr_unref(&rc->refs)) {
+    /* TODO(ctiller): this is dangerous, but I think safe for now:
+       we have no guarantee here that we're at a safe point for creating an
+       execution context, but we have no way of writing this code otherwise.
+       In the future: consider lifting gpr_slice to grpc, and offering an
+       internal_{ref,unref} pair that is execution context aware.
+       Alternatively,
+       make exec_ctx be thread local and 'do the right thing' (whatever that
+       is)
+       if NULL */
+    grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+    grpc_resource_user_free(&exec_ctx, rc->resource_user, rc->size);
+    grpc_exec_ctx_finish(&exec_ctx);
+    gpr_free(rc);
+  }
+}
+
+static gpr_slice ru_slice_create(grpc_resource_user *resource_user,
+                                 size_t size) {
+  ru_slice_refcount *rc = gpr_malloc(sizeof(ru_slice_refcount) + size);
+  rc->base.ref = ru_slice_ref;
+  rc->base.unref = ru_slice_unref;
+  gpr_ref_init(&rc->refs, 1);
+  rc->resource_user = resource_user;
+  rc->size = size;
+  gpr_slice slice;
+  slice.refcount = &rc->base;
+  slice.data.refcounted.bytes = (uint8_t *)(rc + 1);
+  slice.data.refcounted.length = size;
+  return slice;
+}
+
+/*******************************************************************************
+ * grpc_resource_quota internal implementation: resource user manipulation under
+ * the combiner
+ */
+
+static void ru_allocate(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) {
+  grpc_resource_user *resource_user = ru;
+  if (rulist_empty(resource_user->resource_quota,
+                   GRPC_RULIST_AWAITING_ALLOCATION)) {
+    rq_step_sched(exec_ctx, resource_user->resource_quota);
+  }
+  rulist_add_tail(resource_user, GRPC_RULIST_AWAITING_ALLOCATION);
+}
+
+static void ru_add_to_free_pool(grpc_exec_ctx *exec_ctx, void *ru,
+                                grpc_error *error) {
+  grpc_resource_user *resource_user = ru;
+  if (!rulist_empty(resource_user->resource_quota,
+                    GRPC_RULIST_AWAITING_ALLOCATION) &&
+      rulist_empty(resource_user->resource_quota,
+                   GRPC_RULIST_NON_EMPTY_FREE_POOL)) {
+    rq_step_sched(exec_ctx, resource_user->resource_quota);
+  }
+  rulist_add_tail(resource_user, GRPC_RULIST_NON_EMPTY_FREE_POOL);
+}
+
+static void ru_post_benign_reclaimer(grpc_exec_ctx *exec_ctx, void *ru,
+                                     grpc_error *error) {
+  grpc_resource_user *resource_user = ru;
+  if (!rulist_empty(resource_user->resource_quota,
+                    GRPC_RULIST_AWAITING_ALLOCATION) &&
+      rulist_empty(resource_user->resource_quota,
+                   GRPC_RULIST_NON_EMPTY_FREE_POOL) &&
+      rulist_empty(resource_user->resource_quota,
+                   GRPC_RULIST_RECLAIMER_BENIGN)) {
+    rq_step_sched(exec_ctx, resource_user->resource_quota);
+  }
+  rulist_add_tail(resource_user, GRPC_RULIST_RECLAIMER_BENIGN);
+}
+
+static void ru_post_destructive_reclaimer(grpc_exec_ctx *exec_ctx, void *ru,
+                                          grpc_error *error) {
+  grpc_resource_user *resource_user = ru;
+  if (!rulist_empty(resource_user->resource_quota,
+                    GRPC_RULIST_AWAITING_ALLOCATION) &&
+      rulist_empty(resource_user->resource_quota,
+                   GRPC_RULIST_NON_EMPTY_FREE_POOL) &&
+      rulist_empty(resource_user->resource_quota,
+                   GRPC_RULIST_RECLAIMER_BENIGN) &&
+      rulist_empty(resource_user->resource_quota,
+                   GRPC_RULIST_RECLAIMER_DESTRUCTIVE)) {
+    rq_step_sched(exec_ctx, resource_user->resource_quota);
+  }
+  rulist_add_tail(resource_user, GRPC_RULIST_RECLAIMER_DESTRUCTIVE);
+}
+
+static void ru_destroy(grpc_exec_ctx *exec_ctx, void *ru, grpc_error *error) {
+  grpc_resource_user *resource_user = ru;
+  GPR_ASSERT(resource_user->allocated == 0);
+  for (int i = 0; i < GRPC_RULIST_COUNT; i++) {
+    rulist_remove(resource_user, (grpc_rulist)i);
+  }
+  grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[0],
+                      GRPC_ERROR_CANCELLED, NULL);
+  grpc_exec_ctx_sched(exec_ctx, resource_user->reclaimers[1],
+                      GRPC_ERROR_CANCELLED, NULL);
+  grpc_exec_ctx_sched(exec_ctx, (grpc_closure *)gpr_atm_no_barrier_load(
+                                    &resource_user->on_done_destroy_closure),
+                      GRPC_ERROR_NONE, NULL);
+  if (resource_user->free_pool != 0) {
+    resource_user->resource_quota->free_pool += resource_user->free_pool;
+    rq_step_sched(exec_ctx, resource_user->resource_quota);
+  }
+}
+
+static void ru_allocated_slices(grpc_exec_ctx *exec_ctx, void *arg,
+                                grpc_error *error) {
+  grpc_resource_user_slice_allocator *slice_allocator = arg;
+  if (error == GRPC_ERROR_NONE) {
+    for (size_t i = 0; i < slice_allocator->count; i++) {
+      gpr_slice_buffer_add_indexed(
+          slice_allocator->dest, ru_slice_create(slice_allocator->resource_user,
+                                                 slice_allocator->length));
+    }
+  }
+  grpc_closure_run(exec_ctx, &slice_allocator->on_done, GRPC_ERROR_REF(error));
+}
+
+/*******************************************************************************
+ * grpc_resource_quota internal implementation: quota manipulation under the
+ * combiner
+ */
+
+typedef struct {
+  int64_t size;
+  grpc_resource_quota *resource_quota;
+  grpc_closure closure;
+} rq_resize_args;
+
+static void rq_resize(grpc_exec_ctx *exec_ctx, void *args, grpc_error *error) {
+  rq_resize_args *a = args;
+  int64_t delta = a->size - a->resource_quota->size;
+  a->resource_quota->size += delta;
+  a->resource_quota->free_pool += delta;
+  rq_step_sched(exec_ctx, a->resource_quota);
+  grpc_resource_quota_internal_unref(exec_ctx, a->resource_quota);
+  gpr_free(a);
+}
+
+static void rq_reclamation_done(grpc_exec_ctx *exec_ctx, void *rq,
+                                grpc_error *error) {
+  grpc_resource_quota *resource_quota = rq;
+  resource_quota->reclaiming = false;
+  rq_step_sched(exec_ctx, resource_quota);
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
+}
+
+/*******************************************************************************
+ * grpc_resource_quota api
+ */
+
+/* Public API */
+grpc_resource_quota *grpc_resource_quota_create(const char *name) {
+  grpc_resource_quota *resource_quota = gpr_malloc(sizeof(*resource_quota));
+  gpr_ref_init(&resource_quota->refs, 1);
+  resource_quota->combiner = grpc_combiner_create(NULL);
+  resource_quota->free_pool = INT64_MAX;
+  resource_quota->size = INT64_MAX;
+  resource_quota->step_scheduled = false;
+  resource_quota->reclaiming = false;
+  if (name != NULL) {
+    resource_quota->name = gpr_strdup(name);
+  } else {
+    gpr_asprintf(&resource_quota->name, "anonymous_pool_%" PRIxPTR,
+                 (intptr_t)resource_quota);
+  }
+  grpc_closure_init(&resource_quota->rq_step_closure, rq_step, resource_quota);
+  grpc_closure_init(&resource_quota->rq_reclamation_done_closure,
+                    rq_reclamation_done, resource_quota);
+  for (int i = 0; i < GRPC_RULIST_COUNT; i++) {
+    resource_quota->roots[i] = NULL;
+  }
+  return resource_quota;
+}
+
+void grpc_resource_quota_internal_unref(grpc_exec_ctx *exec_ctx,
+                                        grpc_resource_quota *resource_quota) {
+  if (gpr_unref(&resource_quota->refs)) {
+    grpc_combiner_destroy(exec_ctx, resource_quota->combiner);
+    gpr_free(resource_quota->name);
+    gpr_free(resource_quota);
+  }
+}
+
+/* Public API */
+void grpc_resource_quota_unref(grpc_resource_quota *resource_quota) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_resource_quota_internal_unref(&exec_ctx, resource_quota);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+grpc_resource_quota *grpc_resource_quota_internal_ref(
+    grpc_resource_quota *resource_quota) {
+  gpr_ref(&resource_quota->refs);
+  return resource_quota;
+}
+
+/* Public API */
+void grpc_resource_quota_ref(grpc_resource_quota *resource_quota) {
+  grpc_resource_quota_internal_ref(resource_quota);
+}
+
+/* Public API */
+void grpc_resource_quota_resize(grpc_resource_quota *resource_quota,
+                                size_t size) {
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  rq_resize_args *a = gpr_malloc(sizeof(*a));
+  a->resource_quota = grpc_resource_quota_internal_ref(resource_quota);
+  a->size = (int64_t)size;
+  grpc_closure_init(&a->closure, rq_resize, a);
+  grpc_combiner_execute(&exec_ctx, resource_quota->combiner, &a->closure,
+                        GRPC_ERROR_NONE, false);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+/*******************************************************************************
+ * grpc_resource_user channel args api
+ */
+
+grpc_resource_quota *grpc_resource_quota_from_channel_args(
+    const grpc_channel_args *channel_args) {
+  for (size_t i = 0; i < channel_args->num_args; i++) {
+    if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) {
+      if (channel_args->args[i].type == GRPC_ARG_POINTER) {
+        return grpc_resource_quota_internal_ref(
+            channel_args->args[i].value.pointer.p);
+      } else {
+        gpr_log(GPR_DEBUG, GRPC_ARG_RESOURCE_QUOTA " should be a pointer");
+      }
+    }
+  }
+  return grpc_resource_quota_create(NULL);
+}
+
+static void *rq_copy(void *rq) {
+  grpc_resource_quota_ref(rq);
+  return rq;
+}
+
+static void rq_destroy(void *rq) { grpc_resource_quota_unref(rq); }
+
+static int rq_cmp(void *a, void *b) { return GPR_ICMP(a, b); }
+
+const grpc_arg_pointer_vtable *grpc_resource_quota_arg_vtable(void) {
+  static const grpc_arg_pointer_vtable vtable = {rq_copy, rq_destroy, rq_cmp};
+  return &vtable;
+}
+
+/*******************************************************************************
+ * grpc_resource_user api
+ */
+
+void grpc_resource_user_init(grpc_resource_user *resource_user,
+                             grpc_resource_quota *resource_quota,
+                             const char *name) {
+  resource_user->resource_quota =
+      grpc_resource_quota_internal_ref(resource_quota);
+  grpc_closure_init(&resource_user->allocate_closure, &ru_allocate,
+                    resource_user);
+  grpc_closure_init(&resource_user->add_to_free_pool_closure,
+                    &ru_add_to_free_pool, resource_user);
+  grpc_closure_init(&resource_user->post_reclaimer_closure[0],
+                    &ru_post_benign_reclaimer, resource_user);
+  grpc_closure_init(&resource_user->post_reclaimer_closure[1],
+                    &ru_post_destructive_reclaimer, resource_user);
+  grpc_closure_init(&resource_user->destroy_closure, &ru_destroy,
+                    resource_user);
+  gpr_mu_init(&resource_user->mu);
+  resource_user->allocated = 0;
+  resource_user->free_pool = 0;
+  grpc_closure_list_init(&resource_user->on_allocated);
+  resource_user->allocating = false;
+  resource_user->added_to_free_pool = false;
+  gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure, 0);
+  resource_user->reclaimers[0] = NULL;
+  resource_user->reclaimers[1] = NULL;
+  for (int i = 0; i < GRPC_RULIST_COUNT; i++) {
+    resource_user->links[i].next = resource_user->links[i].prev = NULL;
+  }
+  if (name != NULL) {
+    resource_user->name = gpr_strdup(name);
+  } else {
+    gpr_asprintf(&resource_user->name, "anonymous_resource_user_%" PRIxPTR,
+                 (intptr_t)resource_user);
+  }
+}
+
+void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx,
+                                 grpc_resource_user *resource_user,
+                                 grpc_closure *on_done) {
+  gpr_mu_lock(&resource_user->mu);
+  GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->on_done_destroy_closure) ==
+             0);
+  gpr_atm_no_barrier_store(&resource_user->on_done_destroy_closure,
+                           (gpr_atm)on_done);
+  if (resource_user->allocated == 0) {
+    grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner,
+                          &resource_user->destroy_closure, GRPC_ERROR_NONE,
+                          false);
+  }
+  gpr_mu_unlock(&resource_user->mu);
+}
+
+void grpc_resource_user_destroy(grpc_exec_ctx *exec_ctx,
+                                grpc_resource_user *resource_user) {
+  grpc_resource_quota_internal_unref(exec_ctx, resource_user->resource_quota);
+  gpr_mu_destroy(&resource_user->mu);
+  gpr_free(resource_user->name);
+}
+
+void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx,
+                              grpc_resource_user *resource_user, size_t size,
+                              grpc_closure *optional_on_done) {
+  gpr_mu_lock(&resource_user->mu);
+  grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load(
+      &resource_user->on_done_destroy_closure);
+  if (on_done_destroy != NULL) {
+    /* already shutdown */
+    if (grpc_resource_quota_trace) {
+      gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR " after shutdown",
+              resource_user->resource_quota->name, resource_user->name, size);
+    }
+    grpc_exec_ctx_sched(
+        exec_ctx, optional_on_done,
+        GRPC_ERROR_CREATE("Buffer pool user is already shutdown"), NULL);
+    gpr_mu_unlock(&resource_user->mu);
+    return;
+  }
+  resource_user->allocated += (int64_t)size;
+  resource_user->free_pool -= (int64_t)size;
+  if (grpc_resource_quota_trace) {
+    gpr_log(GPR_DEBUG, "RQ %s %s: alloc %" PRIdPTR "; allocated -> %" PRId64
+                       ", free_pool -> %" PRId64,
+            resource_user->resource_quota->name, resource_user->name, size,
+            resource_user->allocated, resource_user->free_pool);
+  }
+  if (resource_user->free_pool < 0) {
+    grpc_closure_list_append(&resource_user->on_allocated, optional_on_done,
+                             GRPC_ERROR_NONE);
+    if (!resource_user->allocating) {
+      resource_user->allocating = true;
+      grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner,
+                            &resource_user->allocate_closure, GRPC_ERROR_NONE,
+                            false);
+    }
+  } else {
+    grpc_exec_ctx_sched(exec_ctx, optional_on_done, GRPC_ERROR_NONE, NULL);
+  }
+  gpr_mu_unlock(&resource_user->mu);
+}
+
+void grpc_resource_user_free(grpc_exec_ctx *exec_ctx,
+                             grpc_resource_user *resource_user, size_t size) {
+  gpr_mu_lock(&resource_user->mu);
+  GPR_ASSERT(resource_user->allocated >= (int64_t)size);
+  bool was_zero_or_negative = resource_user->free_pool <= 0;
+  resource_user->free_pool += (int64_t)size;
+  resource_user->allocated -= (int64_t)size;
+  if (grpc_resource_quota_trace) {
+    gpr_log(GPR_DEBUG, "RQ %s %s: free %" PRIdPTR "; allocated -> %" PRId64
+                       ", free_pool -> %" PRId64,
+            resource_user->resource_quota->name, resource_user->name, size,
+            resource_user->allocated, resource_user->free_pool);
+  }
+  bool is_bigger_than_zero = resource_user->free_pool > 0;
+  if (is_bigger_than_zero && was_zero_or_negative &&
+      !resource_user->added_to_free_pool) {
+    resource_user->added_to_free_pool = true;
+    grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner,
+                          &resource_user->add_to_free_pool_closure,
+                          GRPC_ERROR_NONE, false);
+  }
+  grpc_closure *on_done_destroy = (grpc_closure *)gpr_atm_no_barrier_load(
+      &resource_user->on_done_destroy_closure);
+  if (on_done_destroy != NULL && resource_user->allocated == 0) {
+    grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner,
+                          &resource_user->destroy_closure, GRPC_ERROR_NONE,
+                          false);
+  }
+  gpr_mu_unlock(&resource_user->mu);
+}
+
+void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx,
+                                       grpc_resource_user *resource_user,
+                                       bool destructive,
+                                       grpc_closure *closure) {
+  if (gpr_atm_acq_load(&resource_user->on_done_destroy_closure) == 0) {
+    GPR_ASSERT(resource_user->reclaimers[destructive] == NULL);
+    resource_user->reclaimers[destructive] = closure;
+    grpc_combiner_execute(exec_ctx, resource_user->resource_quota->combiner,
+                          &resource_user->post_reclaimer_closure[destructive],
+                          GRPC_ERROR_NONE, false);
+  } else {
+    grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_CANCELLED, NULL);
+  }
+}
+
+void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx,
+                                           grpc_resource_user *resource_user) {
+  if (grpc_resource_quota_trace) {
+    gpr_log(GPR_DEBUG, "RQ %s %s: reclamation complete",
+            resource_user->resource_quota->name, resource_user->name);
+  }
+  grpc_combiner_execute(
+      exec_ctx, resource_user->resource_quota->combiner,
+      &resource_user->resource_quota->rq_reclamation_done_closure,
+      GRPC_ERROR_NONE, false);
+}
+
+void grpc_resource_user_slice_allocator_init(
+    grpc_resource_user_slice_allocator *slice_allocator,
+    grpc_resource_user *resource_user, grpc_iomgr_cb_func cb, void *p) {
+  grpc_closure_init(&slice_allocator->on_allocated, ru_allocated_slices,
+                    slice_allocator);
+  grpc_closure_init(&slice_allocator->on_done, cb, p);
+  slice_allocator->resource_user = resource_user;
+}
+
+void grpc_resource_user_alloc_slices(
+    grpc_exec_ctx *exec_ctx,
+    grpc_resource_user_slice_allocator *slice_allocator, size_t length,
+    size_t count, gpr_slice_buffer *dest) {
+  slice_allocator->length = length;
+  slice_allocator->count = count;
+  slice_allocator->dest = dest;
+  grpc_resource_user_alloc(exec_ctx, slice_allocator->resource_user,
+                           count * length, &slice_allocator->on_allocated);
+}

src/core/lib/iomgr/resource_quota.h

@@ -0,0 +1,224 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_RESOURCE_QUOTA_H
+#define GRPC_CORE_LIB_IOMGR_RESOURCE_QUOTA_H
+
+#include <grpc/grpc.h>
+
+#include "src/core/lib/iomgr/exec_ctx.h"
+
+/** \file Tracks resource usage against a pool.
+
+    The current implementation tracks only memory usage, but in the future
+    this may be extended to (for example) threads and file descriptors.
+
+    A grpc_resource_quota represents the pooled resources, and
+    grpc_resource_user instances attach to the quota and consume those
+    resources. They also offer a vector for reclamation: if we become
+    resource constrained, grpc_resource_user instances are asked (in turn) to
+    free up whatever they can so that the system as a whole can make progress.
+
+    There are three kinds of reclamation that take place, in order of increasing
+    invasiveness:
+    - an internal reclamation, where cached resource at the resource user level
+      is returned to the quota
+    - a benign reclamation phase, whereby resources that are in use but are not
+      helping anything make progress are reclaimed
+    - a destructive reclamation, whereby resources that are helping something
+      make progress may be enacted so that at least one part of the system can
+      complete.
+
+    Only one reclamation will be outstanding for a given quota at a given time.
+    On each reclamation attempt, the kinds of reclamation are tried in order of
+    increasing invasiveness, stopping at the first one that succeeds. Thus, on a
+    given reclamation attempt, if internal and benign reclamation both fail, it
+    will wind up doing a destructive reclamation. However, the next reclamation
+    attempt may then be able to get what it needs via internal or benign
+    reclamation, due to resources that may have been freed up by the destructive
+    reclamation in the previous attempt.
+
+    Future work will be to expose the current resource pressure so that back
+    pressure can be applied to avoid reclamation phases starting.
+
+    Resource users own references to resource quotas, and resource quotas
+    maintain lists of users (which users arrange to leave before they are
+    destroyed) */
+
+extern int grpc_resource_quota_trace;
+
+grpc_resource_quota *grpc_resource_quota_internal_ref(
+    grpc_resource_quota *resource_quota);
+void grpc_resource_quota_internal_unref(grpc_exec_ctx *exec_ctx,
+                                        grpc_resource_quota *resource_quota);
+grpc_resource_quota *grpc_resource_quota_from_channel_args(
+    const grpc_channel_args *channel_args);
+
+/* Resource users are kept in (potentially) several intrusive linked lists
+   at once. These are the list names. */
+typedef enum {
+  /* Resource users that are waiting for an allocation */
+  GRPC_RULIST_AWAITING_ALLOCATION,
+  /* Resource users that have free memory available for internal reclamation */
+  GRPC_RULIST_NON_EMPTY_FREE_POOL,
+  /* Resource users that have published a benign reclamation is available */
+  GRPC_RULIST_RECLAIMER_BENIGN,
+  /* Resource users that have published a destructive reclamation is
+     available */
+  GRPC_RULIST_RECLAIMER_DESTRUCTIVE,
+  /* Number of lists: must be last */
+  GRPC_RULIST_COUNT
+} grpc_rulist;
+
+typedef struct grpc_resource_user grpc_resource_user;
+
+/* Internal linked list pointers for a resource user */
+typedef struct {
+  grpc_resource_user *next;
+  grpc_resource_user *prev;
+} grpc_resource_user_link;
+
+struct grpc_resource_user {
+  /* The quota this resource user consumes from */
+  grpc_resource_quota *resource_quota;
+
+  /* Closure to schedule an allocation under the resource quota combiner lock */
+  grpc_closure allocate_closure;
+  /* Closure to publish a non empty free pool under the resource quota combiner
+     lock */
+  grpc_closure add_to_free_pool_closure;
+
+  gpr_mu mu;
+  /* Total allocated memory outstanding by this resource user in bytes;
+     always positive */
+  int64_t allocated;
+  /* The amount of memory (in bytes) this user has cached for its own use: to
+     avoid quota contention, each resource user can keep some memory in
+     addition to what it is immediately using (e.g., for caching), and the quota
+     can pull it back under memory pressure.
+     This value can become negative if more memory has been requested than
+     existed in the free pool, at which point the quota is consulted to bring
+     this value non-negative (asynchronously). */
+  int64_t free_pool;
+  /* A list of closures to call once free_pool becomes non-negative - ie when
+     all outstanding allocations have been granted. */
+  grpc_closure_list on_allocated;
+  /* True if we are currently trying to allocate from the quota, false if not */
+  bool allocating;
+  /* True if we are currently trying to add ourselves to the non-free quota
+     list, false otherwise */
+  bool added_to_free_pool;
+
+  /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer
+   */
+  grpc_closure *reclaimers[2];
+  /* Trampoline closures to finish reclamation and re-enter the quota combiner
+     lock */
+  grpc_closure post_reclaimer_closure[2];
+
+  /* Closure to execute under the quota combiner to de-register and shutdown the
+     resource user */
+  grpc_closure destroy_closure;
+  /* User supplied closure to call once the user has finished shutting down AND
+     all outstanding allocations have been freed. Real type is grpc_closure*,
+     but it's stored as an atomic to avoid a mutex on some fast paths. */
+  gpr_atm on_done_destroy_closure;
+
+  /* Links in the various grpc_rulist lists */
+  grpc_resource_user_link links[GRPC_RULIST_COUNT];
+
+  /* The name of this resource user, for debugging/tracing */
+  char *name;
+};
+
+void grpc_resource_user_init(grpc_resource_user *resource_user,
+                             grpc_resource_quota *resource_quota,
+                             const char *name);
+void grpc_resource_user_shutdown(grpc_exec_ctx *exec_ctx,
+                                 grpc_resource_user *resource_user,
+                                 grpc_closure *on_done);
+void grpc_resource_user_destroy(grpc_exec_ctx *exec_ctx,
+                                grpc_resource_user *resource_user);
+
+/* Allocate from the resource user (and its quota).
+   If optional_on_done is NULL, then allocate immediately. This may push the
+   quota over-limit, at which point reclamation will kick in.
+   If optional_on_done is non-NULL, it will be scheduled when the allocation has
+   been granted by the quota. */
+void grpc_resource_user_alloc(grpc_exec_ctx *exec_ctx,
+                              grpc_resource_user *resource_user, size_t size,
+                              grpc_closure *optional_on_done);
+/* Release memory back to the quota */
+void grpc_resource_user_free(grpc_exec_ctx *exec_ctx,
+                             grpc_resource_user *resource_user, size_t size);
+/* Post a memory reclaimer to the resource user. Only one benign and one
+   destructive reclaimer can be posted at once. When executed, the reclaimer
+   MUST call grpc_resource_user_finish_reclamation before it completes, to
+   return control to the resource quota. */
+void grpc_resource_user_post_reclaimer(grpc_exec_ctx *exec_ctx,
+                                       grpc_resource_user *resource_user,
+                                       bool destructive, grpc_closure *closure);
+/* Finish a reclamation step */
+void grpc_resource_user_finish_reclamation(grpc_exec_ctx *exec_ctx,
+                                           grpc_resource_user *resource_user);
+
+/* Helper to allocate slices from a resource user */
+typedef struct grpc_resource_user_slice_allocator {
+  /* Closure for when a resource user allocation completes */
+  grpc_closure on_allocated;
+  /* Closure to call when slices have been allocated */
+  grpc_closure on_done;
+  /* Length of slices to allocate on the current request */
+  size_t length;
+  /* Number of slices to allocate on the current request */
+  size_t count;
+  /* Destination for slices to allocate on the current request */
+  gpr_slice_buffer *dest;
+  /* Parent resource user */
+  grpc_resource_user *resource_user;
+} grpc_resource_user_slice_allocator;
+
+/* Initialize a slice allocator.
+   When an allocation is completed, calls \a cb with arg \p. */
+void grpc_resource_user_slice_allocator_init(
+    grpc_resource_user_slice_allocator *slice_allocator,
+    grpc_resource_user *resource_user, grpc_iomgr_cb_func cb, void *p);
+
+/* Allocate \a count slices of length \a length into \a dest. Only one request
+   can be outstanding at a time. */
+void grpc_resource_user_alloc_slices(
+    grpc_exec_ctx *exec_ctx,
+    grpc_resource_user_slice_allocator *slice_allocator, size_t length,
+    size_t count, gpr_slice_buffer *dest);
+
+#endif /* GRPC_CORE_LIB_IOMGR_RESOURCE_QUOTA_H */

src/core/lib/iomgr/tcp_client.h

@@ -39,6 +39,10 @@
 #include "src/core/lib/iomgr/pollset_set.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 
+/* Channel arg (integer) setting how large a slice to try and read from the wire
+   each time recvmsg (or equivalent) is called */
+#define GRPC_ARG_TCP_READ_CHUNK_SIZE "grpc.experimental.tcp_read_chunk_size"
+
 /* Asynchronously connect to an address (specified as (addr, len)), and call
    cb with arg and the completed connection when done (or call cb with arg and
    NULL on failure).
@@ -47,6 +51,7 @@
 void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_connect,
                              grpc_endpoint **endpoint,
                              grpc_pollset_set *interested_parties,
+                             const grpc_channel_args *channel_args,
                              const grpc_resolved_address *addr,
                              gpr_timespec deadline);
 

src/core/lib/iomgr/tcp_client_posix.c

@@ -35,7 +35,7 @@
 
 #ifdef GRPC_POSIX_SOCKET
 
-#include "src/core/lib/iomgr/tcp_client.h"
+#include "src/core/lib/iomgr/tcp_client_posix.h"
 
 #include <errno.h>
 #include <netinet/in.h>
@@ -47,6 +47,7 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/time.h>
 
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_posix.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -69,6 +70,7 @@ typedef struct {
   char *addr_str;
   grpc_endpoint **ep;
   grpc_closure *closure;
+  grpc_channel_args *channel_args;
 } async_connect;
 
 static grpc_error *prepare_socket(const grpc_resolved_address *addr, int fd) {
@@ -114,10 +116,39 @@ static void tc_on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   if (done) {
     gpr_mu_destroy(&ac->mu);
     gpr_free(ac->addr_str);
+    grpc_channel_args_destroy(ac->channel_args);
     gpr_free(ac);
   }
 }
 
+grpc_endpoint *grpc_tcp_client_create_from_fd(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, const grpc_channel_args *channel_args,
+    const char *addr_str) {
+  size_t tcp_read_chunk_size = GRPC_TCP_DEFAULT_READ_SLICE_SIZE;
+  grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL);
+  if (channel_args != NULL) {
+    for (size_t i = 0; i < channel_args->num_args; i++) {
+      if (0 ==
+          strcmp(channel_args->args[i].key, GRPC_ARG_TCP_READ_CHUNK_SIZE)) {
+        grpc_integer_options options = {(int)tcp_read_chunk_size, 1,
+                                        8 * 1024 * 1024};
+        tcp_read_chunk_size = (size_t)grpc_channel_arg_get_integer(
+            &channel_args->args[i], options);
+      } else if (0 ==
+                 strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) {
+        grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
+        resource_quota = grpc_resource_quota_internal_ref(
+            channel_args->args[i].value.pointer.p);
+      }
+    }
+  }
+
+  grpc_endpoint *ep =
+      grpc_tcp_create(fd, resource_quota, tcp_read_chunk_size, addr_str);
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
+  return ep;
+}
+
 static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   async_connect *ac = acp;
   int so_error = 0;
@@ -165,7 +196,8 @@ static void on_writable(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   switch (so_error) {
     case 0:
       grpc_pollset_set_del_fd(exec_ctx, ac->interested_parties, fd);
-      *ep = grpc_tcp_create(fd, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, ac->addr_str);
+      *ep = grpc_tcp_client_create_from_fd(exec_ctx, fd, ac->channel_args,
+                                           ac->addr_str);
       fd = NULL;
       break;
     case ENOBUFS:
@@ -215,6 +247,7 @@ finish:
   if (done) {
     gpr_mu_destroy(&ac->mu);
     gpr_free(ac->addr_str);
+    grpc_channel_args_destroy(ac->channel_args);
     gpr_free(ac);
   }
   grpc_exec_ctx_sched(exec_ctx, closure, error, NULL);
@@ -223,6 +256,7 @@ finish:
 static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
                                     grpc_closure *closure, grpc_endpoint **ep,
                                     grpc_pollset_set *interested_parties,
+                                    const grpc_channel_args *channel_args,
                                     const grpc_resolved_address *addr,
                                     gpr_timespec deadline) {
   int fd;
@@ -270,7 +304,8 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   fdobj = grpc_fd_create(fd, name);
 
   if (err >= 0) {
-    *ep = grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str);
+    *ep =
+        grpc_tcp_client_create_from_fd(exec_ctx, fdobj, channel_args, addr_str);
     grpc_exec_ctx_sched(exec_ctx, closure, GRPC_ERROR_NONE, NULL);
     goto done;
   }
@@ -295,6 +330,7 @@ static void tcp_client_connect_impl(grpc_exec_ctx *exec_ctx,
   ac->refs = 2;
   ac->write_closure.cb = on_writable;
   ac->write_closure.cb_arg = ac;
+  ac->channel_args = grpc_channel_args_copy(channel_args);
 
   if (grpc_tcp_trace) {
     gpr_log(GPR_DEBUG, "CLIENT_CONNECT: %s: asynchronously connecting",
@@ -316,16 +352,18 @@ done:
 // overridden by api_fuzzer.c
 void (*grpc_tcp_client_connect_impl)(
     grpc_exec_ctx *exec_ctx, grpc_closure *closure, grpc_endpoint **ep,
-    grpc_pollset_set *interested_parties, const grpc_resolved_address *addr,
+    grpc_pollset_set *interested_parties, const grpc_channel_args *channel_args,
+    const grpc_resolved_address *addr,
     gpr_timespec deadline) = tcp_client_connect_impl;
 
 void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
                              grpc_endpoint **ep,
                              grpc_pollset_set *interested_parties,
+                             const grpc_channel_args *channel_args,
                              const grpc_resolved_address *addr,
                              gpr_timespec deadline) {
-  grpc_tcp_client_connect_impl(exec_ctx, closure, ep, interested_parties, addr,
-                               deadline);
+  grpc_tcp_client_connect_impl(exec_ctx, closure, ep, interested_parties,
+                               channel_args, addr, deadline);
 }
 
 #endif

src/core/lib/iomgr/tcp_client_posix.h

@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_IOMGR_TCP_CLIENT_POSIX_H
+#define GRPC_CORE_LIB_IOMGR_TCP_CLIENT_POSIX_H
+
+#include "src/core/lib/iomgr/endpoint.h"
+#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/tcp_client.h"
+
+grpc_endpoint *grpc_tcp_client_create_from_fd(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, const grpc_channel_args *channel_args,
+    const char *addr_str);
+
+#endif /* GRPC_CORE_LIB_IOMGR_TCP_CLIENT_POSIX_H */

src/core/lib/iomgr/tcp_client_windows.c

@@ -43,6 +43,7 @@
 #include <grpc/support/slice_buffer.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/iocp_windows.h"
 #include "src/core/lib/iomgr/sockaddr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
@@ -61,13 +62,16 @@ typedef struct {
   int refs;
   grpc_closure on_connect;
   grpc_endpoint **endpoint;
+  grpc_resource_quota *resource_quota;
 } async_connect;
 
-static void async_connect_unlock_and_cleanup(async_connect *ac,
+static void async_connect_unlock_and_cleanup(grpc_exec_ctx *exec_ctx,
+                                             async_connect *ac,
                                              grpc_winsocket *socket) {
   int done = (--ac->refs == 0);
   gpr_mu_unlock(&ac->mu);
   if (done) {
+    grpc_resource_quota_internal_unref(exec_ctx, ac->resource_quota);
     gpr_mu_destroy(&ac->mu);
     gpr_free(ac->addr_name);
     gpr_free(ac);
@@ -83,7 +87,7 @@ static void on_alarm(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
   if (socket != NULL) {
     grpc_winsocket_shutdown(socket);
   }
-  async_connect_unlock_and_cleanup(ac, socket);
+  async_connect_unlock_and_cleanup(exec_ctx, ac, socket);
 }
 
 static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
@@ -113,12 +117,12 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
     if (!wsa_success) {
       error = GRPC_WSA_ERROR(WSAGetLastError(), "ConnectEx");
     } else {
-      *ep = grpc_tcp_create(socket, ac->addr_name);
+      *ep = grpc_tcp_create(socket, ac->resource_quota, ac->addr_name);
       socket = NULL;
     }
   }
 
-  async_connect_unlock_and_cleanup(ac, socket);
+  async_connect_unlock_and_cleanup(exec_ctx, ac, socket);
   /* If the connection was aborted, the callback was already called when
      the deadline was met. */
   grpc_exec_ctx_sched(exec_ctx, on_done, error, NULL);
@@ -129,6 +133,7 @@ static void on_connect(grpc_exec_ctx *exec_ctx, void *acp, grpc_error *error) {
 void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
                              grpc_endpoint **endpoint,
                              grpc_pollset_set *interested_parties,
+                             const grpc_channel_args *channel_args,
                              const grpc_resolved_address *addr,
                              gpr_timespec deadline) {
   SOCKET sock = INVALID_SOCKET;
@@ -144,6 +149,17 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
   grpc_winsocket_callback_info *info;
   grpc_error *error = GRPC_ERROR_NONE;
 
+  grpc_resource_quota *resource_quota = grpc_resource_quota_create(NULL);
+  if (channel_args != NULL) {
+    for (size_t i = 0; i < channel_args->num_args; i++) {
+      if (0 == strcmp(channel_args->args[i].key, GRPC_ARG_RESOURCE_QUOTA)) {
+        grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
+        resource_quota = grpc_resource_quota_internal_ref(
+            channel_args->args[i].value.pointer.p);
+      }
+    }
+  }
+
   *endpoint = NULL;
 
   /* Use dualstack sockets where available. */
@@ -177,8 +193,8 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
 
   grpc_sockaddr_make_wildcard6(0, &local_address);
 
-  status =
-      bind(sock, (struct sockaddr *)&local_address.addr, local_address.len);
+  status = bind(sock, (struct sockaddr *)&local_address.addr,
+                (int)local_address.len);
   if (status != 0) {
     error = GRPC_WSA_ERROR(WSAGetLastError(), "bind");
     goto failure;
@@ -206,6 +222,7 @@ void grpc_tcp_client_connect(grpc_exec_ctx *exec_ctx, grpc_closure *on_done,
   ac->refs = 2;
   ac->addr_name = grpc_sockaddr_to_uri(addr);
   ac->endpoint = endpoint;
+  ac->resource_quota = resource_quota;
   grpc_closure_init(&ac->on_connect, on_connect, ac);
 
   grpc_timer_init(exec_ctx, &ac->alarm, deadline, on_alarm, ac,
@@ -225,6 +242,7 @@ failure:
   } else if (sock != INVALID_SOCKET) {
     closesocket(sock);
   }
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
   grpc_exec_ctx_sched(exec_ctx, on_done, final_error, NULL);
 }
 

src/core/lib/iomgr/tcp_posix.c

@@ -80,6 +80,7 @@ typedef struct {
   msg_iovlen_type iov_size; /* Number of slices to allocate per read attempt */
   size_t slice_size;
   gpr_refcount refcount;
+  gpr_atm shutdown_count;
 
   /* garbage after the last read */
   gpr_slice_buffer last_read_buffer;
@@ -100,15 +101,29 @@ typedef struct {
   grpc_closure write_closure;
 
   char *peer_string;
+
+  grpc_resource_user resource_user;
+  grpc_resource_user_slice_allocator slice_allocator;
 } grpc_tcp;
 
 static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
                             grpc_error *error);
 static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
                              grpc_error *error);
+static void tcp_unref_closure(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
+                              grpc_error *error);
+
+static void tcp_maybe_shutdown_resource_user(grpc_exec_ctx *exec_ctx,
+                                             grpc_tcp *tcp) {
+  if (gpr_atm_full_fetch_add(&tcp->shutdown_count, 1) == 0) {
+    grpc_resource_user_shutdown(exec_ctx, &tcp->resource_user,
+                                grpc_closure_create(tcp_unref_closure, tcp));
+  }
+}
 
 static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
   grpc_tcp *tcp = (grpc_tcp *)ep;
+  tcp_maybe_shutdown_resource_user(exec_ctx, tcp);
   grpc_fd_shutdown(exec_ctx, tcp->em_fd);
 }
 
@@ -116,6 +131,7 @@ static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   grpc_fd_orphan(exec_ctx, tcp->em_fd, tcp->release_fd_cb, tcp->release_fd,
                  "tcp_unref_orphan");
   gpr_slice_buffer_destroy(&tcp->last_read_buffer);
+  grpc_resource_user_destroy(exec_ctx, &tcp->resource_user);
   gpr_free(tcp->peer_string);
   gpr_free(tcp);
 }
@@ -152,9 +168,16 @@ static void tcp_unref(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
 static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); }
 #endif
 
+static void tcp_unref_closure(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error) {
+  TCP_UNREF(exec_ctx, arg, "resource_user");
+}
+
 static void tcp_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
   grpc_network_status_unregister_endpoint(ep);
   grpc_tcp *tcp = (grpc_tcp *)ep;
+  tcp_maybe_shutdown_resource_user(exec_ctx, tcp);
+  gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer);
   TCP_UNREF(exec_ctx, tcp, "destroy");
 }
 
@@ -181,7 +204,7 @@ static void call_read_cb(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
 }
 
 #define MAX_READ_IOVEC 4
-static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
+static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   struct msghdr msg;
   struct iovec iov[MAX_READ_IOVEC];
   ssize_t read_bytes;
@@ -192,10 +215,6 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   GPR_ASSERT(tcp->incoming_buffer->count <= MAX_READ_IOVEC);
   GPR_TIMER_BEGIN("tcp_continue_read", 0);
 
-  while (tcp->incoming_buffer->count < (size_t)tcp->iov_size) {
-    gpr_slice_buffer_add_indexed(tcp->incoming_buffer,
-                                 gpr_slice_malloc(tcp->slice_size));
-  }
   for (i = 0; i < tcp->incoming_buffer->count; i++) {
     iov[i].iov_base = GPR_SLICE_START_PTR(tcp->incoming_buffer->slices[i]);
     iov[i].iov_len = GPR_SLICE_LENGTH(tcp->incoming_buffer->slices[i]);
@@ -232,7 +251,7 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   } else if (read_bytes == 0) {
     /* 0 read size ==> end of stream */
     gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer);
-    call_read_cb(exec_ctx, tcp, GRPC_ERROR_CREATE("EOF"));
+    call_read_cb(exec_ctx, tcp, GRPC_ERROR_CREATE("Socket closed"));
     TCP_UNREF(exec_ctx, tcp, "read");
   } else {
     GPR_ASSERT((size_t)read_bytes <= tcp->incoming_buffer->length);
@@ -252,6 +271,30 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
   GPR_TIMER_END("tcp_continue_read", 0);
 }
 
+static void tcp_read_allocation_done(grpc_exec_ctx *exec_ctx, void *tcpp,
+                                     grpc_error *error) {
+  grpc_tcp *tcp = tcpp;
+  if (error != GRPC_ERROR_NONE) {
+    gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer);
+    gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer);
+    call_read_cb(exec_ctx, tcp, GRPC_ERROR_REF(error));
+    TCP_UNREF(exec_ctx, tcp, "read");
+  } else {
+    tcp_do_read(exec_ctx, tcp);
+  }
+}
+
+static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
+  if (tcp->incoming_buffer->count < (size_t)tcp->iov_size) {
+    grpc_resource_user_alloc_slices(
+        exec_ctx, &tcp->slice_allocator, tcp->slice_size,
+        (size_t)tcp->iov_size - tcp->incoming_buffer->count,
+        tcp->incoming_buffer);
+  } else {
+    tcp_do_read(exec_ctx, tcp);
+  }
+}
+
 static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
                             grpc_error *error) {
   grpc_tcp *tcp = (grpc_tcp *)arg;
@@ -259,6 +302,7 @@ static void tcp_handle_read(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
 
   if (error != GRPC_ERROR_NONE) {
     gpr_slice_buffer_reset_and_unref(tcp->incoming_buffer);
+    gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer);
     call_read_cb(exec_ctx, tcp, GRPC_ERROR_REF(error));
     TCP_UNREF(exec_ctx, tcp, "read");
   } else {
@@ -469,6 +513,11 @@ static grpc_workqueue *tcp_get_workqueue(grpc_endpoint *ep) {
   return grpc_fd_get_workqueue(tcp->em_fd);
 }
 
+static grpc_resource_user *tcp_get_resource_user(grpc_endpoint *ep) {
+  grpc_tcp *tcp = (grpc_tcp *)ep;
+  return &tcp->resource_user;
+}
+
 static const grpc_endpoint_vtable vtable = {tcp_read,
                                             tcp_write,
                                             tcp_get_workqueue,
@@ -476,10 +525,12 @@ static const grpc_endpoint_vtable vtable = {tcp_read,
                                             tcp_add_to_pollset_set,
                                             tcp_shutdown,
                                             tcp_destroy,
+                                            tcp_get_resource_user,
                                             tcp_get_peer};
 
-grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
-                               const char *peer_string) {
+grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd,
+                               grpc_resource_quota *resource_quota,
+                               size_t slice_size, const char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp));
   tcp->base.vtable = &vtable;
   tcp->peer_string = gpr_strdup(peer_string);
@@ -492,14 +543,20 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size,
   tcp->slice_size = slice_size;
   tcp->iov_size = 1;
   tcp->finished_edge = true;
-  /* paired with unref in grpc_tcp_destroy */
-  gpr_ref_init(&tcp->refcount, 1);
+  /* paired with unref in grpc_tcp_destroy, and with the shutdown for our
+   * resource_user */
+  gpr_ref_init(&tcp->refcount, 2);
+  gpr_atm_no_barrier_store(&tcp->shutdown_count, 0);
   tcp->em_fd = em_fd;
   tcp->read_closure.cb = tcp_handle_read;
   tcp->read_closure.cb_arg = tcp;
   tcp->write_closure.cb = tcp_handle_write;
   tcp->write_closure.cb_arg = tcp;
   gpr_slice_buffer_init(&tcp->last_read_buffer);
+  grpc_resource_user_init(&tcp->resource_user, resource_quota, peer_string);
+  grpc_resource_user_slice_allocator_init(&tcp->slice_allocator,
+                                          &tcp->resource_user,
+                                          tcp_read_allocation_done, tcp);
   /* Tell network status tracker about new endpoint */
   grpc_network_status_register_endpoint(&tcp->base);
 
@@ -514,10 +571,13 @@ int grpc_tcp_fd(grpc_endpoint *ep) {
 
 void grpc_tcp_destroy_and_release_fd(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
                                      int *fd, grpc_closure *done) {
+  grpc_network_status_unregister_endpoint(ep);
   grpc_tcp *tcp = (grpc_tcp *)ep;
   GPR_ASSERT(ep->vtable == &vtable);
   tcp->release_fd = fd;
   tcp->release_fd_cb = done;
+  tcp_maybe_shutdown_resource_user(exec_ctx, tcp);
+  gpr_slice_buffer_reset_and_unref(&tcp->last_read_buffer);
   TCP_UNREF(exec_ctx, tcp, "destroy");
 }
 

src/core/lib/iomgr/tcp_posix.h

@@ -53,8 +53,8 @@ extern int grpc_tcp_trace;
 
 /* Create a tcp endpoint given a file desciptor and a read slice size.
    Takes ownership of fd. */
-grpc_endpoint *grpc_tcp_create(grpc_fd *fd, size_t read_slice_size,
-                               const char *peer_string);
+grpc_endpoint *grpc_tcp_create(grpc_fd *fd, grpc_resource_quota *resource_quota,
+                               size_t read_slice_size, const char *peer_string);
 
 /* Return the tcp endpoint's fd, or -1 if this is not available. Does not
    release the fd.

src/core/lib/iomgr/tcp_server.h

@@ -61,7 +61,8 @@ typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg,
 /* Create a server, initially not bound to any ports. The caller owns one ref.
    If shutdown_complete is not NULL, it will be used by
    grpc_tcp_server_unref() when the ref count reaches zero. */
-grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
+grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx,
+                                   grpc_closure *shutdown_complete,
                                    const grpc_channel_args *args,
                                    grpc_tcp_server **server);
 

src/core/lib/iomgr/tcp_server_posix.c

@@ -134,6 +134,8 @@ struct grpc_tcp_server {
 
   /* next pollset to assign a channel to */
   gpr_atm next_pollset_to_assign;
+
+  grpc_resource_quota *resource_quota;
 };
 
 static gpr_once check_init = GPR_ONCE_INIT;
@@ -150,23 +152,37 @@ static void init(void) {
 #endif
 }
 
-grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
+grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx,
+                                   grpc_closure *shutdown_complete,
                                    const grpc_channel_args *args,
                                    grpc_tcp_server **server) {
   gpr_once_init(&check_init, init);
 
   grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server));
   s->so_reuseport = has_so_reuseport;
+  s->resource_quota = grpc_resource_quota_create(NULL);
   for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) {
     if (0 == strcmp(GRPC_ARG_ALLOW_REUSEPORT, args->args[i].key)) {
       if (args->args[i].type == GRPC_ARG_INTEGER) {
         s->so_reuseport =
             has_so_reuseport && (args->args[i].value.integer != 0);
       } else {
+        grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota);
         gpr_free(s);
         return GRPC_ERROR_CREATE(GRPC_ARG_ALLOW_REUSEPORT
                                  " must be an integer");
       }
+    } else if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) {
+      if (args->args[i].type == GRPC_ARG_POINTER) {
+        grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota);
+        s->resource_quota =
+            grpc_resource_quota_internal_ref(args->args[i].value.pointer.p);
+      } else {
+        grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota);
+        gpr_free(s);
+        return GRPC_ERROR_CREATE(GRPC_ARG_RESOURCE_QUOTA
+                                 " must be a pointer to a buffer pool");
+      }
     }
   }
   gpr_ref_init(&s->refs, 1);
@@ -203,6 +219,8 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
     gpr_free(sp);
   }
 
+  grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota);
+
   gpr_free(s);
 }
 
@@ -419,7 +437,8 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *err) {
 
     sp->server->on_accept_cb(
         exec_ctx, sp->server->on_accept_cb_arg,
-        grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
+        grpc_tcp_create(fdobj, sp->server->resource_quota,
+                        GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
         read_notifier_pollset, &acceptor);
 
     gpr_free(name);

src/core/lib/iomgr/tcp_server_windows.c

@@ -100,14 +100,32 @@ struct grpc_tcp_server {
 
   /* shutdown callback */
   grpc_closure *shutdown_complete;
+
+  grpc_resource_quota *resource_quota;
 };
 
 /* Public function. Allocates the proper data structures to hold a
    grpc_tcp_server. */
-grpc_error *grpc_tcp_server_create(grpc_closure *shutdown_complete,
+grpc_error *grpc_tcp_server_create(grpc_exec_ctx *exec_ctx,
+                                   grpc_closure *shutdown_complete,
                                    const grpc_channel_args *args,
                                    grpc_tcp_server **server) {
   grpc_tcp_server *s = gpr_malloc(sizeof(grpc_tcp_server));
+  s->resource_quota = grpc_resource_quota_create(NULL);
+  for (size_t i = 0; i < (args == NULL ? 0 : args->num_args); i++) {
+    if (0 == strcmp(GRPC_ARG_RESOURCE_QUOTA, args->args[i].key)) {
+      if (args->args[i].type == GRPC_ARG_POINTER) {
+        grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota);
+        s->resource_quota =
+            grpc_resource_quota_internal_ref(args->args[i].value.pointer.p);
+      } else {
+        grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota);
+        gpr_free(s);
+        return GRPC_ERROR_CREATE(GRPC_ARG_RESOURCE_QUOTA
+                                 " must be a pointer to a buffer pool");
+      }
+    }
+  }
   gpr_ref_init(&s->refs, 1);
   gpr_mu_init(&s->mu);
   s->active_ports = 0;
@@ -137,6 +155,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s) {
     grpc_winsocket_destroy(sp->socket);
     gpr_free(sp);
   }
+  grpc_resource_quota_internal_unref(exec_ctx, s->resource_quota);
   gpr_free(s);
 }
 
@@ -207,12 +226,13 @@ static grpc_error *prepare_socket(SOCKET sock,
     goto failure;
   }
 
-  sockname_temp.len = sizeof(struct sockaddr_storage);
+  int sockname_temp_len = sizeof(struct sockaddr_storage);
   if (getsockname(sock, (struct sockaddr *)sockname_temp.addr,
-                  &sockname_temp.len) == SOCKET_ERROR) {
+                  &sockname_temp_len) == SOCKET_ERROR) {
     error = GRPC_WSA_ERROR(WSAGetLastError(), "getsockname");
     goto failure;
   }
+  sockname_temp.len = sockname_temp_len;
 
   *port = grpc_sockaddr_get_port(&sockname_temp);
   return GRPC_ERROR_NONE;
@@ -357,8 +377,10 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
         gpr_log(GPR_ERROR, "setsockopt error: %s", utf8_message);
         gpr_free(utf8_message);
       }
+      int peer_name_len = (int)peer_name.len;
       err =
-          getpeername(sock, (struct sockaddr *)peer_name.addr, &peer_name.len);
+          getpeername(sock, (struct sockaddr *)peer_name.addr, &peer_name_len);
+      peer_name.len = peer_name_len;
       if (!err) {
         peer_name_string = grpc_sockaddr_to_uri(&peer_name);
       } else {
@@ -368,7 +390,7 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
       }
       gpr_asprintf(&fd_name, "tcp_server:%s", peer_name_string);
       ep = grpc_tcp_create(grpc_winsocket_create(sock, fd_name),
-                           peer_name_string);
+                           sp->server->resource_quota, peer_name_string);
       gpr_free(fd_name);
       gpr_free(peer_name_string);
     } else {
@@ -466,10 +488,11 @@ grpc_error *grpc_tcp_server_add_port(grpc_tcp_server *s,
      as some previously created listener. */
   if (grpc_sockaddr_get_port(addr) == 0) {
     for (sp = s->head; sp; sp = sp->next) {
-      sockname_temp.len = sizeof(struct sockaddr_storage);
+      int sockname_temp_len = sizeof(struct sockaddr_storage);
       if (0 == getsockname(sp->socket->socket,
                            (struct sockaddr *)sockname_temp.addr,
-                           &sockname_temp.len)) {
+                           &sockname_temp_len)) {
+        sockname_temp.len = sockname_temp_len;
         *port = grpc_sockaddr_get_port(&sockname_temp);
         if (*port > 0) {
           allocated_addr = gpr_malloc(sizeof(grpc_resolved_address));

src/core/lib/iomgr/tcp_uv.c

@@ -315,6 +315,9 @@ grpc_endpoint *grpc_tcp_create(uv_tcp_t *handle, char *peer_string) {
     gpr_log(GPR_DEBUG, "Creating TCP endpoint %p", tcp);
   }
 
+  /* Disable Nagle's Algorithm */
+  uv_tcp_nodelay(handle, 1);
+
   memset(tcp, 0, sizeof(grpc_tcp));
   tcp->base.vtable = &vtable;
   tcp->handle = handle;

src/core/lib/iomgr/tcp_windows.c

@@ -109,14 +109,29 @@ typedef struct grpc_tcp {
   gpr_slice_buffer *write_slices;
   gpr_slice_buffer *read_slices;
 
+  grpc_resource_user resource_user;
+
   /* The IO Completion Port runs from another thread. We need some mechanism
      to protect ourselves when requesting a shutdown. */
   gpr_mu mu;
   int shutting_down;
 
+  gpr_atm resource_user_shutdown_count;
+
   char *peer_string;
 } grpc_tcp;
 
+static void win_unref_closure(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
+                              grpc_error *error);
+
+static void win_maybe_shutdown_resource_user(grpc_exec_ctx *exec_ctx,
+                                             grpc_tcp *tcp) {
+  if (gpr_atm_full_fetch_add(&tcp->resource_user_shutdown_count, 1) == 0) {
+    grpc_resource_user_shutdown(exec_ctx, &tcp->resource_user,
+                                grpc_closure_create(win_unref_closure, tcp));
+  }
+}
+
 static void tcp_free(grpc_tcp *tcp) {
   grpc_winsocket_destroy(tcp->socket);
   gpr_mu_destroy(&tcp->mu);
@@ -155,6 +170,11 @@ static void tcp_unref(grpc_tcp *tcp) {
 static void tcp_ref(grpc_tcp *tcp) { gpr_ref(&tcp->refcount); }
 #endif
 
+static void win_unref_closure(grpc_exec_ctx *exec_ctx, void *arg,
+                              grpc_error *error) {
+  TCP_UNREF(arg, "resource_user");
+}
+
 /* Asynchronous callback from the IOCP, or the background thread. */
 static void on_read(grpc_exec_ctx *exec_ctx, void *tcpp, grpc_error *error) {
   grpc_tcp *tcp = tcpp;
@@ -376,12 +396,14 @@ static void win_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
      callback. See the comments in on_read and on_write. */
   tcp->shutting_down = 1;
   grpc_winsocket_shutdown(tcp->socket);
+  win_maybe_shutdown_resource_user(exec_ctx, tcp);
   gpr_mu_unlock(&tcp->mu);
 }
 
 static void win_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) {
   grpc_network_status_unregister_endpoint(ep);
   grpc_tcp *tcp = (grpc_tcp *)ep;
+  win_maybe_shutdown_resource_user(exec_ctx, tcp);
   TCP_UNREF(tcp, "destroy");
 }
 
@@ -392,6 +414,11 @@ static char *win_get_peer(grpc_endpoint *ep) {
 
 static grpc_workqueue *win_get_workqueue(grpc_endpoint *ep) { return NULL; }
 
+static grpc_resource_user *win_get_resource_user(grpc_endpoint *ep) {
+  grpc_tcp *tcp = (grpc_tcp *)ep;
+  return &tcp->resource_user;
+}
+
 static grpc_endpoint_vtable vtable = {win_read,
                                       win_write,
                                       win_get_workqueue,
@@ -399,18 +426,22 @@ static grpc_endpoint_vtable vtable = {win_read,
                                       win_add_to_pollset_set,
                                       win_shutdown,
                                       win_destroy,
+                                      win_get_resource_user,
                                       win_get_peer};
 
-grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) {
+grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket,
+                               grpc_resource_quota *resource_quota,
+                               char *peer_string) {
   grpc_tcp *tcp = (grpc_tcp *)gpr_malloc(sizeof(grpc_tcp));
   memset(tcp, 0, sizeof(grpc_tcp));
   tcp->base.vtable = &vtable;
   tcp->socket = socket;
   gpr_mu_init(&tcp->mu);
-  gpr_ref_init(&tcp->refcount, 1);
+  gpr_ref_init(&tcp->refcount, 2);
   grpc_closure_init(&tcp->on_read, on_read, tcp);
   grpc_closure_init(&tcp->on_write, on_write, tcp);
   tcp->peer_string = gpr_strdup(peer_string);
+  grpc_resource_user_init(&tcp->resource_user, resource_quota, peer_string);
   /* Tell network status tracking code about the new endpoint */
   grpc_network_status_register_endpoint(&tcp->base);
 

src/core/lib/iomgr/tcp_windows.h

@@ -50,7 +50,9 @@
 /* Create a tcp endpoint given a winsock handle.
  * Takes ownership of the handle.
  */
-grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string);
+grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket,
+                               grpc_resource_quota *resource_quota,
+                               char *peer_string);
 
 grpc_error *grpc_tcp_prepare_socket(SOCKET sock);
 

src/core/lib/security/credentials/google_default/google_default_credentials.c

@@ -124,11 +124,14 @@ static int is_stack_running_on_compute_engine(void) {
 
   grpc_httpcli_context_init(&context);
 
+  grpc_resource_quota *resource_quota =
+      grpc_resource_quota_create("google_default_credentials");
   grpc_httpcli_get(
-      &exec_ctx, &context, &detector.pollent, &request,
+      &exec_ctx, &context, &detector.pollent, resource_quota, &request,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), max_detection_delay),
       grpc_closure_create(on_compute_engine_detection_http_response, &detector),
       &detector.response);
+  grpc_resource_quota_internal_unref(&exec_ctx, resource_quota);
 
   grpc_exec_ctx_flush(&exec_ctx);
 

src/core/lib/security/credentials/jwt/jwt_verifier.c

@@ -657,11 +657,17 @@ static void on_openid_config_retrieved(grpc_exec_ctx *exec_ctx, void *user_data,
     *(req.host + (req.http.path - jwks_uri)) = '\0';
   }
 
+  /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host
+     channel. This would allow us to cancel an authentication query when under
+     extreme memory pressure. */
+  grpc_resource_quota *resource_quota =
+      grpc_resource_quota_create("jwt_verifier");
   grpc_httpcli_get(
-      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req,
+      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, resource_quota, &req,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay),
       grpc_closure_create(on_keys_retrieved, ctx),
       &ctx->responses[HTTP_RESPONSE_KEYS]);
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
   grpc_json_destroy(json);
   gpr_free(req.host);
   return;
@@ -764,10 +770,16 @@ static void retrieve_key_and_verify(grpc_exec_ctx *exec_ctx,
     rsp_idx = HTTP_RESPONSE_OPENID;
   }
 
+  /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host
+     channel. This would allow us to cancel an authentication query when under
+     extreme memory pressure. */
+  grpc_resource_quota *resource_quota =
+      grpc_resource_quota_create("jwt_verifier");
   grpc_httpcli_get(
-      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, &req,
+      exec_ctx, &ctx->verifier->http_ctx, &ctx->pollent, resource_quota, &req,
       gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), grpc_jwt_verifier_max_delay),
       http_cb, &ctx->responses[rsp_idx]);
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
   gpr_free(req.host);
   gpr_free(req.http.path);
   return;

src/core/lib/security/credentials/oauth2/oauth2_credentials.c

@@ -307,9 +307,15 @@ static void compute_engine_fetch_oauth2(
   request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
   request.http.hdr_count = 1;
   request.http.hdrs = &header;
-  grpc_httpcli_get(exec_ctx, httpcli_context, pollent, &request, deadline,
-                   grpc_closure_create(response_cb, metadata_req),
+  /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host
+     channel. This would allow us to cancel an authentication query when under
+     extreme memory pressure. */
+  grpc_resource_quota *resource_quota =
+      grpc_resource_quota_create("oauth2_credentials");
+  grpc_httpcli_get(exec_ctx, httpcli_context, pollent, resource_quota, &request,
+                   deadline, grpc_closure_create(response_cb, metadata_req),
                    &metadata_req->response);
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
 }
 
 grpc_call_credentials *grpc_google_compute_engine_credentials_create(
@@ -357,10 +363,16 @@ static void refresh_token_fetch_oauth2(
   request.http.hdr_count = 1;
   request.http.hdrs = &header;
   request.handshaker = &grpc_httpcli_ssl;
-  grpc_httpcli_post(exec_ctx, httpcli_context, pollent, &request, body,
-                    strlen(body), deadline,
+  /* TODO(ctiller): Carry the resource_quota in ctx and share it with the host
+     channel. This would allow us to cancel an authentication query when under
+     extreme memory pressure. */
+  grpc_resource_quota *resource_quota =
+      grpc_resource_quota_create("oauth2_credentials_refresh");
+  grpc_httpcli_post(exec_ctx, httpcli_context, pollent, resource_quota,
+                    &request, body, strlen(body), deadline,
                     grpc_closure_create(response_cb, metadata_req),
                     &metadata_req->response);
+  grpc_resource_quota_internal_unref(exec_ctx, resource_quota);
   gpr_free(body);
 }
 

src/core/lib/security/transport/secure_endpoint.c

@@ -370,6 +370,12 @@ static grpc_workqueue *endpoint_get_workqueue(grpc_endpoint *secure_ep) {
   return grpc_endpoint_get_workqueue(ep->wrapped_ep);
 }
 
+static grpc_resource_user *endpoint_get_resource_user(
+    grpc_endpoint *secure_ep) {
+  secure_endpoint *ep = (secure_endpoint *)secure_ep;
+  return grpc_endpoint_get_resource_user(ep->wrapped_ep);
+}
+
 static const grpc_endpoint_vtable vtable = {endpoint_read,
                                             endpoint_write,
                                             endpoint_get_workqueue,
@@ -377,6 +383,7 @@ static const grpc_endpoint_vtable vtable = {endpoint_read,
                                             endpoint_add_to_pollset_set,
                                             endpoint_shutdown,
                                             endpoint_destroy,
+                                            endpoint_get_resource_user,
                                             endpoint_get_peer};
 
 grpc_endpoint *grpc_secure_endpoint_create(

src/core/lib/surface/call.c

@@ -1516,8 +1516,10 @@ static grpc_call_error call_start_batch(grpc_exec_ctx *exec_ctx,
               call, STATUS_FROM_API_OVERRIDE,
               GRPC_MDSTR_REF(call->send_extra_metadata[1].md->value));
         }
-        set_status_code(call, STATUS_FROM_API_OVERRIDE,
-                        (uint32_t)op->data.send_status_from_server.status);
+        if (op->data.send_status_from_server.status != GRPC_STATUS_OK) {
+          set_status_code(call, STATUS_FROM_API_OVERRIDE,
+                          (uint32_t)op->data.send_status_from_server.status);
+        }
         if (!prepare_application_metadata(
                 call,
                 (int)op->data.send_status_from_server.trailing_metadata_count,

src/core/lib/surface/init.c

@@ -52,6 +52,7 @@
 #include "src/core/lib/iomgr/combiner.h"
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
+#include "src/core/lib/iomgr/resource_quota.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
@@ -191,6 +192,7 @@ void grpc_init(void) {
     // Default timeout trace to 1
     grpc_cq_event_timeout_trace = 1;
     grpc_register_tracer("op_failure", &grpc_trace_operation_failures);
+    grpc_register_tracer("resource_quota", &grpc_resource_quota_trace);
 #ifndef NDEBUG
     grpc_register_tracer("pending_tags", &grpc_trace_pending_tags);
 #endif

src/core/lib/tsi/ssl_transport_security.c

@@ -31,9 +31,6 @@
  *
  */
 
-#include "src/core/lib/iomgr/sockaddr.h"
-
-#include "src/core/lib/iomgr/socket_utils.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
 
 #include <grpc/support/port_platform.h>
@@ -41,6 +38,15 @@
 #include <limits.h>
 #include <string.h>
 
+/* TODO(jboeuf): refactor inet_ntop into a portability header. */
+/* Note: for whomever reads this and tries to refactor this, this
+   can't be in grpc, it has to be in gpr. */
+#ifdef GPR_WINDOWS
+#include <ws2tcpip.h>
+#else
+#include <arpa/inet.h>
+#endif
+
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -349,8 +355,8 @@ static tsi_result add_subject_alt_names_properties_to_peer(
         result = TSI_INTERNAL_ERROR;
         break;
       }
-      const char *name = grpc_inet_ntop(af, subject_alt_name->d.iPAddress->data,
-                                        ntop_buf, INET6_ADDRSTRLEN);
+      const char *name = inet_ntop(af, subject_alt_name->d.iPAddress->data,
+                                   ntop_buf, INET6_ADDRSTRLEN);
       if (name == NULL) {
         gpr_log(GPR_ERROR, "Could not get IP string from asn1 octet.");
         result = TSI_INTERNAL_ERROR;

src/cpp/common/channel_arguments.cc

@@ -34,6 +34,7 @@
 
 #include <sstream>
 
+#include <grpc++/resource_quota.h>
 #include <grpc/impl/codegen/grpc_types.h>
 #include <grpc/support/log.h>
 #include "src/core/lib/channel/channel_args.h"
@@ -113,6 +114,13 @@ void ChannelArguments::SetUserAgentPrefix(
   }
 }
 
+void ChannelArguments::SetResourceQuota(
+    const grpc::ResourceQuota& resource_quota) {
+  SetPointerWithVtable(GRPC_ARG_RESOURCE_QUOTA,
+                       resource_quota.c_resource_quota(),
+                       grpc_resource_quota_arg_vtable());
+}
+
 void ChannelArguments::SetLoadBalancingPolicyName(
     const grpc::string& lb_policy_name) {
   SetString(GRPC_ARG_LB_POLICY_NAME, lb_policy_name);
@@ -132,12 +140,18 @@ void ChannelArguments::SetPointer(const grpc::string& key, void* value) {
   static const grpc_arg_pointer_vtable vtable = {
       &PointerVtableMembers::Copy, &PointerVtableMembers::Destroy,
       &PointerVtableMembers::Compare};
+  SetPointerWithVtable(key, value, &vtable);
+}
+
+void ChannelArguments::SetPointerWithVtable(
+    const grpc::string& key, void* value,
+    const grpc_arg_pointer_vtable* vtable) {
   grpc_arg arg;
   arg.type = GRPC_ARG_POINTER;
   strings_.push_back(key);
   arg.key = const_cast<char*>(strings_.back().c_str());
   arg.value.pointer.p = value;
-  arg.value.pointer.vtable = &vtable;
+  arg.value.pointer.vtable = vtable;
   args_.push_back(arg);
 }
 

src/cpp/common/resource_quota_cc.cc

@@ -0,0 +1,51 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/resource_quota.h>
+#include <grpc/grpc.h>
+
+namespace grpc {
+
+ResourceQuota::ResourceQuota() : impl_(grpc_resource_quota_create(nullptr)) {}
+
+ResourceQuota::ResourceQuota(const grpc::string& name)
+    : impl_(grpc_resource_quota_create(name.c_str())) {}
+
+ResourceQuota::~ResourceQuota() { grpc_resource_quota_unref(impl_); }
+
+ResourceQuota& ResourceQuota::Resize(size_t new_size) {
+  grpc_resource_quota_resize(impl_, new_size);
+  return *this;
+}
+
+}  // namespace grpc

src/cpp/server/server_builder.cc

@@ -34,7 +34,9 @@
 #include <grpc++/server_builder.h>
 
 #include <grpc++/impl/service_type.h>
+#include <grpc++/resource_quota.h>
 #include <grpc++/server.h>
+#include <grpc/support/cpu.h>
 #include <grpc/support/log.h>
 #include <grpc/support/useful.h>
 
@@ -54,6 +56,8 @@ static void do_plugin_list_init(void) {
 ServerBuilder::ServerBuilder()
     : max_receive_message_size_(-1),
       max_send_message_size_(-1),
+      sync_server_settings_(SyncServerSettings()),
+      resource_quota_(nullptr),
       generic_service_(nullptr) {
   gpr_once_init(&once_init_plugin_list, do_plugin_list_init);
   for (auto it = g_plugin_factory_list->begin();
@@ -61,6 +65,7 @@ ServerBuilder::ServerBuilder()
     auto& factory = *it;
     plugins_.emplace_back(factory());
   }
+
   // all compression algorithms enabled by default.
   enabled_compression_algorithms_bitset_ =
       (1u << GRPC_COMPRESS_ALGORITHMS_COUNT) - 1;
@@ -70,6 +75,12 @@ ServerBuilder::ServerBuilder()
          sizeof(maybe_default_compression_algorithm_));
 }
 
+ServerBuilder::~ServerBuilder() {
+  if (resource_quota_ != nullptr) {
+    grpc_resource_quota_unref(resource_quota_);
+  }
+}
+
 std::unique_ptr<ServerCompletionQueue> ServerBuilder::AddCompletionQueue(
     bool is_frequently_polled) {
   ServerCompletionQueue* cq = new ServerCompletionQueue(is_frequently_polled);
@@ -94,7 +105,7 @@ ServerBuilder& ServerBuilder::RegisterAsyncGenericService(
     gpr_log(GPR_ERROR,
             "Adding multiple AsyncGenericService is unsupported for now. "
             "Dropping the service %p",
-            service);
+            (void*)service);
   } else {
     generic_service_ = service;
   }
@@ -107,6 +118,25 @@ ServerBuilder& ServerBuilder::SetOption(
   return *this;
 }
 
+ServerBuilder& ServerBuilder::SetSyncServerOption(
+    ServerBuilder::SyncServerOption option, int val) {
+  switch (option) {
+    case NUM_CQS:
+      sync_server_settings_.num_cqs = val;
+      break;
+    case MIN_POLLERS:
+      sync_server_settings_.min_pollers = val;
+      break;
+    case MAX_POLLERS:
+      sync_server_settings_.max_pollers = val;
+      break;
+    case CQ_TIMEOUT_MSEC:
+      sync_server_settings_.cq_timeout_msec = val;
+      break;
+  }
+  return *this;
+}
+
 ServerBuilder& ServerBuilder::SetCompressionAlgorithmSupportStatus(
     grpc_compression_algorithm algorithm, bool enabled) {
   if (enabled) {
@@ -130,6 +160,16 @@ ServerBuilder& ServerBuilder::SetDefaultCompressionAlgorithm(
   return *this;
 }
 
+ServerBuilder& ServerBuilder::SetResourceQuota(
+    const grpc::ResourceQuota& resource_quota) {
+  if (resource_quota_ != nullptr) {
+    grpc_resource_quota_unref(resource_quota_);
+  }
+  resource_quota_ = resource_quota.c_resource_quota();
+  grpc_resource_quota_ref(resource_quota_);
+  return *this;
+}
+
 ServerBuilder& ServerBuilder::AddListeningPort(
     const grpc::string& addr, std::shared_ptr<ServerCredentials> creds,
     int* selected_port) {
@@ -139,35 +179,24 @@ ServerBuilder& ServerBuilder::AddListeningPort(
 }
 
 std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
-  std::unique_ptr<ThreadPoolInterface> thread_pool;
-  bool has_sync_methods = false;
-  for (auto it = services_.begin(); it != services_.end(); ++it) {
-    if ((*it)->service->has_synchronous_methods()) {
-      if (!thread_pool) {
-        thread_pool.reset(CreateDefaultThreadPool());
-        has_sync_methods = true;
-        break;
-      }
-    }
-  }
   ChannelArguments args;
   for (auto option = options_.begin(); option != options_.end(); ++option) {
     (*option)->UpdateArguments(&args);
     (*option)->UpdatePlugins(&plugins_);
   }
+
   for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
-    if (!thread_pool && (*plugin)->has_sync_methods()) {
-      thread_pool.reset(CreateDefaultThreadPool());
-      has_sync_methods = true;
-    }
     (*plugin)->UpdateChannelArguments(&args);
   }
+
   if (max_receive_message_size_ >= 0) {
     args.SetInt(GRPC_ARG_MAX_RECEIVE_MESSAGE_LENGTH, max_receive_message_size_);
   }
+
   if (max_send_message_size_ >= 0) {
     args.SetInt(GRPC_ARG_MAX_SEND_MESSAGE_LENGTH, max_send_message_size_);
   }
+
   args.SetInt(GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET,
               enabled_compression_algorithms_bitset_);
   if (maybe_default_compression_level_.is_set) {
@@ -178,27 +207,89 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
     args.SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM,
                 maybe_default_compression_algorithm_.algorithm);
   }
-  std::unique_ptr<Server> server(new Server(thread_pool.release(), true,
-                                            max_receive_message_size_, &args));
+
+  if (resource_quota_ != nullptr) {
+    args.SetPointerWithVtable(GRPC_ARG_RESOURCE_QUOTA, resource_quota_,
+                              grpc_resource_quota_arg_vtable());
+  }
+
+  // == Determine if the server has any syncrhonous methods ==
+  bool has_sync_methods = false;
+  for (auto it = services_.begin(); it != services_.end(); ++it) {
+    if ((*it)->service->has_synchronous_methods()) {
+      has_sync_methods = true;
+      break;
+    }
+  }
+
+  if (!has_sync_methods) {
+    for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
+      if ((*plugin)->has_sync_methods()) {
+        has_sync_methods = true;
+        break;
+      }
+    }
+  }
+
+  // If this is a Sync server, i.e a server expositing sync API, then the server
+  // needs to create some completion queues to listen for incoming requests.
+  // 'sync_server_cqs' are those internal completion queues.
+  //
+  // This is different from the completion queues added to the server via
+  // ServerBuilder's AddCompletionQueue() method (those completion queues
+  // are in 'cqs_' member variable of ServerBuilder object)
+  std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
+      sync_server_cqs(std::make_shared<
+                      std::vector<std::unique_ptr<ServerCompletionQueue>>>());
+
+  if (has_sync_methods) {
+    // This is a Sync server
+    gpr_log(GPR_INFO,
+            "Synchronous server. Num CQs: %d, Min pollers: %d, Max Pollers: "
+            "%d, CQ timeout (msec): %d",
+            sync_server_settings_.num_cqs, sync_server_settings_.min_pollers,
+            sync_server_settings_.max_pollers,
+            sync_server_settings_.cq_timeout_msec);
+
+    // Create completion queues to listen to incoming rpc requests
+    for (int i = 0; i < sync_server_settings_.num_cqs; i++) {
+      sync_server_cqs->emplace_back(new ServerCompletionQueue());
+    }
+  }
+
+  std::unique_ptr<Server> server(new Server(
+      max_receive_message_size_, &args, sync_server_cqs,
+      sync_server_settings_.min_pollers, sync_server_settings_.max_pollers,
+      sync_server_settings_.cq_timeout_msec));
+
   ServerInitializer* initializer = server->initializer();
 
-  // If the server has atleast one sync methods, we know that this is a Sync
-  // server or a Hybrid server and the completion queue (server->cq_) would be
-  // frequently polled.
-  int num_frequently_polled_cqs = has_sync_methods ? 1 : 0;
-
-  for (auto cq = cqs_.begin(); cq != cqs_.end(); ++cq) {
-    // A completion queue that is not polled frequently (by calling Next() or
-    // AsyncNext()) is not safe to use for listening to incoming channels.
-    // Register all such completion queues as non-listening completion queues
-    // with the GRPC core library.
-    if ((*cq)->IsFrequentlyPolled()) {
-      grpc_server_register_completion_queue(server->server_, (*cq)->cq(),
+  // Register all the completion queues with the server. i.e
+  //  1. sync_server_cqs: internal completion queues created IF this is a sync
+  //     server
+  //  2. cqs_: Completion queues added via AddCompletionQueue() call
+
+  // All sync cqs (if any) are frequently polled by ThreadManager
+  int num_frequently_polled_cqs = sync_server_cqs->size();
+
+  for (auto it = sync_server_cqs->begin(); it != sync_server_cqs->end(); ++it) {
+    grpc_server_register_completion_queue(server->server_, (*it)->cq(),
+                                          nullptr);
+  }
+
+  // cqs_ contains the completion queue added by calling the ServerBuilder's
+  // AddCompletionQueue() API. Some of them may not be frequently polled (i.e by
+  // calling Next() or AsyncNext()) and hence are not safe to be used for
+  // listening to incoming channels. Such completion queues must be registered
+  // as non-listening queues
+  for (auto it = cqs_.begin(); it != cqs_.end(); ++it) {
+    if ((*it)->IsFrequentlyPolled()) {
+      grpc_server_register_completion_queue(server->server_, (*it)->cq(),
                                             nullptr);
       num_frequently_polled_cqs++;
     } else {
       grpc_server_register_non_listening_completion_queue(server->server_,
-                                                          (*cq)->cq(), nullptr);
+                                                          (*it)->cq(), nullptr);
     }
   }
 
@@ -214,9 +305,11 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
       return nullptr;
     }
   }
+
   for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
     (*plugin)->InitServer(initializer);
   }
+
   if (generic_service_) {
     server->RegisterAsyncGenericService(generic_service_);
   } else {
@@ -229,6 +322,7 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
       }
     }
   }
+
   for (auto port = ports_.begin(); port != ports_.end(); port++) {
     int r = server->AddListeningPort(port->addr, port->creds.get());
     if (!r) return nullptr;
@@ -236,13 +330,16 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
       *port->selected_port = r;
     }
   }
+
   auto cqs_data = cqs_.empty() ? nullptr : &cqs_[0];
   if (!server->Start(cqs_data, cqs_.size())) {
     return nullptr;
   }
+
   for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
     (*plugin)->Finish(initializer);
   }
+
   return server;
 }
 

src/cpp/server/server_cc.cc

@@ -1,5 +1,4 @@
 /*
- *
  * Copyright 2015, Google Inc.
  * All rights reserved.
  *
@@ -52,7 +51,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/profiling/timers.h"
-#include "src/cpp/server/thread_pool_interface.h"
+#include "src/cpp/thread_manager/thread_manager.h"
 
 namespace grpc {
 
@@ -118,12 +117,9 @@ class Server::UnimplementedAsyncResponse GRPC_FINAL
   UnimplementedAsyncRequest* const request_;
 };
 
-class Server::ShutdownRequest GRPC_FINAL : public CompletionQueueTag {
+class ShutdownTag : public CompletionQueueTag {
  public:
-  bool FinalizeResult(void** tag, bool* status) {
-    delete this;
-    return false;
-  }
+  bool FinalizeResult(void** tag, bool* status) { return false; }
 };
 
 class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
@@ -147,36 +143,6 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
     grpc_metadata_array_destroy(&request_metadata_);
   }
 
-  static SyncRequest* Wait(CompletionQueue* cq, bool* ok) {
-    void* tag = nullptr;
-    *ok = false;
-    if (!cq->Next(&tag, ok)) {
-      return nullptr;
-    }
-    auto* mrd = static_cast<SyncRequest*>(tag);
-    GPR_ASSERT(mrd->in_flight_);
-    return mrd;
-  }
-
-  static bool AsyncWait(CompletionQueue* cq, SyncRequest** req, bool* ok,
-                        gpr_timespec deadline) {
-    void* tag = nullptr;
-    *ok = false;
-    switch (cq->AsyncNext(&tag, ok, deadline)) {
-      case CompletionQueue::TIMEOUT:
-        *req = nullptr;
-        return true;
-      case CompletionQueue::SHUTDOWN:
-        *req = nullptr;
-        return false;
-      case CompletionQueue::GOT_EVENT:
-        *req = static_cast<SyncRequest*>(tag);
-        GPR_ASSERT((*req)->in_flight_);
-        return true;
-    }
-    GPR_UNREACHABLE_CODE(return false);
-  }
-
   void SetupRequest() { cq_ = grpc_completion_queue_create(nullptr); }
 
   void TeardownRequest() {
@@ -266,7 +232,6 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
   void* const tag_;
   bool in_flight_;
   const bool has_request_payload_;
-  uint32_t incoming_flags_;
   grpc_call* call_;
   grpc_call_details* call_details_;
   gpr_timespec deadline_;
@@ -275,33 +240,141 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
   grpc_completion_queue* cq_;
 };
 
+// Implementation of ThreadManager. Each instance of SyncRequestThreadManager
+// manages a pool of threads that poll for incoming Sync RPCs and call the
+// appropriate RPC handlers
+class Server::SyncRequestThreadManager : public ThreadManager {
+ public:
+  SyncRequestThreadManager(Server* server, CompletionQueue* server_cq,
+                           std::shared_ptr<GlobalCallbacks> global_callbacks,
+                           int min_pollers, int max_pollers,
+                           int cq_timeout_msec)
+      : ThreadManager(min_pollers, max_pollers),
+        server_(server),
+        server_cq_(server_cq),
+        cq_timeout_msec_(cq_timeout_msec),
+        global_callbacks_(global_callbacks) {}
+
+  WorkStatus PollForWork(void** tag, bool* ok) GRPC_OVERRIDE {
+    *tag = nullptr;
+    gpr_timespec deadline =
+        gpr_time_from_millis(cq_timeout_msec_, GPR_TIMESPAN);
+
+    switch (server_cq_->AsyncNext(tag, ok, deadline)) {
+      case CompletionQueue::TIMEOUT:
+        return TIMEOUT;
+      case CompletionQueue::SHUTDOWN:
+        return SHUTDOWN;
+      case CompletionQueue::GOT_EVENT:
+        return WORK_FOUND;
+    }
+
+    GPR_UNREACHABLE_CODE(return TIMEOUT);
+  }
+
+  void DoWork(void* tag, bool ok) GRPC_OVERRIDE {
+    SyncRequest* sync_req = static_cast<SyncRequest*>(tag);
+
+    if (!sync_req) {
+      // No tag. Nothing to work on. This is an unlikley scenario and possibly a
+      // bug in RPC Manager implementation.
+      gpr_log(GPR_ERROR, "Sync server. DoWork() was called with NULL tag");
+      return;
+    }
+
+    if (ok) {
+      // Calldata takes ownership of the completion queue inside sync_req
+      SyncRequest::CallData cd(server_, sync_req);
+      {
+        // Prepare for the next request
+        if (!IsShutdown()) {
+          sync_req->SetupRequest();  // Create new completion queue for sync_req
+          sync_req->Request(server_->c_server(), server_cq_->cq());
+        }
+      }
+
+      GPR_TIMER_SCOPE("cd.Run()", 0);
+      cd.Run(global_callbacks_);
+    }
+    // TODO (sreek) If ok is false here (which it isn't in case of
+    // grpc_request_registered_call), we should still re-queue the request
+    // object
+  }
+
+  void AddSyncMethod(RpcServiceMethod* method, void* tag) {
+    sync_requests_.emplace_back(new SyncRequest(method, tag));
+  }
+
+  void AddUnknownSyncMethod() {
+    if (!sync_requests_.empty()) {
+      unknown_method_.reset(new RpcServiceMethod(
+          "unknown", RpcMethod::BIDI_STREAMING, new UnknownMethodHandler));
+      sync_requests_.emplace_back(
+          new SyncRequest(unknown_method_.get(), nullptr));
+    }
+  }
+
+  void ShutdownAndDrainCompletionQueue() {
+    server_cq_->Shutdown();
+
+    // Drain any pending items from the queue
+    void* tag;
+    bool ok;
+    while (server_cq_->Next(&tag, &ok)) {
+      // Nothing to be done here
+    }
+  }
+
+  void Start() {
+    if (!sync_requests_.empty()) {
+      for (auto m = sync_requests_.begin(); m != sync_requests_.end(); m++) {
+        (*m)->SetupRequest();
+        (*m)->Request(server_->c_server(), server_cq_->cq());
+      }
+
+      Initialize();  // ThreadManager's Initialize()
+    }
+  }
+
+ private:
+  Server* server_;
+  CompletionQueue* server_cq_;
+  int cq_timeout_msec_;
+  std::vector<std::unique_ptr<SyncRequest>> sync_requests_;
+  std::unique_ptr<RpcServiceMethod> unknown_method_;
+  std::shared_ptr<Server::GlobalCallbacks> global_callbacks_;
+};
+
 static internal::GrpcLibraryInitializer g_gli_initializer;
-Server::Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
-               int max_receive_message_size, ChannelArguments* args)
+Server::Server(
+    int max_receive_message_size, ChannelArguments* args,
+    std::shared_ptr<std::vector<std::unique_ptr<ServerCompletionQueue>>>
+        sync_server_cqs,
+    int min_pollers, int max_pollers, int sync_cq_timeout_msec)
     : max_receive_message_size_(max_receive_message_size),
+      sync_server_cqs_(sync_server_cqs),
       started_(false),
       shutdown_(false),
       shutdown_notified_(false),
-      num_running_cb_(0),
-      sync_methods_(new std::list<SyncRequest>),
       has_generic_service_(false),
       server_(nullptr),
-      thread_pool_(thread_pool),
-      thread_pool_owned_(thread_pool_owned),
       server_initializer_(new ServerInitializer(this)) {
   g_gli_initializer.summon();
   gpr_once_init(&g_once_init_callbacks, InitGlobalCallbacks);
   global_callbacks_ = g_callbacks;
   global_callbacks_->UpdateArguments(args);
+
+  for (auto it = sync_server_cqs_->begin(); it != sync_server_cqs_->end();
+       it++) {
+    sync_req_mgrs_.emplace_back(new SyncRequestThreadManager(
+        this, (*it).get(), global_callbacks_, min_pollers, max_pollers,
+        sync_cq_timeout_msec));
+  }
+
   grpc_channel_args channel_args;
   args->SetChannelArgs(&channel_args);
+
   server_ = grpc_server_create(&channel_args, nullptr);
-  if (thread_pool_ == nullptr) {
-    grpc_server_register_non_listening_completion_queue(server_, cq_.cq(),
-                                                        nullptr);
-  } else {
-    grpc_server_register_completion_queue(server_, cq_.cq(), nullptr);
-  }
 }
 
 Server::~Server() {
@@ -311,17 +384,14 @@ Server::~Server() {
       lock.unlock();
       Shutdown();
     } else if (!started_) {
-      cq_.Shutdown();
+      // Shutdown the completion queues
+      for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
+        (*it)->ShutdownAndDrainCompletionQueue();
+      }
     }
   }
-  void* got_tag;
-  bool ok;
-  GPR_ASSERT(!cq_.Next(&got_tag, &ok));
+
   grpc_server_destroy(server_);
-  if (thread_pool_owned_) {
-    delete thread_pool_;
-  }
-  delete sync_methods_;
 }
 
 void Server::SetGlobalCallbacks(GlobalCallbacks* callbacks) {
@@ -352,12 +422,14 @@ bool Server::RegisterService(const grpc::string* host, Service* service) {
                "Can only register an asynchronous service against one server.");
     service->server_ = this;
   }
+
   const char* method_name = nullptr;
   for (auto it = service->methods_.begin(); it != service->methods_.end();
        ++it) {
     if (it->get() == nullptr) {  // Handled by generic service if any.
       continue;
     }
+
     RpcServiceMethod* method = it->get();
     void* tag = grpc_server_register_method(
         server_, method->name(), host ? host->c_str() : nullptr,
@@ -367,11 +439,15 @@ bool Server::RegisterService(const grpc::string* host, Service* service) {
               method->name());
       return false;
     }
-    if (method->handler() == nullptr) {
+
+    if (method->handler() == nullptr) {  // Async method
       method->set_server_tag(tag);
     } else {
-      sync_methods_->emplace_back(method, tag);
+      for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
+        (*it)->AddSyncMethod(method, tag);
+      }
     }
+
     method_name = method->name();
   }
 
@@ -406,28 +482,19 @@ bool Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) {
   grpc_server_start(server_);
 
   if (!has_generic_service_) {
-    if (!sync_methods_->empty()) {
-      unknown_method_.reset(new RpcServiceMethod(
-          "unknown", RpcMethod::BIDI_STREAMING, new UnknownMethodHandler));
-      // Use of emplace_back with just constructor arguments is not accepted
-      // here by gcc-4.4 because it can't match the anonymous nullptr with a
-      // proper constructor implicitly. Construct the object and use push_back.
-      sync_methods_->push_back(SyncRequest(unknown_method_.get(), nullptr));
+    for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
+      (*it)->AddUnknownSyncMethod();
     }
+
     for (size_t i = 0; i < num_cqs; i++) {
       if (cqs[i]->IsFrequentlyPolled()) {
         new UnimplementedAsyncRequest(this, cqs[i]);
       }
     }
   }
-  // Start processing rpcs.
-  if (!sync_methods_->empty()) {
-    for (auto m = sync_methods_->begin(); m != sync_methods_->end(); m++) {
-      m->SetupRequest();
-      m->Request(server_, cq_.cq());
-    }
 
-    ScheduleCallback();
+  for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
+    (*it)->Start();
   }
 
   return true;
@@ -437,29 +504,43 @@ void Server::ShutdownInternal(gpr_timespec deadline) {
   grpc::unique_lock<grpc::mutex> lock(mu_);
   if (started_ && !shutdown_) {
     shutdown_ = true;
-    grpc_server_shutdown_and_notify(server_, cq_.cq(), new ShutdownRequest());
-    cq_.Shutdown();
-    lock.unlock();
-    // Spin, eating requests until the completion queue is completely shutdown.
-    // If the deadline expires then cancel anything that's pending and keep
-    // spinning forever until the work is actually drained.
-    // Since nothing else needs to touch state guarded by mu_, holding it
-    // through this loop is fine.
-    SyncRequest* request;
+
+    /// The completion queue to use for server shutdown completion notification
+    CompletionQueue shutdown_cq;
+    ShutdownTag shutdown_tag;  // Dummy shutdown tag
+    grpc_server_shutdown_and_notify(server_, shutdown_cq.cq(), &shutdown_tag);
+
+    // Shutdown all ThreadManagers. This will try to gracefully stop all the
+    // threads in the ThreadManagers (once they process any inflight requests)
+    for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
+      (*it)->Shutdown();  // ThreadManager's Shutdown()
+    }
+
+    shutdown_cq.Shutdown();
+
+    void* tag;
     bool ok;
-    while (SyncRequest::AsyncWait(&cq_, &request, &ok, deadline)) {
-      if (request == NULL) {  // deadline expired
-        grpc_server_cancel_all_calls(server_);
-        deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
-      } else if (ok) {
-        SyncRequest::CallData call_data(this, request);
-      }
+    CompletionQueue::NextStatus status =
+        shutdown_cq.AsyncNext(&tag, &ok, deadline);
+
+    // If this timed out, it means we are done with the grace period for a clean
+    // shutdown. We should force a shutdown now by cancelling all inflight calls
+    if (status == CompletionQueue::NextStatus::TIMEOUT) {
+      grpc_server_cancel_all_calls(server_);
     }
-    lock.lock();
+    // Else in case of SHUTDOWN or GOT_EVENT, it means that the server has
+    // successfully shutdown
 
-    // Wait for running callbacks to finish.
-    while (num_running_cb_ != 0) {
-      callback_cv_.wait(lock);
+    // Wait for threads in all ThreadManagers to terminate
+    for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) {
+      (*it)->Wait();
+      (*it)->ShutdownAndDrainCompletionQueue();
+    }
+
+    // Drain the shutdown queue (if the previous call to AsyncNext() timed out
+    // and we didn't remove the tag from the queue yet)
+    while (shutdown_cq.Next(&tag, &ok)) {
+      // Nothing to be done here. Just ignore ok and tag values
     }
 
     shutdown_notified_ = true;
@@ -585,47 +666,6 @@ Server::UnimplementedAsyncResponse::UnimplementedAsyncResponse(
   request_->stream()->call_.PerformOps(this);
 }
 
-void Server::ScheduleCallback() {
-  {
-    grpc::unique_lock<grpc::mutex> lock(mu_);
-    num_running_cb_++;
-  }
-  thread_pool_->Add(std::bind(&Server::RunRpc, this));
-}
-
-void Server::RunRpc() {
-  // Wait for one more incoming rpc.
-  bool ok;
-  GPR_TIMER_SCOPE("Server::RunRpc", 0);
-  auto* mrd = SyncRequest::Wait(&cq_, &ok);
-  if (mrd) {
-    ScheduleCallback();
-    if (ok) {
-      SyncRequest::CallData cd(this, mrd);
-      {
-        mrd->SetupRequest();
-        grpc::unique_lock<grpc::mutex> lock(mu_);
-        if (!shutdown_) {
-          mrd->Request(server_, cq_.cq());
-        } else {
-          // destroy the structure that was created
-          mrd->TeardownRequest();
-        }
-      }
-      GPR_TIMER_SCOPE("cd.Run()", 0);
-      cd.Run(global_callbacks_);
-    }
-  }
-
-  {
-    grpc::unique_lock<grpc::mutex> lock(mu_);
-    num_running_cb_--;
-    if (shutdown_) {
-      callback_cv_.notify_all();
-    }
-  }
-}
-
 ServerInitializer* Server::initializer() { return server_initializer_.get(); }
 
 }  // namespace grpc

src/cpp/thread_manager/thread_manager.cc

@@ -0,0 +1,181 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <grpc++/impl/sync.h>
+#include <grpc++/impl/thd.h>
+#include <grpc/support/log.h>
+#include <climits>
+
+#include "src/cpp/thread_manager/thread_manager.h"
+
+namespace grpc {
+
+ThreadManager::WorkerThread::WorkerThread(ThreadManager* thd_mgr)
+    : thd_mgr_(thd_mgr), thd_(&ThreadManager::WorkerThread::Run, this) {}
+
+void ThreadManager::WorkerThread::Run() {
+  thd_mgr_->MainWorkLoop();
+  thd_mgr_->MarkAsCompleted(this);
+}
+
+ThreadManager::WorkerThread::~WorkerThread() { thd_.join(); }
+
+ThreadManager::ThreadManager(int min_pollers, int max_pollers)
+    : shutdown_(false),
+      num_pollers_(0),
+      min_pollers_(min_pollers),
+      max_pollers_(max_pollers == -1 ? INT_MAX : max_pollers),
+      num_threads_(0) {}
+
+ThreadManager::~ThreadManager() {
+  {
+    std::unique_lock<grpc::mutex> lock(mu_);
+    GPR_ASSERT(num_threads_ == 0);
+  }
+
+  CleanupCompletedThreads();
+}
+
+void ThreadManager::Wait() {
+  std::unique_lock<grpc::mutex> lock(mu_);
+  while (num_threads_ != 0) {
+    shutdown_cv_.wait(lock);
+  }
+}
+
+void ThreadManager::Shutdown() {
+  std::unique_lock<grpc::mutex> lock(mu_);
+  shutdown_ = true;
+}
+
+bool ThreadManager::IsShutdown() {
+  std::unique_lock<grpc::mutex> lock(mu_);
+  return shutdown_;
+}
+
+void ThreadManager::MarkAsCompleted(WorkerThread* thd) {
+  {
+    std::unique_lock<grpc::mutex> list_lock(list_mu_);
+    completed_threads_.push_back(thd);
+  }
+
+  grpc::unique_lock<grpc::mutex> lock(mu_);
+  num_threads_--;
+  if (num_threads_ == 0) {
+    shutdown_cv_.notify_one();
+  }
+}
+
+void ThreadManager::CleanupCompletedThreads() {
+  std::unique_lock<grpc::mutex> lock(list_mu_);
+  for (auto thd = completed_threads_.begin(); thd != completed_threads_.end();
+       thd = completed_threads_.erase(thd)) {
+    delete *thd;
+  }
+}
+
+void ThreadManager::Initialize() {
+  for (int i = 0; i < min_pollers_; i++) {
+    MaybeCreatePoller();
+  }
+}
+
+// If the number of pollers (i.e threads currently blocked in PollForWork()) is
+// less than max threshold (i.e max_pollers_) and the total number of threads is
+// below the maximum threshold, we can let the current thread continue as poller
+bool ThreadManager::MaybeContinueAsPoller() {
+  std::unique_lock<grpc::mutex> lock(mu_);
+  if (shutdown_ || num_pollers_ > max_pollers_) {
+    return false;
+  }
+
+  num_pollers_++;
+  return true;
+}
+
+// Create a new poller if the current number of pollers i.e num_pollers_ (i.e
+// threads currently blocked in PollForWork()) is below the threshold (i.e
+// min_pollers_) and the total number of threads is below the maximum threshold
+void ThreadManager::MaybeCreatePoller() {
+  grpc::unique_lock<grpc::mutex> lock(mu_);
+  if (!shutdown_ && num_pollers_ < min_pollers_) {
+    num_pollers_++;
+    num_threads_++;
+
+    // Create a new thread (which ends up calling the MainWorkLoop() function
+    new WorkerThread(this);
+  }
+}
+
+void ThreadManager::MainWorkLoop() {
+  void* tag;
+  bool ok;
+
+  /*
+   1. Poll for work (i.e PollForWork())
+   2. After returning from PollForWork, reduce the number of pollers by 1. If
+      PollForWork() returned a TIMEOUT, then it may indicate that we have more
+      polling threads than needed. Check if the number of pollers is greater
+      than min_pollers and if so, terminate the thread.
+   3. Since we are short of one poller now, see if a new poller has to be
+      created (i.e see MaybeCreatePoller() for more details)
+   4. Do the actual work (DoWork())
+   5. After doing the work, see it this thread can resume polling work (i.e
+      see MaybeContinueAsPoller() for more details) */
+  do {
+    WorkStatus work_status = PollForWork(&tag, &ok);
+
+    {
+      grpc::unique_lock<grpc::mutex> lock(mu_);
+      num_pollers_--;
+
+      if (work_status == TIMEOUT && num_pollers_ > min_pollers_) {
+        break;
+      }
+    }
+
+    // Note that MaybeCreatePoller does check for shutdown and creates a new
+    // thread only if ThreadManager is not shutdown
+    if (work_status == WORK_FOUND) {
+      MaybeCreatePoller();
+      DoWork(tag, ok);
+    }
+  } while (MaybeContinueAsPoller());
+
+  CleanupCompletedThreads();
+
+  // If we are here, either ThreadManager is shutting down or it already has
+  // enough threads.
+}
+
+}  // namespace grpc

src/cpp/thread_manager/thread_manager.h

@@ -0,0 +1,159 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CPP_THREAD_MANAGER_H
+#define GRPC_INTERNAL_CPP_THREAD_MANAGER_H
+
+#include <list>
+#include <memory>
+
+#include <grpc++/impl/sync.h>
+#include <grpc++/impl/thd.h>
+#include <grpc++/support/config.h>
+
+namespace grpc {
+
+class ThreadManager {
+ public:
+  explicit ThreadManager(int min_pollers, int max_pollers);
+  virtual ~ThreadManager();
+
+  // Initializes and Starts the Rpc Manager threads
+  void Initialize();
+
+  // The return type of PollForWork() function
+  enum WorkStatus { WORK_FOUND, SHUTDOWN, TIMEOUT };
+
+  // "Polls" for new work.
+  // If the return value is WORK_FOUND:
+  //  - The implementaion of PollForWork() MAY set some opaque identifier to
+  //    (identify the work item found) via the '*tag' parameter
+  //  - The implementaion MUST set the value of 'ok' to 'true' or 'false'. A
+  //    value of 'false' indicates some implemenation specific error (that is
+  //    neither SHUTDOWN nor TIMEOUT)
+  //  - ThreadManager does not interpret the values of 'tag' and 'ok'
+  //  - ThreadManager WILL call DoWork() and pass '*tag' and 'ok' as input to
+  //    DoWork()
+  //
+  // If the return value is SHUTDOWN:,
+  //  - ThreadManager WILL NOT call DoWork() and terminates the thead
+  //
+  // If the return value is TIMEOUT:,
+  //  - ThreadManager WILL NOT call DoWork()
+  //  - ThreadManager MAY terminate the thread depending on the current number
+  //    of active poller threads and mix_pollers/max_pollers settings
+  //  - Also, the value of timeout is specific to the derived class
+  //    implementation
+  virtual WorkStatus PollForWork(void** tag, bool* ok) = 0;
+
+  // The implementation of DoWork() is supposed to perform the work found by
+  // PollForWork(). The tag and ok parameters are the same as returned by
+  // PollForWork()
+  //
+  // The implementation of DoWork() should also do any setup needed to ensure
+  // that the next call to PollForWork() (not necessarily by the current thread)
+  // actually finds some work
+  virtual void DoWork(void* tag, bool ok) = 0;
+
+  // Mark the ThreadManager as shutdown and begin draining the work. This is a
+  // non-blocking call and the caller should call Wait(), a blocking call which
+  // returns only once the shutdown is complete
+  void Shutdown();
+
+  // Has Shutdown() been called
+  bool IsShutdown();
+
+  // A blocking call that returns only after the ThreadManager has shutdown and
+  // all the threads have drained all the outstanding work
+  void Wait();
+
+ private:
+  // Helper wrapper class around std::thread. This takes a ThreadManager object
+  // and starts a new std::thread to calls the Run() function.
+  //
+  // The Run() function calls ThreadManager::MainWorkLoop() function and once
+  // that completes, it marks the WorkerThread completed by calling
+  // ThreadManager::MarkAsCompleted()
+  class WorkerThread {
+   public:
+    WorkerThread(ThreadManager* thd_mgr);
+    ~WorkerThread();
+
+   private:
+    // Calls thd_mgr_->MainWorkLoop() and once that completes, calls
+    // thd_mgr_>MarkAsCompleted(this) to mark the thread as completed
+    void Run();
+
+    ThreadManager* thd_mgr_;
+    grpc::thread thd_;
+  };
+
+  // The main funtion in ThreadManager
+  void MainWorkLoop();
+
+  // Create a new poller if the number of current pollers is less than the
+  // minimum number of pollers needed (i.e min_pollers).
+  void MaybeCreatePoller();
+
+  // Returns true if the current thread can resume as a poller. i.e if the
+  // current number of pollers is less than the max_pollers.
+  bool MaybeContinueAsPoller();
+
+  void MarkAsCompleted(WorkerThread* thd);
+  void CleanupCompletedThreads();
+
+  // Protects shutdown_, num_pollers_ and num_threads_
+  // TODO: sreek - Change num_pollers and num_threads_ to atomics
+  grpc::mutex mu_;
+
+  bool shutdown_;
+  grpc::condition_variable shutdown_cv_;
+
+  // Number of threads doing polling
+  int num_pollers_;
+
+  // The minimum and maximum number of threads that should be doing polling
+  int min_pollers_;
+  int max_pollers_;
+
+  // The total number of threads (includes threads includes the threads that are
+  // currently polling i.e num_pollers_)
+  int num_threads_;
+
+  grpc::mutex list_mu_;
+  std::list<WorkerThread*> completed_threads_;
+};
+
+}  // namespace grpc
+
+#endif  // GRPC_INTERNAL_CPP_THREAD_MANAGER_H

src/csharp/Grpc.Core/Internal/AsyncCall.cs

@@ -52,9 +52,8 @@ namespace Grpc.Core.Internal
         // Completion of a pending unary response if not null.
         TaskCompletionSource<TResponse> unaryResponseTcs;
 
-        // TODO(jtattermusch): this field doesn't need to be initialized for unary response calls.
-        // Indicates that response streaming call has finished.
-        TaskCompletionSource<object> streamingCallFinishedTcs = new TaskCompletionSource<object>();
+        // Completion of a streaming response call if not null.
+        TaskCompletionSource<object> streamingResponseCallFinishedTcs;
 
         // TODO(jtattermusch): this field could be lazy-initialized (only if someone requests the response headers).
         // Response headers set here once received.
@@ -198,6 +197,7 @@ namespace Grpc.Core.Internal
 
                 byte[] payload = UnsafeSerialize(msg);
 
+                streamingResponseCallFinishedTcs = new TaskCompletionSource<object>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
                     call.StartServerStreaming(HandleFinished, payload, metadataArray, GetWriteFlagsForCall());
@@ -219,6 +219,7 @@ namespace Grpc.Core.Internal
 
                 Initialize(details.Channel.CompletionQueue);
 
+                streamingResponseCallFinishedTcs = new TaskCompletionSource<object>();
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
                     call.StartDuplexStreaming(HandleFinished, metadataArray);
@@ -276,13 +277,13 @@ namespace Grpc.Core.Internal
         }
 
         /// <summary>
-        /// Get the task that completes once if streaming call finishes with ok status and throws RpcException with given status otherwise.
+        /// Get the task that completes once if streaming response call finishes with ok status and throws RpcException with given status otherwise.
         /// </summary>
-        public Task StreamingCallFinishedTask
+        public Task StreamingResponseCallFinishedTask
         {
             get
             {
-                return streamingCallFinishedTcs.Task;
+                return streamingResponseCallFinishedTcs.Task;
             }
         }
 
@@ -529,11 +530,11 @@ namespace Grpc.Core.Internal
             var status = receivedStatus.Status;
             if (status.StatusCode != StatusCode.OK)
             {
-                streamingCallFinishedTcs.SetException(new RpcException(status));
+                streamingResponseCallFinishedTcs.SetException(new RpcException(status));
                 return;
             }
 
-            streamingCallFinishedTcs.SetResult(null);
+            streamingResponseCallFinishedTcs.SetResult(null);
         }
     }
 }

src/csharp/Grpc.Core/Internal/ClientResponseStream.cs

@@ -73,7 +73,7 @@ namespace Grpc.Core.Internal
 
             if (result == null)
             {
-                await call.StreamingCallFinishedTask.ConfigureAwait(false);
+                await call.StreamingResponseCallFinishedTask.ConfigureAwait(false);
                 return false;
             }
             return true;

src/node/ext/byte_buffer.cc

@@ -44,8 +44,8 @@
 namespace grpc {
 namespace node {
 
+using Nan::MaybeLocal;
 
-using v8::Context;
 using v8::Function;
 using v8::Local;
 using v8::Object;
@@ -89,15 +89,19 @@ Local<Value> ByteBufferToBuffer(grpc_byte_buffer *buffer) {
 Local<Value> MakeFastBuffer(Local<Value> slowBuffer) {
   Nan::EscapableHandleScope scope;
   Local<Object> globalObj = Nan::GetCurrentContext()->Global();
+  MaybeLocal<Value> constructorValue = Nan::Get(
+      globalObj, Nan::New("Buffer").ToLocalChecked());
   Local<Function> bufferConstructor = Local<Function>::Cast(
-      globalObj->Get(Nan::New("Buffer").ToLocalChecked()));
-  Local<Value> consArgs[3] = {
+      constructorValue.ToLocalChecked());
+  const int argc = 3;
+  Local<Value> consArgs[argc] = {
     slowBuffer,
     Nan::New<Number>(::node::Buffer::Length(slowBuffer)),
     Nan::New<Number>(0)
   };
-  Local<Object> fastBuffer = bufferConstructor->NewInstance(3, consArgs);
-  return scope.Escape(fastBuffer);
+  MaybeLocal<Object> fastBuffer = Nan::NewInstance(bufferConstructor,
+                                                   argc, consArgs);
+  return scope.Escape(fastBuffer.ToLocalChecked());
 }
 }  // namespace node
 }  // namespace grpc

src/node/ext/call.cc

@@ -669,16 +669,16 @@ NAN_METHOD(Call::New) {
         return Nan::ThrowTypeError("Call's fourth argument must be a string");
       }
       call = new Call(wrapped_call);
-      info.This()->SetHiddenValue(Nan::New("channel_").ToLocalChecked(),
-                                  channel_object);
+      Nan::Set(info.This(), Nan::New("channel_").ToLocalChecked(),
+               channel_object);
     }
     call->Wrap(info.This());
     info.GetReturnValue().Set(info.This());
   } else {
     const int argc = 4;
     Local<Value> argv[argc] = {info[0], info[1], info[2], info[3]};
-    MaybeLocal<Object> maybe_instance = constructor->GetFunction()->NewInstance(
-        argc, argv);
+    MaybeLocal<Object> maybe_instance = Nan::NewInstance(
+        constructor->GetFunction(), argc, argv);
     if (maybe_instance.IsEmpty()) {
       // There's probably a pending exception
       return;

src/node/ext/channel.cc

@@ -208,8 +208,8 @@ NAN_METHOD(Channel::New) {
   } else {
     const int argc = 3;
     Local<Value> argv[argc] = {info[0], info[1], info[2]};
-    MaybeLocal<Object> maybe_instance = constructor->GetFunction()->NewInstance(
-        argc, argv);
+    MaybeLocal<Object> maybe_instance = Nan::NewInstance(
+        constructor->GetFunction(), argc, argv);
     if (maybe_instance.IsEmpty()) {
       // There's probably a pending exception
       return;

src/node/ext/server.cc

@@ -222,7 +222,7 @@ NAN_METHOD(Server::New) {
     const int argc = 1;
     Local<Value> argv[argc] = {info[0]};
     MaybeLocal<Object> maybe_instance =
-        constructor->GetFunction()->NewInstance(argc, argv);
+        Nan::NewInstance(constructor->GetFunction(), argc, argv);
     if (maybe_instance.IsEmpty()) {
       // There's probably a pending exception
       return;

src/node/performance/benchmark_client_express.js

@@ -0,0 +1,291 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/**
+ * Benchmark client module
+ * @module
+ */
+
+'use strict';
+
+var fs = require('fs');
+var path = require('path');
+var util = require('util');
+var EventEmitter = require('events');
+var http = require('http');
+var https = require('https');
+
+var async = require('async');
+var _ = require('lodash');
+var PoissonProcess = require('poisson-process');
+var Histogram = require('./histogram');
+
+/**
+ * Convert a time difference, as returned by process.hrtime, to a number of
+ * nanoseconds.
+ * @param {Array.<number>} time_diff The time diff, represented as
+ *     [seconds, nanoseconds]
+ * @return {number} The total number of nanoseconds
+ */
+function timeDiffToNanos(time_diff) {
+  return time_diff[0] * 1e9 + time_diff[1];
+}
+
+function BenchmarkClient(server_targets, channels, histogram_params,
+    security_params) {
+  var options = {
+    method: 'PUT',
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  };
+  var protocol;
+  if (security_params) {
+    var ca_path;
+    protocol = https;
+    this.request = _.bind(https.request, https);
+    if (security_params.use_test_ca) {
+      ca_path = path.join(__dirname, '../test/data/ca.pem');
+      var ca_data = fs.readFileSync(ca_path);
+      options.ca = ca_data;
+    }
+    if (security_params.server_host_override) {
+      var host_override = security_params.server_host_override;
+      options.servername = host_override;
+    }
+  } else {
+    protocol = http;
+  }
+
+  this.request = _.bind(protocol.request, protocol);
+
+  this.client_options = [];
+
+  for (var i = 0; i < channels; i++) {
+    var host_port;
+    host_port = server_targets[i % server_targets.length].split(':')
+    var new_options = _.assign({hostname: host_port[0], port: +host_port[1]}, options);
+    new_options.agent = new protocol.Agent(new_options);
+    this.client_options[i] = new_options;
+  }
+
+  this.histogram = new Histogram(histogram_params.resolution,
+                                 histogram_params.max_possible);
+
+  this.running = false;
+
+  this.pending_calls = 0;
+}
+
+util.inherits(BenchmarkClient, EventEmitter);
+
+function startAllClients(client_options_list, outstanding_rpcs_per_channel,
+                         makeCall, emitter) {
+  _.each(client_options_list, function(client_options) {
+    _.times(outstanding_rpcs_per_channel, function() {
+      makeCall(client_options);
+    });
+  });
+}
+
+BenchmarkClient.prototype.startClosedLoop = function(
+    outstanding_rpcs_per_channel, rpc_type, req_size, resp_size, generic) {
+  var self = this;
+
+  var options = {};
+
+  self.running = true;
+
+  if (rpc_type == 'UNARY') {
+    options.path = '/serviceProto.BenchmarkService.service/unaryCall';
+  } else {
+    self.emit('error', new Error('Unsupported rpc_type: ' + rpc_type));
+  }
+
+  if (generic) {
+    self.emit('error', new Error('Generic client not supported'));
+  }
+
+  self.last_wall_time = process.hrtime();
+
+  var argument = {
+    response_size: resp_size,
+    payload: {
+      body: '0'.repeat(req_size)
+    }
+  };
+
+  function makeCall(client_options) {
+    if (self.running) {
+      self.pending_calls++;
+      var start_time = process.hrtime();
+      var req = self.request(client_options, function(res) {
+        var res_data = '';
+        res.on('data', function(data) {
+          res_data += data;
+        });
+        res.on('end', function() {
+          JSON.parse(res_data);
+          var time_diff = process.hrtime(start_time);
+          self.histogram.add(timeDiffToNanos(time_diff));
+          makeCall(client_options);
+          self.pending_calls--;
+          if ((!self.running) && self.pending_calls == 0) {
+            self.emit('finished');
+          }
+        });
+      });
+      req.write(JSON.stringify(argument));
+      req.end();
+      req.on('error', function(error) {
+        self.emit('error', new Error('Client error: ' + error.message));
+        self.running = false;
+      });
+    }
+  }
+
+  startAllClients(_.map(self.client_options, _.partial(_.assign, options)),
+                  outstanding_rpcs_per_channel, makeCall, self);
+};
+
+BenchmarkClient.prototype.startPoisson = function(
+    outstanding_rpcs_per_channel, rpc_type, req_size, resp_size, offered_load,
+    generic) {
+  var self = this;
+
+  var options = {};
+
+  self.running = true;
+
+  if (rpc_type == 'UNARY') {
+    options.path = '/serviceProto.BenchmarkService.service/unaryCall';
+  } else {
+    self.emit('error', new Error('Unsupported rpc_type: ' + rpc_type));
+  }
+
+  if (generic) {
+    self.emit('error', new Error('Generic client not supported'));
+  }
+
+  self.last_wall_time = process.hrtime();
+
+  var argument = {
+    response_size: resp_size,
+    payload: {
+      body: '0'.repeat(req_size)
+    }
+  };
+
+  function makeCall(client_options, poisson) {
+    if (self.running) {
+      self.pending_calls++;
+      var start_time = process.hrtime();
+      var req = self.request(client_options, function(res) {
+        var res_data = '';
+        res.on('data', function(data) {
+          res_data += data;
+        });
+        res.on('end', function() {
+          JSON.parse(res_data);
+          var time_diff = process.hrtime(start_time);
+          self.histogram.add(timeDiffToNanos(time_diff));
+          self.pending_calls--;
+          if ((!self.running) && self.pending_calls == 0) {
+            self.emit('finished');
+          }
+        });
+      });
+      req.write(JSON.stringify(argument));
+      req.end();
+      req.on('error', function(error) {
+        self.emit('error', new Error('Client error: ' + error.message));
+        self.running = false;
+      });
+    } else {
+      poisson.stop();
+    }
+  }
+
+  var averageIntervalMs = (1 / offered_load) * 1000;
+
+  startAllClients(_.map(self.client_options, _.partial(_.assign, options)),
+                  outstanding_rpcs_per_channel, function(opts){
+                    var p = PoissonProcess.create(averageIntervalMs, function() {
+                      makeCall(opts, p);
+                    });
+                    p.start();
+                  }, self);
+};
+
+/**
+ * Return curent statistics for the client. If reset is set, restart
+ * statistic collection.
+ * @param {boolean} reset Indicates that statistics should be reset
+ * @return {object} Client statistics
+ */
+BenchmarkClient.prototype.mark = function(reset) {
+  var wall_time_diff = process.hrtime(this.last_wall_time);
+  var histogram = this.histogram;
+  if (reset) {
+    this.last_wall_time = process.hrtime();
+    this.histogram = new Histogram(histogram.resolution,
+                                   histogram.max_possible);
+  }
+
+  return {
+    latencies: {
+      bucket: histogram.getContents(),
+      min_seen: histogram.minimum(),
+      max_seen: histogram.maximum(),
+      sum: histogram.getSum(),
+      sum_of_squares: histogram.sumOfSquares(),
+      count: histogram.getCount()
+    },
+    time_elapsed: wall_time_diff[0] + wall_time_diff[1] / 1e9,
+    // Not sure how to measure these values
+    time_user: 0,
+    time_system: 0
+  };
+};
+
+/**
+ * Stop the clients.
+ * @param {function} callback Called when the clients have finished shutting
+ *     down
+ */
+BenchmarkClient.prototype.stop = function(callback) {
+  this.running = false;
+  this.on('finished', callback);
+};
+
+module.exports = BenchmarkClient;

src/node/performance/benchmark_server.js

@@ -40,6 +40,8 @@
 
 var fs = require('fs');
 var path = require('path');
+var EventEmitter = require('events');
+var util = require('util');
 
 var genericService = require('./generic_service');
 
@@ -138,12 +140,15 @@ function BenchmarkServer(host, port, tls, generic, response_size) {
   this.server = server;
 }
 
+util.inherits(BenchmarkServer, EventEmitter);
+
 /**
  * Start the benchmark server.
  */
 BenchmarkServer.prototype.start = function() {
   this.server.start();
   this.last_wall_time = process.hrtime();
+  this.emit('started');
 };
 
 /**

src/node/performance/benchmark_server_express.js

@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/**
+ * Benchmark server module
+ * @module
+ */
+
+'use strict';
+
+var fs = require('fs');
+var path = require('path');
+var http = require('http');
+var https = require('https');
+var EventEmitter = require('events');
+var util = require('util');
+
+var express = require('express');
+var bodyParser = require('body-parser')
+
+function unaryCall(req, res) {
+  var reqObj = req.body;
+  var payload = {body: '0'.repeat(reqObj.response_size)};
+  res.json(payload);
+}
+
+function BenchmarkServer(host, port, tls, generic, response_size) {
+  var app = express();
+  app.use(bodyParser.json())
+  app.put('/serviceProto.BenchmarkService.service/unaryCall', unaryCall);
+  this.input_host = host;
+  this.input_port = port;
+  if (tls) {
+    var credentials = {};
+    var key_path = path.join(__dirname, '../test/data/server1.key');
+    var pem_path = path.join(__dirname, '../test/data/server1.pem');
+
+    var key_data = fs.readFileSync(key_path);
+    var pem_data = fs.readFileSync(pem_path);
+    credentials['key'] = key_data;
+    credentials['cert'] = pem_data;
+    this.server = https.createServer(credentials, app);
+  } else {
+    this.server = http.createServer(app);
+  }
+}
+
+util.inherits(BenchmarkServer, EventEmitter);
+
+BenchmarkServer.prototype.start = function() {
+  var self = this;
+  this.server.listen(this.input_port, this.input_hostname, function() {
+    self.last_wall_time = process.hrtime();
+    self.emit('started');
+  });
+};
+
+BenchmarkServer.prototype.getPort = function() {
+  return this.server.address().port;
+};
+
+BenchmarkServer.prototype.mark = function(reset) {
+  var wall_time_diff = process.hrtime(this.last_wall_time);
+  if (reset) {
+    this.last_wall_time = process.hrtime();
+  }
+  return {
+    time_elapsed: wall_time_diff[0] + wall_time_diff[1] / 1e9,
+    // Not sure how to measure these values
+    time_user: 0,
+    time_system: 0
+  };
+};
+
+BenchmarkServer.prototype.stop = function(callback) {
+  this.server.close(callback);
+};
+
+module.exports = BenchmarkServer;

src/node/performance/worker.js

@@ -34,18 +34,18 @@
 'use strict';
 
 var console = require('console');
-var worker_service_impl = require('./worker_service_impl');
+var WorkerServiceImpl = require('./worker_service_impl');
 
 var grpc = require('../../../');
 var serviceProto = grpc.load({
   root: __dirname + '/../../..',
   file: 'src/proto/grpc/testing/services.proto'}).grpc.testing;
 
-function runServer(port) {
+function runServer(port, benchmark_impl) {
   var server_creds = grpc.ServerCredentials.createInsecure();
   var server = new grpc.Server();
   server.addProtoService(serviceProto.WorkerService.service,
-                         worker_service_impl);
+                         new WorkerServiceImpl(benchmark_impl, server));
   var address = '0.0.0.0:' + port;
   server.bind(address, server_creds);
   server.start();
@@ -57,9 +57,9 @@ if (require.main === module) {
   Error.stackTraceLimit = Infinity;
   var parseArgs = require('minimist');
   var argv = parseArgs(process.argv, {
-    string: ['driver_port']
+    string: ['driver_port', 'benchmark_impl']
   });
-  runServer(argv.driver_port);
+  runServer(argv.driver_port, argv.benchmark_impl);
 }
 
 exports.runServer = runServer;

src/node/performance/worker_service_impl.js

@@ -38,121 +38,141 @@ var console = require('console');
 var BenchmarkClient = require('./benchmark_client');
 var BenchmarkServer = require('./benchmark_server');
 
-exports.quitWorker = function quitWorker(call, callback) {
-  callback(null, {});
-  process.exit(0);
-}
+module.exports = function WorkerServiceImpl(benchmark_impl, server) {
+  var BenchmarkClient;
+  var BenchmarkServer;
+  switch (benchmark_impl) {
+    case 'grpc':
+    BenchmarkClient = require('./benchmark_client');
+    BenchmarkServer = require('./benchmark_server');
+    break;
+    case 'express':
+    BenchmarkClient = require('./benchmark_client_express');
+    BenchmarkServer = require('./benchmark_server_express');
+    break;
+    default:
+    throw new Error('Unrecognized benchmark impl: ' + benchmark_impl);
+  }
 
-exports.runClient = function runClient(call) {
-  var client;
-  call.on('data', function(request) {
-    var stats;
-    switch (request.argtype) {
-      case 'setup':
-      var setup = request.setup;
-      console.log('ClientConfig %j', setup);
-      client = new BenchmarkClient(setup.server_targets,
-                                   setup.client_channels,
-                                   setup.histogram_params,
-                                   setup.security_params);
-      client.on('error', function(error) {
-        call.emit('error', error);
-      });
-      var req_size, resp_size, generic;
-      switch (setup.payload_config.payload) {
-        case 'bytebuf_params':
-        req_size = setup.payload_config.bytebuf_params.req_size;
-        resp_size = setup.payload_config.bytebuf_params.resp_size;
-        generic = true;
+  this.quitWorker = function quitWorker(call, callback) {
+    server.tryShutdown(function() {
+      callback(null, {});
+    });
+  };
+
+  this.runClient = function runClient(call) {
+    var client;
+    call.on('data', function(request) {
+      var stats;
+      switch (request.argtype) {
+        case 'setup':
+        var setup = request.setup;
+        console.log('ClientConfig %j', setup);
+        client = new BenchmarkClient(setup.server_targets,
+                                     setup.client_channels,
+                                     setup.histogram_params,
+                                     setup.security_params);
+        client.on('error', function(error) {
+          call.emit('error', error);
+        });
+        var req_size, resp_size, generic;
+        switch (setup.payload_config.payload) {
+          case 'bytebuf_params':
+          req_size = setup.payload_config.bytebuf_params.req_size;
+          resp_size = setup.payload_config.bytebuf_params.resp_size;
+          generic = true;
+          break;
+          case 'simple_params':
+          req_size = setup.payload_config.simple_params.req_size;
+          resp_size = setup.payload_config.simple_params.resp_size;
+          generic = false;
+          break;
+          default:
+          call.emit('error', new Error('Unsupported PayloadConfig type' +
+              setup.payload_config.payload));
+        }
+        switch (setup.load_params.load) {
+          case 'closed_loop':
+          client.startClosedLoop(setup.outstanding_rpcs_per_channel,
+                                 setup.rpc_type, req_size, resp_size, generic);
+          break;
+          case 'poisson':
+          client.startPoisson(setup.outstanding_rpcs_per_channel,
+                              setup.rpc_type, req_size, resp_size,
+                              setup.load_params.poisson.offered_load, generic);
+          break;
+          default:
+          call.emit('error', new Error('Unsupported LoadParams type' +
+              setup.load_params.load));
+        }
+        stats = client.mark();
+        call.write({
+          stats: stats
+        });
         break;
-        case 'simple_params':
-        req_size = setup.payload_config.simple_params.req_size;
-        resp_size = setup.payload_config.simple_params.resp_size;
-        generic = false;
+        case 'mark':
+        if (client) {
+          stats = client.mark(request.mark.reset);
+          call.write({
+            stats: stats
+          });
+        } else {
+          call.emit('error', new Error('Got Mark before ClientConfig'));
+        }
         break;
         default:
-        call.emit('error', new Error('Unsupported PayloadConfig type' +
-            setup.payload_config.payload));
+        throw new Error('Nonexistent client argtype option: ' + request.argtype);
       }
-      switch (setup.load_params.load) {
-        case 'closed_loop':
-        client.startClosedLoop(setup.outstanding_rpcs_per_channel,
-                               setup.rpc_type, req_size, resp_size, generic);
+    });
+    call.on('end', function() {
+      client.stop(function() {
+        call.end();
+      });
+    });
+  };
+
+  this.runServer = function runServer(call) {
+    var server;
+    call.on('data', function(request) {
+      var stats;
+      switch (request.argtype) {
+        case 'setup':
+        console.log('ServerConfig %j', request.setup);
+        server = new BenchmarkServer('[::]', request.setup.port,
+                                     request.setup.security_params);
+        server.on('started', function() {
+          stats = server.mark();
+          call.write({
+            stats: stats,
+            port: server.getPort()
+          });
+        });
+        server.start();
         break;
-        case 'poisson':
-        client.startPoisson(setup.outstanding_rpcs_per_channel,
-                            setup.rpc_type, req_size, resp_size,
-                            setup.load_params.poisson.offered_load, generic);
+        case 'mark':
+        if (server) {
+          stats = server.mark(request.mark.reset);
+          call.write({
+            stats: stats,
+            port: server.getPort(),
+            cores: 1
+          });
+        } else {
+          call.emit('error', new Error('Got Mark before ServerConfig'));
+        }
         break;
         default:
-        call.emit('error', new Error('Unsupported LoadParams type' +
-            setup.load_params.load));
+        throw new Error('Nonexistent server argtype option');
       }
-      stats = client.mark();
-      call.write({
-        stats: stats
-      });
-      break;
-      case 'mark':
-      if (client) {
-        stats = client.mark(request.mark.reset);
-        call.write({
-          stats: stats
-        });
-      } else {
-        call.emit('error', new Error('Got Mark before ClientConfig'));
-      }
-      break;
-      default:
-      throw new Error('Nonexistent client argtype option: ' + request.argtype);
-    }
-  });
-  call.on('end', function() {
-    client.stop(function() {
-      call.end();
     });
-  });
-};
-
-exports.runServer = function runServer(call) {
-  var server;
-  call.on('data', function(request) {
-    var stats;
-    switch (request.argtype) {
-      case 'setup':
-      console.log('ServerConfig %j', request.setup);
-      server = new BenchmarkServer('[::]', request.setup.port,
-                                   request.setup.security_params);
-      server.start();
-      stats = server.mark();
-      call.write({
-        stats: stats,
-        port: server.getPort()
+    call.on('end', function() {
+      server.stop(function() {
+        call.end();
       });
-      break;
-      case 'mark':
-      if (server) {
-        stats = server.mark(request.mark.reset);
-        call.write({
-          stats: stats,
-          port: server.getPort(),
-          cores: 1
-        });
-      } else {
-        call.emit('error', new Error('Got Mark before ServerConfig'));
-      }
-      break;
-      default:
-      throw new Error('Nonexistent server argtype option');
-    }
-  });
-  call.on('end', function() {
-    server.stop(function() {
-      call.end();
     });
-  });
-};
+  };
 
-exports.coreCount = function coreCount(call, callback) {
-  callback(null, {cores: os.cpus().length});
+  this.coreCount = function coreCount(call, callback) {
+    callback(null, {cores: os.cpus().length});
+  };
 };

src/node/src/common.js

@@ -141,7 +141,7 @@ exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service,
     binaryAsBase64 = options.binaryAsBase64;
     longsAsStrings = options.longsAsStrings;
   }
-  return _.object(_.map(service.children, function(method) {
+  return _.fromPairs(_.map(service.children, function(method) {
     return [_.camelCase(method.name), {
       path: prefix + method.name,
       requestStream: method.requestStream,

src/objective-c/!ProtoCompiler-gRPCPlugin.podspec

@@ -36,7 +36,7 @@ Pod::Spec.new do |s|
   # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed
   # before them.
   s.name     = '!ProtoCompiler-gRPCPlugin'
-  v = '1.0.0'
+  v = '1.0.1'
   s.version  = v
   s.summary  = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.'
   s.description = <<-DESC
@@ -95,7 +95,7 @@ Pod::Spec.new do |s|
   s.preserve_paths = plugin
 
   # Restrict the protoc version to the one supported by this plugin.
-  s.dependency '!ProtoCompiler', '3.0.0'
+  s.dependency '!ProtoCompiler', '3.0.2'
   # For the Protobuf dependency not to complain:
   s.ios.deployment_target = '7.1'
   s.osx.deployment_target = '10.9'

src/objective-c/!ProtoCompiler.podspec

@@ -36,7 +36,7 @@ Pod::Spec.new do |s|
   # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed
   # before them.
   s.name     = '!ProtoCompiler'
-  v = '3.0.0'
+  v = '3.0.2'
   s.version  = v
   s.summary  = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files'
   s.description = <<-DESC

src/objective-c/BoringSSL.podspec

@@ -31,7 +31,7 @@
 
 Pod::Spec.new do |s|
   s.name     = 'BoringSSL'
-  version = '6.0'
+  version = '7.0'
   s.version  = version
   s.summary  = 'BoringSSL is a fork of OpenSSL that is designed to meet Google’s needs.'
   # Adapted from the homepage:
@@ -70,7 +70,7 @@ Pod::Spec.new do |s|
   s.source = {
     :git => 'https://boringssl.googlesource.com/boringssl',
     :tag => "version_for_cocoapods_#{version}",
-    # :commit => '4ac2dc4c0d48ca45da4f66c40e60d6b425fa94a3',
+    # :commit => '4fec04b48406111cb88fdd8d196253adc54f7a31',
   }
 
   name = 'openssl'
@@ -388,42 +388,42 @@ Pod::Spec.new do |s|
           0x28340c19,
           0x283480ac,
           0x283500ea,
-          0x2c322843,
-          0x2c32a851,
-          0x2c332863,
-          0x2c33a875,
-          0x2c342889,
-          0x2c34a89b,
-          0x2c3528b6,
-          0x2c35a8c8,
-          0x2c3628db,
+          0x2c322910,
+          0x2c32a91e,
+          0x2c332930,
+          0x2c33a942,
+          0x2c342956,
+          0x2c34a968,
+          0x2c352983,
+          0x2c35a995,
+          0x2c3629a8,
           0x2c36832d,
-          0x2c3728e8,
-          0x2c37a8fa,
-          0x2c38290d,
-          0x2c38a924,
-          0x2c392932,
-          0x2c39a942,
-          0x2c3a2954,
-          0x2c3aa968,
-          0x2c3b2979,
-          0x2c3ba998,
-          0x2c3c29ac,
-          0x2c3ca9c2,
-          0x2c3d29db,
-          0x2c3da9f8,
-          0x2c3e2a09,
-          0x2c3eaa17,
-          0x2c3f2a2f,
-          0x2c3faa47,
-          0x2c402a54,
+          0x2c3729b5,
+          0x2c37a9c7,
+          0x2c3829da,
+          0x2c38a9f1,
+          0x2c3929ff,
+          0x2c39aa0f,
+          0x2c3a2a21,
+          0x2c3aaa35,
+          0x2c3b2a46,
+          0x2c3baa65,
+          0x2c3c2a79,
+          0x2c3caa8f,
+          0x2c3d2aa8,
+          0x2c3daac5,
+          0x2c3e2ad6,
+          0x2c3eaae4,
+          0x2c3f2afc,
+          0x2c3fab14,
+          0x2c402b21,
           0x2c4090e7,
-          0x2c412a65,
-          0x2c41aa78,
+          0x2c412b32,
+          0x2c41ab45,
           0x2c4210c0,
-          0x2c42aa89,
+          0x2c42ab56,
           0x2c430720,
-          0x2c43a98a,
+          0x2c43aa57,
           0x30320000,
           0x30328015,
           0x3033001f,
@@ -576,174 +576,183 @@ Pod::Spec.new do |s|
           0x403b9861,
           0x403c0064,
           0x403c8083,
-          0x403d1890,
-          0x403d98a6,
-          0x403e18b5,
-          0x403e98c8,
-          0x403f18e2,
-          0x403f98f0,
-          0x40401905,
-          0x40409919,
-          0x40411936,
-          0x40419951,
-          0x4042196a,
-          0x4042997d,
-          0x40431991,
-          0x404399a9,
-          0x404419c0,
+          0x403d18aa,
+          0x403d98c0,
+          0x403e18cf,
+          0x403e98e2,
+          0x403f18fc,
+          0x403f990a,
+          0x4040191f,
+          0x40409933,
+          0x40411950,
+          0x4041996b,
+          0x40421984,
+          0x40429997,
+          0x404319ab,
+          0x404399c3,
+          0x404419da,
           0x404480ac,
-          0x404519d5,
-          0x404599e7,
-          0x40461a0b,
-          0x40469a2b,
-          0x40471a39,
-          0x40479a60,
-          0x40481a89,
-          0x40489aa2,
-          0x40491ab9,
-          0x40499ad3,
-          0x404a1aea,
-          0x404a9b08,
-          0x404b1b20,
-          0x404b9b37,
-          0x404c1b4d,
-          0x404c9b5f,
-          0x404d1b80,
-          0x404d9ba2,
-          0x404e1bb6,
-          0x404e9bc3,
-          0x404f1bf0,
-          0x404f9c19,
-          0x40501c43,
-          0x40509c57,
-          0x40511c72,
-          0x40519c82,
-          0x40521c99,
-          0x40529cbd,
-          0x40531cd5,
-          0x40539ce8,
-          0x40541cfd,
-          0x40549d20,
-          0x40551d2e,
-          0x40559d4b,
-          0x40561d58,
-          0x40569d71,
-          0x40571d89,
-          0x40579d9c,
-          0x40581db1,
-          0x40589dc3,
-          0x40591df2,
-          0x40599e0b,
-          0x405a1e1f,
-          0x405a9e2f,
-          0x405b1e47,
-          0x405b9e58,
-          0x405c1e6b,
-          0x405c9e7c,
-          0x405d1e89,
-          0x405d9ea0,
-          0x405e1ec0,
+          0x404519ef,
+          0x40459a01,
+          0x40461a25,
+          0x40469a45,
+          0x40471a53,
+          0x40479a7a,
+          0x40481ab7,
+          0x40489ad0,
+          0x40491ae7,
+          0x40499b01,
+          0x404a1b18,
+          0x404a9b36,
+          0x404b1b4e,
+          0x404b9b65,
+          0x404c1b7b,
+          0x404c9b8d,
+          0x404d1bae,
+          0x404d9bd0,
+          0x404e1be4,
+          0x404e9bf1,
+          0x404f1c1e,
+          0x404f9c47,
+          0x40501c71,
+          0x40509c85,
+          0x40511ca0,
+          0x40519cb0,
+          0x40521cc7,
+          0x40529ceb,
+          0x40531d03,
+          0x40539d16,
+          0x40541d2b,
+          0x40549d4e,
+          0x40551d5c,
+          0x40559d79,
+          0x40561d86,
+          0x40569d9f,
+          0x40571db7,
+          0x40579dca,
+          0x40581ddf,
+          0x40589e06,
+          0x40591e35,
+          0x40599e62,
+          0x405a1e76,
+          0x405a9e86,
+          0x405b1e9e,
+          0x405b9eaf,
+          0x405c1ec2,
+          0x405c9ee3,
+          0x405d1ef0,
+          0x405d9f07,
+          0x405e1f27,
           0x405e8a95,
-          0x405f1ee1,
-          0x405f9eee,
-          0x40601efc,
-          0x40609f1e,
-          0x40611f46,
-          0x40619f5b,
-          0x40621f72,
-          0x40629f83,
-          0x40631f94,
-          0x40639fa9,
-          0x40641fc0,
-          0x40649fd1,
-          0x40651fec,
-          0x4065a003,
-          0x4066201b,
-          0x4066a045,
-          0x40672070,
-          0x4067a091,
-          0x406820a4,
-          0x4068a0c5,
-          0x406920f7,
-          0x4069a125,
-          0x406a2146,
-          0x406aa166,
-          0x406b22ee,
-          0x406ba311,
-          0x406c2327,
-          0x406ca553,
-          0x406d2582,
-          0x406da5aa,
-          0x406e25c3,
-          0x406ea5db,
-          0x406f25fa,
-          0x406fa60f,
-          0x40702622,
-          0x4070a63f,
+          0x405f1f48,
+          0x405f9f55,
+          0x40601f63,
+          0x40609f85,
+          0x40611fad,
+          0x40619fc2,
+          0x40621fd9,
+          0x40629fea,
+          0x40631ffb,
+          0x4063a010,
+          0x40642027,
+          0x4064a053,
+          0x4065206e,
+          0x4065a085,
+          0x4066209d,
+          0x4066a0c7,
+          0x406720f2,
+          0x4067a113,
+          0x40682126,
+          0x4068a147,
+          0x40692179,
+          0x4069a1a7,
+          0x406a21c8,
+          0x406aa1e8,
+          0x406b2370,
+          0x406ba393,
+          0x406c23a9,
+          0x406ca60b,
+          0x406d263a,
+          0x406da662,
+          0x406e2690,
+          0x406ea6a8,
+          0x406f26c7,
+          0x406fa6dc,
+          0x407026ef,
+          0x4070a70c,
           0x40710800,
-          0x4071a651,
-          0x40722664,
-          0x4072a67d,
-          0x40732695,
+          0x4071a71e,
+          0x40722731,
+          0x4072a74a,
+          0x40732762,
           0x4073936d,
-          0x407426a9,
-          0x4074a6c3,
-          0x407526d4,
-          0x4075a6e8,
-          0x407626f6,
+          0x40742776,
+          0x4074a790,
+          0x407527a1,
+          0x4075a7b5,
+          0x407627c3,
           0x407691aa,
-          0x4077271b,
-          0x4077a73d,
-          0x40782758,
-          0x4078a791,
-          0x407927a8,
-          0x4079a7be,
-          0x407a27ca,
-          0x407aa7dd,
-          0x407b27f2,
-          0x407ba804,
-          0x407c2819,
-          0x407ca822,
-          0x407d20e0,
-          0x407d9c29,
-          0x407e276d,
-          0x407e9dd3,
-          0x407f1a4d,
-          0x407f986d,
-          0x40801c00,
-          0x40809a75,
-          0x40811cab,
-          0x40819bda,
-          0x41f42219,
-          0x41f922ab,
-          0x41fe219e,
-          0x41fea37a,
-          0x41ff246b,
-          0x42032232,
-          0x42082254,
-          0x4208a290,
-          0x42092182,
-          0x4209a2ca,
-          0x420a21d9,
-          0x420aa1b9,
-          0x420b21f9,
-          0x420ba272,
-          0x420c2487,
-          0x420ca347,
-          0x420d2361,
-          0x420da398,
-          0x421223b2,
-          0x4217244e,
-          0x4217a3f4,
-          0x421c2416,
-          0x421f23d1,
-          0x4221249e,
-          0x42262431,
-          0x422b2537,
-          0x422ba500,
-          0x422c251f,
-          0x422ca4da,
-          0x422d24b9,
+          0x407727e8,
+          0x4077a80a,
+          0x40782825,
+          0x4078a85e,
+          0x40792875,
+          0x4079a88b,
+          0x407a2897,
+          0x407aa8aa,
+          0x407b28bf,
+          0x407ba8d1,
+          0x407c28e6,
+          0x407ca8ef,
+          0x407d2162,
+          0x407d9c57,
+          0x407e283a,
+          0x407e9e16,
+          0x407f1a67,
+          0x407f9887,
+          0x40801c2e,
+          0x40809a8f,
+          0x40811cd9,
+          0x40819c08,
+          0x4082267b,
+          0x4082986d,
+          0x40831df1,
+          0x4083a038,
+          0x40841aa3,
+          0x40849e4e,
+          0x40851ed3,
+          0x41f4229b,
+          0x41f9232d,
+          0x41fe2220,
+          0x41fea3fc,
+          0x41ff24ed,
+          0x420322b4,
+          0x420822d6,
+          0x4208a312,
+          0x42092204,
+          0x4209a34c,
+          0x420a225b,
+          0x420aa23b,
+          0x420b227b,
+          0x420ba2f4,
+          0x420c2509,
+          0x420ca3c9,
+          0x420d23e3,
+          0x420da41a,
+          0x42122434,
+          0x421724d0,
+          0x4217a476,
+          0x421c2498,
+          0x421f2453,
+          0x42212520,
+          0x422624b3,
+          0x422b25ef,
+          0x422ba59d,
+          0x422c25d7,
+          0x422ca55c,
+          0x422d253b,
+          0x422da5bc,
+          0x422e2582,
           0x4432072b,
           0x4432873a,
           0x44330746,
@@ -786,69 +795,69 @@ Pod::Spec.new do |s|
           0x4c3d136d,
           0x4c3d937c,
           0x4c3e1389,
-          0x50322a9b,
-          0x5032aaaa,
-          0x50332ab5,
-          0x5033aac5,
-          0x50342ade,
-          0x5034aaf8,
-          0x50352b06,
-          0x5035ab1c,
-          0x50362b2e,
-          0x5036ab44,
-          0x50372b5d,
-          0x5037ab70,
-          0x50382b88,
-          0x5038ab99,
-          0x50392bae,
-          0x5039abc2,
-          0x503a2be2,
-          0x503aabf8,
-          0x503b2c10,
-          0x503bac22,
-          0x503c2c3e,
-          0x503cac55,
-          0x503d2c6e,
-          0x503dac84,
-          0x503e2c91,
-          0x503eaca7,
-          0x503f2cb9,
+          0x50322b68,
+          0x5032ab77,
+          0x50332b82,
+          0x5033ab92,
+          0x50342bab,
+          0x5034abc5,
+          0x50352bd3,
+          0x5035abe9,
+          0x50362bfb,
+          0x5036ac11,
+          0x50372c2a,
+          0x5037ac3d,
+          0x50382c55,
+          0x5038ac66,
+          0x50392c7b,
+          0x5039ac8f,
+          0x503a2caf,
+          0x503aacc5,
+          0x503b2cdd,
+          0x503bacef,
+          0x503c2d0b,
+          0x503cad22,
+          0x503d2d3b,
+          0x503dad51,
+          0x503e2d5e,
+          0x503ead74,
+          0x503f2d86,
           0x503f8382,
-          0x50402ccc,
-          0x5040acdc,
-          0x50412cf6,
-          0x5041ad05,
-          0x50422d1f,
-          0x5042ad3c,
-          0x50432d4c,
-          0x5043ad5c,
-          0x50442d6b,
+          0x50402d99,
+          0x5040ada9,
+          0x50412dc3,
+          0x5041add2,
+          0x50422dec,
+          0x5042ae09,
+          0x50432e19,
+          0x5043ae29,
+          0x50442e38,
           0x5044843f,
-          0x50452d7f,
-          0x5045ad9d,
-          0x50462db0,
-          0x5046adc6,
-          0x50472dd8,
-          0x5047aded,
-          0x50482e13,
-          0x5048ae21,
-          0x50492e34,
-          0x5049ae49,
-          0x504a2e5f,
-          0x504aae6f,
-          0x504b2e8f,
-          0x504baea2,
-          0x504c2ec5,
-          0x504caef3,
-          0x504d2f05,
-          0x504daf22,
-          0x504e2f3d,
-          0x504eaf59,
-          0x504f2f6b,
-          0x504faf82,
-          0x50502f91,
+          0x50452e4c,
+          0x5045ae6a,
+          0x50462e7d,
+          0x5046ae93,
+          0x50472ea5,
+          0x5047aeba,
+          0x50482ee0,
+          0x5048aeee,
+          0x50492f01,
+          0x5049af16,
+          0x504a2f2c,
+          0x504aaf3c,
+          0x504b2f5c,
+          0x504baf6f,
+          0x504c2f92,
+          0x504cafc0,
+          0x504d2fd2,
+          0x504dafef,
+          0x504e300a,
+          0x504eb026,
+          0x504f3038,
+          0x504fb04f,
+          0x5050305e,
           0x505086ef,
-          0x50512fa4,
+          0x50513071,
           0x58320ec9,
           0x68320e8b,
           0x68328c25,
@@ -1209,6 +1218,7 @@ Pod::Spec.new do |s|
           "BAD_SSL_FILETYPE\\0"
           "BAD_WRITE_RETRY\\0"
           "BIO_NOT_SET\\0"
+          "BLOCK_CIPHER_PAD_IS_WRONG\\0"
           "BUFFERED_MESSAGES_ON_CIPHER_CHANGE\\0"
           "CA_DN_LENGTH_MISMATCH\\0"
           "CA_DN_TOO_LONG\\0"
@@ -1233,6 +1243,7 @@ Pod::Spec.new do |s|
           "DOWNGRADE_DETECTED\\0"
           "DTLS_MESSAGE_TOO_BIG\\0"
           "DUPLICATE_EXTENSION\\0"
+          "DUPLICATE_KEY_SHARE\\0"
           "ECC_CERT_NOT_FOR_SIGNING\\0"
           "EMS_STATE_INCONSISTENT\\0"
           "ENCRYPTED_LENGTH_TOO_LONG\\0"
@@ -1270,15 +1281,18 @@ Pod::Spec.new do |s|
           "NO_CERTIFICATE_SET\\0"
           "NO_CIPHERS_AVAILABLE\\0"
           "NO_CIPHERS_PASSED\\0"
+          "NO_CIPHERS_SPECIFIED\\0"
           "NO_CIPHER_MATCH\\0"
           "NO_COMMON_SIGNATURE_ALGORITHMS\\0"
           "NO_COMPRESSION_SPECIFIED\\0"
+          "NO_GROUPS_SPECIFIED\\0"
           "NO_METHOD_SPECIFIED\\0"
           "NO_P256_SUPPORT\\0"
           "NO_PRIVATE_KEY_ASSIGNED\\0"
           "NO_RENEGOTIATION\\0"
           "NO_REQUIRED_DIGEST\\0"
           "NO_SHARED_CIPHER\\0"
+          "NO_SHARED_GROUP\\0"
           "NULL_SSL_CTX\\0"
           "NULL_SSL_METHOD_PASSED\\0"
           "OLD_SESSION_CIPHER_NOT_RETURNED\\0"
@@ -1294,6 +1308,7 @@ Pod::Spec.new do |s|
           "READ_TIMEOUT_EXPIRED\\0"
           "RECORD_LENGTH_MISMATCH\\0"
           "RECORD_TOO_LARGE\\0"
+          "RENEGOTIATION_EMS_MISMATCH\\0"
           "RENEGOTIATION_ENCODING_ERR\\0"
           "RENEGOTIATION_MISMATCH\\0"
           "REQUIRED_CIPHER_MISSING\\0"
@@ -1338,12 +1353,15 @@ Pod::Spec.new do |s|
           "TLSV1_ALERT_USER_CANCELLED\\0"
           "TLSV1_BAD_CERTIFICATE_HASH_VALUE\\0"
           "TLSV1_BAD_CERTIFICATE_STATUS_RESPONSE\\0"
+          "TLSV1_CERTIFICATE_REQUIRED\\0"
           "TLSV1_CERTIFICATE_UNOBTAINABLE\\0"
+          "TLSV1_UNKNOWN_PSK_IDENTITY\\0"
           "TLSV1_UNRECOGNIZED_NAME\\0"
           "TLSV1_UNSUPPORTED_EXTENSION\\0"
           "TLS_PEER_DID_NOT_RESPOND_WITH_CERTIFICATE_LIST\\0"
           "TLS_RSA_ENCRYPTED_VALUE_LENGTH_IS_WRONG\\0"
           "TOO_MANY_EMPTY_FRAGMENTS\\0"
+          "TOO_MANY_KEY_UPDATES\\0"
           "TOO_MANY_WARNING_ALERTS\\0"
           "UNABLE_TO_FIND_ECDH_PARAMETERS\\0"
           "UNEXPECTED_EXTENSION\\0"

src/objective-c/CronetFramework.podspec

@@ -30,14 +30,47 @@
 
 Pod::Spec.new do |s|
   s.name         = "CronetFramework"
-  s.version      = "0.0.2"
+  s.version      = "0.0.3"
   s.summary      = "Cronet, precompiled and used as a framework."
   s.homepage     = "http://chromium.org"
-  s.license      = { :type => 'BSD' }
+  s.license      = {
+    :type => 'BSD',
+    :text => <<-LICENSE
+      Copyright 2015, Google Inc.
+      All rights reserved.
+
+      Redistribution and use in source and binary forms, with or without
+      modification, are permitted provided that the following conditions are
+      met:
+
+          * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+          * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following disclaimer
+      in the documentation and/or other materials provided with the
+      distribution.
+          * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived from
+      this software without specific prior written permission.
+
+      THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+      "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+      LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+      A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+      OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+      SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+      LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+      DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+      THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+      (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+      OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+    LICENSE
+  }
   s.vendored_framework = "Cronet.framework"
   s.author             = "The Chromium Authors"
-  s.ios.deployment_target = "7.1"
+  s.ios.deployment_target = "8.0"
   s.source       = { :http => 'https://storage.googleapis.com/grpc-precompiled-binaries/cronet/Cronet.framework.zip' }
   s.preserve_paths = "Cronet.framework"
   s.public_header_files = "Cronet.framework/Headers/**/*{.h}"
+  s.source_files = "Cronet.framework/Headers/**/*{.h}"
 end

src/objective-c/GRPCClient/private/GRPCHost.m

@@ -51,7 +51,7 @@ NS_ASSUME_NONNULL_BEGIN
 // TODO(jcanizales): Generate the version in a standalone header, from
 // templates. Like
 // templates/src/core/surface/version.c.template .
-#define GRPC_OBJC_VERSION_STRING @"1.0.0"
+#define GRPC_OBJC_VERSION_STRING @"1.0.1"
 
 static NSMutableDictionary *kHostCache;
 

src/php/lib/Grpc/BaseStub.php

@@ -116,7 +116,7 @@ class BaseStub
     }
 
     /**
-     * @param $timeout in microseconds
+     * @param int $timeout in microseconds
      *
      * @return bool true if channel is ready
      * @throw Exception if channel is in FATAL_ERROR state
@@ -189,7 +189,7 @@ class BaseStub
     /**
      * validate and normalize the metadata array.
      *
-     * @param $metadata The metadata map
+     * @param array $metadata The metadata map
      *
      * @return $metadata Validated and key-normalized metadata map
      * @throw InvalidArgumentException if key contains invalid characters
@@ -216,8 +216,8 @@ class BaseStub
      * Call a remote method that takes a single argument and has a
      * single output.
      *
-     * @param string $method The name of the method to call
-     * @param $argument The argument to the method
+     * @param string   $method      The name of the method to call
+     * @param mixed    $argument    The argument to the method
      * @param callable $deserialize A function that deserializes the response
      * @param array    $metadata    A metadata map to send to the server
      *
@@ -250,8 +250,8 @@ class BaseStub
      * Call a remote method that takes a stream of arguments and has a single
      * output.
      *
-     * @param string $method The name of the method to call
-     * @param $arguments An array or Traversable of arguments to stream to the
+     * @param string   $method      The name of the method to call
+     * @param array    $arguments   An array or Traversable of arguments to stream to the
      *     server
      * @param callable $deserialize A function that deserializes the response
      * @param array    $metadata    A metadata map to send to the server
@@ -284,8 +284,8 @@ class BaseStub
      * Call a remote method that takes a single argument and returns a stream of
      * responses.
      *
-     * @param string $method The name of the method to call
-     * @param $argument The argument to the method
+     * @param string   $method      The name of the method to call
+     * @param mixed    $argument    The argument to the method
      * @param callable $deserialize A function that deserializes the responses
      * @param array    $metadata    A metadata map to send to the server
      *

src/proto/grpc/testing/control.proto

@@ -137,6 +137,11 @@ message ServerConfig {
 
   // If we use an OTHER_SERVER client_type, this string gives more detail
   string other_server_api = 11;
+
+  // c++-only options (for now) --------------------------------
+
+  // Buffer pool size (no buffer pool specified if unset)
+  int32 resource_quota_size = 1001;
 }
 
 message ServerArgs {
@@ -213,6 +218,10 @@ message ScenarioResultSummary
   double latency_95 = 9;
   double latency_99 = 10;
   double latency_999 = 11;
+
+  // Number of requests that succeeded/failed
+  double successful_requests_per_second = 12;
+  double failed_requests_per_second = 13;
 }
 
 // Results of a single benchmark scenario.
@@ -232,4 +241,6 @@ message ScenarioResult {
   // Information on success or failure of each worker
   repeated bool client_success = 7;
   repeated bool server_success = 8;
+  // Number of failed requests (one row per status code seen)
+  repeated RequestResultCount request_results = 9;
 }

src/proto/grpc/testing/proto2/empty2.proto

@@ -0,0 +1,37 @@
+
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+package grpc.testing.proto2;
+
+message EmptyWithExtensions {
+  extensions 100 to 999;
+}

src/proto/grpc/testing/proto2/empty2_extensions.proto

@@ -0,0 +1,43 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+import "src/proto/grpc/testing/proto2/empty2.proto";
+
+package grpc.testing.proto2;
+
+// Fill emptiness with music.
+extend grpc.testing.proto2.EmptyWithExtensions {
+  optional int64 Deadmau5 = 124;
+  optional float Madeon = 125;
+  optional string AboveAndBeyond = 126;
+  optional bool Tycho = 127;
+  optional fixed64 Pendulum = 128;
+}

src/proto/grpc/testing/stats.proto

@@ -59,6 +59,11 @@ message HistogramData {
   double count = 6;
 }
 
+message RequestResultCount {
+  int32 status_code = 1;
+  int64 count = 2;
+}
+
 message ClientStats {
   // Latency histogram. Data points are in nanoseconds.
   HistogramData latencies = 1;
@@ -67,4 +72,7 @@ message ClientStats {
   double time_elapsed = 2;
   double time_user = 3;
   double time_system = 4;
+
+  // Number of failed requests (one row per status code seen)
+  repeated RequestResultCount request_results = 5;
 }

src/python/.gitignore

@@ -1 +1,3 @@
 gens/
+*_pb2.py
+*_pb2_grpc.py

src/python/grpcio/grpc_core_dependencies.py

@@ -122,6 +122,7 @@ CORE_SOURCE_FILES = [
   'src/core/lib/iomgr/resolve_address_posix.c',
   'src/core/lib/iomgr/resolve_address_uv.c',
   'src/core/lib/iomgr/resolve_address_windows.c',
+  'src/core/lib/iomgr/resource_quota.c',
   'src/core/lib/iomgr/sockaddr_utils.c',
   'src/core/lib/iomgr/socket_utils_common_posix.c',
   'src/core/lib/iomgr/socket_utils_linux.c',