Merge github.com:grpc/grpc into split-me-baby-one-more-time

.travis.yml

@@ -34,7 +34,7 @@ script:
   - if [ ! -z "$USE_GCC" ] ; then export CC=gcc-$USE_GCC ; export CXX=g++-$USE_GCC ; fi
   - ./tools/run_tests/run_tests.py -l $TEST -t -j $JOBS -c $CONFIG -s 4.0
 after_success:
-  - if [ "$CONFIG" = "gcov" ] ; then coveralls --exclude third_party --exclude gens --exclude test --exclude src/compiler -b. --gcov-options '\-p' ; fi
+  - if [ "$CONFIG" = "gcov" ] ; then coveralls --exclude third_party --exclude gens --exclude test --exclude tools --exclude src/compiler -b. --gcov-options '\-p' ; fi
 notifications:
   email: false
   webhooks:

BUILD

@@ -589,14 +589,13 @@ cc_library(
     "src/cpp/client/secure_credentials.h",
     "src/cpp/server/secure_server_credentials.h",
     "src/cpp/client/channel.h",
-    "src/cpp/proto/proto_utils.h",
     "src/cpp/server/thread_pool.h",
+    "src/cpp/client/secure_channel_arguments.cc",
     "src/cpp/client/secure_credentials.cc",
     "src/cpp/server/secure_server_credentials.cc",
     "src/cpp/client/channel.cc",
     "src/cpp/client/channel_arguments.cc",
     "src/cpp/client/client_context.cc",
-    "src/cpp/client/client_unary_call.cc",
     "src/cpp/client/create_channel.cc",
     "src/cpp/client/credentials.cc",
     "src/cpp/client/generic_stub.cc",
@@ -628,6 +627,7 @@ cc_library(
     "include/grpc++/client_context.h",
     "include/grpc++/completion_queue.h",
     "include/grpc++/config.h",
+    "include/grpc++/config_protobuf.h",
     "include/grpc++/create_channel.h",
     "include/grpc++/credentials.h",
     "include/grpc++/generic_stub.h",
@@ -635,8 +635,10 @@ cc_library(
     "include/grpc++/impl/client_unary_call.h",
     "include/grpc++/impl/grpc_library.h",
     "include/grpc++/impl/internal_stub.h",
+    "include/grpc++/impl/proto_utils.h",
     "include/grpc++/impl/rpc_method.h",
     "include/grpc++/impl/rpc_service_method.h",
+    "include/grpc++/impl/serialization_traits.h",
     "include/grpc++/impl/service_type.h",
     "include/grpc++/impl/sync.h",
     "include/grpc++/impl/sync_cxx11.h",
@@ -671,12 +673,10 @@ cc_library(
   name = "grpc++_unsecure",
   srcs = [
     "src/cpp/client/channel.h",
-    "src/cpp/proto/proto_utils.h",
     "src/cpp/server/thread_pool.h",
     "src/cpp/client/channel.cc",
     "src/cpp/client/channel_arguments.cc",
     "src/cpp/client/client_context.cc",
-    "src/cpp/client/client_unary_call.cc",
     "src/cpp/client/create_channel.cc",
     "src/cpp/client/credentials.cc",
     "src/cpp/client/generic_stub.cc",
@@ -708,6 +708,7 @@ cc_library(
     "include/grpc++/client_context.h",
     "include/grpc++/completion_queue.h",
     "include/grpc++/config.h",
+    "include/grpc++/config_protobuf.h",
     "include/grpc++/create_channel.h",
     "include/grpc++/credentials.h",
     "include/grpc++/generic_stub.h",
@@ -715,8 +716,10 @@ cc_library(
     "include/grpc++/impl/client_unary_call.h",
     "include/grpc++/impl/grpc_library.h",
     "include/grpc++/impl/internal_stub.h",
+    "include/grpc++/impl/proto_utils.h",
     "include/grpc++/impl/rpc_method.h",
     "include/grpc++/impl/rpc_service_method.h",
+    "include/grpc++/impl/serialization_traits.h",
     "include/grpc++/impl/service_type.h",
     "include/grpc++/impl/sync.h",
     "include/grpc++/impl/sync_cxx11.h",
@@ -750,6 +753,8 @@ cc_library(
 cc_library(
   name = "grpc_plugin_support",
   srcs = [
+    "include/grpc++/config.h",
+    "include/grpc++/config_protobuf.h",
     "src/compiler/config.h",
     "src/compiler/cpp_generator.h",
     "src/compiler/cpp_generator_helpers.h",
@@ -800,6 +805,342 @@ cc_library(
 
 
 
+objc_library(
+  name = "gpr_objc",
+  srcs = [
+    "src/core/support/alloc.c",
+    "src/core/support/cancellable.c",
+    "src/core/support/cmdline.c",
+    "src/core/support/cpu_iphone.c",
+    "src/core/support/cpu_linux.c",
+    "src/core/support/cpu_posix.c",
+    "src/core/support/cpu_windows.c",
+    "src/core/support/env_linux.c",
+    "src/core/support/env_posix.c",
+    "src/core/support/env_win32.c",
+    "src/core/support/file.c",
+    "src/core/support/file_posix.c",
+    "src/core/support/file_win32.c",
+    "src/core/support/histogram.c",
+    "src/core/support/host_port.c",
+    "src/core/support/log.c",
+    "src/core/support/log_android.c",
+    "src/core/support/log_linux.c",
+    "src/core/support/log_posix.c",
+    "src/core/support/log_win32.c",
+    "src/core/support/murmur_hash.c",
+    "src/core/support/slice.c",
+    "src/core/support/slice_buffer.c",
+    "src/core/support/string.c",
+    "src/core/support/string_posix.c",
+    "src/core/support/string_win32.c",
+    "src/core/support/subprocess_posix.c",
+    "src/core/support/sync.c",
+    "src/core/support/sync_posix.c",
+    "src/core/support/sync_win32.c",
+    "src/core/support/thd.c",
+    "src/core/support/thd_posix.c",
+    "src/core/support/thd_win32.c",
+    "src/core/support/time.c",
+    "src/core/support/time_posix.c",
+    "src/core/support/time_win32.c",
+    "src/core/support/tls_pthread.c",
+  ],
+  hdrs = [
+    "include/grpc/support/alloc.h",
+    "include/grpc/support/atm.h",
+    "include/grpc/support/atm_gcc_atomic.h",
+    "include/grpc/support/atm_gcc_sync.h",
+    "include/grpc/support/atm_win32.h",
+    "include/grpc/support/cancellable_platform.h",
+    "include/grpc/support/cmdline.h",
+    "include/grpc/support/cpu.h",
+    "include/grpc/support/histogram.h",
+    "include/grpc/support/host_port.h",
+    "include/grpc/support/log.h",
+    "include/grpc/support/log_win32.h",
+    "include/grpc/support/port_platform.h",
+    "include/grpc/support/slice.h",
+    "include/grpc/support/slice_buffer.h",
+    "include/grpc/support/string_util.h",
+    "include/grpc/support/subprocess.h",
+    "include/grpc/support/sync.h",
+    "include/grpc/support/sync_generic.h",
+    "include/grpc/support/sync_posix.h",
+    "include/grpc/support/sync_win32.h",
+    "include/grpc/support/thd.h",
+    "include/grpc/support/time.h",
+    "include/grpc/support/tls.h",
+    "include/grpc/support/tls_gcc.h",
+    "include/grpc/support/tls_msvc.h",
+    "include/grpc/support/tls_pthread.h",
+    "include/grpc/support/useful.h",
+    "src/core/support/env.h",
+    "src/core/support/file.h",
+    "src/core/support/murmur_hash.h",
+    "src/core/support/string.h",
+    "src/core/support/string_win32.h",
+    "src/core/support/thd_internal.h",
+  ],
+  includes = [
+    "include",
+    ".",
+  ],
+  deps = [
+  ],
+)
+
+
+objc_library(
+  name = "grpc_objc",
+  srcs = [
+    "src/core/httpcli/format_request.c",
+    "src/core/httpcli/httpcli.c",
+    "src/core/httpcli/httpcli_security_connector.c",
+    "src/core/httpcli/parser.c",
+    "src/core/security/base64.c",
+    "src/core/security/client_auth_filter.c",
+    "src/core/security/credentials.c",
+    "src/core/security/credentials_metadata.c",
+    "src/core/security/credentials_posix.c",
+    "src/core/security/credentials_win32.c",
+    "src/core/security/google_default_credentials.c",
+    "src/core/security/json_token.c",
+    "src/core/security/secure_endpoint.c",
+    "src/core/security/secure_transport_setup.c",
+    "src/core/security/security_connector.c",
+    "src/core/security/security_context.c",
+    "src/core/security/server_auth_filter.c",
+    "src/core/security/server_secure_chttp2.c",
+    "src/core/surface/init_secure.c",
+    "src/core/surface/secure_channel_create.c",
+    "src/core/tsi/fake_transport_security.c",
+    "src/core/tsi/ssl_transport_security.c",
+    "src/core/tsi/transport_security.c",
+    "src/core/census/grpc_context.c",
+    "src/core/channel/channel_args.c",
+    "src/core/channel/channel_stack.c",
+    "src/core/channel/child_channel.c",
+    "src/core/channel/client_channel.c",
+    "src/core/channel/client_setup.c",
+    "src/core/channel/connected_channel.c",
+    "src/core/channel/http_client_filter.c",
+    "src/core/channel/http_server_filter.c",
+    "src/core/channel/noop_filter.c",
+    "src/core/compression/algorithm.c",
+    "src/core/compression/message_compress.c",
+    "src/core/debug/trace.c",
+    "src/core/iomgr/alarm.c",
+    "src/core/iomgr/alarm_heap.c",
+    "src/core/iomgr/endpoint.c",
+    "src/core/iomgr/endpoint_pair_posix.c",
+    "src/core/iomgr/endpoint_pair_windows.c",
+    "src/core/iomgr/fd_posix.c",
+    "src/core/iomgr/iocp_windows.c",
+    "src/core/iomgr/iomgr.c",
+    "src/core/iomgr/iomgr_posix.c",
+    "src/core/iomgr/iomgr_windows.c",
+    "src/core/iomgr/pollset_kick_posix.c",
+    "src/core/iomgr/pollset_multipoller_with_epoll.c",
+    "src/core/iomgr/pollset_multipoller_with_poll_posix.c",
+    "src/core/iomgr/pollset_posix.c",
+    "src/core/iomgr/pollset_set_posix.c",
+    "src/core/iomgr/pollset_set_windows.c",
+    "src/core/iomgr/pollset_windows.c",
+    "src/core/iomgr/resolve_address_posix.c",
+    "src/core/iomgr/resolve_address_windows.c",
+    "src/core/iomgr/sockaddr_utils.c",
+    "src/core/iomgr/socket_utils_common_posix.c",
+    "src/core/iomgr/socket_utils_linux.c",
+    "src/core/iomgr/socket_utils_posix.c",
+    "src/core/iomgr/socket_windows.c",
+    "src/core/iomgr/tcp_client_posix.c",
+    "src/core/iomgr/tcp_client_windows.c",
+    "src/core/iomgr/tcp_posix.c",
+    "src/core/iomgr/tcp_server_posix.c",
+    "src/core/iomgr/tcp_server_windows.c",
+    "src/core/iomgr/tcp_windows.c",
+    "src/core/iomgr/time_averaged_stats.c",
+    "src/core/iomgr/wakeup_fd_eventfd.c",
+    "src/core/iomgr/wakeup_fd_nospecial.c",
+    "src/core/iomgr/wakeup_fd_pipe.c",
+    "src/core/iomgr/wakeup_fd_posix.c",
+    "src/core/json/json.c",
+    "src/core/json/json_reader.c",
+    "src/core/json/json_string.c",
+    "src/core/json/json_writer.c",
+    "src/core/profiling/basic_timers.c",
+    "src/core/profiling/stap_timers.c",
+    "src/core/surface/byte_buffer.c",
+    "src/core/surface/byte_buffer_queue.c",
+    "src/core/surface/byte_buffer_reader.c",
+    "src/core/surface/call.c",
+    "src/core/surface/call_details.c",
+    "src/core/surface/call_log_batch.c",
+    "src/core/surface/channel.c",
+    "src/core/surface/channel_create.c",
+    "src/core/surface/client.c",
+    "src/core/surface/completion_queue.c",
+    "src/core/surface/event_string.c",
+    "src/core/surface/init.c",
+    "src/core/surface/lame_client.c",
+    "src/core/surface/metadata_array.c",
+    "src/core/surface/server.c",
+    "src/core/surface/server_chttp2.c",
+    "src/core/surface/server_create.c",
+    "src/core/surface/surface_trace.c",
+    "src/core/transport/chttp2/alpn.c",
+    "src/core/transport/chttp2/bin_encoder.c",
+    "src/core/transport/chttp2/frame_data.c",
+    "src/core/transport/chttp2/frame_goaway.c",
+    "src/core/transport/chttp2/frame_ping.c",
+    "src/core/transport/chttp2/frame_rst_stream.c",
+    "src/core/transport/chttp2/frame_settings.c",
+    "src/core/transport/chttp2/frame_window_update.c",
+    "src/core/transport/chttp2/hpack_parser.c",
+    "src/core/transport/chttp2/hpack_table.c",
+    "src/core/transport/chttp2/huffsyms.c",
+    "src/core/transport/chttp2/incoming_metadata.c",
+    "src/core/transport/chttp2/parsing.c",
+    "src/core/transport/chttp2/status_conversion.c",
+    "src/core/transport/chttp2/stream_encoder.c",
+    "src/core/transport/chttp2/stream_lists.c",
+    "src/core/transport/chttp2/stream_map.c",
+    "src/core/transport/chttp2/timeout_encoding.c",
+    "src/core/transport/chttp2/varint.c",
+    "src/core/transport/chttp2/writing.c",
+    "src/core/transport/chttp2_transport.c",
+    "src/core/transport/metadata.c",
+    "src/core/transport/stream_op.c",
+    "src/core/transport/transport.c",
+    "src/core/transport/transport_op_string.c",
+    "src/core/census/context.c",
+    "src/core/census/initialize.c",
+  ],
+  hdrs = [
+    "include/grpc/grpc_security.h",
+    "include/grpc/byte_buffer.h",
+    "include/grpc/byte_buffer_reader.h",
+    "include/grpc/compression.h",
+    "include/grpc/grpc.h",
+    "include/grpc/status.h",
+    "include/grpc/census.h",
+    "src/core/httpcli/format_request.h",
+    "src/core/httpcli/httpcli.h",
+    "src/core/httpcli/httpcli_security_connector.h",
+    "src/core/httpcli/parser.h",
+    "src/core/security/auth_filters.h",
+    "src/core/security/base64.h",
+    "src/core/security/credentials.h",
+    "src/core/security/json_token.h",
+    "src/core/security/secure_endpoint.h",
+    "src/core/security/secure_transport_setup.h",
+    "src/core/security/security_connector.h",
+    "src/core/security/security_context.h",
+    "src/core/tsi/fake_transport_security.h",
+    "src/core/tsi/ssl_transport_security.h",
+    "src/core/tsi/transport_security.h",
+    "src/core/tsi/transport_security_interface.h",
+    "src/core/census/grpc_context.h",
+    "src/core/channel/census_filter.h",
+    "src/core/channel/channel_args.h",
+    "src/core/channel/channel_stack.h",
+    "src/core/channel/child_channel.h",
+    "src/core/channel/client_channel.h",
+    "src/core/channel/client_setup.h",
+    "src/core/channel/connected_channel.h",
+    "src/core/channel/context.h",
+    "src/core/channel/http_client_filter.h",
+    "src/core/channel/http_server_filter.h",
+    "src/core/channel/noop_filter.h",
+    "src/core/compression/message_compress.h",
+    "src/core/debug/trace.h",
+    "src/core/iomgr/alarm.h",
+    "src/core/iomgr/alarm_heap.h",
+    "src/core/iomgr/alarm_internal.h",
+    "src/core/iomgr/endpoint.h",
+    "src/core/iomgr/endpoint_pair.h",
+    "src/core/iomgr/fd_posix.h",
+    "src/core/iomgr/iocp_windows.h",
+    "src/core/iomgr/iomgr.h",
+    "src/core/iomgr/iomgr_internal.h",
+    "src/core/iomgr/iomgr_posix.h",
+    "src/core/iomgr/pollset.h",
+    "src/core/iomgr/pollset_kick_posix.h",
+    "src/core/iomgr/pollset_posix.h",
+    "src/core/iomgr/pollset_set.h",
+    "src/core/iomgr/pollset_set_posix.h",
+    "src/core/iomgr/pollset_set_windows.h",
+    "src/core/iomgr/pollset_windows.h",
+    "src/core/iomgr/resolve_address.h",
+    "src/core/iomgr/sockaddr.h",
+    "src/core/iomgr/sockaddr_posix.h",
+    "src/core/iomgr/sockaddr_utils.h",
+    "src/core/iomgr/sockaddr_win32.h",
+    "src/core/iomgr/socket_utils_posix.h",
+    "src/core/iomgr/socket_windows.h",
+    "src/core/iomgr/tcp_client.h",
+    "src/core/iomgr/tcp_posix.h",
+    "src/core/iomgr/tcp_server.h",
+    "src/core/iomgr/tcp_windows.h",
+    "src/core/iomgr/time_averaged_stats.h",
+    "src/core/iomgr/wakeup_fd_pipe.h",
+    "src/core/iomgr/wakeup_fd_posix.h",
+    "src/core/json/json.h",
+    "src/core/json/json_common.h",
+    "src/core/json/json_reader.h",
+    "src/core/json/json_writer.h",
+    "src/core/profiling/timers.h",
+    "src/core/profiling/timers_preciseclock.h",
+    "src/core/surface/byte_buffer_queue.h",
+    "src/core/surface/call.h",
+    "src/core/surface/channel.h",
+    "src/core/surface/client.h",
+    "src/core/surface/completion_queue.h",
+    "src/core/surface/event_string.h",
+    "src/core/surface/init.h",
+    "src/core/surface/server.h",
+    "src/core/surface/surface_trace.h",
+    "src/core/transport/chttp2/alpn.h",
+    "src/core/transport/chttp2/bin_encoder.h",
+    "src/core/transport/chttp2/frame.h",
+    "src/core/transport/chttp2/frame_data.h",
+    "src/core/transport/chttp2/frame_goaway.h",
+    "src/core/transport/chttp2/frame_ping.h",
+    "src/core/transport/chttp2/frame_rst_stream.h",
+    "src/core/transport/chttp2/frame_settings.h",
+    "src/core/transport/chttp2/frame_window_update.h",
+    "src/core/transport/chttp2/hpack_parser.h",
+    "src/core/transport/chttp2/hpack_table.h",
+    "src/core/transport/chttp2/http2_errors.h",
+    "src/core/transport/chttp2/huffsyms.h",
+    "src/core/transport/chttp2/incoming_metadata.h",
+    "src/core/transport/chttp2/internal.h",
+    "src/core/transport/chttp2/status_conversion.h",
+    "src/core/transport/chttp2/stream_encoder.h",
+    "src/core/transport/chttp2/stream_map.h",
+    "src/core/transport/chttp2/timeout_encoding.h",
+    "src/core/transport/chttp2/varint.h",
+    "src/core/transport/chttp2_transport.h",
+    "src/core/transport/metadata.h",
+    "src/core/transport/stream_op.h",
+    "src/core/transport/transport.h",
+    "src/core/transport/transport_impl.h",
+    "src/core/census/context.h",
+  ],
+  includes = [
+    "include",
+    ".",
+  ],
+  deps = [
+    ":gpr_objc",
+    "//external:libssl_objc",
+  ],
+  sdk_dylibs = ["libz"],
+)
+
+
+
 cc_binary(
   name = "grpc_cpp_plugin",
   srcs = [
@@ -863,3 +1204,76 @@ cc_binary(
 
 
 
+
+
+
+objc_path = "src/objective-c"
+
+rx_library_path = objc_path + "/RxLibrary"
+
+objc_library(
+  name = "rx_library",
+  hdrs = glob([
+    rx_library_path + "/*.h",
+    rx_library_path + "/transformations/*.h",
+  ]),
+  srcs = glob([
+    rx_library_path + "/*.m",
+    rx_library_path + "/transformations/*.m",
+  ]),
+  includes = [objc_path],
+  deps = [
+    ":rx_library_private",
+  ],
+)
+
+objc_library(
+  name = "rx_library_private",
+  hdrs = glob([rx_library_path + "/private/*.h"]),
+  srcs = glob([rx_library_path + "/private/*.m"]),
+  visibility = ["//visibility:private"],
+)
+
+objc_client_path = objc_path + "/GRPCClient"
+
+objc_library(
+  name = "grpc_client",
+  hdrs = glob([
+    objc_client_path + "/*.h",
+    objc_client_path + "/private/*.h",
+  ]),
+  srcs = glob([
+    objc_client_path + "/*.m",
+    objc_client_path + "/private/*.m",
+  ]),
+  includes = [objc_path],
+  bundles = [":gRPCCertificates"],
+  deps = [
+    ":grpc_objc",
+    ":rx_library",
+  ],
+)
+
+objc_bundle_library(
+    # The choice of name is signicant here, since it determines the bundle name.
+    name = "gRPCCertificates",
+    resources = ["etc/roots.pem"],
+)
+
+proto_objc_rpc_path = objc_path + "/ProtoRPC"
+
+objc_library(
+  name = "proto_objc_rpc",
+  hdrs = glob([
+    proto_objc_rpc_path + "/*.h",
+  ]),
+  srcs = glob([
+    proto_objc_rpc_path + "/*.m",
+  ]),
+  includes = [objc_path],
+  deps = [
+    ":grpc_client",
+    ":rx_library",
+    "//external:protobuf_objc",
+  ],
+)

build.json

@@ -36,6 +36,7 @@
         "include/grpc++/client_context.h",
         "include/grpc++/completion_queue.h",
         "include/grpc++/config.h",
+        "include/grpc++/config_protobuf.h",
         "include/grpc++/create_channel.h",
         "include/grpc++/credentials.h",
         "include/grpc++/generic_stub.h",
@@ -43,8 +44,10 @@
         "include/grpc++/impl/client_unary_call.h",
         "include/grpc++/impl/grpc_library.h",
         "include/grpc++/impl/internal_stub.h",
+        "include/grpc++/impl/proto_utils.h",
         "include/grpc++/impl/rpc_method.h",
         "include/grpc++/impl/rpc_service_method.h",
+        "include/grpc++/impl/serialization_traits.h",
         "include/grpc++/impl/service_type.h",
         "include/grpc++/impl/sync.h",
         "include/grpc++/impl/sync_cxx11.h",
@@ -65,14 +68,12 @@
       ],
       "headers": [
         "src/cpp/client/channel.h",
-        "src/cpp/proto/proto_utils.h",
         "src/cpp/server/thread_pool.h"
       ],
       "src": [
         "src/cpp/client/channel.cc",
         "src/cpp/client/channel_arguments.cc",
         "src/cpp/client/client_context.cc",
-        "src/cpp/client/client_unary_call.cc",
         "src/cpp/client/create_channel.cc",
         "src/cpp/client/credentials.cc",
         "src/cpp/client/generic_stub.cc",
@@ -294,6 +295,14 @@
     },
     {
       "name": "grpc_test_util_base",
+      "headers": [
+        "test/core/end2end/cq_verifier.h",
+        "test/core/iomgr/endpoint_tests.h",
+        "test/core/util/grpc_profiler.h",
+        "test/core/util/parse_hexstring.h",
+        "test/core/util/port.h",
+        "test/core/util/slice_splitter.h"
+      ],
       "src": [
         "test/core/end2end/cq_verifier.c",
         "test/core/iomgr/endpoint_tests.c",
@@ -471,6 +480,9 @@
       "name": "grpc_test_util",
       "build": "private",
       "language": "c",
+      "headers": [
+        "test/core/end2end/data/ssl_test_data.h"
+      ],
       "src": [
         "test/core/end2end/data/server1_cert.c",
         "test/core/end2end/data/server1_key.c",
@@ -528,6 +540,7 @@
         "src/cpp/server/secure_server_credentials.h"
       ],
       "src": [
+        "src/cpp/client/secure_channel_arguments.cc",
         "src/cpp/client/secure_credentials.cc",
         "src/cpp/server/secure_server_credentials.cc"
       ],
@@ -542,20 +555,13 @@
       "secure": "check",
       "vs_project_guid": "{C187A093-A0FE-489D-A40A-6E33DE0F9FEB}"
     },
-    {
-      "name": "grpc++_benchmark_config",
-      "build": "private",
-      "language": "c++",
-      "src": [
-        "test/cpp/qps/qpstest.proto",
-        "test/cpp/qps/report.cc",
-        "test/cpp/util/benchmark_config.cc"
-      ]
-    },
     {
       "name": "grpc++_test_config",
       "build": "private",
       "language": "c++",
+      "headers": [
+        "test/cpp/util/test_config.h"
+      ],
       "src": [
         "test/cpp/util/test_config.cc"
       ]
@@ -564,6 +570,12 @@
       "name": "grpc++_test_util",
       "build": "private",
       "language": "c++",
+      "headers": [
+        "test/cpp/util/cli_call.h",
+        "test/cpp/util/create_test_channel.h",
+        "test/cpp/util/fake_credentials.h",
+        "test/cpp/util/subprocess.h"
+      ],
       "src": [
         "test/cpp/util/messages.proto",
         "test/cpp/util/echo.proto",
@@ -572,6 +584,10 @@
         "test/cpp/util/create_test_channel.cc",
         "test/cpp/util/fake_credentials.cc",
         "test/cpp/util/subprocess.cc"
+      ],
+      "deps": [
+        "grpc++",
+        "grpc_test_util"
       ]
     },
     {
@@ -594,6 +610,8 @@
       "build": "protoc",
       "language": "c++",
       "headers": [
+        "include/grpc++/config.h",
+        "include/grpc++/config_protobuf.h",
         "src/compiler/config.h",
         "src/compiler/cpp_generator.h",
         "src/compiler/cpp_generator_helpers.h",
@@ -623,6 +641,9 @@
       "name": "interop_client_helper",
       "build": "private",
       "language": "c++",
+      "headers": [
+        "test/cpp/interop/client_helper.h"
+      ],
       "src": [
         "test/cpp/interop/client_helper.cc"
       ],
@@ -638,6 +659,9 @@
       "name": "interop_client_main",
       "build": "private",
       "language": "c++",
+      "headers": [
+        "test/cpp/interop/interop_client.h"
+      ],
       "src": [
         "test/proto/empty.proto",
         "test/proto/messages.proto",
@@ -646,6 +670,7 @@
         "test/cpp/interop/interop_client.cc"
       ],
       "deps": [
+        "interop_client_helper",
         "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
@@ -659,6 +684,9 @@
       "name": "interop_server_helper",
       "build": "private",
       "language": "c++",
+      "headers": [
+        "test/cpp/interop/server_helper.h"
+      ],
       "src": [
         "test/cpp/interop/server_helper.cc"
       ],
@@ -680,6 +708,7 @@
         "test/cpp/interop/server.cc"
       ],
       "deps": [
+        "interop_server_helper",
         "grpc++_test_util",
         "grpc_test_util",
         "grpc++",
@@ -693,6 +722,10 @@
       "name": "pubsub_client_lib",
       "build": "do_not_build",
       "language": "c++",
+      "headers": [
+        "examples/pubsub/publisher.h",
+        "examples/pubsub/subscriber.h"
+      ],
       "src": [
         "examples/pubsub/label.proto",
         "examples/pubsub/empty.proto",
@@ -711,11 +744,16 @@
       "build": "private",
       "language": "c++",
       "headers": [
+        "test/cpp/qps/client.h",
         "test/cpp/qps/driver.h",
+        "test/cpp/qps/histogram.h",
         "test/cpp/qps/interarrival.h",
         "test/cpp/qps/qps_worker.h",
         "test/cpp/qps/report.h",
-        "test/cpp/qps/timer.h"
+        "test/cpp/qps/server.h",
+        "test/cpp/qps/stats.h",
+        "test/cpp/qps/timer.h",
+        "test/cpp/util/benchmark_config.h"
       ],
       "src": [
         "test/cpp/qps/qpstest.proto",
@@ -723,13 +761,16 @@
         "test/cpp/qps/client_sync.cc",
         "test/cpp/qps/driver.cc",
         "test/cpp/qps/qps_worker.cc",
+        "test/cpp/qps/report.cc",
         "test/cpp/qps/server_async.cc",
         "test/cpp/qps/server_sync.cc",
-        "test/cpp/qps/timer.cc"
+        "test/cpp/qps/timer.cc",
+        "test/cpp/util/benchmark_config.cc"
       ],
       "deps": [
         "grpc_test_util",
-        "grpc++_test_util"
+        "grpc++_test_util",
+        "grpc++"
       ]
     },
     {
@@ -876,6 +917,23 @@
         "posix"
       ]
     },
+    {
+      "name": "fd_conservation_posix_test",
+      "build": "test",
+      "language": "c",
+      "src": [
+        "test/core/iomgr/fd_conservation_posix_test.c"
+      ],
+      "deps": [
+        "grpc_test_util",
+        "grpc",
+        "gpr_test_util",
+        "gpr"
+      ],
+      "platforms": [
+        "posix"
+      ]
+    },
     {
       "name": "fd_posix_test",
       "build": "test",
@@ -962,10 +1020,9 @@
       "build": "tool",
       "language": "c",
       "src": [
-        "src/core/transport/chttp2/gen_hpack_tables.c"
+        "tools/codegen/core/gen_hpack_tables.c"
       ],
       "deps": [
-        "grpc_test_util",
         "gpr",
         "grpc"
       ]
@@ -1488,6 +1545,20 @@
         "gpr"
       ]
     },
+    {
+      "name": "multiple_server_queues_test",
+      "build": "test",
+      "language": "c",
+      "src": [
+        "test/core/end2end/multiple_server_queues_test.c"
+      ],
+      "deps": [
+        "grpc_test_util",
+        "grpc",
+        "gpr_test_util",
+        "gpr"
+      ]
+    },
     {
       "name": "murmur_hash_test",
       "build": "test",
@@ -1734,7 +1805,6 @@
       "deps": [
         "qps",
         "grpc++_test_util",
-        "grpc++_benchmark_config",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -1752,7 +1822,6 @@
       "deps": [
         "qps",
         "grpc++_test_util",
-        "grpc++_benchmark_config",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -2128,8 +2197,7 @@
         "grpc",
         "gpr_test_util",
         "gpr",
-        "grpc++_test_config",
-        "grpc++_benchmark_config"
+        "grpc++_test_config"
       ]
     },
     {
@@ -2160,7 +2228,6 @@
       "deps": [
         "qps",
         "grpc++_test_util",
-        "grpc++_benchmark_config",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -2179,7 +2246,6 @@
       "deps": [
         "qps",
         "grpc++_test_util",
-        "grpc++_benchmark_config",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -2268,7 +2334,6 @@
       "deps": [
         "qps",
         "grpc++_test_util",
-        "grpc++_benchmark_config",
         "grpc_test_util",
         "grpc++",
         "grpc",
@@ -2286,7 +2351,6 @@
       "deps": [
         "qps",
         "grpc++_test_util",
-        "grpc++_benchmark_config",
         "grpc_test_util",
         "grpc++",
         "grpc",

include/grpc++/async_unary_call.h

@@ -51,47 +51,50 @@ class ClientAsyncResponseReaderInterface {
   virtual ~ClientAsyncResponseReaderInterface() {}
   virtual void ReadInitialMetadata(void* tag) = 0;
   virtual void Finish(R* msg, Status* status, void* tag) = 0;
-
 };
 
 template <class R>
 class ClientAsyncResponseReader GRPC_FINAL
     : public ClientAsyncResponseReaderInterface<R> {
  public:
+  template <class W>
   ClientAsyncResponseReader(ChannelInterface* channel, CompletionQueue* cq,
                             const RpcMethod& method, ClientContext* context,
-                            const grpc::protobuf::Message& request)
+                            const W& request)
       : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    init_buf_.AddSendInitialMetadata(&context->send_initial_metadata_);
-    init_buf_.AddSendMessage(request);
-    init_buf_.AddClientSendClose();
+    init_buf_.SendInitialMetadata(context->send_initial_metadata_);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(init_buf_.SendMessage(request).ok());
+    init_buf_.ClientSendClose();
     call_.PerformOps(&init_buf_);
   }
 
   void ReadInitialMetadata(void* tag) {
     GPR_ASSERT(!context_->initial_metadata_received_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddRecvInitialMetadata(context_);
+    meta_buf_.set_output_tag(tag);
+    meta_buf_.RecvInitialMetadata(context_);
     call_.PerformOps(&meta_buf_);
   }
 
   void Finish(R* msg, Status* status, void* tag) {
-    finish_buf_.Reset(tag);
+    finish_buf_.set_output_tag(tag);
     if (!context_->initial_metadata_received_) {
-      finish_buf_.AddRecvInitialMetadata(context_);
+      finish_buf_.RecvInitialMetadata(context_);
     }
-    finish_buf_.AddRecvMessage(msg);
-    finish_buf_.AddClientRecvStatus(context_, status);
+    finish_buf_.RecvMessage(msg);
+    finish_buf_.ClientRecvStatus(context_, status);
     call_.PerformOps(&finish_buf_);
   }
 
  private:
   ClientContext* context_;
   Call call_;
-  SneakyCallOpBuffer init_buf_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer finish_buf_;
+  SneakyCallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+                  CallOpClientSendClose> init_buf_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_buf_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>,
+            CallOpClientRecvStatus> finish_buf_;
 };
 
 template <class W>
@@ -104,34 +107,36 @@ class ServerAsyncResponseWriter GRPC_FINAL
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
     GPR_ASSERT(!ctx_->sent_initial_metadata_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    meta_buf_.set_output_tag(tag);
+    meta_buf_.SendInitialMetadata(ctx_->initial_metadata_);
     ctx_->sent_initial_metadata_ = true;
     call_.PerformOps(&meta_buf_);
   }
 
   void Finish(const W& msg, const Status& status, void* tag) {
-    finish_buf_.Reset(tag);
+    finish_buf_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      finish_buf_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
     // The response is dropped if the status is not OK.
     if (status.ok()) {
-      finish_buf_.AddSendMessage(msg);
+      finish_buf_.ServerSendStatus(
+          ctx_->trailing_metadata_, finish_buf_.SendMessage(msg));
+    } else {
+      finish_buf_.ServerSendStatus(ctx_->trailing_metadata_, status);
     }
-    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
     call_.PerformOps(&finish_buf_);
   }
 
   void FinishWithError(const Status& status, void* tag) {
     GPR_ASSERT(!status.ok());
-    finish_buf_.Reset(tag);
+    finish_buf_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      finish_buf_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
+    finish_buf_.ServerSendStatus(ctx_->trailing_metadata_, status);
     call_.PerformOps(&finish_buf_);
   }
 
@@ -140,8 +145,9 @@ class ServerAsyncResponseWriter GRPC_FINAL
 
   Call call_;
   ServerContext* ctx_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer finish_buf_;
+  CallOpSet<CallOpSendInitialMetadata> meta_buf_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+            CallOpServerSendStatus> finish_buf_;
 };
 
 }  // namespace grpc

include/grpc++/byte_buffer.h

@@ -39,6 +39,8 @@
 #include <grpc/support/log.h>
 #include <grpc++/config.h>
 #include <grpc++/slice.h>
+#include <grpc++/status.h>
+#include <grpc++/impl/serialization_traits.h>
 
 #include <vector>
 
@@ -62,7 +64,10 @@ class ByteBuffer GRPC_FINAL {
   size_t Length() const;
 
  private:
-  friend class CallOpBuffer;
+  friend class SerializationTraits<ByteBuffer, void>;
+
+  ByteBuffer(const ByteBuffer&);
+  ByteBuffer& operator=(const ByteBuffer&);
 
   // takes ownership
   void set_buffer(grpc_byte_buffer* buf) {
@@ -78,6 +83,22 @@ class ByteBuffer GRPC_FINAL {
   grpc_byte_buffer* buffer_;
 };
 
+template <>
+class SerializationTraits<ByteBuffer, void> {
+ public:
+  static Status Deserialize(grpc_byte_buffer* byte_buffer, ByteBuffer* dest,
+                            int max_message_size) {
+    dest->set_buffer(byte_buffer);
+    return Status::OK;
+  }
+  static Status Serialize(const ByteBuffer& source, grpc_byte_buffer** buffer, 
+                        bool* own_buffer) {
+    *buffer = source.buffer();
+    *own_buffer = false;
+    return Status::OK;
+  }
+};
+
 }  // namespace grpc
 
 #endif  // GRPCXX_BYTE_BUFFER_H

include/grpc++/client_context.h

@@ -49,7 +49,6 @@ struct grpc_completion_queue;
 
 namespace grpc {
 
-class CallOpBuffer;
 class ChannelInterface;
 class CompletionQueue;
 class Credentials;
@@ -115,7 +114,8 @@ class ClientContext {
   ClientContext(const ClientContext&);
   ClientContext& operator=(const ClientContext&);
 
-  friend class CallOpBuffer;
+  friend class CallOpClientRecvStatus;
+  friend class CallOpRecvInitialMetadata;
   friend class Channel;
   template <class R>
   friend class ::grpc::ClientReader;
@@ -131,6 +131,12 @@ class ClientContext {
   friend class ::grpc::ClientAsyncReaderWriter;
   template <class R>
   friend class ::grpc::ClientAsyncResponseReader;
+  template <class InputMessage, class OutputMessage>
+  friend Status BlockingUnaryCall(ChannelInterface* channel,
+                                  const RpcMethod& method,
+                                  ClientContext* context,
+                                  const InputMessage& request,
+                                  OutputMessage* result);
 
   grpc_call* call() { return call_; }
   void set_call(grpc_call* call,

include/grpc++/completion_queue.h

@@ -35,8 +35,8 @@
 #define GRPCXX_COMPLETION_QUEUE_H
 
 #include <grpc/support/time.h>
-#include <grpc++/impl/client_unary_call.h>
 #include <grpc++/impl/grpc_library.h>
+#include <grpc++/status.h>
 #include <grpc++/time.h>
 
 struct grpc_completion_queue;
@@ -55,8 +55,19 @@ template <class W>
 class ServerWriter;
 template <class R, class W>
 class ServerReaderWriter;
-
+template <class ServiceType, class RequestType, class ResponseType>
+class RpcMethodHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class ClientStreamingHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class ServerStreamingHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class BidiStreamingHandler;
+
+class ChannelInterface;
+class ClientContext;
 class CompletionQueue;
+class RpcMethod;
 class Server;
 class ServerBuilder;
 class ServerContext;
@@ -84,7 +95,7 @@ class CompletionQueue : public GrpcLibrary {
 
   // Nonblocking (until deadline) read from queue.
   // Cannot rely on result of tag or ok if return is TIMEOUT
-  template<typename T>
+  template <typename T>
   NextStatus AsyncNext(void** tag, bool* ok, const T& deadline) {
     TimePoint<T> deadline_tp(deadline);
     return AsyncNextInternal(tag, ok, deadline_tp.raw_time());
@@ -118,13 +129,22 @@ class CompletionQueue : public GrpcLibrary {
   friend class ::grpc::ServerWriter;
   template <class R, class W>
   friend class ::grpc::ServerReaderWriter;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class RpcMethodHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class ClientStreamingHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class ServerStreamingHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class BidiStreamingHandler;
   friend class ::grpc::Server;
   friend class ::grpc::ServerContext;
+  template <class InputMessage, class OutputMessage>
   friend Status BlockingUnaryCall(ChannelInterface* channel,
                                   const RpcMethod& method,
                                   ClientContext* context,
-                                  const grpc::protobuf::Message& request,
-                                  grpc::protobuf::Message* result);
+                                  const InputMessage& request,
+                                  OutputMessage* result);
 
   NextStatus AsyncNextInternal(void** tag, bool* ok, gpr_timespec deadline);
 

include/grpc++/config.h

@@ -77,31 +77,6 @@
 #define GRPC_OVERRIDE override
 #endif
 
-#ifndef GRPC_CUSTOM_PROTOBUF_INT64
-#include <google/protobuf/stubs/common.h>
-#define GRPC_CUSTOM_PROTOBUF_INT64 ::google::protobuf::int64
-#endif
-
-#ifndef GRPC_CUSTOM_MESSAGE
-#include <google/protobuf/message.h>
-#define GRPC_CUSTOM_MESSAGE ::google::protobuf::Message
-#endif
-
-#ifndef GRPC_CUSTOM_STRING
-#include <string>
-#define GRPC_CUSTOM_STRING std::string
-#endif
-
-#ifndef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream.h>
-#define GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM \
-  ::google::protobuf::io::ZeroCopyOutputStream
-#define GRPC_CUSTOM_ZEROCOPYINPUTSTREAM \
-  ::google::protobuf::io::ZeroCopyInputStream
-#define GRPC_CUSTOM_CODEDINPUTSTREAM ::google::protobuf::io::CodedInputStream
-#endif
-
 #ifdef GRPC_CXX0X_NO_NULLPTR
 #include <memory>
 const class {
@@ -125,23 +100,15 @@ const class {
 } nullptr = {};
 #endif
 
+#ifndef GRPC_CUSTOM_STRING
+#include <string>
+#define GRPC_CUSTOM_STRING std::string
+#endif
+
 namespace grpc {
 
 typedef GRPC_CUSTOM_STRING string;
 
-namespace protobuf {
-
-typedef GRPC_CUSTOM_MESSAGE Message;
-typedef GRPC_CUSTOM_PROTOBUF_INT64 int64;
-
-namespace io {
-typedef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM ZeroCopyOutputStream;
-typedef GRPC_CUSTOM_ZEROCOPYINPUTSTREAM ZeroCopyInputStream;
-typedef GRPC_CUSTOM_CODEDINPUTSTREAM CodedInputStream;
-}  // namespace io
-
-}  // namespace protobuf
-
 }  // namespace grpc
 
 #endif  // GRPCXX_CONFIG_H

include/grpc++/config_protobuf.h

@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_CONFIG_PROTOBUF_H
+#define GRPCXX_CONFIG_PROTOBUF_H
+
+#ifndef GRPC_CUSTOM_PROTOBUF_INT64
+#include <google/protobuf/stubs/common.h>
+#define GRPC_CUSTOM_PROTOBUF_INT64 ::google::protobuf::int64
+#endif
+
+#ifndef GRPC_CUSTOM_MESSAGE
+#include <google/protobuf/message.h>
+#define GRPC_CUSTOM_MESSAGE ::google::protobuf::Message
+#endif
+
+#ifndef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#define GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM \
+  ::google::protobuf::io::ZeroCopyOutputStream
+#define GRPC_CUSTOM_ZEROCOPYINPUTSTREAM \
+  ::google::protobuf::io::ZeroCopyInputStream
+#define GRPC_CUSTOM_CODEDINPUTSTREAM ::google::protobuf::io::CodedInputStream
+#endif
+
+namespace grpc {
+namespace protobuf {
+
+typedef GRPC_CUSTOM_MESSAGE Message;
+typedef GRPC_CUSTOM_PROTOBUF_INT64 int64;
+
+namespace io {
+typedef GRPC_CUSTOM_ZEROCOPYOUTPUTSTREAM ZeroCopyOutputStream;
+typedef GRPC_CUSTOM_ZEROCOPYINPUTSTREAM ZeroCopyInputStream;
+typedef GRPC_CUSTOM_CODEDINPUTSTREAM CodedInputStream;
+}  // namespace io
+
+}  // namespace protobuf
+}  // namespace grpc
+
+#endif  // GRPCXX_CONFIG_PROTOBUF_H

include/grpc++/impl/call.h

@@ -34,14 +34,19 @@
 #ifndef GRPCXX_IMPL_CALL_H
 #define GRPCXX_IMPL_CALL_H
 
-#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc++/client_context.h>
 #include <grpc++/completion_queue.h>
 #include <grpc++/config.h>
 #include <grpc++/status.h>
+#include <grpc++/impl/serialization_traits.h>
 
+#include <functional>
 #include <memory>
 #include <map>
 
+#include <string.h>
+
 struct grpc_call;
 struct grpc_op;
 
@@ -50,84 +55,383 @@ namespace grpc {
 class ByteBuffer;
 class Call;
 
-class CallOpBuffer : public CompletionQueueTag {
+void FillMetadataMap(grpc_metadata_array* arr,
+                     std::multimap<grpc::string, grpc::string>* metadata);
+grpc_metadata* FillMetadataArray(
+    const std::multimap<grpc::string, grpc::string>& metadata);
+
+/// Default argument for CallOpSet. I is unused by the class, but can be
+/// used for generating multiple names for the same thing.
+template <int I>
+class CallNoOp {
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {}
+  void FinishOp(bool* status, int max_message_size) {}
+};
+
+class CallOpSendInitialMetadata {
  public:
-  CallOpBuffer();
-  ~CallOpBuffer();
-
-  void Reset(void* next_return_tag);
-
-  // Does not take ownership.
-  void AddSendInitialMetadata(
-      std::multimap<grpc::string, grpc::string>* metadata);
-  void AddSendInitialMetadata(ClientContext* ctx);
-  void AddRecvInitialMetadata(ClientContext* ctx);
-  void AddSendMessage(const grpc::protobuf::Message& message);
-  void AddSendMessage(const ByteBuffer& message);
-  void AddRecvMessage(grpc::protobuf::Message* message);
-  void AddRecvMessage(ByteBuffer* message);
-  void AddClientSendClose();
-  void AddClientRecvStatus(ClientContext* ctx, Status* status);
-  void AddServerSendStatus(std::multimap<grpc::string, grpc::string>* metadata,
-                           const Status& status);
-  void AddServerRecvClose(bool* cancelled);
-
-  // INTERNAL API:
-
-  // Convert to an array of grpc_op elements
-  void FillOps(grpc_op* ops, size_t* nops);
-
-  // Called by completion queue just prior to returning from Next() or Pluck()
-  bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE;
+  CallOpSendInitialMetadata() : send_(false) {}
 
-  void set_max_message_size(int max_message_size) {
-    max_message_size_ = max_message_size;
+  void SendInitialMetadata(
+      const std::multimap<grpc::string, grpc::string>& metadata) {
+    send_ = true;
+    initial_metadata_count_ = metadata.size();
+    initial_metadata_ = FillMetadataArray(metadata);
   }
 
-  bool got_message;
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (!send_) return;
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_SEND_INITIAL_METADATA;
+    op->flags = 0;
+    op->data.send_initial_metadata.count = initial_metadata_count_;
+    op->data.send_initial_metadata.metadata = initial_metadata_;
+  }
+  void FinishOp(bool* status, int max_message_size) {
+    if (!send_) return;
+    gpr_free(initial_metadata_);
+    send_ = false;
+  }
 
- private:
-  void* return_tag_;
-  // Send initial metadata
-  bool send_initial_metadata_;
+  bool send_;
   size_t initial_metadata_count_;
   grpc_metadata* initial_metadata_;
-  // Recv initial metadta
-  std::multimap<grpc::string, grpc::string>* recv_initial_metadata_;
-  grpc_metadata_array recv_initial_metadata_arr_;
-  // Send message
-  const grpc::protobuf::Message* send_message_;
-  const ByteBuffer* send_message_buffer_;
+};
+
+class CallOpSendMessage {
+ public:
+  CallOpSendMessage() : send_buf_(nullptr), own_buf_(false) {}
+
+  template <class M>
+  Status SendMessage(const M& message) GRPC_MUST_USE_RESULT;
+
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (send_buf_ == nullptr) return;
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_SEND_MESSAGE;
+    op->flags = 0;
+    op->data.send_message = send_buf_;
+  }
+  void FinishOp(bool* status, int max_message_size) {
+    if (own_buf_) grpc_byte_buffer_destroy(send_buf_);
+    send_buf_ = nullptr;
+  }
+
+ private:
   grpc_byte_buffer* send_buf_;
-  // Recv message
-  grpc::protobuf::Message* recv_message_;
-  ByteBuffer* recv_message_buffer_;
+  bool own_buf_;
+};
+
+template <class M>
+Status CallOpSendMessage::SendMessage(const M& message) {
+  return SerializationTraits<M>::Serialize(message, &send_buf_, &own_buf_);
+}
+
+template <class R>
+class CallOpRecvMessage {
+ public:
+  CallOpRecvMessage() : got_message(false), message_(nullptr) {}
+
+  void RecvMessage(R* message) { message_ = message; }
+
+  bool got_message;
+
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (message_ == nullptr) return;
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_RECV_MESSAGE;
+    op->flags = 0;
+    op->data.recv_message = &recv_buf_;
+  }
+
+  void FinishOp(bool* status, int max_message_size) {
+    if (message_ == nullptr) return;
+    if (recv_buf_) {
+      if (*status) {
+        got_message = true;
+        *status = SerializationTraits<R>::Deserialize(recv_buf_, message_,
+                                                      max_message_size)
+                      .ok();
+      } else {
+        got_message = false;
+        grpc_byte_buffer_destroy(recv_buf_);
+      }
+    } else {
+      got_message = false;
+      *status = false;
+    }
+    message_ = nullptr;
+  }
+
+ private:
+  R* message_;
   grpc_byte_buffer* recv_buf_;
-  int max_message_size_;
-  // Client send close
-  bool client_send_close_;
-  // Client recv status
+};
+
+class CallOpGenericRecvMessage {
+ public:
+  CallOpGenericRecvMessage() : got_message(false) {}
+
+  template <class R>
+  void RecvMessage(R* message) {
+    deserialize_ = [message](grpc_byte_buffer* buf,
+                             int max_message_size) -> Status {
+      return SerializationTraits<R>::Deserialize(buf, message,
+                                                 max_message_size);
+    };
+  }
+
+  bool got_message;
+
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (!deserialize_) return;
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_RECV_MESSAGE;
+    op->flags = 0;
+    op->data.recv_message = &recv_buf_;
+  }
+
+  void FinishOp(bool* status, int max_message_size) {
+    if (!deserialize_) return;
+    if (recv_buf_) {
+      if (*status) {
+        got_message = true;
+        *status = deserialize_(recv_buf_, max_message_size).ok();
+      } else {
+        got_message = false;
+        grpc_byte_buffer_destroy(recv_buf_);
+      }
+    } else {
+      got_message = false;
+      *status = false;
+    }
+    deserialize_ = DeserializeFunc();
+  }
+
+ private:
+  typedef std::function<Status(grpc_byte_buffer*, int)> DeserializeFunc;
+  DeserializeFunc deserialize_;
+  grpc_byte_buffer* recv_buf_;
+};
+
+class CallOpClientSendClose {
+ public:
+  CallOpClientSendClose() : send_(false) {}
+
+  void ClientSendClose() { send_ = true; }
+
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (!send_) return;
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
+    op->flags = 0;
+  }
+  void FinishOp(bool* status, int max_message_size) { send_ = false; }
+
+ private:
+  bool send_;
+};
+
+class CallOpServerSendStatus {
+ public:
+  CallOpServerSendStatus() : send_status_available_(false) {}
+
+  void ServerSendStatus(
+      const std::multimap<grpc::string, grpc::string>& trailing_metadata,
+      const Status& status) {
+    trailing_metadata_count_ = trailing_metadata.size();
+    trailing_metadata_ = FillMetadataArray(trailing_metadata);
+    send_status_available_ = true;
+    send_status_code_ = static_cast<grpc_status_code>(status.error_code());
+    send_status_details_ = status.error_message();
+  }
+
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (!send_status_available_) return;
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
+    op->data.send_status_from_server.trailing_metadata_count =
+        trailing_metadata_count_;
+    op->data.send_status_from_server.trailing_metadata = trailing_metadata_;
+    op->data.send_status_from_server.status = send_status_code_;
+    op->data.send_status_from_server.status_details =
+        send_status_details_.empty() ? nullptr : send_status_details_.c_str();
+    op->flags = 0;
+  }
+
+  void FinishOp(bool* status, int max_message_size) {
+    if (!send_status_available_) return;
+    gpr_free(trailing_metadata_);
+    send_status_available_ = false;
+  }
+
+ private:
+  bool send_status_available_;
+  grpc_status_code send_status_code_;
+  grpc::string send_status_details_;
+  size_t trailing_metadata_count_;
+  grpc_metadata* trailing_metadata_;
+};
+
+class CallOpRecvInitialMetadata {
+ public:
+  CallOpRecvInitialMetadata() : recv_initial_metadata_(nullptr) {}
+
+  void RecvInitialMetadata(ClientContext* context) {
+    context->initial_metadata_received_ = true;
+    recv_initial_metadata_ = &context->recv_initial_metadata_;
+  }
+
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (!recv_initial_metadata_) return;
+    memset(&recv_initial_metadata_arr_, 0, sizeof(recv_initial_metadata_arr_));
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_RECV_INITIAL_METADATA;
+    op->data.recv_initial_metadata = &recv_initial_metadata_arr_;
+    op->flags = 0;
+  }
+  void FinishOp(bool* status, int max_message_size) {
+    if (recv_initial_metadata_ == nullptr) return;
+    FillMetadataMap(&recv_initial_metadata_arr_, recv_initial_metadata_);
+    recv_initial_metadata_ = nullptr;
+  }
+
+ private:
+  std::multimap<grpc::string, grpc::string>* recv_initial_metadata_;
+  grpc_metadata_array recv_initial_metadata_arr_;
+};
+
+class CallOpClientRecvStatus {
+ public:
+  CallOpClientRecvStatus() : recv_status_(nullptr) {}
+
+  void ClientRecvStatus(ClientContext* context, Status* status) {
+    recv_trailing_metadata_ = &context->trailing_metadata_;
+    recv_status_ = status;
+  }
+
+ protected:
+  void AddOp(grpc_op* ops, size_t* nops) {
+    if (recv_status_ == nullptr) return;
+    memset(&recv_trailing_metadata_arr_, 0,
+           sizeof(recv_trailing_metadata_arr_));
+    status_details_ = nullptr;
+    status_details_capacity_ = 0;
+    grpc_op* op = &ops[(*nops)++];
+    op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
+    op->data.recv_status_on_client.trailing_metadata =
+        &recv_trailing_metadata_arr_;
+    op->data.recv_status_on_client.status = &status_code_;
+    op->data.recv_status_on_client.status_details = &status_details_;
+    op->data.recv_status_on_client.status_details_capacity =
+        &status_details_capacity_;
+    op->flags = 0;
+  }
+
+  void FinishOp(bool* status, int max_message_size) {
+    if (recv_status_ == nullptr) return;
+    FillMetadataMap(&recv_trailing_metadata_arr_, recv_trailing_metadata_);
+    *recv_status_ = Status(
+        static_cast<StatusCode>(status_code_),
+        status_details_ ? grpc::string(status_details_) : grpc::string());
+    gpr_free(status_details_);
+    recv_status_ = nullptr;
+  }
+
+ private:
   std::multimap<grpc::string, grpc::string>* recv_trailing_metadata_;
   Status* recv_status_;
   grpc_metadata_array recv_trailing_metadata_arr_;
   grpc_status_code status_code_;
   char* status_details_;
   size_t status_details_capacity_;
-  // Server send status
-  bool send_status_available_;
-  grpc_status_code send_status_code_;
-  grpc::string send_status_details_;
-  size_t trailing_metadata_count_;
-  grpc_metadata* trailing_metadata_;
-  int cancelled_buf_;
-  bool* recv_closed_;
 };
 
-// SneakyCallOpBuffer does not post completions to the completion queue
-class SneakyCallOpBuffer GRPC_FINAL : public CallOpBuffer {
+/// An abstract collection of call ops, used to generate the
+/// grpc_call_op structure to pass down to the lower layers,
+/// and as it is-a CompletionQueueTag, also massages the final
+/// completion into the correct form for consumption in the C++
+/// API.
+class CallOpSetInterface : public CompletionQueueTag {
+ public:
+  CallOpSetInterface() : max_message_size_(0) {}
+  /// Fills in grpc_op, starting from ops[*nops] and moving
+  /// upwards.
+  virtual void FillOps(grpc_op* ops, size_t* nops) = 0;
+
+  void set_max_message_size(int max_message_size) {
+    max_message_size_ = max_message_size;
+  }
+
+ protected:
+  int max_message_size_;
+};
+
+/// Primary implementaiton of CallOpSetInterface.
+/// Since we cannot use variadic templates, we declare slots up to
+/// the maximum count of ops we'll need in a set. We leverage the
+/// empty base class optimization to slim this class (especially
+/// when there are many unused slots used). To avoid duplicate base classes,
+/// the template parmeter for CallNoOp is varied by argument position.
+template <class Op1 = CallNoOp<1>, class Op2 = CallNoOp<2>,
+          class Op3 = CallNoOp<3>, class Op4 = CallNoOp<4>,
+          class Op5 = CallNoOp<5>, class Op6 = CallNoOp<6>>
+class CallOpSet : public CallOpSetInterface,
+                  public Op1,
+                  public Op2,
+                  public Op3,
+                  public Op4,
+                  public Op5,
+                  public Op6 {
+ public:
+  CallOpSet() : return_tag_(this) {}
+  void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE {
+    this->Op1::AddOp(ops, nops);
+    this->Op2::AddOp(ops, nops);
+    this->Op3::AddOp(ops, nops);
+    this->Op4::AddOp(ops, nops);
+    this->Op5::AddOp(ops, nops);
+    this->Op6::AddOp(ops, nops);
+  }
+
+  bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE {
+    this->Op1::FinishOp(status, max_message_size_);
+    this->Op2::FinishOp(status, max_message_size_);
+    this->Op3::FinishOp(status, max_message_size_);
+    this->Op4::FinishOp(status, max_message_size_);
+    this->Op5::FinishOp(status, max_message_size_);
+    this->Op6::FinishOp(status, max_message_size_);
+    *tag = return_tag_;
+    return true;
+  }
+
+  void set_output_tag(void* return_tag) { return_tag_ = return_tag; }
+
+ private:
+  void* return_tag_;
+};
+
+/// A CallOpSet that does not post completions to the completion queue.
+///
+/// Allows hiding some completions that the C core must generate from
+/// C++ users.
+template <class Op1 = CallNoOp<1>, class Op2 = CallNoOp<2>,
+          class Op3 = CallNoOp<3>, class Op4 = CallNoOp<4>,
+          class Op5 = CallNoOp<5>, class Op6 = CallNoOp<6>>
+class SneakyCallOpSet GRPC_FINAL
+    : public CallOpSet<Op1, Op2, Op3, Op4, Op5, Op6> {
  public:
   bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE {
-    return CallOpBuffer::FinalizeResult(tag, status) && false;
+    typedef CallOpSet<Op1, Op2, Op3, Op4, Op5, Op6> Base;
+    return Base::FinalizeResult(tag, status) && false;
   }
 };
 
@@ -135,7 +439,7 @@ class SneakyCallOpBuffer GRPC_FINAL : public CallOpBuffer {
 class CallHook {
  public:
   virtual ~CallHook() {}
-  virtual void PerformOpsOnCall(CallOpBuffer* ops, Call* call) = 0;
+  virtual void PerformOpsOnCall(CallOpSetInterface* ops, Call* call) = 0;
 };
 
 // Straightforward wrapping of the C call object
@@ -146,7 +450,7 @@ class Call GRPC_FINAL {
   Call(grpc_call* call, CallHook* call_hook_, CompletionQueue* cq,
        int max_message_size);
 
-  void PerformOps(CallOpBuffer* buffer);
+  void PerformOps(CallOpSetInterface* ops);
 
   grpc_call* call() { return call_; }
   CompletionQueue* cq() { return cq_; }

include/grpc++/impl/client_unary_call.h

@@ -37,6 +37,8 @@
 #include <grpc++/config.h>
 #include <grpc++/status.h>
 
+#include <grpc++/impl/call.h>
+
 namespace grpc {
 
 class ChannelInterface;
@@ -45,10 +47,28 @@ class CompletionQueue;
 class RpcMethod;
 
 // Wrapper that performs a blocking unary call
+template <class InputMessage, class OutputMessage>
 Status BlockingUnaryCall(ChannelInterface* channel, const RpcMethod& method,
-                         ClientContext* context,
-                         const grpc::protobuf::Message& request,
-                         grpc::protobuf::Message* result);
+                         ClientContext* context, const InputMessage& request,
+                         OutputMessage* result) {
+  CompletionQueue cq;
+  Call call(channel->CreateCall(method, context, &cq));
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+            CallOpRecvInitialMetadata, CallOpRecvMessage<OutputMessage>,
+            CallOpClientSendClose, CallOpClientRecvStatus> ops;
+  Status status = ops.SendMessage(request);
+  if (!status.ok()) {
+    return status;
+  }
+  ops.SendInitialMetadata(context->send_initial_metadata_);
+  ops.RecvInitialMetadata(context);
+  ops.RecvMessage(result);
+  ops.ClientSendClose();
+  ops.ClientRecvStatus(context, &status);
+  call.PerformOps(&ops);
+  GPR_ASSERT((cq.Pluck(&ops) && ops.got_message) || !status.ok());
+  return status;
+}
 
 }  // namespace grpc
 

include/grpc++/impl/proto_utils.h

@@ -0,0 +1,76 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
+#define GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
+
+#include <type_traits>
+
+#include <grpc/grpc.h>
+#include <grpc++/impl/serialization_traits.h>
+#include <grpc++/config_protobuf.h>
+#include <grpc++/status.h>
+
+namespace grpc {
+
+// Serialize the msg into a buffer created inside the function. The caller
+// should destroy the returned buffer when done with it. If serialization fails,
+// false is returned and buffer is left unchanged.
+Status SerializeProto(const grpc::protobuf::Message& msg,
+                      grpc_byte_buffer** buffer);
+
+// The caller keeps ownership of buffer and msg.
+Status DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
+                        int max_message_size);
+
+template <class T>
+class SerializationTraits<T, typename std::enable_if<std::is_base_of<
+                                 grpc::protobuf::Message, T>::value>::type> {
+ public:
+  static Status Serialize(const grpc::protobuf::Message& msg,
+                          grpc_byte_buffer** buffer, bool* own_buffer) {
+    *own_buffer = true;
+    return SerializeProto(msg, buffer);
+  }
+  static Status Deserialize(grpc_byte_buffer* buffer,
+                            grpc::protobuf::Message* msg,
+                            int max_message_size) {
+    auto status = DeserializeProto(buffer, msg, max_message_size);
+    grpc_byte_buffer_destroy(buffer);
+    return status;
+  }
+};
+
+}  // namespace grpc
+
+#endif  // GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H

include/grpc++/impl/rpc_service_method.h

@@ -55,16 +55,19 @@ class MethodHandler {
  public:
   virtual ~MethodHandler() {}
   struct HandlerParameter {
-    HandlerParameter(Call* c, ServerContext* context,
-                     const grpc::protobuf::Message* req,
-                     grpc::protobuf::Message* resp)
-        : call(c), server_context(context), request(req), response(resp) {}
+    HandlerParameter(Call* c, ServerContext* context, grpc_byte_buffer* req,
+                     int max_size)
+        : call(c),
+          server_context(context),
+          request(req),
+          max_message_size(max_size) {}
     Call* call;
     ServerContext* server_context;
-    const grpc::protobuf::Message* request;
-    grpc::protobuf::Message* response;
+    // Handler required to grpc_byte_buffer_destroy this
+    grpc_byte_buffer* request;
+    int max_message_size;
   };
-  virtual Status RunHandler(const HandlerParameter& param) = 0;
+  virtual void RunHandler(const HandlerParameter& param) = 0;
 };
 
 // A wrapper class of an application provided rpc method handler.
@@ -77,11 +80,25 @@ class RpcMethodHandler : public MethodHandler {
       ServiceType* service)
       : func_(func), service_(service) {}
 
-  Status RunHandler(const HandlerParameter& param) GRPC_FINAL {
-    // Invoke application function, cast proto messages to their actual types.
-    return func_(service_, param.server_context,
-                 dynamic_cast<const RequestType*>(param.request),
-                 dynamic_cast<ResponseType*>(param.response));
+  void RunHandler(const HandlerParameter& param) GRPC_FINAL {
+    RequestType req;
+    Status status = SerializationTraits<RequestType>::Deserialize(
+        param.request, &req, param.max_message_size);
+    ResponseType rsp;
+    if (status.ok()) {
+      status = func_(service_, param.server_context, &req, &rsp);
+    }
+
+    GPR_ASSERT(!param.server_context->sent_initial_metadata_);
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+              CallOpServerSendStatus> ops;
+    ops.SendInitialMetadata(param.server_context->initial_metadata_);
+    if (status.ok()) {
+      status = ops.SendMessage(rsp);
+    }
+    ops.ServerSendStatus(param.server_context->trailing_metadata_, status);
+    param.call->PerformOps(&ops);
+    param.call->cq()->Pluck(&ops);
   }
 
  private:
@@ -102,10 +119,21 @@ class ClientStreamingHandler : public MethodHandler {
       ServiceType* service)
       : func_(func), service_(service) {}
 
-  Status RunHandler(const HandlerParameter& param) GRPC_FINAL {
+  void RunHandler(const HandlerParameter& param) GRPC_FINAL {
     ServerReader<RequestType> reader(param.call, param.server_context);
-    return func_(service_, param.server_context, &reader,
-                 dynamic_cast<ResponseType*>(param.response));
+    ResponseType rsp;
+    Status status = func_(service_, param.server_context, &reader, &rsp);
+
+    GPR_ASSERT(!param.server_context->sent_initial_metadata_);
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+              CallOpServerSendStatus> ops;
+    ops.SendInitialMetadata(param.server_context->initial_metadata_);
+    if (status.ok()) {
+      status = ops.SendMessage(rsp);
+    }
+    ops.ServerSendStatus(param.server_context->trailing_metadata_, status);
+    param.call->PerformOps(&ops);
+    param.call->cq()->Pluck(&ops);
   }
 
  private:
@@ -124,10 +152,23 @@ class ServerStreamingHandler : public MethodHandler {
       ServiceType* service)
       : func_(func), service_(service) {}
 
-  Status RunHandler(const HandlerParameter& param) GRPC_FINAL {
-    ServerWriter<ResponseType> writer(param.call, param.server_context);
-    return func_(service_, param.server_context,
-                 dynamic_cast<const RequestType*>(param.request), &writer);
+  void RunHandler(const HandlerParameter& param) GRPC_FINAL {
+    RequestType req;
+    Status status = SerializationTraits<RequestType>::Deserialize(
+        param.request, &req, param.max_message_size);
+
+    if (status.ok()) {
+      ServerWriter<ResponseType> writer(param.call, param.server_context);
+      status = func_(service_, param.server_context, &req, &writer);
+    }
+
+    CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> ops;
+    if (!param.server_context->sent_initial_metadata_) {
+      ops.SendInitialMetadata(param.server_context->initial_metadata_);
+    }
+    ops.ServerSendStatus(param.server_context->trailing_metadata_, status);
+    param.call->PerformOps(&ops);
+    param.call->cq()->Pluck(&ops);
   }
 
  private:
@@ -147,10 +188,18 @@ class BidiStreamingHandler : public MethodHandler {
       ServiceType* service)
       : func_(func), service_(service) {}
 
-  Status RunHandler(const HandlerParameter& param) GRPC_FINAL {
+  void RunHandler(const HandlerParameter& param) GRPC_FINAL {
     ServerReaderWriter<ResponseType, RequestType> stream(param.call,
                                                          param.server_context);
-    return func_(service_, param.server_context, &stream);
+    Status status = func_(service_, param.server_context, &stream);
+
+    CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> ops;
+    if (!param.server_context->sent_initial_metadata_) {
+      ops.SendInitialMetadata(param.server_context->initial_metadata_);
+    }
+    ops.ServerSendStatus(param.server_context->trailing_metadata_, status);
+    param.call->PerformOps(&ops);
+    param.call->cq()->Pluck(&ops);
   }
 
  private:
@@ -162,29 +211,15 @@ class BidiStreamingHandler : public MethodHandler {
 // Server side rpc method class
 class RpcServiceMethod : public RpcMethod {
  public:
-  // Takes ownership of the handler and two prototype objects.
+  // Takes ownership of the handler
   RpcServiceMethod(const char* name, RpcMethod::RpcType type,
-                   MethodHandler* handler,
-                   grpc::protobuf::Message* request_prototype,
-                   grpc::protobuf::Message* response_prototype)
-      : RpcMethod(name, type, nullptr),
-        handler_(handler),
-        request_prototype_(request_prototype),
-        response_prototype_(response_prototype) {}
+                   MethodHandler* handler)
+      : RpcMethod(name, type, nullptr), handler_(handler) {}
 
   MethodHandler* handler() { return handler_.get(); }
 
-  grpc::protobuf::Message* AllocateRequestProto() {
-    return request_prototype_->New();
-  }
-  grpc::protobuf::Message* AllocateResponseProto() {
-    return response_prototype_->New();
-  }
-
  private:
   std::unique_ptr<MethodHandler> handler_;
-  std::unique_ptr<grpc::protobuf::Message> request_prototype_;
-  std::unique_ptr<grpc::protobuf::Message> response_prototype_;
 };
 
 // This class contains all the method information for an rpc service. It is

include/grpc++/impl/serialization_traits.h

@@ -0,0 +1,68 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPCXX_IMPL_SERIALIZATION_TRAITS_H
+#define GRPCXX_IMPL_SERIALIZATION_TRAITS_H
+
+namespace grpc {
+
+/// Defines how to serialize and deserialize some type.
+/// 
+/// Used for hooking different message serialization API's into GRPC.
+/// Each SerializationTraits implementation must provide the following
+/// functions:
+///   static Status Serialize(const Message& msg,
+///                           grpc_byte_buffer** buffer, 
+//                            bool* own_buffer);
+///   static Status Deserialize(grpc_byte_buffer* buffer,
+///                             Message* msg,
+///                             int max_message_size);
+///
+/// Serialize is required to convert message to a grpc_byte_buffer, and
+/// to store a pointer to that byte buffer at *buffer. *own_buffer should
+/// be set to true if the caller owns said byte buffer, or false if
+/// ownership is retained elsewhere.
+///
+/// Deserialize is required to convert buffer into the message stored at
+/// msg. max_message_size is passed in as a bound on the maximum number of
+/// message bytes Deserialize should accept.
+///
+/// Both functions return a Status, allowing them to explain what went 
+/// wrong if required.
+template <class Message,
+          class UnusedButHereForPartialTemplateSpecialization = void>
+class SerializationTraits;
+
+}  // namespace grpc
+
+#endif  // GRPCXX_IMPL_SERIALIZATION_TRAITS_H

include/grpc++/impl/service_type.h

@@ -35,6 +35,8 @@
 #define GRPCXX_IMPL_SERVICE_TYPE_H
 
 #include <grpc++/config.h>
+#include <grpc++/impl/serialization_traits.h>
+#include <grpc++/server.h>
 #include <grpc++/status.h>
 
 namespace grpc {
@@ -65,20 +67,8 @@ class ServerAsyncStreamingInterface {
 
 class AsynchronousService {
  public:
-  // this is Server, but in disguise to avoid a link dependency
-  class DispatchImpl {
-   public:
-    virtual void RequestAsyncCall(void* registered_method,
-                                  ServerContext* context,
-                                  ::grpc::protobuf::Message* request,
-                                  ServerAsyncStreamingInterface* stream,
-                                  CompletionQueue* call_cq,
-                                  ServerCompletionQueue* notification_cq,
-                                  void* tag) = 0;
-  };
-
   AsynchronousService(const char** method_names, size_t method_count)
-      : dispatch_impl_(nullptr),
+      : server_(nullptr),
         method_names_(method_names),
         method_count_(method_count),
         request_args_(nullptr) {}
@@ -86,42 +76,43 @@ class AsynchronousService {
   ~AsynchronousService() { delete[] request_args_; }
 
  protected:
-  void RequestAsyncUnary(int index, ServerContext* context,
-                         grpc::protobuf::Message* request,
+  template <class Message>
+  void RequestAsyncUnary(int index, ServerContext* context, Message* request,
                          ServerAsyncStreamingInterface* stream,
                          CompletionQueue* call_cq,
                          ServerCompletionQueue* notification_cq, void* tag) {
-    dispatch_impl_->RequestAsyncCall(request_args_[index], context, request,
-                                     stream, call_cq, notification_cq, tag);
+    server_->RequestAsyncCall(request_args_[index], context, stream, call_cq,
+                              notification_cq, tag, request);
   }
   void RequestClientStreaming(int index, ServerContext* context,
                               ServerAsyncStreamingInterface* stream,
                               CompletionQueue* call_cq,
                               ServerCompletionQueue* notification_cq,
                               void* tag) {
-    dispatch_impl_->RequestAsyncCall(request_args_[index], context, nullptr,
-                                     stream, call_cq, notification_cq, tag);
+    server_->RequestAsyncCall(request_args_[index], context, stream, call_cq,
+                              notification_cq, tag);
   }
+  template <class Message>
   void RequestServerStreaming(int index, ServerContext* context,
-                              grpc::protobuf::Message* request,
+                              Message* request,
                               ServerAsyncStreamingInterface* stream,
                               CompletionQueue* call_cq,
                               ServerCompletionQueue* notification_cq,
                               void* tag) {
-    dispatch_impl_->RequestAsyncCall(request_args_[index], context, request,
-                                     stream, call_cq, notification_cq, tag);
+    server_->RequestAsyncCall(request_args_[index], context, stream, call_cq,
+                              notification_cq, tag, request);
   }
   void RequestBidiStreaming(int index, ServerContext* context,
                             ServerAsyncStreamingInterface* stream,
                             CompletionQueue* call_cq,
                             ServerCompletionQueue* notification_cq, void* tag) {
-    dispatch_impl_->RequestAsyncCall(request_args_[index], context, nullptr,
-                                     stream, call_cq, notification_cq, tag);
+    server_->RequestAsyncCall(request_args_[index], context, stream, call_cq,
+                              notification_cq, tag);
   }
 
  private:
   friend class Server;
-  DispatchImpl* dispatch_impl_;
+  Server* server_;
   const char** const method_names_;
   size_t method_count_;
   void** request_args_;

include/grpc++/server.h

@@ -41,25 +41,24 @@
 #include <grpc++/config.h>
 #include <grpc++/impl/call.h>
 #include <grpc++/impl/grpc_library.h>
-#include <grpc++/impl/service_type.h>
 #include <grpc++/impl/sync.h>
 #include <grpc++/status.h>
 
 struct grpc_server;
 
 namespace grpc {
+
 class AsynchronousService;
 class GenericServerContext;
 class AsyncGenericService;
 class RpcService;
 class RpcServiceMethod;
+class ServerAsyncStreamingInterface;
 class ServerCredentials;
 class ThreadPoolInterface;
 
 // Currently it only supports handling rpcs in a single thread.
-class Server GRPC_FINAL : public GrpcLibrary,
-                          private CallHook,
-                          private AsynchronousService::DispatchImpl {
+class Server GRPC_FINAL : public GrpcLibrary, private CallHook {
  public:
   ~Server();
 
@@ -73,6 +72,7 @@ class Server GRPC_FINAL : public GrpcLibrary,
 
  private:
   friend class AsyncGenericService;
+  friend class AsynchronousService;
   friend class ServerBuilder;
 
   class SyncRequest;
@@ -96,21 +96,123 @@ class Server GRPC_FINAL : public GrpcLibrary,
   void RunRpc();
   void ScheduleCallback();
 
-  void PerformOpsOnCall(CallOpBuffer* ops, Call* call) GRPC_OVERRIDE;
+  void PerformOpsOnCall(CallOpSetInterface* ops, Call* call) GRPC_OVERRIDE;
+
+  class BaseAsyncRequest : public CompletionQueueTag {
+   public:
+    BaseAsyncRequest(Server* server, ServerContext* context,
+                     ServerAsyncStreamingInterface* stream,
+                     CompletionQueue* call_cq, void* tag);
+    virtual ~BaseAsyncRequest();
+
+    bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE;
+
+   protected:
+    Server* const server_;
+    ServerContext* const context_;
+    ServerAsyncStreamingInterface* const stream_;
+    CompletionQueue* const call_cq_;
+    void* const tag_;
+    grpc_call* call_;
+    grpc_metadata_array initial_metadata_array_;
+  };
+
+  class RegisteredAsyncRequest : public BaseAsyncRequest {
+   public:
+    RegisteredAsyncRequest(Server* server, ServerContext* context,
+                           ServerAsyncStreamingInterface* stream,
+                           CompletionQueue* call_cq, void* tag);
+
+    // uses BaseAsyncRequest::FinalizeResult
+
+   protected:
+    void IssueRequest(void* registered_method, grpc_byte_buffer** payload,
+                      ServerCompletionQueue* notification_cq);
+  };
+
+  class NoPayloadAsyncRequest GRPC_FINAL : public RegisteredAsyncRequest {
+   public:
+    NoPayloadAsyncRequest(void* registered_method, Server* server,
+                          ServerContext* context,
+                          ServerAsyncStreamingInterface* stream,
+                          CompletionQueue* call_cq,
+                          ServerCompletionQueue* notification_cq, void* tag)
+        : RegisteredAsyncRequest(server, context, stream, call_cq, tag) {
+      IssueRequest(registered_method, nullptr, notification_cq);
+    }
+
+    // uses RegisteredAsyncRequest::FinalizeResult
+  };
+
+  template <class Message>
+  class PayloadAsyncRequest GRPC_FINAL : public RegisteredAsyncRequest {
+   public:
+    PayloadAsyncRequest(void* registered_method, Server* server,
+                        ServerContext* context,
+                        ServerAsyncStreamingInterface* stream,
+                        CompletionQueue* call_cq,
+                        ServerCompletionQueue* notification_cq, void* tag,
+                        Message* request)
+        : RegisteredAsyncRequest(server, context, stream, call_cq, tag),
+          request_(request) {
+      IssueRequest(registered_method, &payload_, notification_cq);
+    }
+
+    bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE {
+      bool serialization_status =
+          *status && payload_ &&
+          SerializationTraits<Message>::Deserialize(payload_, request_,
+                                                    server_->max_message_size_)
+              .ok();
+      bool ret = RegisteredAsyncRequest::FinalizeResult(tag, status);
+      *status = serialization_status && *status;
+      return ret;
+    }
+
+   private:
+    grpc_byte_buffer* payload_;
+    Message* const request_;
+  };
+
+  class GenericAsyncRequest GRPC_FINAL : public BaseAsyncRequest {
+   public:
+    GenericAsyncRequest(Server* server, GenericServerContext* context,
+                        ServerAsyncStreamingInterface* stream,
+                        CompletionQueue* call_cq,
+                        ServerCompletionQueue* notification_cq, void* tag);
+
+    bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE;
+
+   private:
+    grpc_call_details call_details_;
+  };
+
+  template <class Message>
+  void RequestAsyncCall(void* registered_method, ServerContext* context,
+                        ServerAsyncStreamingInterface* stream,
+                        CompletionQueue* call_cq,
+                        ServerCompletionQueue* notification_cq, void* tag,
+                        Message* message) {
+    new PayloadAsyncRequest<Message>(registered_method, this, context, stream,
+                                     call_cq, notification_cq, tag, message);
+  }
 
-  // DispatchImpl
   void RequestAsyncCall(void* registered_method, ServerContext* context,
-                        grpc::protobuf::Message* request,
                         ServerAsyncStreamingInterface* stream,
                         CompletionQueue* call_cq,
-                        ServerCompletionQueue* notification_cq,
-                        void* tag) GRPC_OVERRIDE;
+                        ServerCompletionQueue* notification_cq, void* tag) {
+    new NoPayloadAsyncRequest(registered_method, this, context, stream, call_cq,
+                              notification_cq, tag);
+  }
 
   void RequestAsyncGenericCall(GenericServerContext* context,
                                ServerAsyncStreamingInterface* stream,
-                               CompletionQueue* cq,
+                               CompletionQueue* call_cq,
                                ServerCompletionQueue* notification_cq,
-                               void* tag);
+                               void* tag) {
+    new GenericAsyncRequest(this, context, stream, call_cq, notification_cq,
+                            tag);
+  }
 
   const int max_message_size_;
 
@@ -133,8 +235,6 @@ class Server GRPC_FINAL : public GrpcLibrary,
   ThreadPoolInterface* thread_pool_;
   // Whether the thread pool is created and owned by the server.
   bool thread_pool_owned_;
- private:
-  Server() : max_message_size_(-1), server_(NULL) { abort(); }
 };
 
 }  // namespace grpc

include/grpc++/server_context.h

@@ -60,6 +60,14 @@ template <class W>
 class ServerWriter;
 template <class R, class W>
 class ServerReaderWriter;
+template <class ServiceType, class RequestType, class ResponseType>
+class RpcMethodHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class ClientStreamingHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class ServerStreamingHandler;
+template <class ServiceType, class RequestType, class ResponseType>
+class BidiStreamingHandler;
 
 class Call;
 class CallOpBuffer;
@@ -105,6 +113,14 @@ class ServerContext {
   friend class ::grpc::ServerWriter;
   template <class R, class W>
   friend class ::grpc::ServerReaderWriter;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class RpcMethodHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class ClientStreamingHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class ServerStreamingHandler;
+  template <class ServiceType, class RequestType, class ResponseType>
+  friend class BidiStreamingHandler;
 
   // Prevent copying.
   ServerContext(const ServerContext&);

include/grpc++/stream.h

@@ -93,15 +93,18 @@ template <class R>
 class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
  public:
   // Blocking create a stream and write the first request out.
+  template <class W>
   ClientReader(ChannelInterface* channel, const RpcMethod& method,
-               ClientContext* context, const grpc::protobuf::Message& request)
+               ClientContext* context, const W& request)
       : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
-    CallOpBuffer buf;
-    buf.AddSendInitialMetadata(&context->send_initial_metadata_);
-    buf.AddSendMessage(request);
-    buf.AddClientSendClose();
-    call_.PerformOps(&buf);
-    cq_.Pluck(&buf);
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+              CallOpClientSendClose> ops;
+    ops.SendInitialMetadata(context->send_initial_metadata_);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(ops.SendMessage(request).ok());
+    ops.ClientSendClose();
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);
   }
 
   // Blocking wait for initial metadata from server. The received metadata
@@ -111,28 +114,28 @@ class ClientReader GRPC_FINAL : public ClientReaderInterface<R> {
   void WaitForInitialMetadata() {
     GPR_ASSERT(!context_->initial_metadata_received_);
 
-    CallOpBuffer buf;
-    buf.AddRecvInitialMetadata(context_);
-    call_.PerformOps(&buf);
-    cq_.Pluck(&buf);  // status ignored
+    CallOpSet<CallOpRecvInitialMetadata> ops;
+    ops.RecvInitialMetadata(context_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);  // status ignored
   }
 
   bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
+    CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> ops;
     if (!context_->initial_metadata_received_) {
-      buf.AddRecvInitialMetadata(context_);
+      ops.RecvInitialMetadata(context_);
     }
-    buf.AddRecvMessage(msg);
-    call_.PerformOps(&buf);
-    return cq_.Pluck(&buf) && buf.got_message;
+    ops.RecvMessage(msg);
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops) && ops.got_message;
   }
 
   Status Finish() GRPC_OVERRIDE {
-    CallOpBuffer buf;
+    CallOpSet<CallOpClientRecvStatus> ops;
     Status status;
-    buf.AddClientRecvStatus(context_, &status);
-    call_.PerformOps(&buf);
-    GPR_ASSERT(cq_.Pluck(&buf));
+    ops.ClientRecvStatus(context_, &status);
+    call_.PerformOps(&ops);
+    GPR_ASSERT(cq_.Pluck(&ops));
     return status;
   }
 
@@ -150,48 +153,49 @@ class ClientWriterInterface : public ClientStreamingInterface,
 };
 
 template <class W>
-class ClientWriter GRPC_FINAL : public ClientWriterInterface<W> {
+class ClientWriter : public ClientWriterInterface<W> {
  public:
   // Blocking create a stream.
+  template <class R>
   ClientWriter(ChannelInterface* channel, const RpcMethod& method,
-               ClientContext* context, grpc::protobuf::Message* response)
-      : context_(context),
-        response_(response),
-        call_(channel->CreateCall(method, context, &cq_)) {
-    CallOpBuffer buf;
-    buf.AddSendInitialMetadata(&context->send_initial_metadata_);
-    call_.PerformOps(&buf);
-    cq_.Pluck(&buf);
+               ClientContext* context, R* response)
+      : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
+    finish_ops_.RecvMessage(response);
+
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);
   }
 
   bool Write(const W& msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
-    buf.AddSendMessage(msg);
-    call_.PerformOps(&buf);
-    return cq_.Pluck(&buf);
+    CallOpSet<CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg).ok()) {
+      return false;
+    }
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
   }
 
   bool WritesDone() GRPC_OVERRIDE {
-    CallOpBuffer buf;
-    buf.AddClientSendClose();
-    call_.PerformOps(&buf);
-    return cq_.Pluck(&buf);
+    CallOpSet<CallOpClientSendClose> ops;
+    ops.ClientSendClose();
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
   }
 
   // Read the final response and wait for the final status.
   Status Finish() GRPC_OVERRIDE {
-    CallOpBuffer buf;
     Status status;
-    buf.AddRecvMessage(response_);
-    buf.AddClientRecvStatus(context_, &status);
-    call_.PerformOps(&buf);
-    GPR_ASSERT(cq_.Pluck(&buf));
+    finish_ops_.ClientRecvStatus(context_, &status);
+    call_.PerformOps(&finish_ops_);
+    GPR_ASSERT(cq_.Pluck(&finish_ops_));
     return status;
   }
 
  private:
   ClientContext* context_;
-  grpc::protobuf::Message* const response_;
+  CallOpSet<CallOpGenericRecvMessage, CallOpClientRecvStatus> finish_ops_;
   CompletionQueue cq_;
   Call call_;
 };
@@ -213,10 +217,10 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
   ClientReaderWriter(ChannelInterface* channel, const RpcMethod& method,
                      ClientContext* context)
       : context_(context), call_(channel->CreateCall(method, context, &cq_)) {
-    CallOpBuffer buf;
-    buf.AddSendInitialMetadata(&context->send_initial_metadata_);
-    call_.PerformOps(&buf);
-    cq_.Pluck(&buf);
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);
   }
 
   // Blocking wait for initial metadata from server. The received metadata
@@ -226,42 +230,42 @@ class ClientReaderWriter GRPC_FINAL : public ClientReaderWriterInterface<W, R> {
   void WaitForInitialMetadata() {
     GPR_ASSERT(!context_->initial_metadata_received_);
 
-    CallOpBuffer buf;
-    buf.AddRecvInitialMetadata(context_);
-    call_.PerformOps(&buf);
-    cq_.Pluck(&buf);  // status ignored
+    CallOpSet<CallOpRecvInitialMetadata> ops;
+    ops.RecvInitialMetadata(context_);
+    call_.PerformOps(&ops);
+    cq_.Pluck(&ops);  // status ignored
   }
 
   bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
+    CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> ops;
     if (!context_->initial_metadata_received_) {
-      buf.AddRecvInitialMetadata(context_);
+      ops.RecvInitialMetadata(context_);
     }
-    buf.AddRecvMessage(msg);
-    call_.PerformOps(&buf);
-    return cq_.Pluck(&buf) && buf.got_message;
+    ops.RecvMessage(msg);
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops) && ops.got_message;
   }
 
   bool Write(const W& msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
-    buf.AddSendMessage(msg);
-    call_.PerformOps(&buf);
-    return cq_.Pluck(&buf);
+    CallOpSet<CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg).ok()) return false;
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
   }
 
   bool WritesDone() GRPC_OVERRIDE {
-    CallOpBuffer buf;
-    buf.AddClientSendClose();
-    call_.PerformOps(&buf);
-    return cq_.Pluck(&buf);
+    CallOpSet<CallOpClientSendClose> ops;
+    ops.ClientSendClose();
+    call_.PerformOps(&ops);
+    return cq_.Pluck(&ops);
   }
 
   Status Finish() GRPC_OVERRIDE {
-    CallOpBuffer buf;
+    CallOpSet<CallOpClientRecvStatus> ops;
     Status status;
-    buf.AddClientRecvStatus(context_, &status);
-    call_.PerformOps(&buf);
-    GPR_ASSERT(cq_.Pluck(&buf));
+    ops.ClientRecvStatus(context_, &status);
+    call_.PerformOps(&ops);
+    GPR_ASSERT(cq_.Pluck(&ops));
     return status;
   }
 
@@ -279,18 +283,18 @@ class ServerReader GRPC_FINAL : public ReaderInterface<R> {
   void SendInitialMetadata() {
     GPR_ASSERT(!ctx_->sent_initial_metadata_);
 
-    CallOpBuffer buf;
-    buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(ctx_->initial_metadata_);
     ctx_->sent_initial_metadata_ = true;
-    call_->PerformOps(&buf);
-    call_->cq()->Pluck(&buf);
+    call_->PerformOps(&ops);
+    call_->cq()->Pluck(&ops);
   }
 
   bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
-    buf.AddRecvMessage(msg);
-    call_->PerformOps(&buf);
-    return call_->cq()->Pluck(&buf) && buf.got_message;
+    CallOpSet<CallOpRecvMessage<R>> ops;
+    ops.RecvMessage(msg);
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops) && ops.got_message;
   }
 
  private:
@@ -306,22 +310,24 @@ class ServerWriter GRPC_FINAL : public WriterInterface<W> {
   void SendInitialMetadata() {
     GPR_ASSERT(!ctx_->sent_initial_metadata_);
 
-    CallOpBuffer buf;
-    buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(ctx_->initial_metadata_);
     ctx_->sent_initial_metadata_ = true;
-    call_->PerformOps(&buf);
-    call_->cq()->Pluck(&buf);
+    call_->PerformOps(&ops);
+    call_->cq()->Pluck(&ops);
   }
 
   bool Write(const W& msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg).ok()) {
+      return false;
+    }
     if (!ctx_->sent_initial_metadata_) {
-      buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ops.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    buf.AddSendMessage(msg);
-    call_->PerformOps(&buf);
-    return call_->cq()->Pluck(&buf);
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops);
   }
 
  private:
@@ -339,29 +345,31 @@ class ServerReaderWriter GRPC_FINAL : public WriterInterface<W>,
   void SendInitialMetadata() {
     GPR_ASSERT(!ctx_->sent_initial_metadata_);
 
-    CallOpBuffer buf;
-    buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    CallOpSet<CallOpSendInitialMetadata> ops;
+    ops.SendInitialMetadata(ctx_->initial_metadata_);
     ctx_->sent_initial_metadata_ = true;
-    call_->PerformOps(&buf);
-    call_->cq()->Pluck(&buf);
+    call_->PerformOps(&ops);
+    call_->cq()->Pluck(&ops);
   }
 
   bool Read(R* msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
-    buf.AddRecvMessage(msg);
-    call_->PerformOps(&buf);
-    return call_->cq()->Pluck(&buf) && buf.got_message;
+    CallOpSet<CallOpRecvMessage<R>> ops;
+    ops.RecvMessage(msg);
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops) && ops.got_message;
   }
 
   bool Write(const W& msg) GRPC_OVERRIDE {
-    CallOpBuffer buf;
+    CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> ops;
+    if (!ops.SendMessage(msg).ok()) {
+      return false;
+    }
     if (!ctx_->sent_initial_metadata_) {
-      buf.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      ops.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    buf.AddSendMessage(msg);
-    call_->PerformOps(&buf);
-    return call_->cq()->Pluck(&buf);
+    call_->PerformOps(&ops);
+    return call_->cq()->Pluck(&ops);
   }
 
  private:
@@ -400,57 +408,59 @@ class AsyncWriterInterface {
 
 template <class R>
 class ClientAsyncReaderInterface : public ClientAsyncStreamingInterface,
-                                   public AsyncReaderInterface<R> {
-};
+                                   public AsyncReaderInterface<R> {};
 
 template <class R>
 class ClientAsyncReader GRPC_FINAL : public ClientAsyncReaderInterface<R> {
  public:
   // Create a stream and write the first request out.
+  template <class W>
   ClientAsyncReader(ChannelInterface* channel, CompletionQueue* cq,
                     const RpcMethod& method, ClientContext* context,
-                    const grpc::protobuf::Message& request, void* tag)
+                    const W& request, void* tag)
       : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    init_buf_.Reset(tag);
-    init_buf_.AddSendInitialMetadata(&context->send_initial_metadata_);
-    init_buf_.AddSendMessage(request);
-    init_buf_.AddClientSendClose();
-    call_.PerformOps(&init_buf_);
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(init_ops_.SendMessage(request).ok());
+    init_ops_.ClientSendClose();
+    call_.PerformOps(&init_ops_);
   }
 
   void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
     GPR_ASSERT(!context_->initial_metadata_received_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddRecvInitialMetadata(context_);
-    call_.PerformOps(&meta_buf_);
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
   }
 
   void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_buf_.Reset(tag);
+    read_ops_.set_output_tag(tag);
     if (!context_->initial_metadata_received_) {
-      read_buf_.AddRecvInitialMetadata(context_);
+      read_ops_.RecvInitialMetadata(context_);
     }
-    read_buf_.AddRecvMessage(msg);
-    call_.PerformOps(&read_buf_);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
   }
 
   void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_buf_.Reset(tag);
+    finish_ops_.set_output_tag(tag);
     if (!context_->initial_metadata_received_) {
-      finish_buf_.AddRecvInitialMetadata(context_);
+      finish_ops_.RecvInitialMetadata(context_);
     }
-    finish_buf_.AddClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_buf_);
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
   }
 
  private:
   ClientContext* context_;
   Call call_;
-  CallOpBuffer init_buf_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer read_buf_;
-  CallOpBuffer finish_buf_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage, CallOpClientSendClose>
+      init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
 };
 
 template <class W>
@@ -463,56 +473,57 @@ class ClientAsyncWriterInterface : public ClientAsyncStreamingInterface,
 template <class W>
 class ClientAsyncWriter GRPC_FINAL : public ClientAsyncWriterInterface<W> {
  public:
+  template <class R>
   ClientAsyncWriter(ChannelInterface* channel, CompletionQueue* cq,
                     const RpcMethod& method, ClientContext* context,
-                    grpc::protobuf::Message* response, void* tag)
-      : context_(context),
-        response_(response),
-        call_(channel->CreateCall(method, context, cq)) {
-    init_buf_.Reset(tag);
-    init_buf_.AddSendInitialMetadata(&context->send_initial_metadata_);
-    call_.PerformOps(&init_buf_);
+                    R* response, void* tag)
+      : context_(context), call_(channel->CreateCall(method, context, cq)) {
+    finish_ops_.RecvMessage(response);
+
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&init_ops_);
   }
 
   void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
     GPR_ASSERT(!context_->initial_metadata_received_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddRecvInitialMetadata(context_);
-    call_.PerformOps(&meta_buf_);
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
   }
 
   void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_buf_.Reset(tag);
-    write_buf_.AddSendMessage(msg);
-    call_.PerformOps(&write_buf_);
+    write_ops_.set_output_tag(tag);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
   }
 
   void WritesDone(void* tag) GRPC_OVERRIDE {
-    writes_done_buf_.Reset(tag);
-    writes_done_buf_.AddClientSendClose();
-    call_.PerformOps(&writes_done_buf_);
+    writes_done_ops_.set_output_tag(tag);
+    writes_done_ops_.ClientSendClose();
+    call_.PerformOps(&writes_done_ops_);
   }
 
   void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_buf_.Reset(tag);
+    finish_ops_.set_output_tag(tag);
     if (!context_->initial_metadata_received_) {
-      finish_buf_.AddRecvInitialMetadata(context_);
+      finish_ops_.RecvInitialMetadata(context_);
     }
-    finish_buf_.AddRecvMessage(response_);
-    finish_buf_.AddClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_buf_);
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
   }
 
  private:
   ClientContext* context_;
-  grpc::protobuf::Message* const response_;
   Call call_;
-  CallOpBuffer init_buf_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer write_buf_;
-  CallOpBuffer writes_done_buf_;
-  CallOpBuffer finish_buf_;
+  CallOpSet<CallOpSendInitialMetadata> init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpClientSendClose> writes_done_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpGenericRecvMessage,
+            CallOpClientRecvStatus> finish_ops_;
 };
 
 // Client-side interface for bi-directional streaming.
@@ -532,58 +543,59 @@ class ClientAsyncReaderWriter GRPC_FINAL
                           const RpcMethod& method, ClientContext* context,
                           void* tag)
       : context_(context), call_(channel->CreateCall(method, context, cq)) {
-    init_buf_.Reset(tag);
-    init_buf_.AddSendInitialMetadata(&context->send_initial_metadata_);
-    call_.PerformOps(&init_buf_);
+    init_ops_.set_output_tag(tag);
+    init_ops_.SendInitialMetadata(context->send_initial_metadata_);
+    call_.PerformOps(&init_ops_);
   }
 
   void ReadInitialMetadata(void* tag) GRPC_OVERRIDE {
     GPR_ASSERT(!context_->initial_metadata_received_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddRecvInitialMetadata(context_);
-    call_.PerformOps(&meta_buf_);
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.RecvInitialMetadata(context_);
+    call_.PerformOps(&meta_ops_);
   }
 
   void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_buf_.Reset(tag);
+    read_ops_.set_output_tag(tag);
     if (!context_->initial_metadata_received_) {
-      read_buf_.AddRecvInitialMetadata(context_);
+      read_ops_.RecvInitialMetadata(context_);
     }
-    read_buf_.AddRecvMessage(msg);
-    call_.PerformOps(&read_buf_);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
   }
 
   void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_buf_.Reset(tag);
-    write_buf_.AddSendMessage(msg);
-    call_.PerformOps(&write_buf_);
+    write_ops_.set_output_tag(tag);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
   }
 
   void WritesDone(void* tag) GRPC_OVERRIDE {
-    writes_done_buf_.Reset(tag);
-    writes_done_buf_.AddClientSendClose();
-    call_.PerformOps(&writes_done_buf_);
+    writes_done_ops_.set_output_tag(tag);
+    writes_done_ops_.ClientSendClose();
+    call_.PerformOps(&writes_done_ops_);
   }
 
   void Finish(Status* status, void* tag) GRPC_OVERRIDE {
-    finish_buf_.Reset(tag);
+    finish_ops_.set_output_tag(tag);
     if (!context_->initial_metadata_received_) {
-      finish_buf_.AddRecvInitialMetadata(context_);
+      finish_ops_.RecvInitialMetadata(context_);
     }
-    finish_buf_.AddClientRecvStatus(context_, status);
-    call_.PerformOps(&finish_buf_);
+    finish_ops_.ClientRecvStatus(context_, status);
+    call_.PerformOps(&finish_ops_);
   }
 
  private:
   ClientContext* context_;
   Call call_;
-  CallOpBuffer init_buf_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer read_buf_;
-  CallOpBuffer write_buf_;
-  CallOpBuffer writes_done_buf_;
-  CallOpBuffer finish_buf_;
+  CallOpSet<CallOpSendInitialMetadata> init_ops_;
+  CallOpSet<CallOpRecvInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpClientSendClose> writes_done_ops_;
+  CallOpSet<CallOpRecvInitialMetadata, CallOpClientRecvStatus> finish_ops_;
 };
 
 template <class W, class R>
@@ -596,41 +608,44 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
     GPR_ASSERT(!ctx_->sent_initial_metadata_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
     ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_buf_);
+    call_.PerformOps(&meta_ops_);
   }
 
   void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_buf_.Reset(tag);
-    read_buf_.AddRecvMessage(msg);
-    call_.PerformOps(&read_buf_);
+    read_ops_.set_output_tag(tag);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
   }
 
   void Finish(const W& msg, const Status& status, void* tag) {
-    finish_buf_.Reset(tag);
+    finish_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
     // The response is dropped if the status is not OK.
     if (status.ok()) {
-      finish_buf_.AddSendMessage(msg);
+      finish_ops_.ServerSendStatus(
+          ctx_->trailing_metadata_,
+          finish_ops_.SendMessage(msg));
+    } else {
+      finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
     }
-    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_buf_);
+    call_.PerformOps(&finish_ops_);
   }
 
   void FinishWithError(const Status& status, void* tag) {
     GPR_ASSERT(!status.ok());
-    finish_buf_.Reset(tag);
+    finish_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_buf_);
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
   }
 
  private:
@@ -638,9 +653,10 @@ class ServerAsyncReader GRPC_FINAL : public ServerAsyncStreamingInterface,
 
   Call call_;
   ServerContext* ctx_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer read_buf_;
-  CallOpBuffer finish_buf_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage,
+            CallOpServerSendStatus> finish_ops_;
 };
 
 template <class W>
@@ -653,30 +669,31 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
     GPR_ASSERT(!ctx_->sent_initial_metadata_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
     ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_buf_);
+    call_.PerformOps(&meta_ops_);
   }
 
   void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_buf_.Reset(tag);
+    write_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      write_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    write_buf_.AddSendMessage(msg);
-    call_.PerformOps(&write_buf_);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
   }
 
   void Finish(const Status& status, void* tag) {
-    finish_buf_.Reset(tag);
+    finish_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_buf_);
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
   }
 
  private:
@@ -684,9 +701,9 @@ class ServerAsyncWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
 
   Call call_;
   ServerContext* ctx_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer write_buf_;
-  CallOpBuffer finish_buf_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
 };
 
 // Server-side interface for bi-directional streaming.
@@ -701,36 +718,37 @@ class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
   void SendInitialMetadata(void* tag) GRPC_OVERRIDE {
     GPR_ASSERT(!ctx_->sent_initial_metadata_);
 
-    meta_buf_.Reset(tag);
-    meta_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+    meta_ops_.set_output_tag(tag);
+    meta_ops_.SendInitialMetadata(ctx_->initial_metadata_);
     ctx_->sent_initial_metadata_ = true;
-    call_.PerformOps(&meta_buf_);
+    call_.PerformOps(&meta_ops_);
   }
 
   void Read(R* msg, void* tag) GRPC_OVERRIDE {
-    read_buf_.Reset(tag);
-    read_buf_.AddRecvMessage(msg);
-    call_.PerformOps(&read_buf_);
+    read_ops_.set_output_tag(tag);
+    read_ops_.RecvMessage(msg);
+    call_.PerformOps(&read_ops_);
   }
 
   void Write(const W& msg, void* tag) GRPC_OVERRIDE {
-    write_buf_.Reset(tag);
+    write_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      write_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      write_ops_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    write_buf_.AddSendMessage(msg);
-    call_.PerformOps(&write_buf_);
+    // TODO(ctiller): don't assert
+    GPR_ASSERT(write_ops_.SendMessage(msg).ok());
+    call_.PerformOps(&write_ops_);
   }
 
   void Finish(const Status& status, void* tag) {
-    finish_buf_.Reset(tag);
+    finish_ops_.set_output_tag(tag);
     if (!ctx_->sent_initial_metadata_) {
-      finish_buf_.AddSendInitialMetadata(&ctx_->initial_metadata_);
+      finish_ops_.SendInitialMetadata(ctx_->initial_metadata_);
       ctx_->sent_initial_metadata_ = true;
     }
-    finish_buf_.AddServerSendStatus(&ctx_->trailing_metadata_, status);
-    call_.PerformOps(&finish_buf_);
+    finish_ops_.ServerSendStatus(ctx_->trailing_metadata_, status);
+    call_.PerformOps(&finish_ops_);
   }
 
  private:
@@ -738,10 +756,10 @@ class ServerAsyncReaderWriter GRPC_FINAL : public ServerAsyncStreamingInterface,
 
   Call call_;
   ServerContext* ctx_;
-  CallOpBuffer meta_buf_;
-  CallOpBuffer read_buf_;
-  CallOpBuffer write_buf_;
-  CallOpBuffer finish_buf_;
+  CallOpSet<CallOpSendInitialMetadata> meta_ops_;
+  CallOpSet<CallOpRecvMessage<R>> read_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpSendMessage> write_ops_;
+  CallOpSet<CallOpSendInitialMetadata, CallOpServerSendStatus> finish_ops_;
 };
 
 }  // namespace grpc

include/grpc/grpc.h

@@ -396,7 +396,9 @@ grpc_call *grpc_channel_create_registered_call(
    completion of type 'tag' to the completion queue bound to the call.
    The order of ops specified in the batch has no significance.
    Only one operation of each type can be active at once in any given
-   batch. 
+   batch. You must call grpc_completion_queue_next or
+   grpc_completion_queue_pluck on the completion queue associated with 'call'
+   for work to be performed.
    THREAD SAFETY: access to grpc_call_start_batch in multi-threaded environment
    needs to be synchronized. As an optimization, you may synchronize batches
    containing just send operations independently from batches containing just

include/grpc/support/port_platform.h

@@ -223,7 +223,9 @@
 #endif
 
 /* Validate platform combinations */
-#if defined(GPR_GCC_ATOMIC) + defined(GPR_GCC_SYNC) + defined(GPR_WIN32_ATOMIC) != 1
+#if defined(GPR_GCC_ATOMIC) + defined(GPR_GCC_SYNC) + \
+        defined(GPR_WIN32_ATOMIC) !=                  \
+    1
 #error Must define exactly one of GPR_GCC_ATOMIC, GPR_GCC_SYNC, GPR_WIN32_ATOMIC
 #endif
 
@@ -231,7 +233,9 @@
 #error Must define exactly one of GPR_ARCH_32, GPR_ARCH_64
 #endif
 
-#if defined(GPR_CPU_LINUX) + defined(GPR_CPU_POSIX) + defined(GPR_WIN32) + defined(GPR_CPU_IPHONE) + defined(GPR_CPU_CUSTOM) != 1
+#if defined(GPR_CPU_LINUX) + defined(GPR_CPU_POSIX) + defined(GPR_WIN32) + \
+        defined(GPR_CPU_IPHONE) + defined(GPR_CPU_CUSTOM) !=               \
+    1
 #error Must define exactly one of GPR_CPU_LINUX, GPR_CPU_POSIX, GPR_WIN32, GPR_CPU_IPHONE, GPR_CPU_CUSTOM
 #endif
 
@@ -239,11 +243,15 @@
 #error Must define GPR_POSIX_SOCKET to use GPR_POSIX_MULTIPOLL_WITH_POLL
 #endif
 
-#if defined(GPR_POSIX_SOCKET) + defined(GPR_WINSOCK_SOCKET) + defined(GPR_CUSTOM_SOCKET) != 1
+#if defined(GPR_POSIX_SOCKET) + defined(GPR_WINSOCK_SOCKET) + \
+        defined(GPR_CUSTOM_SOCKET) !=                         \
+    1
 #error Must define exactly one of GPR_POSIX_SOCKET, GPR_WINSOCK_SOCKET, GPR_CUSTOM_SOCKET
 #endif
 
-#if defined(GPR_MSVC_TLS) + defined(GPR_GCC_TLS) + defined(GPR_PTHREAD_TLS) + defined(GPR_CUSTOM_TLS) != 1
+#if defined(GPR_MSVC_TLS) + defined(GPR_GCC_TLS) + defined(GPR_PTHREAD_TLS) + \
+        defined(GPR_CUSTOM_TLS) !=                                            \
+    1
 #error Must define exactly one of GPR_MSVC_TLS, GPR_GCC_TLS, GPR_PTHREAD_TLS, GPR_CUSTOM_TLS
 #endif
 
@@ -266,4 +274,12 @@ typedef uintptr_t gpr_uintptr;
    power of two */
 #define GPR_MAX_ALIGNMENT 16
 
-#endif  /* GRPC_SUPPORT_PORT_PLATFORM_H */
+#ifndef GRPC_MUST_USE_RESULT
+#ifdef __GNUC__
+#define GRPC_MUST_USE_RESULT __attribute__((warn_unused_result))
+#else
+#define GRPC_MUST_USE_RESULT
+#endif
+#endif
+
+#endif /* GRPC_SUPPORT_PORT_PLATFORM_H */

src/compiler/config.h

@@ -35,6 +35,7 @@
 #define SRC_COMPILER_CONFIG_H
 
 #include <grpc++/config.h>
+#include <grpc++/config_protobuf.h>
 
 #ifndef GRPC_CUSTOM_DESCRIPTOR
 #include <google/protobuf/descriptor.h>
@@ -48,7 +49,8 @@
 #ifndef GRPC_CUSTOM_CODEGENERATOR
 #include <google/protobuf/compiler/code_generator.h>
 #define GRPC_CUSTOM_CODEGENERATOR ::google::protobuf::compiler::CodeGenerator
-#define GRPC_CUSTOM_GENERATORCONTEXT ::google::protobuf::compiler::GeneratorContext
+#define GRPC_CUSTOM_GENERATORCONTEXT \
+  ::google::protobuf::compiler::GeneratorContext
 #endif
 
 #ifndef GRPC_CUSTOM_PRINTER
@@ -57,7 +59,8 @@
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #define GRPC_CUSTOM_PRINTER ::google::protobuf::io::Printer
 #define GRPC_CUSTOM_CODEDOUTPUTSTREAM ::google::protobuf::io::CodedOutputStream
-#define GRPC_CUSTOM_STRINGOUTPUTSTREAM ::google::protobuf::io::StringOutputStream
+#define GRPC_CUSTOM_STRINGOUTPUTSTREAM \
+  ::google::protobuf::io::StringOutputStream
 #endif
 
 #ifndef GRPC_CUSTOM_PLUGINMAIN

src/compiler/cpp_generator.cc

@@ -97,7 +97,8 @@ grpc::string GetHeaderPrologue(const grpc::protobuf::FileDescriptor *file,
     vars["filename_base"] = grpc_generator::StripProto(file->name());
 
     printer.Print(vars, "// Generated by the gRPC protobuf plugin.\n");
-    printer.Print(vars, "// If you make any local change, they will be lost.\n");
+    printer.Print(vars,
+                  "// If you make any local change, they will be lost.\n");
     printer.Print(vars, "// source: $filename$\n");
     printer.Print(vars, "#ifndef GRPC_$filename_identifier$__INCLUDED\n");
     printer.Print(vars, "#define GRPC_$filename_identifier$__INCLUDED\n");
@@ -113,6 +114,7 @@ grpc::string GetHeaderIncludes(const grpc::protobuf::FileDescriptor *file,
   grpc::string temp =
       "#include <grpc++/impl/internal_stub.h>\n"
       "#include <grpc++/impl/rpc_method.h>\n"
+      "#include <grpc++/impl/proto_utils.h>\n"
       "#include <grpc++/impl/service_type.h>\n"
       "#include <grpc++/async_unary_call.h>\n"
       "#include <grpc++/status.h>\n"
@@ -141,10 +143,10 @@ grpc::string GetHeaderIncludes(const grpc::protobuf::FileDescriptor *file,
   return temp;
 }
 
-void PrintHeaderClientMethodInterfaces(grpc::protobuf::io::Printer *printer,
-                             const grpc::protobuf::MethodDescriptor *method,
-                             std::map<grpc::string, grpc::string> *vars,
-                             bool is_public) {
+void PrintHeaderClientMethodInterfaces(
+    grpc::protobuf::io::Printer *printer,
+    const grpc::protobuf::MethodDescriptor *method,
+    std::map<grpc::string, grpc::string> *vars, bool is_public) {
   (*vars)["Method"] = method->name();
   (*vars)["Request"] =
       grpc_cpp_generator::ClassName(method->input_type(), true);
@@ -157,19 +159,17 @@ void PrintHeaderClientMethodInterfaces(grpc::protobuf::io::Printer *printer,
           *vars,
           "virtual ::grpc::Status $Method$(::grpc::ClientContext* context, "
           "const $Request$& request, $Response$* response) = 0;\n");
-      printer->Print(
-          *vars,
-          "std::unique_ptr< "
-          "::grpc::ClientAsyncResponseReaderInterface< $Response$>> "
-          "Async$Method$(::grpc::ClientContext* context, "
-          "const $Request$& request, "
-          "::grpc::CompletionQueue* cq) {\n");
+      printer->Print(*vars,
+                     "std::unique_ptr< "
+                     "::grpc::ClientAsyncResponseReaderInterface< $Response$>> "
+                     "Async$Method$(::grpc::ClientContext* context, "
+                     "const $Request$& request, "
+                     "::grpc::CompletionQueue* cq) {\n");
       printer->Indent();
-      printer->Print(
-          *vars,
-          "return std::unique_ptr< "
-          "::grpc::ClientAsyncResponseReaderInterface< $Response$>>("
-          "Async$Method$Raw(context, request, cq));\n");
+      printer->Print(*vars,
+                     "return std::unique_ptr< "
+                     "::grpc::ClientAsyncResponseReaderInterface< $Response$>>("
+                     "Async$Method$Raw(context, request, cq));\n");
       printer->Outdent();
       printer->Print("}\n");
     } else if (ClientOnlyStreaming(method)) {
@@ -188,14 +188,14 @@ void PrintHeaderClientMethodInterfaces(grpc::protobuf::io::Printer *printer,
       printer->Print(
           *vars,
           "std::unique_ptr< ::grpc::ClientAsyncWriterInterface< $Request$>>"
-          " Async$Method$(::grpc::ClientContext* context, $Response$* response, "
+          " Async$Method$(::grpc::ClientContext* context, $Response$* "
+          "response, "
           "::grpc::CompletionQueue* cq, void* tag) {\n");
       printer->Indent();
-      printer->Print(
-          *vars,
-          "return std::unique_ptr< "
-          "::grpc::ClientAsyncWriterInterface< $Request$>>("
-          "Async$Method$Raw(context, response, cq, tag));\n");
+      printer->Print(*vars,
+                     "return std::unique_ptr< "
+                     "::grpc::ClientAsyncWriterInterface< $Request$>>("
+                     "Async$Method$Raw(context, response, cq, tag));\n");
       printer->Outdent();
       printer->Print("}\n");
     } else if (ServerOnlyStreaming(method)) {
@@ -218,18 +218,17 @@ void PrintHeaderClientMethodInterfaces(grpc::protobuf::io::Printer *printer,
           "::grpc::ClientContext* context, const $Request$& request, "
           "::grpc::CompletionQueue* cq, void* tag) {\n");
       printer->Indent();
-      printer->Print(
-          *vars,
-          "return std::unique_ptr< "
-          "::grpc::ClientAsyncReaderInterface< $Response$>>("
-          "Async$Method$Raw(context, request, cq, tag));\n");
+      printer->Print(*vars,
+                     "return std::unique_ptr< "
+                     "::grpc::ClientAsyncReaderInterface< $Response$>>("
+                     "Async$Method$Raw(context, request, cq, tag));\n");
       printer->Outdent();
       printer->Print("}\n");
     } else if (BidiStreaming(method)) {
-      printer->Print(
-          *vars,
-          "std::unique_ptr< ::grpc::ClientReaderWriterInterface< $Request$, $Response$>> "
-          "$Method$(::grpc::ClientContext* context) {\n");
+      printer->Print(*vars,
+                     "std::unique_ptr< ::grpc::ClientReaderWriterInterface< "
+                     "$Request$, $Response$>> "
+                     "$Method$(::grpc::ClientContext* context) {\n");
       printer->Indent();
       printer->Print(
           *vars,
@@ -267,12 +266,11 @@ void PrintHeaderClientMethodInterfaces(grpc::protobuf::io::Printer *printer,
           "virtual ::grpc::ClientWriterInterface< $Request$>*"
           " $Method$Raw("
           "::grpc::ClientContext* context, $Response$* response) = 0;\n");
-      printer->Print(
-          *vars,
-          "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*"
-          " Async$Method$Raw(::grpc::ClientContext* context, "
-          "$Response$* response, "
-          "::grpc::CompletionQueue* cq, void* tag) = 0;\n");
+      printer->Print(*vars,
+                     "virtual ::grpc::ClientAsyncWriterInterface< $Request$>*"
+                     " Async$Method$Raw(::grpc::ClientContext* context, "
+                     "$Response$* response, "
+                     "::grpc::CompletionQueue* cq, void* tag) = 0;\n");
     } else if (ServerOnlyStreaming(method)) {
       printer->Print(
           *vars,
@@ -285,16 +283,15 @@ void PrintHeaderClientMethodInterfaces(grpc::protobuf::io::Printer *printer,
           "::grpc::ClientContext* context, const $Request$& request, "
           "::grpc::CompletionQueue* cq, void* tag) = 0;\n");
     } else if (BidiStreaming(method)) {
-      printer->Print(
-          *vars,
-          "virtual ::grpc::ClientReaderWriterInterface< $Request$, $Response$>* "
-          "$Method$Raw(::grpc::ClientContext* context) = 0;\n");
-      printer->Print(
-          *vars,
-          "virtual ::grpc::ClientAsyncReaderWriterInterface< "
-          "$Request$, $Response$>* "
-          "Async$Method$Raw(::grpc::ClientContext* context, "
-          "::grpc::CompletionQueue* cq, void* tag) = 0;\n");
+      printer->Print(*vars,
+                     "virtual ::grpc::ClientReaderWriterInterface< $Request$, "
+                     "$Response$>* "
+                     "$Method$Raw(::grpc::ClientContext* context) = 0;\n");
+      printer->Print(*vars,
+                     "virtual ::grpc::ClientAsyncReaderWriterInterface< "
+                     "$Request$, $Response$>* "
+                     "Async$Method$Raw(::grpc::ClientContext* context, "
+                     "::grpc::CompletionQueue* cq, void* tag) = 0;\n");
     }
   }
 }
@@ -321,11 +318,10 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer,
           "const $Request$& request, "
           "::grpc::CompletionQueue* cq) {\n");
       printer->Indent();
-      printer->Print(
-          *vars,
-          "return std::unique_ptr< "
-          "::grpc::ClientAsyncResponseReader< $Response$>>("
-          "Async$Method$Raw(context, request, cq));\n");
+      printer->Print(*vars,
+                     "return std::unique_ptr< "
+                     "::grpc::ClientAsyncResponseReader< $Response$>>("
+                     "Async$Method$Raw(context, request, cq));\n");
       printer->Outdent();
       printer->Print("}\n");
     } else if (ClientOnlyStreaming(method)) {
@@ -335,17 +331,16 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer,
           " $Method$("
           "::grpc::ClientContext* context, $Response$* response) {\n");
       printer->Indent();
-      printer->Print(
-          *vars,
-          "return std::unique_ptr< ::grpc::ClientWriter< $Request$>>"
-          "($Method$Raw(context, response));\n");
+      printer->Print(*vars,
+                     "return std::unique_ptr< ::grpc::ClientWriter< $Request$>>"
+                     "($Method$Raw(context, response));\n");
       printer->Outdent();
       printer->Print("}\n");
-      printer->Print(
-          *vars,
-          "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>"
-          " Async$Method$(::grpc::ClientContext* context, $Response$* response, "
-          "::grpc::CompletionQueue* cq, void* tag) {\n");
+      printer->Print(*vars,
+                     "std::unique_ptr< ::grpc::ClientAsyncWriter< $Request$>>"
+                     " Async$Method$(::grpc::ClientContext* context, "
+                     "$Response$* response, "
+                     "::grpc::CompletionQueue* cq, void* tag) {\n");
       printer->Indent();
       printer->Print(
           *vars,
@@ -385,53 +380,47 @@ void PrintHeaderClientMethod(grpc::protobuf::io::Printer *printer,
           "std::unique_ptr< ::grpc::ClientReaderWriter< $Request$, $Response$>>"
           " $Method$(::grpc::ClientContext* context) {\n");
       printer->Indent();
-      printer->Print(
-          *vars,
-          "return std::unique_ptr< "
-          "::grpc::ClientReaderWriter< $Request$, $Response$>>("
-          "$Method$Raw(context));\n");
+      printer->Print(*vars,
+                     "return std::unique_ptr< "
+                     "::grpc::ClientReaderWriter< $Request$, $Response$>>("
+                     "$Method$Raw(context));\n");
       printer->Outdent();
       printer->Print("}\n");
-      printer->Print(
-          *vars,
-          "std::unique_ptr<  ::grpc::ClientAsyncReaderWriter< "
-          "$Request$, $Response$>> "
-          "Async$Method$(::grpc::ClientContext* context, "
-          "::grpc::CompletionQueue* cq, void* tag) {\n");
+      printer->Print(*vars,
+                     "std::unique_ptr<  ::grpc::ClientAsyncReaderWriter< "
+                     "$Request$, $Response$>> "
+                     "Async$Method$(::grpc::ClientContext* context, "
+                     "::grpc::CompletionQueue* cq, void* tag) {\n");
       printer->Indent();
-      printer->Print(
-          *vars,
-          "return std::unique_ptr< "
-          "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>("
-          "Async$Method$Raw(context, cq, tag));\n");
+      printer->Print(*vars,
+                     "return std::unique_ptr< "
+                     "::grpc::ClientAsyncReaderWriter< $Request$, $Response$>>("
+                     "Async$Method$Raw(context, cq, tag));\n");
       printer->Outdent();
       printer->Print("}\n");
     }
   } else {
     if (NoStreaming(method)) {
-      printer->Print(
-          *vars,
-          "::grpc::ClientAsyncResponseReader< $Response$>* "
-          "Async$Method$Raw(::grpc::ClientContext* context, "
-          "const $Request$& request, "
-          "::grpc::CompletionQueue* cq) GRPC_OVERRIDE;\n");
+      printer->Print(*vars,
+                     "::grpc::ClientAsyncResponseReader< $Response$>* "
+                     "Async$Method$Raw(::grpc::ClientContext* context, "
+                     "const $Request$& request, "
+                     "::grpc::CompletionQueue* cq) GRPC_OVERRIDE;\n");
     } else if (ClientOnlyStreaming(method)) {
-      printer->Print(
-          *vars,
-          "::grpc::ClientWriter< $Request$>* $Method$Raw("
-          "::grpc::ClientContext* context, $Response$* response) "
-          "GRPC_OVERRIDE;\n");
+      printer->Print(*vars,
+                     "::grpc::ClientWriter< $Request$>* $Method$Raw("
+                     "::grpc::ClientContext* context, $Response$* response) "
+                     "GRPC_OVERRIDE;\n");
       printer->Print(
           *vars,
           "::grpc::ClientAsyncWriter< $Request$>* Async$Method$Raw("
           "::grpc::ClientContext* context, $Response$* response, "
           "::grpc::CompletionQueue* cq, void* tag) GRPC_OVERRIDE;\n");
     } else if (ServerOnlyStreaming(method)) {
-      printer->Print(
-          *vars,
-          "::grpc::ClientReader< $Response$>* $Method$Raw("
-          "::grpc::ClientContext* context, const $Request$& request)"
-          " GRPC_OVERRIDE;\n");
+      printer->Print(*vars,
+                     "::grpc::ClientReader< $Response$>* $Method$Raw("
+                     "::grpc::ClientContext* context, const $Request$& request)"
+                     " GRPC_OVERRIDE;\n");
       printer->Print(
           *vars,
           "::grpc::ClientAsyncReader< $Response$>* Async$Method$Raw("
@@ -629,7 +618,7 @@ grpc::string GetHeaderServices(const grpc::protobuf::FileDescriptor *file,
                                const Parameters &params) {
   grpc::string output;
   {
-	// Scope the output stream so it closes and finalizes output to the string.
+    // Scope the output stream so it closes and finalizes output to the string.
     grpc::protobuf::io::StringOutputStream output_stream(&output);
     grpc::protobuf::io::Printer printer(&output_stream, '$');
     std::map<grpc::string, grpc::string> vars;
@@ -693,7 +682,8 @@ grpc::string GetSourcePrologue(const grpc::protobuf::FileDescriptor *file,
     vars["filename_base"] = grpc_generator::StripProto(file->name());
 
     printer.Print(vars, "// Generated by the gRPC protobuf plugin.\n");
-    printer.Print(vars, "// If you make any local change, they will be lost.\n");
+    printer.Print(vars,
+                  "// If you make any local change, they will be lost.\n");
     printer.Print(vars, "// source: $filename$\n\n");
     printer.Print(vars, "#include \"$filename_base$.pb.h\"\n");
     printer.Print(vars, "#include \"$filename_base$.grpc.pb.h\"\n");
@@ -1056,8 +1046,7 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer,
           "    new ::grpc::RpcMethodHandler< $ns$$Service$::Service, "
           "$Request$, "
           "$Response$>(\n"
-          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this),\n"
-          "    new $Request$, new $Response$));\n");
+          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this)));\n");
     } else if (ClientOnlyStreaming(method)) {
       printer->Print(
           *vars,
@@ -1066,8 +1055,7 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer,
           "    ::grpc::RpcMethod::CLIENT_STREAMING,\n"
           "    new ::grpc::ClientStreamingHandler< "
           "$ns$$Service$::Service, $Request$, $Response$>(\n"
-          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this),\n"
-          "    new $Request$, new $Response$));\n");
+          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this)));\n");
     } else if (ServerOnlyStreaming(method)) {
       printer->Print(
           *vars,
@@ -1076,8 +1064,7 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer,
           "    ::grpc::RpcMethod::SERVER_STREAMING,\n"
           "    new ::grpc::ServerStreamingHandler< "
           "$ns$$Service$::Service, $Request$, $Response$>(\n"
-          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this),\n"
-          "    new $Request$, new $Response$));\n");
+          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this)));\n");
     } else if (BidiStreaming(method)) {
       printer->Print(
           *vars,
@@ -1086,8 +1073,7 @@ void PrintSourceService(grpc::protobuf::io::Printer *printer,
           "    ::grpc::RpcMethod::BIDI_STREAMING,\n"
           "    new ::grpc::BidiStreamingHandler< "
           "$ns$$Service$::Service, $Request$, $Response$>(\n"
-          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this),\n"
-          "    new $Request$, new $Response$));\n");
+          "        std::mem_fn(&$ns$$Service$::Service::$Method$), this)));\n");
     }
   }
   printer->Print("return service_;\n");

src/compiler/objective_c_generator.cc

@@ -68,18 +68,19 @@ void PrintMethodSignature(Printer *printer,
 
   printer->Print(vars, "- ($return_type$)$method_name$With");
   if (method->client_streaming()) {
-    printer->Print("RequestsWriter:(id<GRXWriter>)request");
+    printer->Print("RequestsWriter:(id<GRXWriter>)requestWriter");
   } else {
     printer->Print(vars, "Request:($request_class$ *)request");
   }
 
   // TODO(jcanizales): Put this on a new line and align colons.
-  // TODO(jcanizales): eventHandler for server streaming?
-  printer->Print(" handler:(void(^)(");
   if (method->server_streaming()) {
-    printer->Print("BOOL done, ");
+    printer->Print(vars, " eventHandler:(void(^)(BOOL done, "
+      "$response_class$ *response, NSError *error))eventHandler");
+  } else {
+    printer->Print(vars, " handler:(void(^)($response_class$ *response, "
+      "NSError *error))handler");
   }
-  printer->Print(vars, "$response_class$ *response, NSError *error))handler");
 }
 
 void PrintSimpleSignature(Printer *printer,
@@ -125,11 +126,15 @@ void PrintSimpleImplementation(Printer *printer,
   printer->Print("{\n");
   printer->Print(vars, "  [[self RPCTo$method_name$With");
   if (method->client_streaming()) {
-    printer->Print("RequestsWriter:request");
+    printer->Print("RequestsWriter:requestWriter");
   } else {
     printer->Print("Request:request");
   }
-  printer->Print(" handler:handler] start];\n");
+  if (method->server_streaming()) {
+    printer->Print(" eventHandler:eventHandler] start];\n");
+  } else {
+    printer->Print(" handler:handler] start];\n");
+  }
   printer->Print("}\n");
 }
 
@@ -141,7 +146,7 @@ void PrintAdvancedImplementation(Printer *printer,
 
   printer->Print("            requestsWriter:");
   if (method->client_streaming()) {
-    printer->Print("request\n");
+    printer->Print("requestWriter\n");
   } else {
     printer->Print("[GRXWriter writerWithValue:request]\n");
   }
@@ -150,7 +155,7 @@ void PrintAdvancedImplementation(Printer *printer,
 
   printer->Print("        responsesWriteable:[GRXWriteable ");
   if (method->server_streaming()) {
-    printer->Print("writeableWithStreamHandler:handler]];\n");
+    printer->Print("writeableWithStreamHandler:eventHandler]];\n");
   } else {
     printer->Print("writeableWithSingleValueHandler:handler]];\n");
   }

src/compiler/objective_c_plugin.cc

@@ -62,8 +62,8 @@ class ObjectiveCGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
     {
       // Generate .pbrpc.h
 
-      string imports = string("#import \"") + file_name + ".pbobjc.h\"\n"
-        "#import <gRPC/ProtoService.h>\n";
+      string imports = string("#import \"") + file_name + ".pbobjc.h\"\n\n"
+        "#import <ProtoRPC/ProtoService.h>\n";
 
       // TODO(jcanizales): Instead forward-declare the input and output types
       // and import the files in the .pbrpc.m
@@ -87,10 +87,10 @@ class ObjectiveCGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator {
     {
       // Generate .pbrpc.m
 
-      string imports = string("#import \"") + file_name + ".pbrpc.h\"\n"
-        "#import <gRPC/GRXWriteable.h>\n"
-        "#import <gRPC/GRXWriter+Immediate.h>\n"
-        "#import <gRPC/ProtoRPC.h>\n";
+      string imports = string("#import \"") + file_name + ".pbrpc.h\"\n\n"
+        "#import <ProtoRPC/ProtoRPC.h>\n"
+        "#import <RxLibrary/GRXWriteable.h>\n"
+        "#import <RxLibrary/GRXWriter+Immediate.h>\n";
 
       string definitions;
       for (int i = 0; i < file->service_count(); i++) {

src/compiler/python_generator.cc

@@ -42,7 +42,7 @@
 #include <tuple>
 #include <vector>
 
-#include "grpc++/config.h"
+#include <grpc++/config.h>
 #include "src/compiler/config.h"
 #include "src/compiler/generator_helpers.h"
 #include "src/compiler/python_generator.h"

src/core/census/context.c

@@ -31,7 +31,7 @@
  *
  */
 
-#include "context.h"
+#include "src/core/census/context.h"
 
 #include <string.h>
 #include <grpc/census.h>

src/core/channel/http_client_filter.c

@@ -43,8 +43,12 @@ typedef struct call_data {
 
   int got_initial_metadata;
   grpc_stream_op_buffer *recv_ops;
-  grpc_iomgr_closure *on_done_recv;
 
+  /** Closure to call when finished with the hc_on_recv hook */
+  grpc_iomgr_closure *on_done_recv;
+  /** Receive closures are chained: we inject this closure as the on_done_recv
+      up-call on transport_op, and remember to call our on_done_recv member
+      after handling it. */
   grpc_iomgr_closure hc_on_recv;
 } call_data;
 

src/core/channel/http_server_filter.c

@@ -47,7 +47,11 @@ typedef struct call_data {
   grpc_linked_mdelem status;
 
   grpc_stream_op_buffer *recv_ops;
+  /** Closure to call when finished with the hs_on_recv hook */
   grpc_iomgr_closure *on_done_recv;
+  /** Receive closures are chained: we inject this closure as the on_done_recv
+      up-call on transport_op, and remember to call our on_done_recv member
+      after handling it. */
   grpc_iomgr_closure hs_on_recv;
 } call_data;
 

src/core/iomgr/fd_posix.c

@@ -74,6 +74,7 @@ static void freelist_fd(grpc_fd *fd) {
   gpr_mu_lock(&fd_freelist_mu);
   fd->freelist_next = fd_freelist;
   fd_freelist = fd;
+  grpc_iomgr_unregister_object(&fd->iomgr_object);
   gpr_mu_unlock(&fd_freelist_mu);
 }
 
@@ -100,6 +101,7 @@ static grpc_fd *alloc_fd(int fd) {
       &r->inactive_watcher_root;
   r->freelist_next = NULL;
   r->read_watcher = r->write_watcher = NULL;
+  r->on_done_closure = NULL;
   return r;
 }
 
@@ -138,10 +140,6 @@ static void unref_by(grpc_fd *fd, int n) {
 #endif
   old = gpr_atm_full_fetch_add(&fd->refst, -n);
   if (old == n) {
-    if (fd->on_done_closure) {
-      grpc_iomgr_add_callback(fd->on_done_closure);
-    }
-    grpc_iomgr_unregister_object(&fd->iomgr_object);
     freelist_fd(fd);
   } else {
     GPR_ASSERT(old > n);
@@ -199,13 +197,24 @@ static void wake_all_watchers_locked(grpc_fd *fd) {
   }
 }
 
+static int has_watchers(grpc_fd *fd) {
+  return fd->read_watcher != NULL || fd->write_watcher != NULL || fd->inactive_watcher_root.next != &fd->inactive_watcher_root;
+}
+
 void grpc_fd_orphan(grpc_fd *fd, grpc_iomgr_closure *on_done,
                     const char *reason) {
   fd->on_done_closure = on_done;
   shutdown(fd->fd, SHUT_RDWR);
   REF_BY(fd, 1, reason); /* remove active status, but keep referenced */
   gpr_mu_lock(&fd->watcher_mu);
-  wake_all_watchers_locked(fd);
+  if (!has_watchers(fd)) {
+    close(fd->fd);
+    if (fd->on_done_closure) {
+      grpc_iomgr_add_callback(fd->on_done_closure);
+    }
+  } else {
+    wake_all_watchers_locked(fd);
+  }
   gpr_mu_unlock(&fd->watcher_mu);
   UNREF_BY(fd, 2, reason); /* drop the reference */
 }
@@ -354,14 +363,21 @@ gpr_uint32 grpc_fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
   GRPC_FD_REF(fd, "poll");
 
   gpr_mu_lock(&fd->watcher_mu);
+  /* if we are shutdown, then don't add to the watcher set */
+  if (gpr_atm_no_barrier_load(&fd->shutdown)) {
+    watcher->fd = NULL;
+    watcher->pollset = NULL;
+    gpr_mu_unlock(&fd->watcher_mu);
+    return 0;
+  }
   /* if there is nobody polling for read, but we need to, then start doing so */
-  if (!fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) {
+  if (read_mask && !fd->read_watcher && gpr_atm_acq_load(&fd->readst) > READY) {
     fd->read_watcher = watcher;
     mask |= read_mask;
   }
   /* if there is nobody polling for write, but we need to, then start doing so
    */
-  if (!fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) {
+  if (write_mask && !fd->write_watcher && gpr_atm_acq_load(&fd->writest) > READY) {
     fd->write_watcher = watcher;
     mask |= write_mask;
   }
@@ -383,6 +399,10 @@ void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) {
   int kick = 0;
   grpc_fd *fd = watcher->fd;
 
+  if (fd == NULL) {
+    return;
+  }
+
   gpr_mu_lock(&fd->watcher_mu);
   if (watcher == fd->read_watcher) {
     /* remove read watcher, kick if we still need a read */
@@ -404,6 +424,12 @@ void grpc_fd_end_poll(grpc_fd_watcher *watcher, int got_read, int got_write) {
   if (kick) {
     maybe_wake_one_watcher_locked(fd);
   }
+  if (grpc_fd_is_orphaned(fd) && !has_watchers(fd)) {
+    close(fd->fd);
+    if (fd->on_done_closure != NULL) {
+      grpc_iomgr_add_callback(fd->on_done_closure);
+    }
+  }
   gpr_mu_unlock(&fd->watcher_mu);
 
   GRPC_FD_UNREF(fd, "poll");

src/core/iomgr/iomgr.h

@@ -73,6 +73,8 @@ void grpc_iomgr_shutdown(void);
  * Can be called from within a callback or from anywhere else */
 void grpc_iomgr_add_callback(grpc_iomgr_closure *closure);
 
+/** As per grpc_iomgr_add_callback, with the ability to set the success
+    argument. */
 void grpc_iomgr_add_delayed_callback(grpc_iomgr_closure *iocb, int success);
 
 #endif  /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_H */

src/core/iomgr/pollset_multipoller_with_epoll.c

@@ -54,17 +54,25 @@ static void multipoll_with_epoll_pollset_add_fd(grpc_pollset *pollset,
   pollset_hdr *h = pollset->data.ptr;
   struct epoll_event ev;
   int err;
-
-  ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
-  ev.data.ptr = fd;
-  err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev);
-  if (err < 0) {
-    /* FDs may be added to a pollset multiple times, so EEXIST is normal. */
-    if (errno != EEXIST) {
-      gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd,
-              strerror(errno));
+  grpc_fd_watcher watcher;
+
+  /* We pretend to be polling whilst adding an fd to keep the fd from being
+     closed during the add. This may result in a spurious wakeup being assigned
+     to this pollset whilst adding, but that should be benign. */
+  GPR_ASSERT(grpc_fd_begin_poll(fd, pollset, 0, 0, &watcher) == 0);
+  if (watcher.fd != NULL) {
+    ev.events = EPOLLIN | EPOLLOUT | EPOLLET;
+    ev.data.ptr = fd;
+    err = epoll_ctl(h->epoll_fd, EPOLL_CTL_ADD, fd->fd, &ev);
+    if (err < 0) {
+      /* FDs may be added to a pollset multiple times, so EEXIST is normal. */
+      if (errno != EEXIST) {
+        gpr_log(GPR_ERROR, "epoll_ctl add for %d failed: %s", fd->fd,
+                strerror(errno));
+      }
     }
   }
+  grpc_fd_end_poll(&watcher, 0, 0);
 }
 
 static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset,
@@ -83,7 +91,7 @@ static void multipoll_with_epoll_pollset_del_fd(grpc_pollset *pollset,
 /* TODO(klempner): We probably want to turn this down a bit */
 #define GRPC_EPOLL_MAX_EVENTS 1000
 
-static int multipoll_with_epoll_pollset_maybe_work(
+static void multipoll_with_epoll_pollset_maybe_work(
     grpc_pollset *pollset, gpr_timespec deadline, gpr_timespec now,
     int allow_synchronous_callback) {
   struct epoll_event ep_ev[GRPC_EPOLL_MAX_EVENTS];
@@ -133,7 +141,6 @@ static int multipoll_with_epoll_pollset_maybe_work(
 
   gpr_mu_lock(&pollset->mu);
   pollset->counter -= 1;
-  return 1;
 }
 
 static void multipoll_with_epoll_pollset_finish_shutdown(

src/core/iomgr/pollset_multipoller_with_poll_posix.c

@@ -103,7 +103,7 @@ static void end_polling(grpc_pollset *pollset) {
   }
 }
 
-static int multipoll_with_poll_pollset_maybe_work(
+static void multipoll_with_poll_pollset_maybe_work(
     grpc_pollset *pollset, gpr_timespec deadline, gpr_timespec now,
     int allow_synchronous_callback) {
   int timeout;
@@ -126,7 +126,7 @@ static int multipoll_with_poll_pollset_maybe_work(
   kfd = grpc_pollset_kick_pre_poll(&pollset->kick_state);
   if (kfd == NULL) {
     /* Already kicked */
-    return 1;
+    return;
   }
   h->pfds[0].fd = GRPC_POLLSET_KICK_GET_FD(kfd);
   h->pfds[0].events = POLLIN;
@@ -154,7 +154,7 @@ static int multipoll_with_poll_pollset_maybe_work(
   h->del_count = 0;
   if (h->pfd_count == 0) {
     end_polling(pollset);
-    return 0;
+    return;
   }
   pollset->counter++;
   gpr_mu_unlock(&pollset->mu);
@@ -179,6 +179,9 @@ static int multipoll_with_poll_pollset_maybe_work(
       grpc_pollset_kick_consume(&pollset->kick_state, kfd);
     }
     for (i = 1; i < np; i++) {
+      if (h->watchers[i].fd == NULL) {
+        continue;
+      }
       if (h->pfds[i].revents & (POLLIN | POLLHUP | POLLERR)) {
         grpc_fd_become_readable(h->watchers[i].fd, allow_synchronous_callback);
       }
@@ -191,8 +194,6 @@ static int multipoll_with_poll_pollset_maybe_work(
 
   gpr_mu_lock(&pollset->mu);
   pollset->counter--;
-
-  return 1;
 }
 
 static void multipoll_with_poll_pollset_kick(grpc_pollset *p) {

src/core/iomgr/pollset_posix.c

@@ -123,7 +123,6 @@ static void finish_shutdown(grpc_pollset *pollset) {
 int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) {
   /* pollset->mu already held */
   gpr_timespec now = gpr_now();
-  int r;
   if (gpr_time_cmp(now, deadline) > 0) {
     return 0;
   }
@@ -137,7 +136,7 @@ int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) {
     return 1;
   }
   gpr_tls_set(&g_current_thread_poller, (gpr_intptr)pollset);
-  r = pollset->vtable->maybe_work(pollset, deadline, now, 1);
+  pollset->vtable->maybe_work(pollset, deadline, now, 1);
   gpr_tls_set(&g_current_thread_poller, 0);
   if (pollset->shutting_down) {
     if (pollset->counter > 0) {
@@ -153,7 +152,7 @@ int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) {
       gpr_mu_lock(&pollset->mu);
     }
   }
-  return r;
+  return 1;
 }
 
 void grpc_pollset_shutdown(grpc_pollset *pollset,
@@ -338,9 +337,9 @@ static void basic_pollset_del_fd(grpc_pollset *pollset, grpc_fd *fd) {
   }
 }
 
-static int basic_pollset_maybe_work(grpc_pollset *pollset,
-                                    gpr_timespec deadline, gpr_timespec now,
-                                    int allow_synchronous_callback) {
+static void basic_pollset_maybe_work(grpc_pollset *pollset,
+                                     gpr_timespec deadline, gpr_timespec now,
+                                     int allow_synchronous_callback) {
   struct pollfd pfd[2];
   grpc_fd *fd;
   grpc_fd_watcher fd_watcher;
@@ -353,7 +352,7 @@ static int basic_pollset_maybe_work(grpc_pollset *pollset,
     /* Give do_promote priority so we don't starve it out */
     gpr_mu_unlock(&pollset->mu);
     gpr_mu_lock(&pollset->mu);
-    return 1;
+    return;
   }
   fd = pollset->data.ptr;
   if (fd && grpc_fd_is_orphaned(fd)) {
@@ -364,7 +363,7 @@ static int basic_pollset_maybe_work(grpc_pollset *pollset,
   kfd = grpc_pollset_kick_pre_poll(&pollset->kick_state);
   if (kfd == NULL) {
     /* Already kicked */
-    return 1;
+    return;
   }
   pfd[0].fd = GRPC_POLLSET_KICK_GET_FD(kfd);
   pfd[0].events = POLLIN;
@@ -418,7 +417,6 @@ static int basic_pollset_maybe_work(grpc_pollset *pollset,
 
   gpr_mu_lock(&pollset->mu);
   pollset->counter--;
-  return 1;
 }
 
 static void basic_pollset_destroy(grpc_pollset *pollset) {

src/core/iomgr/pollset_posix.h

@@ -68,8 +68,8 @@ typedef struct grpc_pollset {
 struct grpc_pollset_vtable {
   void (*add_fd)(grpc_pollset *pollset, struct grpc_fd *fd);
   void (*del_fd)(grpc_pollset *pollset, struct grpc_fd *fd);
-  int (*maybe_work)(grpc_pollset *pollset, gpr_timespec deadline,
-                    gpr_timespec now, int allow_synchronous_callback);
+  void (*maybe_work)(grpc_pollset *pollset, gpr_timespec deadline,
+                     gpr_timespec now, int allow_synchronous_callback);
   void (*kick)(grpc_pollset *pollset);
   void (*finish_shutdown)(grpc_pollset *pollset);
   void (*destroy)(grpc_pollset *pollset);

src/core/iomgr/pollset_windows.c

@@ -39,6 +39,7 @@
 
 #include "src/core/iomgr/alarm_internal.h"
 #include "src/core/iomgr/iomgr_internal.h"
+#include "src/core/iomgr/pollset.h"
 #include "src/core/iomgr/pollset_windows.h"
 
 /* There isn't really any such thing as a pollset under Windows, due to the
@@ -47,6 +48,7 @@
    won't actually do any polling, and return as quickly as possible. */
 
 void grpc_pollset_init(grpc_pollset *pollset) {
+  memset(pollset, 0, sizeof(*pollset));
   gpr_mu_init(&pollset->mu);
   gpr_cv_init(&pollset->cv);
 }
@@ -54,6 +56,10 @@ void grpc_pollset_init(grpc_pollset *pollset) {
 void grpc_pollset_shutdown(grpc_pollset *pollset,
                            void (*shutdown_done)(void *arg),
                            void *shutdown_done_arg) {
+  gpr_mu_lock(&pollset->mu);
+  pollset->shutting_down = 1;
+  gpr_cv_broadcast(&pollset->cv);
+  gpr_mu_unlock(&pollset->mu);
   shutdown_done(shutdown_done_arg);
 }
 
@@ -68,13 +74,15 @@ int grpc_pollset_work(grpc_pollset *pollset, gpr_timespec deadline) {
   if (gpr_time_cmp(now, deadline) > 0) {
     return 0 /* GPR_FALSE */;
   }
-  if (grpc_maybe_call_delayed_callbacks(NULL, 1 /* GPR_TRUE */)) {
+  if (grpc_maybe_call_delayed_callbacks(&pollset->mu, 1 /* GPR_TRUE */)) {
     return 1 /* GPR_TRUE */;
   }
-  if (grpc_alarm_check(NULL, now, &deadline)) {
+  if (grpc_alarm_check(&pollset->mu, now, &deadline)) {
     return 1 /* GPR_TRUE */;
   }
-  gpr_cv_wait(&pollset->cv, &pollset->mu, deadline);
+  if (!pollset->shutting_down) {
+    gpr_cv_wait(&pollset->cv, &pollset->mu, deadline);
+  }
   return 1 /* GPR_TRUE */;
 }
 

src/core/iomgr/pollset_windows.h

@@ -46,6 +46,7 @@
 typedef struct grpc_pollset {
   gpr_mu mu;
   gpr_cv cv;
+  int shutting_down;
 } grpc_pollset;
 
 #define GRPC_POLLSET_MU(pollset) (&(pollset)->mu)

src/core/iomgr/resolve_address_posix.c

@@ -155,9 +155,9 @@ static void do_request(void *rp) {
   grpc_resolve_cb cb = r->cb;
   gpr_free(r->name);
   gpr_free(r->default_port);
+  cb(arg, resolved);
   grpc_iomgr_unregister_object(&r->iomgr_object);
   gpr_free(r);
-  cb(arg, resolved);
 }
 
 void grpc_resolved_addresses_destroy(grpc_resolved_addresses *addrs) {

src/core/security/base64.c

@@ -120,7 +120,68 @@ char *grpc_base64_encode(const void *vdata, size_t data_size, int url_safe,
 }
 
 gpr_slice grpc_base64_decode(const char *b64, int url_safe) {
-  size_t b64_len = strlen(b64);
+  return grpc_base64_decode_with_len(b64, strlen(b64), url_safe);
+}
+
+static void decode_one_char(const unsigned char *codes, unsigned char *result,
+                            size_t *result_offset) {
+  gpr_uint32 packed = (codes[0] << 2) | (codes[1] >> 4);
+  result[(*result_offset)++] = (unsigned char)packed;
+}
+
+static void decode_two_chars(const unsigned char *codes, unsigned char *result,
+                             size_t *result_offset) {
+  gpr_uint32 packed = (codes[0] << 10) | (codes[1] << 4) | (codes[2] >> 2);
+  result[(*result_offset)++] = (unsigned char)(packed >> 8);
+  result[(*result_offset)++] = (unsigned char)(packed);
+}
+
+static int decode_group(const unsigned char *codes, size_t num_codes,
+                        unsigned char *result, size_t *result_offset) {
+  GPR_ASSERT(num_codes <= 4);
+
+  /* Short end groups that may not have padding. */
+  if (num_codes == 1) {
+    gpr_log(GPR_ERROR, "Invalid group. Must be at least 2 bytes.");
+    return 0;
+  }
+  if (num_codes == 2) {
+    decode_one_char(codes, result, result_offset);
+    return 1;
+  }
+  if (num_codes == 3) {
+    decode_two_chars(codes, result, result_offset);
+    return 1;
+  }
+
+  /* Regular 4 byte groups with padding or not. */
+  GPR_ASSERT(num_codes == 4);
+  if (codes[0] == GRPC_BASE64_PAD_BYTE || codes[1] == GRPC_BASE64_PAD_BYTE) {
+    gpr_log(GPR_ERROR, "Invalid padding detected.");
+    return 0;
+  }
+  if (codes[2] == GRPC_BASE64_PAD_BYTE) {
+    if (codes[3] == GRPC_BASE64_PAD_BYTE) {
+      decode_one_char(codes, result, result_offset);
+    } else {
+      gpr_log(GPR_ERROR, "Invalid padding detected.");
+      return 0;
+    }
+  } else if (codes[3] == GRPC_BASE64_PAD_BYTE) {
+    decode_two_chars(codes, result, result_offset);
+  } else {
+    /* No padding. */
+    gpr_uint32 packed =
+        (codes[0] << 18) | (codes[1] << 12) | (codes[2] << 6) | codes[3];
+    result[(*result_offset)++] = (unsigned char)(packed >> 16);
+    result[(*result_offset)++] = (unsigned char)(packed >> 8);
+    result[(*result_offset)++] = (unsigned char)(packed);
+  }
+  return 1;
+}
+
+gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len,
+                                      int url_safe) {
   gpr_slice result = gpr_slice_malloc(b64_len);
   unsigned char *current = GPR_SLICE_START_PTR(result);
   size_t result_size = 0;
@@ -128,7 +189,7 @@ gpr_slice grpc_base64_decode(const char *b64, int url_safe) {
   size_t num_codes = 0;
 
   while (b64_len--) {
-    unsigned char c = *b64++;
+    unsigned char c = (unsigned char)(*b64++);
     signed char code;
     if (c >= GPR_ARRAY_SIZE(base64_bytes)) continue;
     if (url_safe) {
@@ -149,45 +210,17 @@ gpr_slice grpc_base64_decode(const char *b64, int url_safe) {
         goto fail;
       }
     } else {
-      codes[num_codes++] = code;
+      codes[num_codes++] = (unsigned char)code;
       if (num_codes == 4) {
-        if (codes[0] == GRPC_BASE64_PAD_BYTE ||
-            codes[1] == GRPC_BASE64_PAD_BYTE) {
-          gpr_log(GPR_ERROR, "Invalid padding detected.");
-          goto fail;
-        }
-        if (codes[2] == GRPC_BASE64_PAD_BYTE) {
-          if (codes[3] == GRPC_BASE64_PAD_BYTE) {
-            /* Double padding. */
-            gpr_uint32 packed = (codes[0] << 2) | (codes[1] >> 4);
-            current[result_size++] = (unsigned char)packed;
-          } else {
-            gpr_log(GPR_ERROR, "Invalid padding detected.");
-            goto fail;
-          }
-        } else if (codes[3] == GRPC_BASE64_PAD_BYTE) {
-          /* Single padding. */
-          gpr_uint32 packed =
-              (codes[0] << 10) | (codes[1] << 4) | (codes[2] >> 2);
-          current[result_size++] = (unsigned char)(packed >> 8);
-          current[result_size++] = (unsigned char)(packed);
-        } else {
-          /* No padding. */
-          gpr_uint32 packed =
-              (codes[0] << 18) | (codes[1] << 12) | (codes[2] << 6) | codes[3];
-          current[result_size++] = (unsigned char)(packed >> 16);
-          current[result_size++] = (unsigned char)(packed >> 8);
-          current[result_size++] = (unsigned char)(packed);
-        }
+        if (!decode_group(codes, num_codes, current, &result_size)) goto fail;
         num_codes = 0;
       }
     }
   }
 
-  if (num_codes != 0) {
-    gpr_log(GPR_ERROR, "Invalid base64.");
-    gpr_slice_unref(result);
-    return gpr_empty_slice();
+  if (num_codes != 0 &&
+      !decode_group(codes, num_codes, current, &result_size)) {
+    goto fail;
   }
   GPR_SLICE_SET_LENGTH(result, result_size);
   return result;

src/core/security/base64.h

@@ -45,4 +45,8 @@ char *grpc_base64_encode(const void *data, size_t data_size, int url_safe,
    slice in case of failure. */
 gpr_slice grpc_base64_decode(const char *b64, int url_safe);
 
+/* Same as above except that the length is provided by the caller. */
+gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len,
+                                      int url_safe);
+
 #endif  /* GRPC_INTERNAL_CORE_SECURITY_BASE64_H */

src/core/security/secure_endpoint.c

@@ -116,7 +116,7 @@ static void on_read(void *user_data, gpr_slice *slices, size_t nslices,
                     grpc_endpoint_cb_status error) {
   unsigned i;
   gpr_uint8 keep_looping = 0;
-  int input_buffer_count = 0;
+  size_t input_buffer_count = 0;
   tsi_result result = TSI_OK;
   secure_endpoint *ep = (secure_endpoint *)user_data;
   gpr_uint8 *cur = GPR_SLICE_START_PTR(ep->read_staging_buffer);
@@ -129,7 +129,7 @@ static void on_read(void *user_data, gpr_slice *slices, size_t nslices,
     size_t message_size = GPR_SLICE_LENGTH(encrypted);
 
     while (message_size > 0 || keep_looping) {
-      size_t unprotected_buffer_size_written = end - cur;
+      size_t unprotected_buffer_size_written = (size_t)(end - cur);
       size_t processed_message_size = message_size;
       gpr_mu_lock(&ep->protector_mu);
       result = tsi_frame_protector_unprotect(ep->protector, message_bytes,
@@ -166,7 +166,7 @@ static void on_read(void *user_data, gpr_slice *slices, size_t nslices,
         &ep->input_buffer,
         gpr_slice_split_head(
             &ep->read_staging_buffer,
-            cur - GPR_SLICE_START_PTR(ep->read_staging_buffer)));
+            (size_t)(cur - GPR_SLICE_START_PTR(ep->read_staging_buffer))));
   }
 
   /* TODO(yangg) experiment with moving this block after read_cb to see if it
@@ -225,7 +225,7 @@ static grpc_endpoint_write_status endpoint_write(grpc_endpoint *secure_ep,
                                                  grpc_endpoint_write_cb cb,
                                                  void *user_data) {
   unsigned i;
-  int output_buffer_count = 0;
+  size_t output_buffer_count = 0;
   tsi_result result = TSI_OK;
   secure_endpoint *ep = (secure_endpoint *)secure_ep;
   gpr_uint8 *cur = GPR_SLICE_START_PTR(ep->write_staging_buffer);
@@ -248,7 +248,7 @@ static grpc_endpoint_write_status endpoint_write(grpc_endpoint *secure_ep,
     gpr_uint8 *message_bytes = GPR_SLICE_START_PTR(plain);
     size_t message_size = GPR_SLICE_LENGTH(plain);
     while (message_size > 0) {
-      size_t protected_buffer_size_to_send = end - cur;
+      size_t protected_buffer_size_to_send = (size_t)(end - cur);
       size_t processed_message_size = message_size;
       gpr_mu_lock(&ep->protector_mu);
       result = tsi_frame_protector_protect(ep->protector, message_bytes,
@@ -273,7 +273,7 @@ static grpc_endpoint_write_status endpoint_write(grpc_endpoint *secure_ep,
   if (result == TSI_OK) {
     size_t still_pending_size;
     do {
-      size_t protected_buffer_size_to_send = end - cur;
+      size_t protected_buffer_size_to_send = (size_t)(end - cur);
       gpr_mu_lock(&ep->protector_mu);
       result = tsi_frame_protector_protect_flush(ep->protector, cur,
                                                  &protected_buffer_size_to_send,
@@ -290,7 +290,7 @@ static grpc_endpoint_write_status endpoint_write(grpc_endpoint *secure_ep,
           &ep->output_buffer,
           gpr_slice_split_head(
               &ep->write_staging_buffer,
-              cur - GPR_SLICE_START_PTR(ep->write_staging_buffer)));
+              (size_t)(cur - GPR_SLICE_START_PTR(ep->write_staging_buffer))));
     }
   }
 

src/core/security/secure_transport_setup.c

@@ -235,7 +235,7 @@ static void on_handshake_data_received_from_peer(
     gpr_slice_unref(slices[i]); /* split_tail above increments refcount. */
   }
   gpr_slice_buffer_addn(&s->left_overs, &slices[i + 1],
-                        num_left_overs - has_left_overs_in_current_slice);
+                        num_left_overs - (size_t)has_left_overs_in_current_slice);
   check_peer(s);
 }
 

src/core/security/security_connector.c

@@ -196,12 +196,12 @@ typedef struct {
 static void fake_channel_destroy(grpc_security_connector *sc) {
   grpc_channel_security_connector *c = (grpc_channel_security_connector *)sc;
   grpc_credentials_unref(c->request_metadata_creds);
-  grpc_auth_context_unref(sc->auth_context);
+  GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector");
   gpr_free(sc);
 }
 
 static void fake_server_destroy(grpc_security_connector *sc) {
-  grpc_auth_context_unref(sc->auth_context);
+  GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector");
   gpr_free(sc);
 }
 
@@ -242,7 +242,7 @@ static grpc_security_status fake_check_peer(grpc_security_connector *sc,
     status = GRPC_SECURITY_ERROR;
     goto end;
   }
-  grpc_auth_context_unref(sc->auth_context);
+  GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector");
   sc->auth_context = grpc_auth_context_create(NULL, 1);
   sc->auth_context->properties[0] = grpc_auth_property_init_from_cstring(
       GRPC_TRANSPORT_SECURITY_TYPE_PROPERTY_NAME,
@@ -323,7 +323,7 @@ static void ssl_channel_destroy(grpc_security_connector *sc) {
   if (c->target_name != NULL) gpr_free(c->target_name);
   if (c->overridden_target_name != NULL) gpr_free(c->overridden_target_name);
   tsi_peer_destruct(&c->peer);
-  grpc_auth_context_unref(sc->auth_context);
+  GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector");
   gpr_free(sc);
 }
 
@@ -333,7 +333,7 @@ static void ssl_server_destroy(grpc_security_connector *sc) {
   if (c->handshaker_factory != NULL) {
     tsi_ssl_handshaker_factory_destroy(c->handshaker_factory);
   }
-  grpc_auth_context_unref(sc->auth_context);
+  GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector");
   gpr_free(sc);
 }
 
@@ -437,6 +437,9 @@ static grpc_security_status ssl_check_peer(grpc_security_connector *sc,
     gpr_log(GPR_ERROR, "Peer name %s is not in peer certificate", peer_name);
     return GRPC_SECURITY_ERROR;
   }
+  if (sc->auth_context != NULL) {
+    GRPC_AUTH_CONTEXT_UNREF(sc->auth_context, "connector");
+  }
   sc->auth_context = tsi_ssl_peer_to_auth_context(peer);
   return GRPC_SECURITY_OK;
 }
@@ -538,7 +541,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
     alpn_protocol_strings[i] =
         (const unsigned char *)grpc_chttp2_get_alpn_version_index(i);
     alpn_protocol_string_lengths[i] =
-        strlen(grpc_chttp2_get_alpn_version_index(i));
+        (unsigned char)strlen(grpc_chttp2_get_alpn_version_index(i));
   }
 
   if (config == NULL || target_name == NULL) {
@@ -577,7 +580,7 @@ grpc_security_status grpc_ssl_channel_security_connector_create(
       config->pem_private_key, config->pem_private_key_size,
       config->pem_cert_chain, config->pem_cert_chain_size, pem_root_certs,
       pem_root_certs_size, ssl_cipher_suites(), alpn_protocol_strings,
-      alpn_protocol_string_lengths, num_alpn_protocols, &c->handshaker_factory);
+      alpn_protocol_string_lengths, (uint16_t)num_alpn_protocols, &c->handshaker_factory);
   if (result != TSI_OK) {
     gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.",
             tsi_result_to_string(result));
@@ -611,7 +614,7 @@ grpc_security_status grpc_ssl_server_security_connector_create(
     alpn_protocol_strings[i] =
         (const unsigned char *)grpc_chttp2_get_alpn_version_index(i);
     alpn_protocol_string_lengths[i] =
-        strlen(grpc_chttp2_get_alpn_version_index(i));
+        (unsigned char)strlen(grpc_chttp2_get_alpn_version_index(i));
   }
 
   if (config == NULL || config->num_key_cert_pairs == 0) {
@@ -630,7 +633,7 @@ grpc_security_status grpc_ssl_server_security_connector_create(
       (const unsigned char **)config->pem_cert_chains,
       config->pem_cert_chains_sizes, config->num_key_cert_pairs,
       config->pem_root_certs, config->pem_root_certs_size, ssl_cipher_suites(),
-      alpn_protocol_strings, alpn_protocol_string_lengths, num_alpn_protocols,
+      alpn_protocol_strings, alpn_protocol_string_lengths, (uint16_t)num_alpn_protocols,
       &c->handshaker_factory);
   if (result != TSI_OK) {
     gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.",
@@ -649,4 +652,3 @@ error:
   gpr_free(alpn_protocol_string_lengths);
   return GRPC_SECURITY_ERROR;
 }
-

src/core/security/security_context.c

@@ -89,7 +89,7 @@ grpc_client_security_context *grpc_client_security_context_create(void) {
 void grpc_client_security_context_destroy(void *ctx) {
   grpc_client_security_context *c = (grpc_client_security_context *)ctx;
   grpc_credentials_unref(c->creds);
-  grpc_auth_context_unref(c->auth_context);
+  GRPC_AUTH_CONTEXT_UNREF(c->auth_context, "client_security_context");
   gpr_free(ctx);
 }
 
@@ -104,7 +104,7 @@ grpc_server_security_context *grpc_server_security_context_create(void) {
 
 void grpc_server_security_context_destroy(void *ctx) {
   grpc_server_security_context *c = (grpc_server_security_context *)ctx;
-  grpc_auth_context_unref(c->auth_context);
+  GRPC_AUTH_CONTEXT_UNREF(c->auth_context, "server_security_context");
   gpr_free(ctx);
 }
 
@@ -120,21 +120,40 @@ grpc_auth_context *grpc_auth_context_create(grpc_auth_context *chained,
   memset(ctx->properties, 0, property_count * sizeof(grpc_auth_property));
   ctx->property_count = property_count;
   gpr_ref_init(&ctx->refcount, 1);
-  if (chained != NULL) ctx->chained = grpc_auth_context_ref(chained);
+  if (chained != NULL) ctx->chained = GRPC_AUTH_CONTEXT_REF(chained, "chained");
   return ctx;
 }
 
+#ifdef GRPC_AUTH_CONTEXT_REFCOUNT_DEBUG
+grpc_auth_context *grpc_auth_context_ref(grpc_auth_context *ctx,
+                                         const char *file, int line,
+                                         const char *reason) {
+  if (ctx == NULL) return NULL;
+  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+          "AUTH_CONTEXT:%p   ref %d -> %d %s", ctx, (int)ctx->refcount.count,
+          (int)ctx->refcount.count + 1, reason);
+#else
 grpc_auth_context *grpc_auth_context_ref(grpc_auth_context *ctx) {
   if (ctx == NULL) return NULL;
+#endif
   gpr_ref(&ctx->refcount);
   return ctx;
 }
 
+#ifdef GRPC_AUTH_CONTEXT_REFCOUNT_DEBUG
+void grpc_auth_context_unref(grpc_auth_context *ctx, const char *file, int line,
+                             const char *reason) {
+  if (ctx == NULL) return;
+  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG,
+          "AUTH_CONTEXT:%p unref %d -> %d %s", ctx, (int)ctx->refcount.count,
+          (int)ctx->refcount.count - 1, reason);
+#else
 void grpc_auth_context_unref(grpc_auth_context *ctx) {
   if (ctx == NULL) return;
+#endif
   if (gpr_unref(&ctx->refcount)) {
     size_t i;
-    grpc_auth_context_unref(ctx->chained);
+    GRPC_AUTH_CONTEXT_UNREF(ctx->chained, "chained");
     if (ctx->properties != NULL) {
       for (i = 0; i < ctx->property_count; i++) {
         grpc_auth_property_reset(&ctx->properties[i]);
@@ -223,8 +242,8 @@ grpc_auth_property grpc_auth_property_init(const char *name, const char *value,
 }
 
 void grpc_auth_property_reset(grpc_auth_property *property) {
-  if (property->name != NULL) gpr_free(property->name);
-  if (property->value != NULL) gpr_free(property->value);
+  gpr_free(property->name);
+  gpr_free(property->value);
   memset(property, 0, sizeof(grpc_auth_property));
 }
 

src/core/security/security_context.h

@@ -55,9 +55,22 @@ grpc_auth_context *grpc_auth_context_create(grpc_auth_context *chained,
                                             size_t property_count);
 
 /* Refcounting. */
-grpc_auth_context *grpc_auth_context_ref(
-    grpc_auth_context *ctx);
-void grpc_auth_context_unref(grpc_auth_context *ctx);
+#ifdef GRPC_AUTH_CONTEXT_REFCOUNT_DEBUG
+#define GRPC_AUTH_CONTEXT_REF(p, r) \
+  grpc_auth_context_ref((p), __FILE__, __LINE__, (r))
+#define GRPC_AUTH_CONTEXT_UNREF(p, r) \
+  grpc_auth_context_unref((p), __FILE__, __LINE__, (r))
+grpc_auth_context *grpc_auth_context_ref(grpc_auth_context *policy,
+                                         const char *file, int line,
+                                         const char *reason);
+void grpc_auth_context_unref(grpc_auth_context *policy, const char *file,
+                             int line, const char *reason);
+#else
+#define GRPC_AUTH_CONTEXT_REF(p, r) grpc_auth_context_ref((p))
+#define GRPC_AUTH_CONTEXT_UNREF(p, r) grpc_auth_context_unref((p))
+grpc_auth_context *grpc_auth_context_ref(grpc_auth_context *policy);
+void grpc_auth_context_unref(grpc_auth_context *policy);
+#endif
 
 grpc_auth_property grpc_auth_property_init_from_cstring(const char *name,
                                                         const char *value);

src/core/security/server_auth_filter.c

@@ -82,9 +82,13 @@ static void init_call_elem(grpc_call_element *elem,
 
   /* Create a security context for the call and reference the auth context from
      the channel. */
+  if (initial_op->context[GRPC_CONTEXT_SECURITY].value != NULL) {
+    initial_op->context[GRPC_CONTEXT_SECURITY].destroy(
+        initial_op->context[GRPC_CONTEXT_SECURITY].value);
+  }
   server_ctx = grpc_server_security_context_create();
-  server_ctx->auth_context =
-      grpc_auth_context_ref(chand->security_connector->auth_context);
+  server_ctx->auth_context = GRPC_AUTH_CONTEXT_REF(
+      chand->security_connector->auth_context, "server_security_context");
   initial_op->context[GRPC_CONTEXT_SECURITY].value = server_ctx;
   initial_op->context[GRPC_CONTEXT_SECURITY].destroy =
       grpc_server_security_context_destroy;

src/core/support/histogram.c

@@ -164,7 +164,9 @@ static double threshold_for_count_below(gpr_histogram *h, double count_below) {
   size_t lower_idx;
   size_t upper_idx;
 
-  GPR_ASSERT(h->count >= 1);
+  if (h->count == 0) {
+    return 0.0;
+  }
 
   if (count_below <= 0) {
     return h->min_seen;

src/core/surface/call.c

@@ -158,6 +158,11 @@ struct grpc_call {
   gpr_uint8 reading_message;
   /* have we bound a pollset yet? */
   gpr_uint8 bound_pollset;
+  /* is an error status set */
+  gpr_uint8 error_status_set;
+  /** should the alarm be cancelled */
+  gpr_uint8 cancel_alarm;
+
   /* flags with bits corresponding to write states allowing us to determine
      what was sent */
   gpr_uint16 last_send_contains;
@@ -216,7 +221,7 @@ struct grpc_call {
   /* Received call statuses from various sources */
   received_status status[STATUS_SOURCE_COUNT];
 
-  /** Compression level for the call */
+  /* Compression level for the call */
   grpc_compression_level compression_level;
 
   /* Contexts for various subsystems (security, tracing, ...). */
@@ -417,6 +422,7 @@ static void set_status_code(grpc_call *call, status_source source,
 
   call->status[source].is_set = 1;
   call->status[source].code = status;
+  call->error_status_set = status != GRPC_STATUS_OK;
 
   if (status != GRPC_STATUS_OK && !grpc_bbq_empty(&call->incoming_queue)) {
     grpc_bbq_flush(&call->incoming_queue);
@@ -468,6 +474,7 @@ static void unlock(grpc_call *call) {
   int completing_requests = 0;
   int start_op = 0;
   int i;
+  int cancel_alarm = 0;
 
   memset(&op, 0, sizeof(op));
 
@@ -475,6 +482,9 @@ static void unlock(grpc_call *call) {
   start_op = op.cancel_with_status != GRPC_STATUS_OK;
   call->cancel_with_status = GRPC_STATUS_OK; /* reset */
 
+  cancel_alarm = call->cancel_alarm;
+  call->cancel_alarm = 0;
+
   if (!call->receiving && need_more_data(call)) {
     op.recv_ops = &call->recv_ops;
     op.recv_state = &call->recv_state;
@@ -509,6 +519,10 @@ static void unlock(grpc_call *call) {
 
   gpr_mu_unlock(&call->mu);
 
+  if (cancel_alarm) {
+    grpc_alarm_cancel(&call->alarm);
+  }
+
   if (start_op) {
     execute_op(call, &op);
   }
@@ -694,13 +708,13 @@ static void call_on_done_send(void *pc, int success) {
 }
 
 static void finish_message(grpc_call *call) {
-  /* TODO(ctiller): this could be a lot faster if coded directly */
-  grpc_byte_buffer *byte_buffer = grpc_raw_byte_buffer_create(
-      call->incoming_message.slices, call->incoming_message.count);
+  if (call->error_status_set == 0) {
+    /* TODO(ctiller): this could be a lot faster if coded directly */
+    grpc_byte_buffer *byte_buffer = grpc_raw_byte_buffer_create(
+        call->incoming_message.slices, call->incoming_message.count);
+    grpc_bbq_push(&call->incoming_queue, byte_buffer);
+  }
   gpr_slice_buffer_reset_and_unref(&call->incoming_message);
-
-  grpc_bbq_push(&call->incoming_queue, byte_buffer);
-
   GPR_ASSERT(call->incoming_message.count == 0);
   call->reading_message = 0;
 }
@@ -801,10 +815,7 @@ static void call_on_done_recv(void *pc, int success) {
     if (call->recv_state == GRPC_STREAM_CLOSED) {
       GPR_ASSERT(call->read_state <= READ_STATE_STREAM_CLOSED);
       call->read_state = READ_STATE_STREAM_CLOSED;
-      if (call->have_alarm) {
-        grpc_alarm_cancel(&call->alarm);
-        call->have_alarm = 0;
-      }
+      call->cancel_alarm |= call->have_alarm;
       GRPC_CALL_INTERNAL_UNREF(call, "closed", 0);
     }
     finish_read_ops(call);
@@ -983,7 +994,7 @@ static void finish_read_ops(grpc_call *call) {
 
   switch (call->read_state) {
     case READ_STATE_STREAM_CLOSED:
-      if (empty) {
+      if (empty && !call->have_alarm) {
         finish_ioreq_op(call, GRPC_IOREQ_RECV_CLOSE, 1);
       }
     /* fallthrough */
@@ -1081,10 +1092,7 @@ void grpc_call_destroy(grpc_call *c) {
   lock(c);
   GPR_ASSERT(!c->destroy_called);
   c->destroy_called = 1;
-  if (c->have_alarm) {
-    grpc_alarm_cancel(&c->alarm);
-    c->have_alarm = 0;
-  }
+  c->cancel_alarm |= c->have_alarm;
   cancel = c->read_state != READ_STATE_STREAM_CLOSED;
   unlock(c);
   if (cancel) grpc_call_cancel(c);
@@ -1163,12 +1171,14 @@ grpc_call *grpc_call_from_top_element(grpc_call_element *elem) {
 
 static void call_alarm(void *arg, int success) {
   grpc_call *call = arg;
+  lock(call);
+  call->have_alarm = 0;
   if (success) {
-    lock(call);
     cancel_with_status(call, GRPC_STATUS_DEADLINE_EXCEEDED,
                        "Deadline Exceeded");
-    unlock(call);
   }
+  finish_read_ops(call);
+  unlock(call);
   GRPC_CALL_INTERNAL_UNREF(call, "alarm", 1);
 }
 

src/core/surface/completion_queue.c

@@ -88,9 +88,11 @@ grpc_completion_queue *grpc_completion_queue_create(void) {
 }
 
 #ifdef GRPC_CQ_REF_COUNT_DEBUG
-void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason) {
-  gpr_log(GPR_DEBUG, "CQ:%p   ref %d -> %d %s", cc, (int)cc->owning_refs.count,
-          (int)cc->owning_refs.count + 1, reason);
+void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason,
+                          const char *file, int line) {
+  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "CQ:%p   ref %d -> %d %s",
+          cc, (int)cc->owning_refs.count, (int)cc->owning_refs.count + 1,
+          reason);
 #else
 void grpc_cq_internal_ref(grpc_completion_queue *cc) {
 #endif
@@ -103,9 +105,11 @@ static void on_pollset_destroy_done(void *arg) {
 }
 
 #ifdef GRPC_CQ_REF_COUNT_DEBUG
-void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason) {
-  gpr_log(GPR_DEBUG, "CQ:%p unref %d -> %d %s", cc, (int)cc->owning_refs.count,
-          (int)cc->owning_refs.count - 1, reason);
+void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason,
+                            const char *file, int line) {
+  gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, "CQ:%p unref %d -> %d %s",
+          cc, (int)cc->owning_refs.count, (int)cc->owning_refs.count - 1,
+          reason);
 #else
 void grpc_cq_internal_unref(grpc_completion_queue *cc) {
 #endif

src/core/surface/completion_queue.h

@@ -40,10 +40,14 @@
 #include <grpc/grpc.h>
 
 #ifdef GRPC_CQ_REF_COUNT_DEBUG
-void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason);
-void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason);
-#define GRPC_CQ_INTERNAL_REF(cc, reason) grpc_cq_internal_ref(cc, reason)
-#define GRPC_CQ_INTERNAL_UNREF(cc, reason) grpc_cq_internal_unref(cc, reason)
+void grpc_cq_internal_ref(grpc_completion_queue *cc, const char *reason,
+                          const char *file, int line);
+void grpc_cq_internal_unref(grpc_completion_queue *cc, const char *reason,
+                            const char *file, int line);
+#define GRPC_CQ_INTERNAL_REF(cc, reason) \
+  grpc_cq_internal_ref(cc, reason, __FILE__, __LINE__)
+#define GRPC_CQ_INTERNAL_UNREF(cc, reason) \
+  grpc_cq_internal_unref(cc, reason, __FILE__, __LINE__)
 #else
 void grpc_cq_internal_ref(grpc_completion_queue *cc);
 void grpc_cq_internal_unref(grpc_completion_queue *cc);

src/core/transport/chttp2/incoming_metadata.c

@@ -47,6 +47,10 @@ void grpc_chttp2_incoming_metadata_buffer_init(
 
 void grpc_chttp2_incoming_metadata_buffer_destroy(
     grpc_chttp2_incoming_metadata_buffer *buffer) {
+  size_t i;
+  for (i = 0; i < buffer->count; i++) {
+    grpc_mdelem_unref(buffer->elems[i].md);
+  }
   gpr_free(buffer->elems);
 }
 
@@ -99,11 +103,7 @@ void grpc_incoming_metadata_buffer_move_to_referencing_sopb(
     grpc_chttp2_incoming_metadata_buffer *dst, grpc_stream_op_buffer *sopb) {
   size_t delta;
   size_t i;
-  if (gpr_time_cmp(dst->deadline, gpr_inf_future) == 0) {
-    dst->deadline = src->deadline;
-  } else if (gpr_time_cmp(src->deadline, gpr_inf_future) != 0) {
-    dst->deadline = gpr_time_min(src->deadline, dst->deadline);
-  }
+  dst->deadline = gpr_time_min(src->deadline, dst->deadline);
 
   if (src->count == 0) {
     return;
@@ -179,29 +179,3 @@ void grpc_chttp2_incoming_metadata_buffer_postprocess_sopb_and_begin_live_op(
     }
   }
 }
-
-#if 0
-void grpc_chttp2_parsing_add_metadata_batch(
-    grpc_chttp2_transport_parsing *transport_parsing,
-    grpc_chttp2_stream_parsing *stream_parsing) {
-  grpc_metadata_batch b;
-
-  b.list.head = NULL;
-  /* Store away the last element of the list, so that in patch_metadata_ops
-     we can reconstitute the list.
-     We can't do list building here as later incoming metadata may reallocate
-     the underlying array. */
-  b.list.tail = (void *)(gpr_intptr)stream_parsing->incoming_metadata_count;
-  b.garbage.head = b.garbage.tail = NULL;
-  b.deadline = stream_parsing->incoming_deadline;
-  stream_parsing->incoming_deadline = gpr_inf_future;
-
-  grpc_sopb_add_metadata(&stream_parsing->data_parser.incoming_sopb, b);
-}
-#endif
-
-#if 0
-static void patch_metadata_ops(grpc_chttp2_stream_global *stream_global,
-                               grpc_chttp2_stream_parsing *stream_parsing) {
-}
-#endif

src/core/transport/chttp2/internal.h

@@ -72,56 +72,56 @@ typedef enum {
 /* deframer state for the overall http2 stream of bytes */
 typedef enum {
   /* prefix: one entry per http2 connection prefix byte */
-  DTS_CLIENT_PREFIX_0 = 0,
-  DTS_CLIENT_PREFIX_1,
-  DTS_CLIENT_PREFIX_2,
-  DTS_CLIENT_PREFIX_3,
-  DTS_CLIENT_PREFIX_4,
-  DTS_CLIENT_PREFIX_5,
-  DTS_CLIENT_PREFIX_6,
-  DTS_CLIENT_PREFIX_7,
-  DTS_CLIENT_PREFIX_8,
-  DTS_CLIENT_PREFIX_9,
-  DTS_CLIENT_PREFIX_10,
-  DTS_CLIENT_PREFIX_11,
-  DTS_CLIENT_PREFIX_12,
-  DTS_CLIENT_PREFIX_13,
-  DTS_CLIENT_PREFIX_14,
-  DTS_CLIENT_PREFIX_15,
-  DTS_CLIENT_PREFIX_16,
-  DTS_CLIENT_PREFIX_17,
-  DTS_CLIENT_PREFIX_18,
-  DTS_CLIENT_PREFIX_19,
-  DTS_CLIENT_PREFIX_20,
-  DTS_CLIENT_PREFIX_21,
-  DTS_CLIENT_PREFIX_22,
-  DTS_CLIENT_PREFIX_23,
+  GRPC_DTS_CLIENT_PREFIX_0 = 0,
+  GRPC_DTS_CLIENT_PREFIX_1,
+  GRPC_DTS_CLIENT_PREFIX_2,
+  GRPC_DTS_CLIENT_PREFIX_3,
+  GRPC_DTS_CLIENT_PREFIX_4,
+  GRPC_DTS_CLIENT_PREFIX_5,
+  GRPC_DTS_CLIENT_PREFIX_6,
+  GRPC_DTS_CLIENT_PREFIX_7,
+  GRPC_DTS_CLIENT_PREFIX_8,
+  GRPC_DTS_CLIENT_PREFIX_9,
+  GRPC_DTS_CLIENT_PREFIX_10,
+  GRPC_DTS_CLIENT_PREFIX_11,
+  GRPC_DTS_CLIENT_PREFIX_12,
+  GRPC_DTS_CLIENT_PREFIX_13,
+  GRPC_DTS_CLIENT_PREFIX_14,
+  GRPC_DTS_CLIENT_PREFIX_15,
+  GRPC_DTS_CLIENT_PREFIX_16,
+  GRPC_DTS_CLIENT_PREFIX_17,
+  GRPC_DTS_CLIENT_PREFIX_18,
+  GRPC_DTS_CLIENT_PREFIX_19,
+  GRPC_DTS_CLIENT_PREFIX_20,
+  GRPC_DTS_CLIENT_PREFIX_21,
+  GRPC_DTS_CLIENT_PREFIX_22,
+  GRPC_DTS_CLIENT_PREFIX_23,
   /* frame header byte 0... */
   /* must follow from the prefix states */
-  DTS_FH_0,
-  DTS_FH_1,
-  DTS_FH_2,
-  DTS_FH_3,
-  DTS_FH_4,
-  DTS_FH_5,
-  DTS_FH_6,
-  DTS_FH_7,
+  GRPC_DTS_FH_0,
+  GRPC_DTS_FH_1,
+  GRPC_DTS_FH_2,
+  GRPC_DTS_FH_3,
+  GRPC_DTS_FH_4,
+  GRPC_DTS_FH_5,
+  GRPC_DTS_FH_6,
+  GRPC_DTS_FH_7,
   /* ... frame header byte 8 */
-  DTS_FH_8,
+  GRPC_DTS_FH_8,
   /* inside a http2 frame */
-  DTS_FRAME
+  GRPC_DTS_FRAME
 } grpc_chttp2_deframe_transport_state;
 
 typedef enum {
-  WRITE_STATE_OPEN,
-  WRITE_STATE_QUEUED_CLOSE,
-  WRITE_STATE_SENT_CLOSE
+  GRPC_WRITE_STATE_OPEN,
+  GRPC_WRITE_STATE_QUEUED_CLOSE,
+  GRPC_WRITE_STATE_SENT_CLOSE
 } grpc_chttp2_write_state;
 
 typedef enum {
-  DONT_SEND_CLOSED = 0,
-  SEND_CLOSED,
-  SEND_CLOSED_WITH_RST_STREAM
+  GRPC_DONT_SEND_CLOSED = 0,
+  GRPC_SEND_CLOSED,
+  GRPC_SEND_CLOSED_WITH_RST_STREAM
 } grpc_chttp2_send_closed;
 
 typedef struct {
@@ -143,14 +143,14 @@ typedef enum {
 /* We keep several sets of connection wide parameters */
 typedef enum {
   /* The settings our peer has asked for (and we have acked) */
-  PEER_SETTINGS = 0,
+  GRPC_PEER_SETTINGS = 0,
   /* The settings we'd like to have */
-  LOCAL_SETTINGS,
+  GRPC_LOCAL_SETTINGS,
   /* The settings we've published to our peer */
-  SENT_SETTINGS,
+  GRPC_SENT_SETTINGS,
   /* The settings the peer has acked */
-  ACKED_SETTINGS,
-  NUM_SETTING_SETS
+  GRPC_ACKED_SETTINGS,
+  GRPC_NUM_SETTING_SETS
 } grpc_chttp2_setting_set;
 
 /* Outstanding ping request data */
@@ -183,7 +183,7 @@ typedef struct {
   /** bitmask of setting indexes to send out */
   gpr_uint32 force_send_settings;
   /** settings values */
-  gpr_uint32 settings[NUM_SETTING_SETS][GRPC_CHTTP2_NUM_SETTINGS];
+  gpr_uint32 settings[GRPC_NUM_SETTING_SETS][GRPC_CHTTP2_NUM_SETTINGS];
 
   /** has there been a connection level error, and have we notified
       anyone about it? */
@@ -372,34 +372,6 @@ struct grpc_chttp2_transport {
     /** closure for notifying transport closure */
     grpc_iomgr_closure notify_closed;
   } channel_callback;
-
-#if 0
-  /* basic state management - what are we doing at the moment? */
-  gpr_uint8 reading;
-  /** are we calling back any grpc_transport_op completion events */
-  gpr_uint8 calling_back_ops;
-  gpr_uint8 destroying;
-  gpr_uint8 closed;
-
-  /* stream indexing */
-  gpr_uint32 next_stream_id;
-
-  /* window management */
-  gpr_uint32 outgoing_window_update;
-
-  /* state for a stream that's not yet been created */
-  grpc_stream_op_buffer new_stream_sopb;
-
-  /* stream ops that need to be destroyed, but outside of the lock */
-  grpc_stream_op_buffer nuke_later_sopb;
-
-  /* pings */
-  gpr_int64 ping_counter;
-
-
-  grpc_chttp2_stream **accepting_stream;
-
-#endif
 };
 
 typedef struct {
@@ -471,14 +443,6 @@ struct grpc_chttp2_stream_parsing {
 
   /** incoming metadata */
   grpc_chttp2_incoming_metadata_buffer incoming_metadata;
-
-  /*
-    grpc_linked_mdelem *incoming_metadata;
-    size_t incoming_metadata_count;
-    size_t incoming_metadata_capacity;
-    grpc_linked_mdelem *old_incoming_metadata;
-    gpr_timespec incoming_deadline;
-  */
 };
 
 struct grpc_chttp2_stream {
@@ -488,14 +452,6 @@ struct grpc_chttp2_stream {
 
   grpc_chttp2_stream_link links[STREAM_LIST_COUNT];
   gpr_uint8 included[STREAM_LIST_COUNT];
-
-#if 0
-  gpr_uint32 outgoing_window_update;
-  gpr_uint8 cancelled;
-
-  grpc_stream_state callback_state;
-  grpc_stream_op_buffer callback_sopb;
-#endif
 };
 
 /** Transport writing call flow:
@@ -522,14 +478,15 @@ void grpc_chttp2_cleanup_writing(grpc_chttp2_transport_global *global,
 
 void grpc_chttp2_prepare_to_read(grpc_chttp2_transport_global *global,
                                  grpc_chttp2_transport_parsing *parsing);
-/** Process one slice of incoming data */
+/** Process one slice of incoming data; return 1 if the connection is still
+    viable after reading, or 0 if the connection should be torn down */
 int grpc_chttp2_perform_read(grpc_chttp2_transport_parsing *transport_parsing,
                              gpr_slice slice);
 void grpc_chttp2_publish_reads(grpc_chttp2_transport_global *global,
                                grpc_chttp2_transport_parsing *parsing);
 
 /** Get a writable stream
-    \return non-zero if there was a stream available */
+    returns non-zero if there was a stream available */
 void grpc_chttp2_list_add_writable_stream(
     grpc_chttp2_transport_global *transport_global,
     grpc_chttp2_stream_global *stream_global);
@@ -647,10 +604,10 @@ void grpc_chttp2_run_with_global_lock(
 extern int grpc_http_trace;
 extern int grpc_flowctl_trace;
 
-#define IF_TRACING(stmt)  \
-  if (!(grpc_http_trace)) \
-    ;                     \
-  else                    \
+#define GRPC_CHTTP2_IF_TRACING(stmt) \
+  if (!(grpc_http_trace))            \
+    ;                                \
+  else                               \
   stmt
 
 #define GRPC_CHTTP2_FLOWCTL_TRACE_STREAM(reason, transport, context, var,      \

src/core/transport/chttp2/parsing.c

@@ -117,15 +117,15 @@ void grpc_chttp2_publish_reads(
 
   /* update global settings */
   if (transport_parsing->settings_updated) {
-    memcpy(transport_global->settings[PEER_SETTINGS],
+    memcpy(transport_global->settings[GRPC_PEER_SETTINGS],
            transport_parsing->settings, sizeof(transport_parsing->settings));
     transport_parsing->settings_updated = 0;
   }
 
   /* update settings based on ack if received */
   if (transport_parsing->settings_ack_received) {
-    memcpy(transport_global->settings[ACKED_SETTINGS],
-           transport_global->settings[SENT_SETTINGS],
+    memcpy(transport_global->settings[GRPC_ACKED_SETTINGS],
+           transport_global->settings[GRPC_SENT_SETTINGS],
            GRPC_CHTTP2_NUM_SETTINGS * sizeof(gpr_uint32));
     transport_parsing->settings_ack_received = 0;
   }
@@ -238,34 +238,34 @@ int grpc_chttp2_perform_read(grpc_chttp2_transport_parsing *transport_parsing,
   if (cur == end) return 1;
 
   switch (transport_parsing->deframe_state) {
-    case DTS_CLIENT_PREFIX_0:
-    case DTS_CLIENT_PREFIX_1:
-    case DTS_CLIENT_PREFIX_2:
-    case DTS_CLIENT_PREFIX_3:
-    case DTS_CLIENT_PREFIX_4:
-    case DTS_CLIENT_PREFIX_5:
-    case DTS_CLIENT_PREFIX_6:
-    case DTS_CLIENT_PREFIX_7:
-    case DTS_CLIENT_PREFIX_8:
-    case DTS_CLIENT_PREFIX_9:
-    case DTS_CLIENT_PREFIX_10:
-    case DTS_CLIENT_PREFIX_11:
-    case DTS_CLIENT_PREFIX_12:
-    case DTS_CLIENT_PREFIX_13:
-    case DTS_CLIENT_PREFIX_14:
-    case DTS_CLIENT_PREFIX_15:
-    case DTS_CLIENT_PREFIX_16:
-    case DTS_CLIENT_PREFIX_17:
-    case DTS_CLIENT_PREFIX_18:
-    case DTS_CLIENT_PREFIX_19:
-    case DTS_CLIENT_PREFIX_20:
-    case DTS_CLIENT_PREFIX_21:
-    case DTS_CLIENT_PREFIX_22:
-    case DTS_CLIENT_PREFIX_23:
-      while (cur != end && transport_parsing->deframe_state != DTS_FH_0) {
+    case GRPC_DTS_CLIENT_PREFIX_0:
+    case GRPC_DTS_CLIENT_PREFIX_1:
+    case GRPC_DTS_CLIENT_PREFIX_2:
+    case GRPC_DTS_CLIENT_PREFIX_3:
+    case GRPC_DTS_CLIENT_PREFIX_4:
+    case GRPC_DTS_CLIENT_PREFIX_5:
+    case GRPC_DTS_CLIENT_PREFIX_6:
+    case GRPC_DTS_CLIENT_PREFIX_7:
+    case GRPC_DTS_CLIENT_PREFIX_8:
+    case GRPC_DTS_CLIENT_PREFIX_9:
+    case GRPC_DTS_CLIENT_PREFIX_10:
+    case GRPC_DTS_CLIENT_PREFIX_11:
+    case GRPC_DTS_CLIENT_PREFIX_12:
+    case GRPC_DTS_CLIENT_PREFIX_13:
+    case GRPC_DTS_CLIENT_PREFIX_14:
+    case GRPC_DTS_CLIENT_PREFIX_15:
+    case GRPC_DTS_CLIENT_PREFIX_16:
+    case GRPC_DTS_CLIENT_PREFIX_17:
+    case GRPC_DTS_CLIENT_PREFIX_18:
+    case GRPC_DTS_CLIENT_PREFIX_19:
+    case GRPC_DTS_CLIENT_PREFIX_20:
+    case GRPC_DTS_CLIENT_PREFIX_21:
+    case GRPC_DTS_CLIENT_PREFIX_22:
+    case GRPC_DTS_CLIENT_PREFIX_23:
+      while (cur != end && transport_parsing->deframe_state != GRPC_DTS_FH_0) {
         if (*cur != GRPC_CHTTP2_CLIENT_CONNECT_STRING[transport_parsing
                                                           ->deframe_state]) {
-          gpr_log(GPR_ERROR,
+          gpr_log(GPR_INFO,
                   "Connect string mismatch: expected '%c' (%d) got '%c' (%d) "
                   "at byte %d",
                   GRPC_CHTTP2_CLIENT_CONNECT_STRING[transport_parsing
@@ -283,74 +283,74 @@ int grpc_chttp2_perform_read(grpc_chttp2_transport_parsing *transport_parsing,
       }
     /* fallthrough */
     dts_fh_0:
-    case DTS_FH_0:
+    case GRPC_DTS_FH_0:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_frame_size = ((gpr_uint32)*cur) << 16;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_1;
+        transport_parsing->deframe_state = GRPC_DTS_FH_1;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_1:
+    case GRPC_DTS_FH_1:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_frame_size |= ((gpr_uint32)*cur) << 8;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_2;
+        transport_parsing->deframe_state = GRPC_DTS_FH_2;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_2:
+    case GRPC_DTS_FH_2:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_frame_size |= *cur;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_3;
+        transport_parsing->deframe_state = GRPC_DTS_FH_3;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_3:
+    case GRPC_DTS_FH_3:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_frame_type = *cur;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_4;
+        transport_parsing->deframe_state = GRPC_DTS_FH_4;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_4:
+    case GRPC_DTS_FH_4:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_frame_flags = *cur;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_5;
+        transport_parsing->deframe_state = GRPC_DTS_FH_5;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_5:
+    case GRPC_DTS_FH_5:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_stream_id = (((gpr_uint32)*cur) & 0x7f) << 24;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_6;
+        transport_parsing->deframe_state = GRPC_DTS_FH_6;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_6:
+    case GRPC_DTS_FH_6:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_stream_id |= ((gpr_uint32)*cur) << 16;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_7;
+        transport_parsing->deframe_state = GRPC_DTS_FH_7;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_7:
+    case GRPC_DTS_FH_7:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_stream_id |= ((gpr_uint32)*cur) << 8;
       if (++cur == end) {
-        transport_parsing->deframe_state = DTS_FH_8;
+        transport_parsing->deframe_state = GRPC_DTS_FH_8;
         return 1;
       }
     /* fallthrough */
-    case DTS_FH_8:
+    case GRPC_DTS_FH_8:
       GPR_ASSERT(cur < end);
       transport_parsing->incoming_stream_id |= ((gpr_uint32)*cur);
-      transport_parsing->deframe_state = DTS_FRAME;
+      transport_parsing->deframe_state = GRPC_DTS_FRAME;
       if (!init_frame_parser(transport_parsing)) {
         return 0;
       }
@@ -364,7 +364,7 @@ int grpc_chttp2_perform_read(grpc_chttp2_transport_parsing *transport_parsing,
         }
         transport_parsing->incoming_stream = NULL;
         if (++cur == end) {
-          transport_parsing->deframe_state = DTS_FH_0;
+          transport_parsing->deframe_state = GRPC_DTS_FH_0;
           return 1;
         }
         goto dts_fh_0; /* loop */
@@ -373,7 +373,7 @@ int grpc_chttp2_perform_read(grpc_chttp2_transport_parsing *transport_parsing,
         return 1;
       }
     /* fallthrough */
-    case DTS_FRAME:
+    case GRPC_DTS_FRAME:
       GPR_ASSERT(cur < end);
       if ((gpr_uint32)(end - cur) == transport_parsing->incoming_frame_size) {
         if (!parse_frame_slice(
@@ -381,7 +381,7 @@ int grpc_chttp2_perform_read(grpc_chttp2_transport_parsing *transport_parsing,
                 gpr_slice_sub_no_ref(slice, cur - beg, end - beg), 1)) {
           return 0;
         }
-        transport_parsing->deframe_state = DTS_FH_0;
+        transport_parsing->deframe_state = GRPC_DTS_FH_0;
         transport_parsing->incoming_stream = NULL;
         return 1;
       } else if ((gpr_uint32)(end - cur) >
@@ -582,10 +582,10 @@ static void on_header(void *tp, grpc_mdelem *md) {
 
   GPR_ASSERT(stream_parsing);
 
-  IF_TRACING(gpr_log(GPR_INFO, "HTTP:%d:HDR: %s: %s", stream_parsing->id,
-                     transport_parsing->is_client ? "CLI" : "SVR",
-                     grpc_mdstr_as_c_string(md->key),
-                     grpc_mdstr_as_c_string(md->value)));
+  GRPC_CHTTP2_IF_TRACING(gpr_log(
+      GPR_INFO, "HTTP:%d:HDR: %s: %s", stream_parsing->id,
+      transport_parsing->is_client ? "CLI" : "SVR",
+      grpc_mdstr_as_c_string(md->key), grpc_mdstr_as_c_string(md->value)));
 
   if (md->key == transport_parsing->str_grpc_timeout) {
     gpr_timespec *cached_timeout = grpc_mdelem_get_user_data(md, free_timeout);

src/core/transport/chttp2/stream_lists.c

@@ -344,10 +344,9 @@ void grpc_chttp2_for_all_streams(
     void (*cb)(grpc_chttp2_transport_global *transport_global, void *user_data,
                grpc_chttp2_stream_global *stream_global)) {
   grpc_chttp2_stream *s;
-  for (s = TRANSPORT_FROM_GLOBAL(transport_global)
-               ->lists[GRPC_CHTTP2_LIST_ALL_STREAMS]
-               .head;
-       s; s = s->links[GRPC_CHTTP2_LIST_ALL_STREAMS].next) {
+  grpc_chttp2_transport *t = TRANSPORT_FROM_GLOBAL(transport_global);
+  for (s = t->lists[GRPC_CHTTP2_LIST_ALL_STREAMS].head; s != NULL;
+       s = s->links[GRPC_CHTTP2_LIST_ALL_STREAMS].next) {
     cb(transport_global, user_data, &s->global);
   }
 }

src/core/transport/chttp2/writing.c

@@ -54,10 +54,10 @@ int grpc_chttp2_unlocking_check_writes(
       !transport_global->sent_local_settings) {
     gpr_slice_buffer_add(
         &transport_writing->outbuf,
-        grpc_chttp2_settings_create(transport_global->settings[SENT_SETTINGS],
-                                    transport_global->settings[LOCAL_SETTINGS],
-                                    transport_global->force_send_settings,
-                                    GRPC_CHTTP2_NUM_SETTINGS));
+        grpc_chttp2_settings_create(
+            transport_global->settings[GRPC_SENT_SETTINGS],
+            transport_global->settings[GRPC_LOCAL_SETTINGS],
+            transport_global->force_send_settings, GRPC_CHTTP2_NUM_SETTINGS));
     transport_global->force_send_settings = 0;
     transport_global->dirtied_local_settings = 0;
     transport_global->sent_local_settings = 1;
@@ -84,16 +84,16 @@ int grpc_chttp2_unlocking_check_writes(
     transport_global->outgoing_window -= window_delta;
     stream_global->outgoing_window -= window_delta;
 
-    if (stream_global->write_state == WRITE_STATE_QUEUED_CLOSE &&
+    if (stream_global->write_state == GRPC_WRITE_STATE_QUEUED_CLOSE &&
         stream_global->outgoing_sopb->nops == 0) {
       if (!transport_global->is_client && !stream_global->read_closed) {
-        stream_writing->send_closed = SEND_CLOSED_WITH_RST_STREAM;
+        stream_writing->send_closed = GRPC_SEND_CLOSED_WITH_RST_STREAM;
       } else {
-        stream_writing->send_closed = SEND_CLOSED;
+        stream_writing->send_closed = GRPC_SEND_CLOSED;
       }
     }
     if (stream_writing->sopb.nops > 0 ||
-        stream_writing->send_closed != DONT_SEND_CLOSED) {
+        stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) {
       grpc_chttp2_list_add_writing_stream(transport_writing, stream_writing);
     }
 
@@ -112,7 +112,7 @@ int grpc_chttp2_unlocking_check_writes(
   while (grpc_chttp2_list_pop_writable_window_update_stream(transport_global,
                                                             &stream_global)) {
     window_delta =
-        transport_global->settings[LOCAL_SETTINGS]
+        transport_global->settings[GRPC_LOCAL_SETTINGS]
                                   [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] -
         stream_global->incoming_window;
     if (!stream_global->read_closed && window_delta > 0) {
@@ -128,7 +128,7 @@ int grpc_chttp2_unlocking_check_writes(
   }
 
   /* if the grpc_chttp2_transport is ready to send a window update, do so here
-   * also */
+     also; 3/4 is a magic number that will likely get tuned soon */
   if (transport_global->incoming_window <
       transport_global->connection_window_target * 3 / 4) {
     window_delta = transport_global->connection_window_target -
@@ -174,11 +174,11 @@ static void finalize_outbuf(grpc_chttp2_transport_writing *transport_writing) {
   while (
       grpc_chttp2_list_pop_writing_stream(transport_writing, &stream_writing)) {
     grpc_chttp2_encode(stream_writing->sopb.ops, stream_writing->sopb.nops,
-                       stream_writing->send_closed != DONT_SEND_CLOSED,
+                       stream_writing->send_closed != GRPC_DONT_SEND_CLOSED,
                        stream_writing->id, &transport_writing->hpack_compressor,
                        &transport_writing->outbuf);
     stream_writing->sopb.nops = 0;
-    if (stream_writing->send_closed == SEND_CLOSED_WITH_RST_STREAM) {
+    if (stream_writing->send_closed == GRPC_SEND_CLOSED_WITH_RST_STREAM) {
       gpr_slice_buffer_add(&transport_writing->outbuf,
                            grpc_chttp2_rst_stream_create(stream_writing->id,
                                                          GRPC_CHTTP2_NO_ERROR));
@@ -201,8 +201,8 @@ void grpc_chttp2_cleanup_writing(
 
   while (grpc_chttp2_list_pop_written_stream(
       transport_global, transport_writing, &stream_global, &stream_writing)) {
-    if (stream_writing->send_closed != DONT_SEND_CLOSED) {
-      stream_global->write_state = WRITE_STATE_SENT_CLOSE;
+    if (stream_writing->send_closed != GRPC_DONT_SEND_CLOSED) {
+      stream_global->write_state = GRPC_WRITE_STATE_SENT_CLOSE;
       if (!transport_global->is_client) {
         stream_global->read_closed = 1;
       }

src/core/transport/chttp2_transport.c

@@ -50,8 +50,6 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-/* #define REFCOUNTING_DEBUG */
-
 #define DEFAULT_WINDOW 65535
 #define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024)
 #define MAX_WINDOW 0x7fffffffu
@@ -236,7 +234,8 @@ static void init_transport(grpc_chttp2_transport *t,
   t->parsing.is_client = is_client;
   t->parsing.str_grpc_timeout =
       grpc_mdstr_from_string(t->metadata_context, "grpc-timeout");
-  t->parsing.deframe_state = is_client ? DTS_FH_0 : DTS_CLIENT_PREFIX_0;
+  t->parsing.deframe_state =
+      is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0;
   t->writing.is_client = is_client;
 
   gpr_slice_buffer_init(&t->global.qbuf);
@@ -268,7 +267,7 @@ static void init_transport(grpc_chttp2_transport *t,
   /* copy in initial settings to all setting sets */
   for (i = 0; i < GRPC_CHTTP2_NUM_SETTINGS; i++) {
     t->parsing.settings[i] = grpc_chttp2_settings_parameters[i].default_value;
-    for (j = 0; j < NUM_SETTING_SETS; j++) {
+    for (j = 0; j < GRPC_NUM_SETTING_SETS; j++) {
       t->global.settings[j][i] =
           grpc_chttp2_settings_parameters[i].default_value;
     }
@@ -407,11 +406,11 @@ static int init_stream(grpc_transport *gt, grpc_stream *gs,
     GPR_ASSERT(t->executor.parsing_active);
     s->global.id = (gpr_uint32)(gpr_uintptr)server_data;
     s->global.outgoing_window =
-        t->global
-            .settings[PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+        t->global.settings[GRPC_PEER_SETTINGS]
+                          [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
     s->parsing.incoming_window = s->global.incoming_window =
-        t->global
-            .settings[SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+        t->global.settings[GRPC_SENT_SETTINGS]
+                          [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
     *t->accepting_stream = s;
     grpc_chttp2_stream_map_add(&t->parsing_stream_map, s->global.id, s);
     s->global.in_stream_map = 1;
@@ -436,9 +435,12 @@ static void destroy_stream(grpc_transport *gt, grpc_stream *gs) {
   GPR_ASSERT(s->global.outgoing_sopb == NULL);
   GPR_ASSERT(s->global.publish_sopb == NULL);
   grpc_sopb_destroy(&s->writing.sopb);
+  grpc_sopb_destroy(&s->global.incoming_sopb);
   grpc_chttp2_data_parser_destroy(&s->parsing.data_parser);
   grpc_chttp2_incoming_metadata_buffer_destroy(&s->parsing.incoming_metadata);
   grpc_chttp2_incoming_metadata_buffer_destroy(&s->global.incoming_metadata);
+  grpc_chttp2_incoming_metadata_live_op_buffer_end(
+      &s->global.outstanding_metadata);
 
   UNREF_TRANSPORT(t, "stream");
 }
@@ -573,8 +575,8 @@ static void push_setting(grpc_chttp2_transport *t, grpc_chttp2_setting_id id,
     gpr_log(GPR_INFO, "Requested parameter %s clamped from %d to %d", sp->name,
             value, use_value);
   }
-  if (use_value != t->global.settings[LOCAL_SETTINGS][id]) {
-    t->global.settings[LOCAL_SETTINGS][id] = use_value;
+  if (use_value != t->global.settings[GRPC_LOCAL_SETTINGS][id]) {
+    t->global.settings[GRPC_LOCAL_SETTINGS][id] = use_value;
     t->global.dirtied_local_settings = 1;
   }
 }
@@ -634,14 +636,15 @@ static void maybe_start_some_streams(
    * concurrency */
   while (transport_global->next_stream_id <= MAX_CLIENT_STREAM_ID &&
          transport_global->concurrent_stream_count <
-             transport_global->settings
-                 [PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS] &&
+             transport_global
+                 ->settings[GRPC_PEER_SETTINGS]
+                           [GRPC_CHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS] &&
          grpc_chttp2_list_pop_waiting_for_concurrency(transport_global,
                                                       &stream_global)) {
-    IF_TRACING(gpr_log(GPR_DEBUG,
-                       "HTTP:%s: Allocating new grpc_chttp2_stream %p to id %d",
-                       transport_global->is_client ? "CLI" : "SVR",
-                       stream_global, transport_global->next_stream_id));
+    GRPC_CHTTP2_IF_TRACING(gpr_log(
+        GPR_DEBUG, "HTTP:%s: Allocating new grpc_chttp2_stream %p to id %d",
+        transport_global->is_client ? "CLI" : "SVR", stream_global,
+        transport_global->next_stream_id));
 
     GPR_ASSERT(stream_global->id == 0);
     stream_global->id = transport_global->next_stream_id;
@@ -654,11 +657,11 @@ static void maybe_start_some_streams(
     }
 
     stream_global->outgoing_window =
-        transport_global
-            ->settings[PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+        transport_global->settings[GRPC_PEER_SETTINGS]
+                                  [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
     stream_global->incoming_window =
-        transport_global
-            ->settings[SENT_SETTINGS][GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
+        transport_global->settings[GRPC_SENT_SETTINGS]
+                                  [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE];
     grpc_chttp2_stream_map_add(
         &TRANSPORT_FROM_GLOBAL(transport_global)->new_stream_map,
         stream_global->id, STREAM_FROM_GLOBAL(stream_global));
@@ -692,11 +695,12 @@ static void perform_op_locked(grpc_chttp2_transport *t,
     stream_global->send_done_closure = op->on_done_send;
     if (!stream_global->cancelled) {
       stream_global->outgoing_sopb = op->send_ops;
-      if (op->is_last_send && stream_global->write_state == WRITE_STATE_OPEN) {
-        stream_global->write_state = WRITE_STATE_QUEUED_CLOSE;
+      if (op->is_last_send &&
+          stream_global->write_state == GRPC_WRITE_STATE_OPEN) {
+        stream_global->write_state = GRPC_WRITE_STATE_QUEUED_CLOSE;
       }
       if (stream_global->id == 0) {
-        IF_TRACING(gpr_log(
+        GRPC_CHTTP2_IF_TRACING(gpr_log(
             GPR_DEBUG,
             "HTTP:%s: New grpc_chttp2_stream %p waiting for concurrency",
             transport_global->is_client ? "CLI" : "SVR", stream_global));
@@ -814,7 +818,7 @@ static void unlock_check_read_write_state(grpc_chttp2_transport *t) {
     while (grpc_chttp2_list_pop_closed_waiting_for_parsing(transport_global,
                                                            &stream_global)) {
       GPR_ASSERT(stream_global->in_stream_map);
-      GPR_ASSERT(stream_global->write_state != WRITE_STATE_OPEN);
+      GPR_ASSERT(stream_global->write_state != GRPC_WRITE_STATE_OPEN);
       GPR_ASSERT(stream_global->read_closed);
       remove_stream(t, stream_global->id);
       grpc_chttp2_list_add_read_write_state_changed(transport_global,
@@ -825,7 +829,7 @@ static void unlock_check_read_write_state(grpc_chttp2_transport *t) {
   while (grpc_chttp2_list_pop_read_write_state_changed(transport_global,
                                                        &stream_global)) {
     if (stream_global->cancelled) {
-      stream_global->write_state = WRITE_STATE_SENT_CLOSE;
+      stream_global->write_state = GRPC_WRITE_STATE_SENT_CLOSE;
       stream_global->read_closed = 1;
       if (!stream_global->published_cancelled) {
         char buffer[GPR_LTOA_MIN_BUFSIZE];
@@ -838,7 +842,7 @@ static void unlock_check_read_write_state(grpc_chttp2_transport *t) {
         stream_global->published_cancelled = 1;
       }
     }
-    if (stream_global->write_state == WRITE_STATE_SENT_CLOSE &&
+    if (stream_global->write_state == GRPC_WRITE_STATE_SENT_CLOSE &&
         stream_global->read_closed && stream_global->in_stream_map) {
       if (t->executor.parsing_active) {
         grpc_chttp2_list_add_closed_waiting_for_parsing(transport_global,
@@ -857,7 +861,8 @@ static void unlock_check_read_write_state(grpc_chttp2_transport *t) {
        To fix this will require having an edge after stream-closed
        indicating that the stream is closed AND safe to delete. */
     state = compute_state(
-        stream_global->write_state == WRITE_STATE_SENT_CLOSE && !stream_global->in_stream_map,
+        stream_global->write_state == GRPC_WRITE_STATE_SENT_CLOSE &&
+            !stream_global->in_stream_map,
         stream_global->read_closed);
     if (stream_global->incoming_sopb.nops == 0 &&
         state == stream_global->published_state) {
@@ -915,17 +920,21 @@ static void drop_connection(grpc_chttp2_transport *t) {
   end_all_the_calls(t);
 }
 
-static void recv_data_error_locked(grpc_chttp2_transport *t, grpc_chttp2_stream *s, void *a) {
-  size_t i;
-
-  drop_connection(t);
+static void read_error_locked(grpc_chttp2_transport *t) {
   t->endpoint_reading = 0;
   if (!t->executor.writing_active && t->ep) {
     grpc_endpoint_destroy(t->ep);
     t->ep = NULL;
-    UNREF_TRANSPORT(
-        t, "disconnect"); /* safe as we still have a ref for read */
+    /* safe as we still have a ref for read */
+    UNREF_TRANSPORT(t, "disconnect");
   }
+}
+
+static void recv_data_error_locked(grpc_chttp2_transport *t, grpc_chttp2_stream *s, void *a) {
+  size_t i;
+
+  drop_connection(t);
+  read_error_locked(t);
   for (i = 0; i < t->executor_parsing.nslices; i++) gpr_slice_unref(t->executor_parsing.slices[i]);
   memset(&t->executor_parsing, 0, sizeof(t->executor_parsing));
   UNREF_TRANSPORT(t, "recv_data");
@@ -947,10 +956,14 @@ static void finish_parsing_locked(grpc_chttp2_transport *t, grpc_chttp2_stream *
 
   for (; i < t->executor_parsing.nslices; i++) gpr_slice_unref(t->executor_parsing.slices[i]);
 
+  memset(&t->executor_parsing, 0, sizeof(t->executor_parsing));
+
   if (i == t->executor_parsing.nslices) {
     grpc_chttp2_schedule_closure(&t->global, &t->reading_action, 1);
+  } else {
+    read_error_locked(t);
+    UNREF_TRANSPORT(t, "recv_data");
   }
-  memset(&t->executor_parsing, 0, sizeof(t->executor_parsing));
 }
 
 static void parsing_action(void *pt, int iomgr_success_ignored) {
@@ -980,6 +993,19 @@ static void recv_data_ok_locked(grpc_chttp2_transport *t, grpc_chttp2_stream *s,
   }
 }
 
+/** update window from a settings change */
+static void update_global_window(void *args, gpr_uint32 id, void *stream) {
+  grpc_chttp2_transport *t = args;
+  grpc_chttp2_stream *s = stream;
+  grpc_chttp2_transport_global *transport_global = &t->global;
+  grpc_chttp2_stream_global *stream_global = &s->global;
+
+  GRPC_CHTTP2_FLOWCTL_TRACE_STREAM("settings", transport_global, stream_global,
+                                   outgoing_window,
+                                   t->parsing.initial_window_update);
+  stream_global->outgoing_window += t->parsing.initial_window_update;
+}
+
 /* tcp read callback */
 static void recv_data(void *tp, gpr_slice *slices, size_t nslices,
                       grpc_endpoint_cb_status error) {
@@ -998,6 +1024,9 @@ static void recv_data(void *tp, gpr_slice *slices, size_t nslices,
       grpc_chttp2_run_with_global_lock(t, NULL, recv_data_ok_locked, NULL, 0);
       break;
   }
+  if (unref) {
+    UNREF_TRANSPORT(t, "recv_data");
+  }
 }
 
 static void reading_action(void *pt, int iomgr_success_ignored) {

src/core/transport/metadata.c

@@ -120,7 +120,7 @@ static void unlock(grpc_mdctx *ctx) {
   if (ctx->refs == 0) {
     /* uncomment if you're having trouble diagnosing an mdelem leak to make
        things clearer (slows down destruction a lot, however) */
-    /* gc_mdtab(ctx); */
+    gc_mdtab(ctx);
     if (ctx->mdtab_count && ctx->mdtab_count == ctx->mdtab_free) {
       discard_metadata(ctx);
     }

src/core/transport/transport.c

@@ -112,8 +112,8 @@ void grpc_transport_op_add_cancellation(grpc_transport_op *op,
                                         grpc_mdstr *message) {
   if (op->cancel_with_status == GRPC_STATUS_OK) {
     op->cancel_with_status = status;
-    op->cancel_message = message;
-  } else if (message) {
+  }
+  if (message) {
     grpc_mdstr_unref(message);
   }
 }

src/core/transport/transport.h

@@ -77,7 +77,6 @@ typedef struct grpc_transport_op {
   grpc_pollset *bind_pollset;
 
   grpc_status_code cancel_with_status;
-  grpc_mdstr *cancel_message;
 
   /* Indexes correspond to grpc_context_index enum values */
   grpc_call_context_element *context;

src/core/transport/transport_op_string.c

@@ -144,11 +144,6 @@ char *grpc_transport_op_string(grpc_transport_op *op) {
     first = 0;
     gpr_asprintf(&tmp, "CANCEL:%d", op->cancel_with_status);
     gpr_strvec_add(&b, tmp);
-    if (op->cancel_message) {
-      gpr_asprintf(&tmp, ";msg='%s'",
-                   grpc_mdstr_as_c_string(op->cancel_message));
-      gpr_strvec_add(&b, tmp);
-    }
   }
 
   out = gpr_strvec_flatten(&b, NULL);

src/cpp/client/channel.cc

@@ -36,12 +36,10 @@
 #include <memory>
 
 #include <grpc/grpc.h>
-#include <grpc/grpc_security.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice.h>
 
 #include "src/core/profiling/timers.h"
-#include "src/cpp/proto/proto_utils.h"
 #include <grpc++/channel_arguments.h>
 #include <grpc++/client_context.h>
 #include <grpc++/completion_queue.h>
@@ -75,14 +73,14 @@ Call Channel::CreateCall(const RpcMethod& method, ClientContext* context,
   return Call(c_call, this, cq);
 }
 
-void Channel::PerformOpsOnCall(CallOpBuffer* buf, Call* call) {
+void Channel::PerformOpsOnCall(CallOpSetInterface* ops, Call* call) {
   static const size_t MAX_OPS = 8;
-  size_t nops = MAX_OPS;
-  grpc_op ops[MAX_OPS];
+  size_t nops = 0;
+  grpc_op cops[MAX_OPS];
   GRPC_TIMER_BEGIN(GRPC_PTAG_CPP_PERFORM_OPS, call->call());
-  buf->FillOps(ops, &nops);
+  ops->FillOps(cops, &nops);
   GPR_ASSERT(GRPC_CALL_OK ==
-             grpc_call_start_batch(call->call(), ops, nops, buf));
+             grpc_call_start_batch(call->call(), cops, nops, ops));
   GRPC_TIMER_END(GRPC_PTAG_CPP_PERFORM_OPS, call->call());
 }
 

src/cpp/client/channel.h

@@ -44,22 +44,22 @@ struct grpc_channel;
 
 namespace grpc {
 class Call;
-class CallOpBuffer;
+class CallOpSetInterface;
 class ChannelArguments;
 class CompletionQueue;
 class Credentials;
 class StreamContextInterface;
 
-class Channel GRPC_FINAL : public GrpcLibrary,
-                           public ChannelInterface {
+class Channel GRPC_FINAL : public GrpcLibrary, public ChannelInterface {
  public:
   Channel(const grpc::string& target, grpc_channel* c_channel);
   ~Channel() GRPC_OVERRIDE;
 
-  virtual void *RegisterMethod(const char *method) GRPC_OVERRIDE;
+  virtual void* RegisterMethod(const char* method) GRPC_OVERRIDE;
   virtual Call CreateCall(const RpcMethod& method, ClientContext* context,
                           CompletionQueue* cq) GRPC_OVERRIDE;
-  virtual void PerformOpsOnCall(CallOpBuffer* ops, Call* call) GRPC_OVERRIDE;
+  virtual void PerformOpsOnCall(CallOpSetInterface* ops,
+                                Call* call) GRPC_OVERRIDE;
 
  private:
   const grpc::string target_;

src/cpp/client/channel_arguments.cc

@@ -33,28 +33,14 @@
 
 #include <grpc++/channel_arguments.h>
 
-#include <grpc/grpc_security.h>
 #include "src/core/channel/channel_args.h"
 
 namespace grpc {
 
-void ChannelArguments::SetSslTargetNameOverride(const grpc::string& name) {
-  SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, name);
-}
-
 void ChannelArguments::SetCompressionLevel(grpc_compression_level level) {
   SetInt(GRPC_COMPRESSION_LEVEL_ARG, level);
 }
 
-grpc::string ChannelArguments::GetSslTargetNameOverride() const {
-  for (unsigned int i = 0; i < args_.size(); i++) {
-    if (grpc::string(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == args_[i].key) {
-      return args_[i].value.string;
-    }
-  }
-  return "";
-}
-
 void ChannelArguments::SetInt(const grpc::string& key, int value) {
   grpc_arg arg;
   arg.type = GRPC_ARG_INTEGER;

src/cpp/proto/proto_utils.h → src/cpp/client/secure_channel_arguments.cc

@@ -31,25 +31,24 @@
  *
  */
 
-#ifndef GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
-#define GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H
+#include <grpc++/channel_arguments.h>
+#include <grpc/grpc_security.h>
 
-#include <grpc++/config.h>
-
-struct grpc_byte_buffer;
+#include "src/core/channel/channel_args.h"
 
 namespace grpc {
 
-// Serialize the msg into a buffer created inside the function. The caller
-// should destroy the returned buffer when done with it. If serialization fails,
-// false is returned and buffer is left unchanged.
-bool SerializeProto(const grpc::protobuf::Message& msg,
-                    grpc_byte_buffer** buffer);
+void ChannelArguments::SetSslTargetNameOverride(const grpc::string& name) {
+  SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, name);
+}
 
-// The caller keeps ownership of buffer and msg.
-bool DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
-                      int max_message_size);
+grpc::string ChannelArguments::GetSslTargetNameOverride() const {
+  for (unsigned int i = 0; i < args_.size(); i++) {
+    if (grpc::string(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == args_[i].key) {
+      return args_[i].value.string;
+    }
+  }
+  return "";
+}
 
 }  // namespace grpc
-
-#endif  // GRPC_INTERNAL_CPP_PROTO_PROTO_UTILS_H

src/cpp/common/call.cc

@@ -39,107 +39,32 @@
 #include <grpc++/channel_interface.h>
 
 #include "src/core/profiling/timers.h"
-#include "src/cpp/proto/proto_utils.h"
 
 namespace grpc {
 
-CallOpBuffer::CallOpBuffer()
-    : return_tag_(this),
-      send_initial_metadata_(false),
-      initial_metadata_count_(0),
-      initial_metadata_(nullptr),
-      recv_initial_metadata_(nullptr),
-      send_message_(nullptr),
-      send_message_buffer_(nullptr),
-      send_buf_(nullptr),
-      recv_message_(nullptr),
-      recv_message_buffer_(nullptr),
-      recv_buf_(nullptr),
-      max_message_size_(-1),
-      client_send_close_(false),
-      recv_trailing_metadata_(nullptr),
-      recv_status_(nullptr),
-      status_code_(GRPC_STATUS_OK),
-      status_details_(nullptr),
-      status_details_capacity_(0),
-      send_status_available_(false),
-      send_status_code_(GRPC_STATUS_OK),
-      trailing_metadata_count_(0),
-      trailing_metadata_(nullptr),
-      cancelled_buf_(0),
-      recv_closed_(nullptr) {
-  memset(&recv_trailing_metadata_arr_, 0, sizeof(recv_trailing_metadata_arr_));
-  memset(&recv_initial_metadata_arr_, 0, sizeof(recv_initial_metadata_arr_));
-  recv_trailing_metadata_arr_.metadata = nullptr;
-  recv_initial_metadata_arr_.metadata = nullptr;
-}
-
-void CallOpBuffer::Reset(void* next_return_tag) {
-  return_tag_ = next_return_tag;
-
-  send_initial_metadata_ = false;
-  initial_metadata_count_ = 0;
-  gpr_free(initial_metadata_);
-
-  recv_initial_metadata_ = nullptr;
-  recv_initial_metadata_arr_.count = 0;
-
-  if (send_buf_ && send_message_) {
-    grpc_byte_buffer_destroy(send_buf_);
-  }
-  send_message_ = nullptr;
-  send_message_buffer_ = nullptr;
-  send_buf_ = nullptr;
-
-  got_message = false;
-  if (recv_buf_ && recv_message_) {
-    grpc_byte_buffer_destroy(recv_buf_);
-  }
-  recv_message_ = nullptr;
-  recv_message_buffer_ = nullptr;
-  recv_buf_ = nullptr;
-
-  client_send_close_ = false;
-
-  recv_trailing_metadata_ = nullptr;
-  recv_status_ = nullptr;
-  recv_trailing_metadata_arr_.count = 0;
-
-  status_code_ = GRPC_STATUS_OK;
-
-  send_status_available_ = false;
-  send_status_code_ = GRPC_STATUS_OK;
-  send_status_details_.clear();
-  trailing_metadata_count_ = 0;
-  trailing_metadata_ = nullptr;
-
-  recv_closed_ = nullptr;
-}
-
-CallOpBuffer::~CallOpBuffer() {
-  gpr_free(status_details_);
-  gpr_free(recv_initial_metadata_arr_.metadata);
-  gpr_free(recv_trailing_metadata_arr_.metadata);
-  if (recv_buf_ && recv_message_) {
-    grpc_byte_buffer_destroy(recv_buf_);
-  }
-  if (send_buf_ && send_message_) {
-    grpc_byte_buffer_destroy(send_buf_);
+void FillMetadataMap(grpc_metadata_array* arr,
+                     std::multimap<grpc::string, grpc::string>* metadata) {
+  for (size_t i = 0; i < arr->count; i++) {
+    // TODO(yangg) handle duplicates?
+    metadata->insert(std::pair<grpc::string, grpc::string>(
+        arr->metadata[i].key,
+        grpc::string(arr->metadata[i].value, arr->metadata[i].value_length)));
   }
+  grpc_metadata_array_destroy(arr);
+  grpc_metadata_array_init(arr);
 }
 
-namespace {
 // TODO(yangg) if the map is changed before we send, the pointers will be a
 // mess. Make sure it does not happen.
 grpc_metadata* FillMetadataArray(
-    std::multimap<grpc::string, grpc::string>* metadata) {
-  if (metadata->empty()) {
+    const std::multimap<grpc::string, grpc::string>& metadata) {
+  if (metadata.empty()) {
     return nullptr;
   }
   grpc_metadata* metadata_array =
-      (grpc_metadata*)gpr_malloc(metadata->size() * sizeof(grpc_metadata));
+      (grpc_metadata*)gpr_malloc(metadata.size() * sizeof(grpc_metadata));
   size_t i = 0;
-  for (auto iter = metadata->cbegin(); iter != metadata->cend(); ++iter, ++i) {
+  for (auto iter = metadata.cbegin(); iter != metadata.cend(); ++iter, ++i) {
     metadata_array[i].key = iter->first.c_str();
     metadata_array[i].value = iter->second.c_str();
     metadata_array[i].value_length = iter->second.size();
@@ -147,206 +72,6 @@ grpc_metadata* FillMetadataArray(
   return metadata_array;
 }
 
-void FillMetadataMap(grpc_metadata_array* arr,
-                     std::multimap<grpc::string, grpc::string>* metadata) {
-  for (size_t i = 0; i < arr->count; i++) {
-    // TODO(yangg) handle duplicates?
-    metadata->insert(std::pair<grpc::string, grpc::string>(
-        arr->metadata[i].key,
-        grpc::string(arr->metadata[i].value, arr->metadata[i].value_length)));
-  }
-  grpc_metadata_array_destroy(arr);
-  grpc_metadata_array_init(arr);
-}
-}  // namespace
-
-void CallOpBuffer::AddSendInitialMetadata(
-    std::multimap<grpc::string, grpc::string>* metadata) {
-  send_initial_metadata_ = true;
-  initial_metadata_count_ = metadata->size();
-  initial_metadata_ = FillMetadataArray(metadata);
-}
-
-void CallOpBuffer::AddRecvInitialMetadata(ClientContext* ctx) {
-  ctx->initial_metadata_received_ = true;
-  recv_initial_metadata_ = &ctx->recv_initial_metadata_;
-}
-
-void CallOpBuffer::AddSendInitialMetadata(ClientContext* ctx) {
-  AddSendInitialMetadata(&ctx->send_initial_metadata_);
-}
-
-void CallOpBuffer::AddSendMessage(const grpc::protobuf::Message& message) {
-  send_message_ = &message;
-}
-
-void CallOpBuffer::AddSendMessage(const ByteBuffer& message) {
-  send_message_buffer_ = &message;
-}
-
-void CallOpBuffer::AddRecvMessage(grpc::protobuf::Message* message) {
-  recv_message_ = message;
-  recv_message_->Clear();
-}
-
-void CallOpBuffer::AddRecvMessage(ByteBuffer* message) {
-  recv_message_buffer_ = message;
-  recv_message_buffer_->Clear();
-}
-
-void CallOpBuffer::AddClientSendClose() { client_send_close_ = true; }
-
-void CallOpBuffer::AddServerRecvClose(bool* cancelled) {
-  recv_closed_ = cancelled;
-}
-
-void CallOpBuffer::AddClientRecvStatus(ClientContext* context, Status* status) {
-  recv_trailing_metadata_ = &context->trailing_metadata_;
-  recv_status_ = status;
-}
-
-void CallOpBuffer::AddServerSendStatus(
-    std::multimap<grpc::string, grpc::string>* metadata, const Status& status) {
-  if (metadata != NULL) {
-    trailing_metadata_count_ = metadata->size();
-    trailing_metadata_ = FillMetadataArray(metadata);
-  } else {
-    trailing_metadata_count_ = 0;
-  }
-  send_status_available_ = true;
-  send_status_code_ = static_cast<grpc_status_code>(status.error_code());
-  send_status_details_ = status.error_message();
-}
-
-void CallOpBuffer::FillOps(grpc_op* ops, size_t* nops) {
-  *nops = 0;
-  if (send_initial_metadata_) {
-    ops[*nops].op = GRPC_OP_SEND_INITIAL_METADATA;
-    ops[*nops].data.send_initial_metadata.count = initial_metadata_count_;
-    ops[*nops].data.send_initial_metadata.metadata = initial_metadata_;
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-  if (recv_initial_metadata_) {
-    ops[*nops].op = GRPC_OP_RECV_INITIAL_METADATA;
-    ops[*nops].data.recv_initial_metadata = &recv_initial_metadata_arr_;
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-  if (send_message_ || send_message_buffer_) {
-    if (send_message_) {
-      GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_SERIALIZE, 0);
-      bool success = SerializeProto(*send_message_, &send_buf_);
-      if (!success) {
-        abort();
-        // TODO handle parse failure
-      }
-      GRPC_TIMER_END(GRPC_PTAG_PROTO_SERIALIZE, 0);
-    } else {
-      send_buf_ = send_message_buffer_->buffer();
-    }
-    ops[*nops].op = GRPC_OP_SEND_MESSAGE;
-    ops[*nops].data.send_message = send_buf_;
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-  if (recv_message_ || recv_message_buffer_) {
-    ops[*nops].op = GRPC_OP_RECV_MESSAGE;
-    ops[*nops].data.recv_message = &recv_buf_;
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-  if (client_send_close_) {
-    ops[*nops].op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-  if (recv_status_) {
-    ops[*nops].op = GRPC_OP_RECV_STATUS_ON_CLIENT;
-    ops[*nops].data.recv_status_on_client.trailing_metadata =
-        &recv_trailing_metadata_arr_;
-    ops[*nops].data.recv_status_on_client.status = &status_code_;
-    ops[*nops].data.recv_status_on_client.status_details = &status_details_;
-    ops[*nops].data.recv_status_on_client.status_details_capacity =
-        &status_details_capacity_;
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-  if (send_status_available_) {
-    ops[*nops].op = GRPC_OP_SEND_STATUS_FROM_SERVER;
-    ops[*nops].data.send_status_from_server.trailing_metadata_count =
-        trailing_metadata_count_;
-    ops[*nops].data.send_status_from_server.trailing_metadata =
-        trailing_metadata_;
-    ops[*nops].data.send_status_from_server.status = send_status_code_;
-    ops[*nops].data.send_status_from_server.status_details =
-        send_status_details_.empty() ? nullptr : send_status_details_.c_str();
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-  if (recv_closed_) {
-    ops[*nops].op = GRPC_OP_RECV_CLOSE_ON_SERVER;
-    ops[*nops].data.recv_close_on_server.cancelled = &cancelled_buf_;
-    ops[*nops].flags = 0;
-    (*nops)++;
-  }
-}
-
-bool CallOpBuffer::FinalizeResult(void** tag, bool* status) {
-  // Release send buffers.
-  if (send_buf_ && send_message_) {
-    if (send_message_) {
-      grpc_byte_buffer_destroy(send_buf_);
-    }
-    send_buf_ = nullptr;
-  }
-  if (initial_metadata_) {
-    gpr_free(initial_metadata_);
-    initial_metadata_ = nullptr;
-  }
-  if (trailing_metadata_count_) {
-    gpr_free(trailing_metadata_);
-    trailing_metadata_ = nullptr;
-  }
-  // Set user-facing tag.
-  *tag = return_tag_;
-  // Process received initial metadata
-  if (recv_initial_metadata_) {
-    FillMetadataMap(&recv_initial_metadata_arr_, recv_initial_metadata_);
-  }
-  // Parse received message if any.
-  if (recv_message_ || recv_message_buffer_) {
-    if (recv_buf_) {
-      got_message = *status;
-      if (recv_message_) {
-        GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_DESERIALIZE, 0);
-        *status = *status &&
-                  DeserializeProto(recv_buf_, recv_message_, max_message_size_);
-        grpc_byte_buffer_destroy(recv_buf_);
-        GRPC_TIMER_END(GRPC_PTAG_PROTO_DESERIALIZE, 0);
-      } else {
-        recv_message_buffer_->set_buffer(recv_buf_);
-      }
-      recv_buf_ = nullptr;
-    } else {
-      // Read failed
-      got_message = false;
-      *status = false;
-    }
-  }
-  // Parse received status.
-  if (recv_status_) {
-    FillMetadataMap(&recv_trailing_metadata_arr_, recv_trailing_metadata_);
-    *recv_status_ = Status(
-        static_cast<StatusCode>(status_code_),
-        status_details_ ? grpc::string(status_details_) : grpc::string());
-  }
-  if (recv_closed_) {
-    *recv_closed_ = cancelled_buf_ != 0;
-  }
-  return true;
-}
-
 Call::Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq)
     : call_hook_(call_hook), cq_(cq), call_(call), max_message_size_(-1) {}
 
@@ -357,11 +82,11 @@ Call::Call(grpc_call* call, CallHook* call_hook, CompletionQueue* cq,
       call_(call),
       max_message_size_(max_message_size) {}
 
-void Call::PerformOps(CallOpBuffer* buffer) {
+void Call::PerformOps(CallOpSetInterface* ops) {
   if (max_message_size_ > 0) {
-    buffer->set_max_message_size(max_message_size_);
+    ops->set_max_message_size(max_message_size_);
   }
-  call_hook_->PerformOpsOnCall(buffer, this);
+  call_hook_->PerformOpsOnCall(ops, this);
 }
 
 }  // namespace grpc

src/cpp/proto/proto_utils.cc

@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/cpp/proto/proto_utils.h"
+#include <grpc++/impl/proto_utils.h>
 #include <grpc++/config.h>
 
 #include <grpc/grpc.h>
@@ -67,7 +67,7 @@ class GrpcBufferWriter GRPC_FINAL
       slice_ = gpr_slice_malloc(block_size_);
     }
     *data = GPR_SLICE_START_PTR(slice_);
-    byte_count_ += *size = GPR_SLICE_LENGTH(slice_);
+    byte_count_ += * size = GPR_SLICE_LENGTH(slice_);
     gpr_slice_buffer_add(slice_buffer_, slice_);
     return true;
   }
@@ -118,7 +118,7 @@ class GrpcBufferReader GRPC_FINAL
     }
     gpr_slice_unref(slice_);
     *data = GPR_SLICE_START_PTR(slice_);
-    byte_count_ += *size = GPR_SLICE_LENGTH(slice_);
+    byte_count_ += * size = GPR_SLICE_LENGTH(slice_);
     return true;
   }
 
@@ -152,20 +152,32 @@ class GrpcBufferReader GRPC_FINAL
 
 namespace grpc {
 
-bool SerializeProto(const grpc::protobuf::Message& msg, grpc_byte_buffer** bp) {
+Status SerializeProto(const grpc::protobuf::Message& msg, grpc_byte_buffer** bp) {
   GrpcBufferWriter writer(bp);
-  return msg.SerializeToZeroCopyStream(&writer);
+  return msg.SerializeToZeroCopyStream(&writer)
+             ? Status::OK
+             : Status(StatusCode::INVALID_ARGUMENT,
+                      "Failed to serialize message");
 }
 
-bool DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
-                      int max_message_size) {
-  if (!buffer) return false;
+Status DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg,
+                        int max_message_size) {
+  if (!buffer) {
+    return Status(StatusCode::INVALID_ARGUMENT, "No payload");
+  }
   GrpcBufferReader reader(buffer);
   ::grpc::protobuf::io::CodedInputStream decoder(&reader);
   if (max_message_size > 0) {
     decoder.SetTotalBytesLimit(max_message_size, max_message_size);
   }
-  return msg->ParseFromCodedStream(&decoder) && decoder.ConsumedEntireMessage();
+  if (!msg->ParseFromCodedStream(&decoder)) {
+    return Status(StatusCode::INVALID_ARGUMENT,
+                  msg->InitializationErrorString());
+  }
+  if (!decoder.ConsumedEntireMessage()) {
+    return Status(StatusCode::INVALID_ARGUMENT, "Did not read entire message");
+  }
+  return Status::OK;
 }
 
 }  // namespace grpc

src/cpp/server/insecure_server_credentials.cc

@@ -31,9 +31,10 @@
  *
  */
 
-#include <grpc/grpc_security.h>
 #include <grpc++/server_credentials.h>
 
+#include <grpc/grpc.h>
+
 namespace grpc {
 namespace {
 class InsecureServerCredentialsImpl GRPC_FINAL : public ServerCredentials {

src/cpp/server/server.cc

@@ -35,7 +35,6 @@
 #include <utility>
 
 #include <grpc/grpc.h>
-#include <grpc/grpc_security.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc++/completion_queue.h>
@@ -48,7 +47,6 @@
 #include <grpc++/time.h>
 
 #include "src/core/profiling/timers.h"
-#include "src/cpp/proto/proto_utils.h"
 
 namespace grpc {
 
@@ -69,16 +67,11 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
         has_request_payload_(method->method_type() == RpcMethod::NORMAL_RPC ||
                              method->method_type() ==
                                  RpcMethod::SERVER_STREAMING),
-        has_response_payload_(method->method_type() == RpcMethod::NORMAL_RPC ||
-                              method->method_type() ==
-                              RpcMethod::CLIENT_STREAMING),
         cq_(nullptr) {
     grpc_metadata_array_init(&request_metadata_);
   }
 
-  ~SyncRequest() {
-    grpc_metadata_array_destroy(&request_metadata_);
-  }
+  ~SyncRequest() { grpc_metadata_array_destroy(&request_metadata_); }
 
   static SyncRequest* Wait(CompletionQueue* cq, bool* ok) {
     void* tag = nullptr;
@@ -91,9 +84,7 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
     return mrd;
   }
 
-  void SetupRequest() {
-    cq_ = grpc_completion_queue_create();
-  }
+  void SetupRequest() { cq_ = grpc_completion_queue_create(); }
 
   void TeardownRequest() {
     grpc_completion_queue_destroy(cq_);
@@ -125,7 +116,6 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
           ctx_(mrd->deadline_, mrd->request_metadata_.metadata,
                mrd->request_metadata_.count),
           has_request_payload_(mrd->has_request_payload_),
-          has_response_payload_(mrd->has_response_payload_),
           request_payload_(mrd->request_payload_),
           method_(mrd->method_) {
       ctx_.call_ = mrd->call_;
@@ -142,35 +132,10 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
     }
 
     void Run() {
-      std::unique_ptr<grpc::protobuf::Message> req;
-      std::unique_ptr<grpc::protobuf::Message> res;
-      if (has_request_payload_) {
-        GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_DESERIALIZE, call_.call());
-        req.reset(method_->AllocateRequestProto());
-        if (!DeserializeProto(request_payload_, req.get(),
-                              call_.max_message_size())) {
-          // FIXME(yangg) deal with deserialization failure
-          cq_.Shutdown();
-          return;
-        }
-        GRPC_TIMER_END(GRPC_PTAG_PROTO_DESERIALIZE, call_.call());
-      }
-      if (has_response_payload_) {
-        res.reset(method_->AllocateResponseProto());
-      }
       ctx_.BeginCompletionOp(&call_);
-      auto status = method_->handler()->RunHandler(
-          MethodHandler::HandlerParameter(&call_, &ctx_, req.get(), res.get()));
-      CallOpBuffer buf;
-      if (!ctx_.sent_initial_metadata_) {
-        buf.AddSendInitialMetadata(&ctx_.initial_metadata_);
-      }
-      if (has_response_payload_) {
-        buf.AddSendMessage(*res);
-      }
-      buf.AddServerSendStatus(&ctx_.trailing_metadata_, status);
-      call_.PerformOps(&buf);
-      cq_.Pluck(&buf);  /* status ignored */
+      method_->handler()->RunHandler(MethodHandler::HandlerParameter(
+          &call_, &ctx_, request_payload_, call_.max_message_size()));
+      request_payload_ = nullptr;
       void* ignored_tag;
       bool ignored_ok;
       cq_.Shutdown();
@@ -182,7 +147,6 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
     Call call_;
     ServerContext ctx_;
     const bool has_request_payload_;
-    const bool has_response_payload_;
     grpc_byte_buffer* request_payload_;
     RpcServiceMethod* const method_;
   };
@@ -192,7 +156,6 @@ class Server::SyncRequest GRPC_FINAL : public CompletionQueueTag {
   void* const tag_;
   bool in_flight_;
   const bool has_request_payload_;
-  const bool has_response_payload_;
   grpc_call* call_;
   gpr_timespec deadline_;
   grpc_metadata_array request_metadata_;
@@ -260,9 +223,9 @@ bool Server::RegisterService(RpcService* service) {
 }
 
 bool Server::RegisterAsyncService(AsynchronousService* service) {
-  GPR_ASSERT(service->dispatch_impl_ == nullptr &&
+  GPR_ASSERT(service->server_ == nullptr &&
              "Can only register an asynchronous service against one server.");
-  service->dispatch_impl_ = this;
+  service->server_ = this;
   service->request_args_ = new void*[service->method_count_];
   for (size_t i = 0; i < service->method_count_; ++i) {
     void* tag = grpc_server_register_method(server_, service->method_names_[i],
@@ -328,141 +291,90 @@ void Server::Wait() {
   }
 }
 
-void Server::PerformOpsOnCall(CallOpBuffer* buf, Call* call) {
+void Server::PerformOpsOnCall(CallOpSetInterface* ops, Call* call) {
   static const size_t MAX_OPS = 8;
-  size_t nops = MAX_OPS;
-  grpc_op ops[MAX_OPS];
-  buf->FillOps(ops, &nops);
+  size_t nops = 0;
+  grpc_op cops[MAX_OPS];
+  ops->FillOps(cops, &nops);
   GPR_ASSERT(GRPC_CALL_OK ==
-             grpc_call_start_batch(call->call(), ops, nops, buf));
+             grpc_call_start_batch(call->call(), cops, nops, ops));
 }
 
-class Server::AsyncRequest GRPC_FINAL : public CompletionQueueTag {
- public:
-  AsyncRequest(Server* server, void* registered_method, ServerContext* ctx,
-               grpc::protobuf::Message* request,
-               ServerAsyncStreamingInterface* stream, CompletionQueue* call_cq,
-               ServerCompletionQueue* notification_cq, void* tag)
-      : tag_(tag),
-        request_(request),
-        stream_(stream),
-        call_cq_(call_cq),
-        ctx_(ctx),
-        generic_ctx_(nullptr),
-        server_(server),
-        call_(nullptr),
-        payload_(nullptr) {
-    memset(&array_, 0, sizeof(array_));
-    grpc_call_details_init(&call_details_);
-    GPR_ASSERT(notification_cq);
-    GPR_ASSERT(call_cq);
-    grpc_server_request_registered_call(
-        server->server_, registered_method, &call_, &call_details_.deadline,
-        &array_, request ? &payload_ : nullptr, call_cq->cq(),
-        notification_cq->cq(), this);
-  }
+Server::BaseAsyncRequest::BaseAsyncRequest(
+    Server* server, ServerContext* context,
+    ServerAsyncStreamingInterface* stream, CompletionQueue* call_cq, void* tag)
+    : server_(server),
+      context_(context),
+      stream_(stream),
+      call_cq_(call_cq),
+      tag_(tag),
+      call_(nullptr) {
+  memset(&initial_metadata_array_, 0, sizeof(initial_metadata_array_));
+}
 
-  AsyncRequest(Server* server, GenericServerContext* ctx,
-               ServerAsyncStreamingInterface* stream, CompletionQueue* call_cq,
-               ServerCompletionQueue* notification_cq, void* tag)
-      : tag_(tag),
-        request_(nullptr),
-        stream_(stream),
-        call_cq_(call_cq),
-        ctx_(nullptr),
-        generic_ctx_(ctx),
-        server_(server),
-        call_(nullptr),
-        payload_(nullptr) {
-    memset(&array_, 0, sizeof(array_));
-    grpc_call_details_init(&call_details_);
-    GPR_ASSERT(notification_cq);
-    GPR_ASSERT(call_cq);
-    grpc_server_request_call(server->server_, &call_, &call_details_, &array_,
-                             call_cq->cq(), notification_cq->cq(), this);
-  }
+Server::BaseAsyncRequest::~BaseAsyncRequest() {}
 
-  ~AsyncRequest() {
-    if (payload_) {
-      grpc_byte_buffer_destroy(payload_);
+bool Server::BaseAsyncRequest::FinalizeResult(void** tag, bool* status) {
+  if (*status) {
+    for (size_t i = 0; i < initial_metadata_array_.count; i++) {
+      context_->client_metadata_.insert(std::make_pair(
+          grpc::string(initial_metadata_array_.metadata[i].key),
+          grpc::string(initial_metadata_array_.metadata[i].value,
+                       initial_metadata_array_.metadata[i].value +
+                           initial_metadata_array_.metadata[i].value_length)));
     }
-    grpc_metadata_array_destroy(&array_);
   }
-
-  bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE {
-    *tag = tag_;
-    bool orig_status = *status;
-    if (*status && request_) {
-      if (payload_) {
-        GRPC_TIMER_BEGIN(GRPC_PTAG_PROTO_DESERIALIZE, call_);
-        *status =
-            DeserializeProto(payload_, request_, server_->max_message_size_);
-        GRPC_TIMER_END(GRPC_PTAG_PROTO_DESERIALIZE, call_);
-      } else {
-        *status = false;
-      }
-    }
-    ServerContext* ctx = ctx_ ? ctx_ : generic_ctx_;
-    GPR_ASSERT(ctx);
-    if (*status) {
-      ctx->deadline_ = call_details_.deadline;
-      for (size_t i = 0; i < array_.count; i++) {
-        ctx->client_metadata_.insert(std::make_pair(
-            grpc::string(array_.metadata[i].key),
-            grpc::string(
-                array_.metadata[i].value,
-                array_.metadata[i].value + array_.metadata[i].value_length)));
-      }
-      if (generic_ctx_) {
-        // TODO(yangg) remove the copy here.
-        generic_ctx_->method_ = call_details_.method;
-        generic_ctx_->host_ = call_details_.host;
-        gpr_free(call_details_.method);
-        gpr_free(call_details_.host);
-      }
-    }
-    ctx->call_ = call_;
-    ctx->cq_ = call_cq_;
-    Call call(call_, server_, call_cq_, server_->max_message_size_);
-    if (orig_status && call_) {
-      ctx->BeginCompletionOp(&call);
-    }
-    // just the pointers inside call are copied here
-    stream_->BindCall(&call);
-    delete this;
-    return true;
+  grpc_metadata_array_destroy(&initial_metadata_array_);
+  context_->call_ = call_;
+  context_->cq_ = call_cq_;
+  Call call(call_, server_, call_cq_, server_->max_message_size_);
+  if (*status && call_) {
+    context_->BeginCompletionOp(&call);
   }
+  // just the pointers inside call are copied here
+  stream_->BindCall(&call);
+  *tag = tag_;
+  delete this;
+  return true;
+}
 
- private:
-  void* const tag_;
-  grpc::protobuf::Message* const request_;
-  ServerAsyncStreamingInterface* const stream_;
-  CompletionQueue* const call_cq_;
-  ServerContext* const ctx_;
-  GenericServerContext* const generic_ctx_;
-  Server* const server_;
-  grpc_call* call_;
-  grpc_call_details call_details_;
-  grpc_metadata_array array_;
-  grpc_byte_buffer* payload_;
-};
+Server::RegisteredAsyncRequest::RegisteredAsyncRequest(
+    Server* server, ServerContext* context,
+    ServerAsyncStreamingInterface* stream, CompletionQueue* call_cq, void* tag)
+    : BaseAsyncRequest(server, context, stream, call_cq, tag) {}
+
+void Server::RegisteredAsyncRequest::IssueRequest(
+    void* registered_method, grpc_byte_buffer** payload,
+    ServerCompletionQueue* notification_cq) {
+  grpc_server_request_registered_call(
+      server_->server_, registered_method, &call_, &context_->deadline_,
+      &initial_metadata_array_, payload, call_cq_->cq(), notification_cq->cq(),
+      this);
+}
 
-void Server::RequestAsyncCall(void* registered_method, ServerContext* context,
-                              grpc::protobuf::Message* request,
-                              ServerAsyncStreamingInterface* stream,
-                              CompletionQueue* call_cq,
-                              ServerCompletionQueue* notification_cq,
-                              void* tag) {
-  new AsyncRequest(this, registered_method, context, request, stream, call_cq,
-                   notification_cq, tag);
+Server::GenericAsyncRequest::GenericAsyncRequest(
+    Server* server, GenericServerContext* context,
+    ServerAsyncStreamingInterface* stream, CompletionQueue* call_cq,
+    ServerCompletionQueue* notification_cq, void* tag)
+    : BaseAsyncRequest(server, context, stream, call_cq, tag) {
+  grpc_call_details_init(&call_details_);
+  GPR_ASSERT(notification_cq);
+  GPR_ASSERT(call_cq);
+  grpc_server_request_call(server->server_, &call_, &call_details_,
+                           &initial_metadata_array_, call_cq->cq(),
+                           notification_cq->cq(), this);
 }
 
-void Server::RequestAsyncGenericCall(GenericServerContext* context,
-                                     ServerAsyncStreamingInterface* stream,
-                                     CompletionQueue* call_cq,
-                                     ServerCompletionQueue* notification_cq,
-                                     void* tag) {
-  new AsyncRequest(this, context, stream, call_cq, notification_cq, tag);
+bool Server::GenericAsyncRequest::FinalizeResult(void** tag, bool* status) {
+  // TODO(yangg) remove the copy here.
+  if (*status) {
+    static_cast<GenericServerContext*>(context_)->method_ =
+        call_details_.method;
+    static_cast<GenericServerContext*>(context_)->host_ = call_details_.host;
+  }
+  gpr_free(call_details_.method);
+  gpr_free(call_details_.host);
+  return BaseAsyncRequest::FinalizeResult(tag, status);
 }
 
 void Server::ScheduleCallback() {

src/cpp/server/server_context.cc

@@ -43,12 +43,12 @@ namespace grpc {
 
 // CompletionOp
 
-class ServerContext::CompletionOp GRPC_FINAL : public CallOpBuffer {
+class ServerContext::CompletionOp GRPC_FINAL : public CallOpSetInterface {
  public:
   // initial refs: one in the server context, one in the cq
-  CompletionOp() : refs_(2), finalized_(false), cancelled_(false) {
-    AddServerRecvClose(&cancelled_);
-  }
+  CompletionOp() : refs_(2), finalized_(false), cancelled_(0) {}
+
+  void FillOps(grpc_op* ops, size_t* nops) GRPC_OVERRIDE;
   bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE;
 
   bool CheckCancelled(CompletionQueue* cq);
@@ -59,7 +59,7 @@ class ServerContext::CompletionOp GRPC_FINAL : public CallOpBuffer {
   grpc::mutex mu_;
   int refs_;
   bool finalized_;
-  bool cancelled_;
+  int cancelled_;
 };
 
 void ServerContext::CompletionOp::Unref() {
@@ -73,14 +73,20 @@ void ServerContext::CompletionOp::Unref() {
 bool ServerContext::CompletionOp::CheckCancelled(CompletionQueue* cq) {
   cq->TryPluck(this);
   grpc::lock_guard<grpc::mutex> g(mu_);
-  return finalized_ ? cancelled_ : false;
+  return finalized_ ? cancelled_ != 0 : false;
+}
+
+void ServerContext::CompletionOp::FillOps(grpc_op* ops, size_t* nops) {
+  ops->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
+  ops->data.recv_close_on_server.cancelled = &cancelled_;
+  ops->flags = 0;
+  *nops = 1;
 }
 
 bool ServerContext::CompletionOp::FinalizeResult(void** tag, bool* status) {
-  GPR_ASSERT(CallOpBuffer::FinalizeResult(tag, status));
   grpc::unique_lock<grpc::mutex> lock(mu_);
   finalized_ = true;
-  if (!*status) cancelled_ = true;
+  if (!*status) cancelled_ = 1;
   if (--refs_ == 0) {
     lock.unlock();
     delete this;

src/cpp/server/server_credentials.cc

@@ -31,8 +31,6 @@
  *
  */
 
-#include <grpc/grpc_security.h>
-
 #include <grpc++/server_credentials.h>
 
 namespace grpc {

src/objective-c/GRPCClient/GRPCCall.h

@@ -46,7 +46,7 @@
 // transparently on the same TCP connection.
 
 #import <Foundation/Foundation.h>
-#import <gRPC/GRXWriter.h>
+#import <RxLibrary/GRXWriter.h>
 
 @class GRPCMethodName;
 

src/objective-c/GRPCClient/GRPCCall.m

@@ -34,7 +34,7 @@
 #import "GRPCCall.h"
 
 #include <grpc/grpc.h>
-#include <grpc/support/grpc_time.h>
+#include <grpc/support/time.h>
 
 #import "GRPCMethodName.h"
 #import "private/GRPCChannel.h"

src/objective-c/GRPCClient/private/GRPCDelegateWrapper.m

@@ -33,7 +33,7 @@
 
 #import "GRPCDelegateWrapper.h"
 
-#import <gRPC/GRXWriteable.h>
+#import <RxLibrary/GRXWriteable.h>
 
 @interface GRPCDelegateWrapper ()
 // These are atomic so that cancellation can nillify them from any thread.

src/objective-c/GRPCClient/private/GRPCMethodName+HTTP2Encoding.h

@@ -33,7 +33,7 @@
 
 #import <Foundation/Foundation.h>
 
-#import "GRPCMethodName.h"
+#import "GRPCClient/GRPCMethodName.h"
 
 @interface GRPCMethodName (HTTP2Encoding)
 - (NSString *)HTTP2Path;

src/objective-c/GRPCClient/private/GRPCSecureChannel.m

@@ -38,13 +38,17 @@
 @implementation GRPCSecureChannel
 
 - (instancetype)initWithHost:(NSString *)host {
-  static const grpc_credentials *kCredentials;
+  static grpc_credentials *kCredentials;
   static dispatch_once_t loading;
   dispatch_once(&loading, ^{
     // Do not use NSBundle.mainBundle, as it's nil for tests of library projects.
     NSBundle *bundle = [NSBundle bundleForClass:self.class];
-    NSString *certsPath = [bundle pathForResource:@"gRPC.bundle/roots" ofType:@"pem"];
+    NSString *certsPath = [bundle pathForResource:@"gRPCCertificates.bundle/roots" ofType:@"pem"];
+    NSAssert(certsPath.length,
+             @"gRPCCertificates.bundle/roots.pem not found under %@. This file, with the root "
+             "certificates, is needed to establish TLS (HTTPS) connections.", bundle.bundlePath);
     NSData *certsData = [NSData dataWithContentsOfFile:certsPath];
+    NSAssert(certsData.length, @"No data read from %@", certsPath);
     NSString *certsString = [[NSString alloc] initWithData:certsData encoding:NSUTF8StringEncoding];
     kCredentials = grpc_ssl_credentials_create(certsString.UTF8String, NULL);
   });

src/objective-c/GRPCClient/private/GRPCWrappedCall.m

@@ -132,8 +132,12 @@
     grpc_metadata_array_init(&_headers);
     _op.data.recv_initial_metadata = &_headers;
     if (handler) {
+      // Prevent reference cycle with _handler
+      __weak typeof(self) weakSelf = self;
       _handler = ^{
-        NSDictionary *metadata = [NSDictionary grpc_dictionaryFromMetadataArray:_headers];
+        __strong typeof(self) strongSelf = weakSelf;
+        NSDictionary *metadata = [NSDictionary
+                                  grpc_dictionaryFromMetadataArray:strongSelf->_headers];
         handler(metadata);
       };
     }
@@ -160,8 +164,11 @@
     _op.op = GRPC_OP_RECV_MESSAGE;
     _op.data.recv_message = &_receivedMessage;
     if (handler) {
+      // Prevent reference cycle with _handler
+      __weak typeof(self) weakSelf = self;
       _handler = ^{
-        handler(_receivedMessage);
+        __strong typeof(self) strongSelf = weakSelf;
+        handler(strongSelf->_receivedMessage);
       };
     }
   }
@@ -190,9 +197,14 @@
     grpc_metadata_array_init(&_trailers);
     _op.data.recv_status_on_client.trailing_metadata = &_trailers;
     if (handler) {
+      // Prevent reference cycle with _handler
+      __weak typeof(self) weakSelf = self;
       _handler = ^{
-        NSError *error = [NSError grpc_errorFromStatusCode:_statusCode details:_details];
-        NSDictionary *trailers = [NSDictionary grpc_dictionaryFromMetadataArray:_trailers];
+        __strong typeof(self) strongSelf = weakSelf;
+        NSError *error = [NSError grpc_errorFromStatusCode:strongSelf->_statusCode
+                                                   details:strongSelf->_details];
+        NSDictionary *trailers = [NSDictionary
+                                  grpc_dictionaryFromMetadataArray:strongSelf->_trailers];
         handler(error, trailers);
       };
     }

src/objective-c/GRPCClient/private/NSError+GRPC.m

@@ -33,7 +33,7 @@
 
 #import "NSError+GRPC.h"
 
-#include <grpc.h>
+#include <grpc/grpc.h>
 
 NSString * const kGRPCErrorDomain = @"io.grpc";
 

src/objective-c/ProtoRPC/ProtoRPC.h

@@ -32,7 +32,7 @@
  */
 
 #import <Foundation/Foundation.h>
-#import <gRPC/GRPCCall.h>
+#import <GRPCClient/GRPCCall.h>
 
 @interface ProtoRPC : GRPCCall
 

src/objective-c/ProtoRPC/ProtoRPC.m

@@ -33,10 +33,10 @@
 
 #import "ProtoRPC.h"
 
-#import <gRPC/GRXWriteable.h>
-#import <gRPC/GRXWriter.h>
-#import <gRPC/GRXWriter+Transformations.h>
-#import <Protobuf/GPBProtocolBuffers.h>
+#import <GPBProtocolBuffers.h>
+#import <RxLibrary/GRXWriteable.h>
+#import <RxLibrary/GRXWriter.h>
+#import <RxLibrary/GRXWriter+Transformations.h>
 
 @implementation ProtoRPC {
   id<GRXWriteable> _responseWriteable;
@@ -66,6 +66,8 @@
   // A writer that serializes the proto messages to send.
   id<GRXWriter> bytesWriter =
       [[[GRXWriter alloc] initWithWriter:requestsWriter] map:^id(GPBMessage *proto) {
+        // TODO(jcanizales): Fail with an understandable error message if the requestsWriter isn't
+        // sending GPBMessages.
         return [proto data];
       }];
   if ((self = [super initWithHost:host method:method requestsWriter:bytesWriter])) {

src/objective-c/ProtoRPC/ProtoService.m

@@ -33,9 +33,9 @@
 
 #import "ProtoService.h"
 
-#import <gRPC/GRPCMethodName.h>
-#import <gRPC/GRXWriteable.h>
-#import <gRPC/GRXWriter.h>
+#import <GRPCClient/GRPCMethodName.h>
+#import <RxLibrary/GRXWriteable.h>
+#import <RxLibrary/GRXWriter.h>
 
 #import "ProtoRPC.h"
 

src/objective-c/RxLibrary/transformations/GRXMappingWriter.h

@@ -31,7 +31,7 @@
  *
  */
 
-#import "GRXWriter.h"
+#import "RxLibrary/GRXWriter.h"
 
 // A "proxy" writer that transforms all the values of its input writer by using a mapping function.
 @interface GRXMappingWriter : GRXWriter

src/objective-c/examples/Sample/Sample/ViewController.m

@@ -33,12 +33,12 @@
 
 #import "ViewController.h"
 
-#import <gRPC/GRPCCall.h>
-#import <gRPC/GRPCMethodName.h>
-#import <gRPC/GRXWriter+Immediate.h>
-#import <gRPC/GRXWriteable.h>
+#import <GRPCClient/GRPCCall.h>
+#import <GRPCClient/GRPCMethodName.h>
 #import <RemoteTest/Messages.pbobjc.h>
 #import <RemoteTest/Test.pbrpc.h>
+#import <RxLibrary/GRXWriter+Immediate.h>
+#import <RxLibrary/GRXWriteable.h>
 
 @implementation ViewController
 

src/objective-c/tests/GRPCClientTests.m

@@ -34,11 +34,11 @@
 #import <UIKit/UIKit.h>
 #import <XCTest/XCTest.h>
 
-#import <gRPC/GRPCCall.h>
-#import <gRPC/GRPCMethodName.h>
-#import <gRPC/GRXWriter+Immediate.h>
-#import <gRPC/GRXWriteable.h>
+#import <GRPCClient/GRPCCall.h>
+#import <GRPCClient/GRPCMethodName.h>
 #import <RemoteTest/Messages.pbobjc.h>
+#import <RxLibrary/GRXWriteable.h>
+#import <RxLibrary/GRXWriter+Immediate.h>
 
 // These are a few tests similar to InteropTests, but which use the generic gRPC client (GRPCCall)
 // rather than a generated proto library on top of it.

src/objective-c/tests/InteropTests.m

@@ -36,13 +36,13 @@
 #import <UIKit/UIKit.h>
 #import <XCTest/XCTest.h>
 
-#import <gRPC/GRXWriter+Immediate.h>
-#import <gRPC/GRXBufferedPipe.h>
-#import <gRPC/ProtoRPC.h>
+#import <ProtoRPC/ProtoRPC.h>
 #import <RemoteTest/Empty.pbobjc.h>
 #import <RemoteTest/Messages.pbobjc.h>
 #import <RemoteTest/Test.pbobjc.h>
 #import <RemoteTest/Test.pbrpc.h>
+#import <RxLibrary/GRXBufferedPipe.h>
+#import <RxLibrary/GRXWriter+Immediate.h>
 
 // Convenience constructors for the generated proto messages:
 
@@ -174,7 +174,7 @@
 
   __block int index = 0;
   [_service streamingOutputCallWithRequest:request
-                                   handler:^(BOOL done,
+                              eventHandler:^(BOOL done,
                                              RMTStreamingOutputCallResponse *response,
                                              NSError *error){
     XCTAssertNil(error, @"Finished with unexpected error: %@", error);
@@ -211,7 +211,7 @@
   [requestsBuffer writeValue:request];
 
   [_service fullDuplexCallWithRequestsWriter:requestsBuffer
-                                     handler:^(BOOL done,
+                                eventHandler:^(BOOL done,
                                                RMTStreamingOutputCallResponse *response,
                                                NSError *error) {
     XCTAssertNil(error, @"Finished with unexpected error: %@", error);
@@ -242,7 +242,7 @@
 - (void)testEmptyStreamRPC {
   __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyStream"];
   [_service fullDuplexCallWithRequestsWriter:[GRXWriter emptyWriter]
-                                     handler:^(BOOL done,
+                                eventHandler:^(BOOL done,
                                                RMTStreamingOutputCallResponse *response,
                                                NSError *error) {
     XCTAssertNil(error, @"Finished with unexpected error: %@", error);
@@ -283,7 +283,7 @@
   [requestsBuffer writeValue:request];
   
   __block ProtoRPC *call = [_service RPCToFullDuplexCallWithRequestsWriter:requestsBuffer
-                                                                   handler:^(BOOL done,
+                                                              eventHandler:^(BOOL done,
                                                                              RMTStreamingOutputCallResponse *response,
                                                                              NSError *error) {
     if (receivedResponse) {

src/objective-c/tests/LocalClearTextTests.m

@@ -34,12 +34,12 @@
 #import <UIKit/UIKit.h>
 #import <XCTest/XCTest.h>
 
-#import <gRPC/GRPCCall.h>
-#import <gRPC/GRPCMethodName.h>
-#import <gRPC/GRXWriter+Immediate.h>
-#import <gRPC/GRXWriteable.h>
+#import <GRPCClient/GRPCCall.h>
+#import <GRPCClient/GRPCMethodName.h>
 #import <RouteGuide/RouteGuide.pbobjc.h>
 #import <RouteGuide/RouteGuide.pbrpc.h>
+#import <RxLibrary/GRXWriteable.h>
+#import <RxLibrary/GRXWriter+Immediate.h>
 
 // These tests require a gRPC "RouteGuide" sample server to be running locally. You can compile and
 // run one by following the instructions here: https://github.com/grpc/grpc-common/blob/master/cpp/cpptutorial.md#try-it-out

src/objective-c/tests/RxLibraryUnitTests.m

@@ -34,9 +34,9 @@
 #import <UIKit/UIKit.h>
 #import <XCTest/XCTest.h>
 
-#import <gRPC/GRXBufferedPipe.h>
-#import <gRPC/GRXWriter.h>
-#import <gRPC/GRXWriteable.h>
+#import <RxLibrary/GRXBufferedPipe.h>
+#import <RxLibrary/GRXWriteable.h>
+#import <RxLibrary/GRXWriter.h>
 
 // A mock of a GRXSingleValueHandler block that can be queried for how many times it was called and
 // what were the last values passed to it.

src/php/README.md

@@ -5,14 +5,27 @@ This directory contains source code for PHP implementation of gRPC layered on sh
 
 #Status
 
-Pre-Alpha : This gRPC PHP implementation is work-in-progress and is not expected to work yet.
+Alpha : Ready for early adopters
 
 ## ENVIRONMENT
 
-Prerequisite: PHP 5.5 or later, PHPUnit, pecl
+Prerequisite: PHP 5.5 or later, `phpunit`, `pecl`
+
+Linux:
+
+```sh
+$ sudo apt-get install php5 php5-dev phpunit php-pear
+```
+
+OS X:
 
 ```sh
-sudo apt-get install php5 php5-dev phpunit php-pear
+$ curl https://phar.phpunit.de/phpunit.phar -o phpunit.phar
+$ chmod +x phpunit.phar
+$ sudo mv phpunit.phar /usr/local/bin/phpunit
+
+$ curl -O http://pear.php.net/go-pear.phar
+$ sudo php -d detect_unicode=0 go-pear.phar
 ```
 
 ## Build from Homebrew

src/php/bin/determine_extension_dir.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set -e
+default_extension_dir=$(php-config --extension-dir)
+if command -v brew >/dev/null && [ -d $(brew --prefix)/opt/grpc-php ]; then
+  # homebrew and the grpc-php formula are installed
+  extension_dir="-d extension_dir="$(brew --prefix)/opt/grpc-php
+elif [ ! -e $default_extension_dir/grpc.so ]; then
+  # the grpc extension is not found in the default PHP extension dir
+  # try the source modules directory
+  module_dir=../ext/grpc/modules
+  if [ ! -e $module_dir/grpc.so ]; then
+    echo "Please run 'phpize && ./configure && make' from ext/grpc first"
+    exit 1
+  fi
+  # sym-link in system supplied extensions
+  for f in $default_extension_dir/*.so; do
+    ln -s $f $module_dir/$(basename $f) &> /dev/null || true
+  done
+  extension_dir="-d extension_dir="$module_dir
+fi

src/php/bin/interop_client.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -28,11 +28,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
-set +e
+set -e
 cd $(dirname $0)
-
-module_dir=../ext/grpc/modules
-
-php -d extension_dir=$module_dir -d extension=grpc.so \
+source ./determine_extension_dir.sh
+php $extension_dir -d extension=grpc.so \
   ../tests/interop/interop_client.php $@ 1>&2

src/php/bin/run_gen_code_test.sh

@@ -1,4 +1,4 @@
-# Runs the generated code test against the ruby server
+#!/bin/bash
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -28,10 +28,11 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+set -e
 cd $(dirname $0)
-GRPC_TEST_HOST=localhost:50051 php -d extension_dir=../ext/grpc/modules/ \
-  -d extension=grpc.so `which phpunit` -v --debug --strict \
+source ./determine_extension_dir.sh
+export GRPC_TEST_HOST=localhost:7071
+php $extension_dir -d extension=grpc.so $(which phpunit) -v --debug --strict \
   ../tests/generated_code/GeneratedCodeTest.php
-GRPC_TEST_HOST=localhost:50051 php -d extension_dir=../ext/grpc/modules/ \
-  -d extension=grpc.so `which phpunit` -v --debug --strict \
+php $extension_dir -d extension=grpc.so $(which phpunit) -v --debug --strict \
   ../tests/generated_code/GeneratedCodeWithCallbackTest.php

src/php/bin/run_tests.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2015, Google Inc.
 # All rights reserved.
 #
@@ -32,33 +32,6 @@
 # against it
 set -e
 cd $(dirname $0)
-default_extension_dir=`php -i | grep extension_dir | sed 's/.*=> //g'`
-
-if command -v brew >/dev/null && [ -d `brew --prefix`/opt/grpc-php ]
-then
-  # homebrew and the grpc-php formula are installed
-  extension_dir="-d extension_dir="`brew --prefix`/opt/grpc-php
-elif [ ! -e $default_extension_dir/grpc.so ]
-then
-  # the grpc extension is not found in the default PHP extension dir
-  # try the source modules directory
-  module_dir=../ext/grpc/modules
-  if [ ! -d $module_dir ]
-  then
-    echo "Please run 'phpize && ./configure && make' from ext/grpc first"
-    exit 1
-  fi
-
-  # sym-link in system supplied extensions
-  for f in $default_extension_dir/*.so
-  do
-    ln -s $f $module_dir/$(basename $f) &> /dev/null || true
-  done
-
-  extension_dir='-d extension_dir='$module_dir
-fi
-
-php \
-  $extension_dir \
-  -d extension=grpc.so \
-  `which phpunit` -v --debug --strict ../tests/unit_tests
+source ./determine_extension_dir.sh
+php $extension_dir -d extension=grpc.so $(which phpunit) -v --debug --strict \
+  ../tests/unit_tests

src/php/ext/grpc/config.m4

@@ -35,8 +35,13 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_LIBRARY(dl,,GRPC_SHARED_LIBADD)
   PHP_ADD_LIBRARY(dl)
 
-  PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD)
-  PHP_ADD_LIBRARY(rt)
+  case $host in
+    *darwin*) ;;
+    *)
+      PHP_ADD_LIBRARY(rt,,GRPC_SHARED_LIBADD)
+      PHP_ADD_LIBRARY(rt)
+      ;;
+  esac
 
   GRPC_LIBDIR=$GRPC_DIR/${GRPC_LIB_SUBDIR-lib}
 

src/python/src/grpc/_adapter/_c/types.h

@@ -241,10 +241,10 @@ double pygrpc_cast_gpr_timespec_to_double(gpr_timespec timespec);
 gpr_timespec pygrpc_cast_double_to_gpr_timespec(double seconds);
 
 /* Returns true on success, false on failure. */
-int pygrpc_cast_pylist_to_send_metadata(
-    PyObject *pylist, grpc_metadata **metadata, size_t *count);
+int pygrpc_cast_pyseq_to_send_metadata(
+    PyObject *pyseq, grpc_metadata **metadata, size_t *count);
 /* Returns a metadata array as a Python object on success, else NULL. */
-PyObject *pygrpc_cast_metadata_array_to_pylist(grpc_metadata_array metadata);
+PyObject *pygrpc_cast_metadata_array_to_pyseq(grpc_metadata_array metadata);
 
 /* Transliterate from a list of python channel arguments (2-tuples of string
    and string|integer|None) to a grpc_channel_args object. The strings placed

src/python/src/grpc/_adapter/_c/utility.c

@@ -32,6 +32,7 @@
  */
 
 #include <math.h>
+#include <string.h>
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
@@ -118,7 +119,7 @@ PyObject *pygrpc_consume_event(grpc_event event) {
           tag->request_call_details.method, tag->request_call_details.host,
           pygrpc_cast_gpr_timespec_to_double(tag->request_call_details.deadline),
           GRPC_OP_RECV_INITIAL_METADATA,
-          pygrpc_cast_metadata_array_to_pylist(tag->request_metadata), Py_None,
+          pygrpc_cast_metadata_array_to_pyseq(tag->request_metadata), Py_None,
           Py_None, Py_None, Py_None,
           event.success ? Py_True : Py_False);
     } else {
@@ -172,7 +173,7 @@ int pygrpc_produce_op(PyObject *op, grpc_op *result) {
   c_op.flags = 0;
   switch (type) {
   case GRPC_OP_SEND_INITIAL_METADATA:
-    if (!pygrpc_cast_pylist_to_send_metadata(
+    if (!pygrpc_cast_pyseq_to_send_metadata(
             PyTuple_GetItem(op, INITIAL_METADATA_INDEX),
             &c_op.data.send_initial_metadata.metadata,
             &c_op.data.send_initial_metadata.count)) {
@@ -190,7 +191,7 @@ int pygrpc_produce_op(PyObject *op, grpc_op *result) {
     /* Don't need to fill in any other fields. */
     break;
   case GRPC_OP_SEND_STATUS_FROM_SERVER:
-    if (!pygrpc_cast_pylist_to_send_metadata(
+    if (!pygrpc_cast_pyseq_to_send_metadata(
             PyTuple_GetItem(op, TRAILING_METADATA_INDEX),
             &c_op.data.send_status_from_server.trailing_metadata,
             &c_op.data.send_status_from_server.trailing_metadata_count)) {
@@ -247,8 +248,16 @@ int pygrpc_produce_op(PyObject *op, grpc_op *result) {
 }
 
 void pygrpc_discard_op(grpc_op op) {
+  size_t i;
   switch(op.op) {
   case GRPC_OP_SEND_INITIAL_METADATA:
+    /* Whenever we produce send-metadata, we allocate new strings (to handle
+       arbitrary sequence input as opposed to just lists or just tuples). We
+       thus must free those elements. */
+    for (i = 0; i < op.data.send_initial_metadata.count; ++i) {
+      gpr_free((void *)op.data.send_initial_metadata.metadata[i].key);
+      gpr_free((void *)op.data.send_initial_metadata.metadata[i].value);
+    }
     gpr_free(op.data.send_initial_metadata.metadata);
     break;
   case GRPC_OP_SEND_MESSAGE:
@@ -258,6 +267,16 @@ void pygrpc_discard_op(grpc_op op) {
     /* Don't need to free any fields. */
     break;
   case GRPC_OP_SEND_STATUS_FROM_SERVER:
+    /* Whenever we produce send-metadata, we allocate new strings (to handle
+       arbitrary sequence input as opposed to just lists or just tuples). We
+       thus must free those elements. */
+    for (i = 0; i < op.data.send_status_from_server.trailing_metadata_count;
+         ++i) {
+      gpr_free(
+          (void *)op.data.send_status_from_server.trailing_metadata[i].key);
+      gpr_free(
+          (void *)op.data.send_status_from_server.trailing_metadata[i].value);
+    }
     gpr_free(op.data.send_status_from_server.trailing_metadata);
     gpr_free((char *)op.data.send_status_from_server.status_details);
     break;
@@ -419,31 +438,41 @@ void pygrpc_discard_channel_args(grpc_channel_args args) {
   gpr_free(args.args);
 }
 
-int pygrpc_cast_pylist_to_send_metadata(
-    PyObject *pylist, grpc_metadata **metadata, size_t *count) {
+int pygrpc_cast_pyseq_to_send_metadata(
+    PyObject *pyseq, grpc_metadata **metadata, size_t *count) {
   size_t i;
   Py_ssize_t value_length;
-  *count = PyList_Size(pylist);
+  char *key;
+  char *value;
+  if (!PySequence_Check(pyseq)) {
+    return 0;
+  }
+  *count = PySequence_Size(pyseq);
   *metadata = gpr_malloc(sizeof(grpc_metadata) * *count);
   for (i = 0; i < *count; ++i) {
-    if (!PyArg_ParseTuple(
-        PyList_GetItem(pylist, i), "ss#",
-        &(*metadata)[i].key, &(*metadata)[i].value, &value_length)) {
+    PyObject *item = PySequence_GetItem(pyseq, i);
+    if (!PyArg_ParseTuple(item, "ss#", &key, &value, &value_length)) {
+      Py_DECREF(item);
       gpr_free(*metadata);
       *count = 0;
       *metadata = NULL;
       return 0;
+    } else {
+      (*metadata)[i].key = gpr_strdup(key);
+      (*metadata)[i].value = gpr_malloc(value_length);
+      memcpy((void *)(*metadata)[i].value, value, value_length);
+      Py_DECREF(item);
     }
     (*metadata)[i].value_length = value_length;
   }
   return 1;
 }
 
-PyObject *pygrpc_cast_metadata_array_to_pylist(grpc_metadata_array metadata) {
-  PyObject *result = PyList_New(metadata.count);
+PyObject *pygrpc_cast_metadata_array_to_pyseq(grpc_metadata_array metadata) {
+  PyObject *result = PyTuple_New(metadata.count);
   size_t i;
   for (i = 0; i < metadata.count; ++i) {
-    PyList_SetItem(
+    PyTuple_SetItem(
         result, i, Py_BuildValue(
             "ss#", metadata.metadata[i].key, metadata.metadata[i].value,
             (Py_ssize_t)metadata.metadata[i].value_length));

src/python/src/grpc/_adapter/_intermediary_low.py

@@ -144,10 +144,11 @@ class Call(object):
     self._metadata.append((key, value))
 
   def premetadata(self):
-    return self._internal.start_batch([
+    result = self._internal.start_batch([
           _types.OpArgs.send_initial_metadata(self._metadata)
       ], _IGNORE_ME_TAG)
     self._metadata = []
+    return result
 
   def read(self, tag):
     return self._internal.start_batch([

src/python/src/grpc/_adapter/_intermediary_low_test.py

@@ -282,6 +282,9 @@ class EchoTest(unittest.TestCase):
     self.assertIn(server_trailing_binary_metadata_key, metadata)
     self.assertEqual(server_trailing_binary_metadata_value,
                      metadata[server_trailing_binary_metadata_key])
+    self.assertSetEqual(set(key for key, _ in finish_accepted.metadata),
+                        set((server_trailing_metadata_key,
+                             server_trailing_binary_metadata_key,)))
 
     server_timeout_none_event = self.server_completion_queue.get(0)
     self.assertIsNone(server_timeout_none_event)

src/ruby/lib/grpc/version.rb

@@ -29,5 +29,5 @@
 
 # GRPC contains the General RPC module.
 module GRPC
-  VERSION = '0.9.3'
+  VERSION = '0.9.4'
 end

templates/BUILD.template

@@ -63,6 +63,12 @@ ${cc_library(lib)}
 % endif
 % endfor
 
+% for lib in libs:
+% if lib.name in ("grpc", "gpr"):
+${objc_library(lib)}
+% endif
+% endfor
+
 % for tgt in targets:
 % if tgt.build == 'protoc':
 ${cc_binary(tgt)}
@@ -97,6 +103,40 @@ cc_library(
 )
 </%def>
 
+<%def name="objc_library(lib)">
+objc_library(
+  name = "${lib.name}_objc",
+  srcs = [
+% for src in lib.src:
+    "${src}",
+% endfor
+  ],
+  hdrs = [
+% for hdr in lib.get("public_headers", []):
+    "${hdr}",
+% endfor
+% for hdr in lib.get("headers", []):
+    "${hdr}",
+% endfor
+  ],
+  includes = [
+    "include",
+    ".",
+  ],
+  deps = [
+% for dep in lib.get("deps", []):
+    ":${dep}_objc",
+% endfor
+% if lib.get('secure', 'no') == 'yes':
+    "//external:libssl_objc",
+% endif
+  ],
+% if lib.get("baselib", false):
+  sdk_dylibs = ["libz"],
+% endif
+)
+</%def>
+
 <%def name="cc_binary(tgt)">
 cc_binary(
   name = "${tgt.name}",
@@ -112,3 +152,74 @@ cc_binary(
   ],
 )
 </%def>
+
+objc_path = "src/objective-c"
+
+rx_library_path = objc_path + "/RxLibrary"
+
+objc_library(
+  name = "rx_library",
+  hdrs = glob([
+    rx_library_path + "/*.h",
+    rx_library_path + "/transformations/*.h",
+  ]),
+  srcs = glob([
+    rx_library_path + "/*.m",
+    rx_library_path + "/transformations/*.m",
+  ]),
+  includes = [objc_path],
+  deps = [
+    ":rx_library_private",
+  ],
+)
+
+objc_library(
+  name = "rx_library_private",
+  hdrs = glob([rx_library_path + "/private/*.h"]),
+  srcs = glob([rx_library_path + "/private/*.m"]),
+  visibility = ["//visibility:private"],
+)
+
+objc_client_path = objc_path + "/GRPCClient"
+
+objc_library(
+  name = "grpc_client",
+  hdrs = glob([
+    objc_client_path + "/*.h",
+    objc_client_path + "/private/*.h",
+  ]),
+  srcs = glob([
+    objc_client_path + "/*.m",
+    objc_client_path + "/private/*.m",
+  ]),
+  includes = [objc_path],
+  bundles = [":gRPCCertificates"],
+  deps = [
+    ":grpc_objc",
+    ":rx_library",
+  ],
+)
+
+objc_bundle_library(
+    # The choice of name is signicant here, since it determines the bundle name.
+    name = "gRPCCertificates",
+    resources = ["etc/roots.pem"],
+)
+
+proto_objc_rpc_path = objc_path + "/ProtoRPC"
+
+objc_library(
+  name = "proto_objc_rpc",
+  hdrs = glob([
+    proto_objc_rpc_path + "/*.h",
+  ]),
+  srcs = glob([
+    proto_objc_rpc_path + "/*.m",
+  ]),
+  includes = [objc_path],
+  deps = [
+    ":grpc_client",
+    ":rx_library",
+    "//external:protobuf_objc",
+  ],
+)

templates/Makefile.template

@@ -171,7 +171,7 @@ LD_asan = clang
 LDXX_asan = clang++
 CPPFLAGS_asan = -O0 -fsanitize=address -fno-omit-frame-pointer
 LDFLAGS_asan = -fsanitize=address
-DEFINES_asan = GRPC_TEST_SLOWDOWN_BUILD_FACTOR=5
+DEFINES_asan = GRPC_TEST_SLOWDOWN_BUILD_FACTOR=3
 
 VALID_CONFIG_msan = 1
 REQUIRE_CUSTOM_LIBRARIES_msan = 1
@@ -182,7 +182,7 @@ LDXX_msan = clang++-libc++
 CPPFLAGS_msan = -O0 -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1
 OPENSSL_CFLAGS_msan = -DPURIFY
 LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1
-DEFINES_msan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20
+DEFINES_msan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=4
 
 VALID_CONFIG_ubsan = 1
 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1
@@ -193,7 +193,7 @@ LDXX_ubsan = clang++
 CPPFLAGS_ubsan = -O1 -fsanitize=undefined -fno-omit-frame-pointer
 OPENSSL_CFLAGS_ubsan = -DPURIFY
 LDFLAGS_ubsan = -fsanitize=undefined
-DEFINES_ubsan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=10
+DEFINES_ubsan = NDEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=3
 
 VALID_CONFIG_gcov = 1
 CC_gcov = gcc
@@ -345,6 +345,32 @@ HOST_LDLIBS = $(LDLIBS)
 # These are automatically computed variables.
 # There shouldn't be any need to change anything from now on.
 
+HAS_PKG_CONFIG = $(shell command -v pkg-config >/dev/null 2>&1 && echo true || echo false)
+
+PC_TEMPLATE = prefix=$(prefix)\n\
+exec_prefix=${'\$${prefix}'}\n\
+includedir=${'\$${prefix}'}/include\n\
+libdir=${'\$${exec_prefix}'}/lib\n\
+\n\
+Name: $(PC_NAME)\n\
+Description: $(PC_DESCRIPTION)\n\
+Version: $(VERSION)\n\
+Cflags: -I${'\$${includedir}'} $(PC_CFLAGS)\n\
+Requires.private: $(PC_REQUIRES_PRIVATE)\n\
+Libs: -L${'\$${libdir}'}\n\
+Libs.private: $(PC_LIBS_PRIVATE)
+
+# gpr .pc file
+PC_NAME = gRPC Portable Runtime
+PC_DESCRIPTION = gRPC Portable Runtime
+PC_CFLAGS = -pthread
+PC_REQUIRES_PRIVATE =
+PC_LIBS_PRIVATE = -lpthread
+ifeq ($(SYSTEM),Darwin)
+PC_LIBS_PRIVATE += -lrt
+endif
+GPR_PC_FILE := $(PC_TEMPLATE)
+
 ifeq ($(SYSTEM),MINGW32)
 SHARED_EXT = dll
 endif
@@ -369,6 +395,13 @@ ifeq ($(SYSTEM),Darwin)
 OPENSSL_REQUIRES_DL = true
 endif
 
+ifeq ($(HAS_PKG_CONFIG),true)
+OPENSSL_ALPN_CHECK_CMD = pkg-config --atleast-version=1.0.2 openssl
+ZLIB_CHECK_CMD = pkg-config --exists zlib
+PERFTOOLS_CHECK_CMD = pkg-config --exists profiler
+PROTOBUF_CHECK_CMD = pkg-config --atleast-version=3.0.0-alpha-3 protobuf
+else # HAS_PKG_CONFIG
+
 ifeq ($(SYSTEM),MINGW32)
 OPENSSL_LIBS = ssl32 eay32
 else
@@ -379,15 +412,18 @@ OPENSSL_ALPN_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/ope
 ZLIB_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/zlib.c -lz $(LDFLAGS)
 PERFTOOLS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/perftools.c -lprofiler $(LDFLAGS)
 PROTOBUF_CHECK_CMD = $(CXX) $(CXXFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/protobuf.cc -lprotobuf $(LDFLAGS)
-PROTOC_CHECK_CMD = which protoc > /dev/null
-PROTOC_CHECK_VERSION_CMD = protoc --version | grep -q libprotoc.3
-DTRACE_CHECK_CMD = which dtrace > /dev/null
-SYSTEMTAP_HEADERS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/systemtap.c $(LDFLAGS)
 
 ifeq ($(OPENSSL_REQUIRES_DL),true)
 OPENSSL_ALPN_CHECK_CMD += -ldl
 endif
 
+endif # HAS_PKG_CONFIG
+
+PROTOC_CHECK_CMD = which protoc > /dev/null
+PROTOC_CHECK_VERSION_CMD = protoc --version | grep -q libprotoc.3
+DTRACE_CHECK_CMD = which dtrace > /dev/null
+SYSTEMTAP_HEADERS_CHECK_CMD = $(CC) $(CFLAGS) $(CPPFLAGS) -o $(TMPOUT) test/build/systemtap.c $(LDFLAGS)
+
 ifndef REQUIRE_CUSTOM_LIBRARIES_$(CONFIG)
 HAS_SYSTEM_PERFTOOLS = $(shell $(PERFTOOLS_CHECK_CMD) 2> /dev/null && echo true || echo false)
 ifeq ($(HAS_SYSTEM_PERFTOOLS),true)
@@ -448,6 +484,9 @@ else
 HAS_EMBEDDED_PROTOBUF = true
 endif
 
+PC_REQUIRES_GRPC = gpr
+PC_LIBS_GRPC =
+
 ifeq ($(HAS_SYSTEM_ZLIB),false)
 ifeq ($(HAS_EMBEDDED_ZLIB),true)
 ZLIB_DEP = $(LIBDIR)/$(CONFIG)/zlib/libz.a
@@ -456,9 +495,41 @@ LDFLAGS += -L$(LIBDIR)/$(CONFIG)/zlib
 else
 DEP_MISSING += zlib
 endif
+else
+ifeq ($(HAS_PKG_CONFIG),true)
+CPPFLAGS += $(shell pkg-config --cflags zlib)
+LDFLAGS += $(shell pkg-config --libs-only-L zlib)
+PC_REQUIRES_GRPC += zlib
+else
+PC_LIBS_GRPC += -lz
+endif
 endif
 
-ifeq ($(HAS_SYSTEM_OPENSSL_ALPN),false)
+OPENSSL_PKG_CONFIG = false
+
+PC_REQUIRES_SECURE =
+PC_LIBS_SECURE =
+
+ifeq ($(HAS_SYSTEM_OPENSSL_ALPN),true)
+ifeq ($(HAS_PKG_CONFIG),true)
+OPENSSL_PKG_CONFIG = true
+PC_REQUIRES_SECURE = openssl
+CPPFLAGS := $(shell pkg-config --cflags openssl) $(CPPFLAGS)
+LDFLAGS_OPENSSL_PKG_CONFIG = $(shell pkg-config --libs-only-L openssl)
+ifeq ($(SYSTEM),Linux)
+ifneq ($(LDFLAGS_OPENSSL_PKG_CONFIG),)
+LDFLAGS_OPENSSL_PKG_CONFIG += $(shell pkg-config --libs-only-L openssl | sed s/L/Wl,-rpath,/)
+endif
+endif
+LDFLAGS := $(LDFLAGS_OPENSSL_PKG_CONFIG) $(LDFLAGS)
+else
+LIBS_SECURE = $(OPENSSL_LIBS)
+ifeq ($(OPENSSL_REQUIRES_DL),true)
+LIBS_SECURE += dl
+PC_LIBS_SECURE = $(addprefix -l, $(LIBS_SECURE))
+endif
+endif
+else
 ifeq ($(HAS_EMBEDDED_OPENSSL_ALPN),true)
 OPENSSL_DEP = $(LIBDIR)/$(CONFIG)/openssl/libssl.a
 OPENSSL_MERGE_LIBS += $(LIBDIR)/$(CONFIG)/openssl/libssl.a $(LIBDIR)/$(CONFIG)/openssl/libcrypto.a
@@ -471,16 +542,50 @@ endif
 else
 NO_SECURE = true
 endif
-else
-LIBS_SECURE = $(OPENSSL_LIBS)
-ifeq ($(OPENSSL_REQUIRES_DL),true)
-LIBS_SECURE += dl
-endif
 endif
 
+ifeq ($(OPENSSL_PKG_CONFIG),true)
+LDLIBS_SECURE += $(shell pkg-config --libs-only-l openssl)
+else
 LDLIBS_SECURE += $(addprefix -l, $(LIBS_SECURE))
+endif
 
-ifeq ($(HAS_SYSTEM_PROTOBUF),false)
+# grpc .pc file
+PC_NAME = gRPC
+PC_DESCRIPTION = high performance general RPC framework
+PC_CFLAGS =
+PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GRPC) $(PC_REQUIRES_SECURE)
+PC_LIBS_PRIVATE = $(PC_LIBS_GRPC) $(PC_LIBS_SECURE)
+GRPC_PC_FILE := $(PC_TEMPLATE)
+
+# gprc_unsecure .pc file
+PC_NAME = gRPC unsecure
+PC_DESCRIPTION = high performance general RPC framework without SSL
+PC_CFLAGS =
+PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GRPC)
+PC_LIBS_PRIVATE = $(PC_LIBS_GRPC)
+GRPC_UNSECURE_PC_FILE := $(PC_TEMPLATE)
+
+PROTOBUF_PKG_CONFIG = false
+
+PC_REQUIRES_GRPCXX =
+PC_LIBS_GRPCXX =
+
+ifeq ($(HAS_SYSTEM_PROTOBUF),true)
+ifeq ($(HAS_PKG_CONFIG),true)
+PROTOBUF_PKG_CONFIG = true
+PC_REQUIRES_GRPCXX = protobuf
+CPPFLAGS := $(shell pkg-config --cflags protobuf) $(CPPFLAGS)
+LDFLAGS_PROTOBUF_PKG_CONFIG = $(shell pkg-config --libs-only-L protobuf)
+ifeq ($(SYSTEM),Linux)
+ifneq ($(LDFLAGS_PROTOBUF_PKG_CONFIG),)
+LDFLAGS_PROTOBUF_PKG_CONFIG += $(shell pkg-config --libs-only-L protobuf | sed s/L/Wl,-rpath,/)
+endif
+endif
+else
+PC_LIBS_GRPCXX = -lprotobuf
+endif
+else
 ifeq ($(HAS_EMBEDDED_PROTOBUF),true)
 PROTOBUF_DEP = $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a
 CPPFLAGS := -Ithird_party/protobuf/src $(CPPFLAGS)
@@ -489,15 +594,35 @@ PROTOC = $(BINDIR)/$(CONFIG)/protobuf/protoc
 else
 NO_PROTOBUF = true
 endif
-else
 endif
 
 LIBS_PROTOBUF = protobuf
 LIBS_PROTOC = protoc protobuf
 
-LDLIBS_PROTOBUF += $(addprefix -l, $(LIBS_PROTOBUF))
 HOST_LDLIBS_PROTOC += $(addprefix -l, $(LIBS_PROTOC))
 
+ifeq ($(PROTOBUF_PKG_CONFIG),true)
+LDLIBS_PROTOBUF += $(shell pkg-config --libs-only-l protobuf)
+else
+LDLIBS_PROTOBUF += $(addprefix -l, $(LIBS_PROTOBUF))
+endif
+
+# grpc++ .pc file
+PC_NAME = gRPC++
+PC_DESCRIPTION = C++ wrapper for gRPC
+PC_CFLAGS =
+PC_REQUIRES_PRIVATE = grpc $(PC_REQUIRES_GRPCXX)
+PC_LIBS_PRIVATE = $(PC_LIBS_GRPCXX)
+GRPCXX_PC_FILE := $(PC_TEMPLATE)
+
+# grpc++_unsecure .pc file
+PC_NAME = gRPC++ unsecure
+PC_DESCRIPTION = C++ wrapper for gRPC without SSL
+PC_CFLAGS =
+PC_REQUIRES_PRIVATE = grpc_unsecure $(PC_REQUIRES_GRPCXX)
+PC_LIBS_PRIVATE = $(PC_LIBS_GRPCXX)
+GRPCXX_UNSECURE_PC_FILE := $(PC_TEMPLATE)
+
 ifeq ($(MAKECMDGOALS),clean)
 NO_DEPS = true
 endif
@@ -688,7 +813,7 @@ $(LIBDIR)/$(CONFIG)/protobuf/libprotobuf.a: third_party/protobuf/configure
 
 static: static_c static_cxx
 
-static_c: \
+static_c: pc_c pc_c_unsecure \
 % for lib in libs:
 % if lib.build == 'all' and lib.language == 'c':
  $(LIBDIR)/$(CONFIG)/lib${lib.name}.a\
@@ -696,7 +821,7 @@ static_c: \
 % endfor
 
 
-static_cxx: \
+static_cxx: pc_cxx pc_cxx_unsecure pc_gpr\
 % for lib in libs:
 % if lib.build == 'all' and lib.language == 'c++':
  $(LIBDIR)/$(CONFIG)/lib${lib.name}.a\
@@ -706,7 +831,7 @@ static_cxx: \
 
 shared: shared_c shared_cxx
 
-shared_c: \
+shared_c: pc_c pc_c_unsecure pc_gpr\
 % for lib in libs:
 % if lib.build == 'all' and lib.language == 'c':
  $(LIBDIR)/$(CONFIG)/lib${lib.name}.$(SHARED_EXT)\
@@ -714,7 +839,7 @@ shared_c: \
 % endfor
 
 
-shared_cxx: \
+shared_cxx: pc_cxx pc_cxx_unsecure \
 % for lib in libs:
 % if lib.build == 'all' and lib.language == 'c++':
  $(LIBDIR)/$(CONFIG)/lib${lib.name}.$(SHARED_EXT)\
@@ -742,6 +867,15 @@ privatelibs_c: \
 % endif
 % endfor
 
+pc_gpr: $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc
+
+pc_c: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc
+
+pc_c_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc
+
+pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc
+
+pc_cxx_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc
 
 privatelibs_cxx: \
 % for lib in libs:
@@ -914,6 +1048,31 @@ ifeq ($(CONFIG),opt)
 % endfor
 endif
 
+$(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc:
+	$(E) "[MAKE]    Generating $@"
+	$(Q) mkdir -p $(@D)
+	$(Q) echo -e "$(GPR_PC_FILE)" >$@
+
+$(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc:
+	$(E) "[MAKE]    Generating $@"
+	$(Q) mkdir -p $(@D)
+	$(Q) echo -e "$(GRPC_PC_FILE)" >$@
+
+$(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc:
+	$(E) "[MAKE]    Generating $@"
+	$(Q) mkdir -p $(@D)
+	$(Q) echo -e "$(GRPC_UNSECURE_PC_FILE)" >$@
+
+$(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc:
+	$(E) "[MAKE]    Generating $@"
+	$(Q) mkdir -p $(@D)
+	$(Q) echo -e "$(GRPCXX_PC_FILE)" >$@
+
+$(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc:
+	$(E) "[MAKE]    Generating $@"
+	$(Q) mkdir -p $(@D)
+	$(Q) echo -e "$(GRPCXX_UNSECURE_PC_FILE)" >$@
+
 % for p in protos:
 ifeq ($(NO_PROTOC),true)
 $(GENDIR)/${p}.pb.cc: protoc_dep_error
@@ -988,7 +1147,7 @@ install-headers_cxx:
 
 install-static: install-static_c install-static_cxx
 
-install-static_c: static_c strip-static_c
+install-static_c: static_c strip-static_c install-pkg-config_c
 % for lib in libs:
 % if lib.language == "c":
 % if lib.build == "all":
@@ -999,7 +1158,7 @@ install-static_c: static_c strip-static_c
 % endif
 % endfor
 
-install-static_cxx: static_cxx strip-static_cxx
+install-static_cxx: static_cxx strip-static_cxx install-pkg-config_cxx
 % for lib in libs:
 % if lib.language == "c++":
 % if lib.build == "all":
@@ -1038,10 +1197,10 @@ endif
 endif
 </%def>
 
-install-shared_c: shared_c strip-shared_c
+install-shared_c: shared_c strip-shared_c install-pkg-config_c
 ${install_shared("c")}
 
-install-shared_cxx: shared_cxx strip-shared_cxx install-shared_c
+install-shared_cxx: shared_cxx strip-shared_cxx install-shared_c install-pkg-config_cxx
 ${install_shared("c++")}
 
 install-shared_csharp: shared_csharp strip-shared_csharp
@@ -1060,6 +1219,19 @@ else
 % endfor
 endif
 
+install-pkg-config_c: pc_gpr pc_c pc_c_unsecure
+	$(E) "[INSTALL] Installing C pkg-config files"
+	$(Q) $(INSTALL) -d $(prefix)/lib/pkgconfig
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc $(prefix)/lib/pkgconfig/gpr.pc
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc $(prefix)/lib/pkgconfig/grpc.pc
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc $(prefix)/lib/pkgconfig/grpc_unsecure.pc
+
+install-pkg-config_cxx: pc_cxx pc_cxx_unsecure
+	$(E) "[INSTALL] Installing C++ pkg-config files"
+	$(Q) $(INSTALL) -d $(prefix)/lib/pkgconfig
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc $(prefix)/lib/pkgconfig/grpc++.pc
+	$(Q) $(INSTALL) $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++_unsecure.pc $(prefix)/lib/pkgconfig/grpc++_unsecure.pc
+
 install-certs: etc/roots.pem
 	$(E) "[INSTALL] Installing root certificates"
 	$(Q) $(INSTALL) -d $(prefix)/share/grpc