Merge github.com:grpc/grpc into wc

BUILD

@@ -691,6 +691,8 @@ grpc_cc_library(
         "src/core/lib/transport/transport.c",
         "src/core/lib/transport/transport_op_string.c",
         "src/core/lib/backoff/backoff.c",
+        "src/core/lib/debug/stats.c",
+        "src/core/lib/debug/stats_data.c",
     ],
     hdrs = [
         "src/core/lib/channel/channel_args.h",
@@ -809,6 +811,8 @@ grpc_cc_library(
         "src/core/lib/transport/transport.h",
         "src/core/lib/transport/transport_impl.h",
         "src/core/lib/backoff/backoff.h",
+        "src/core/lib/debug/stats.h",
+        "src/core/lib/debug/stats_data.h",
     ],
     external_deps = [
         "zlib",

CMakeLists.txt

@@ -962,6 +962,8 @@ add_library(grpc
   src/core/lib/compression/compression.c
   src/core/lib/compression/message_compress.c
   src/core/lib/compression/stream_compression.c
+  src/core/lib/debug/stats.c
+  src/core/lib/debug/stats_data.c
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
@@ -1313,6 +1315,8 @@ add_library(grpc_cronet
   src/core/lib/compression/compression.c
   src/core/lib/compression/message_compress.c
   src/core/lib/compression/stream_compression.c
+  src/core/lib/debug/stats.c
+  src/core/lib/debug/stats_data.c
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
@@ -1632,6 +1636,8 @@ add_library(grpc_test_util
   src/core/lib/compression/compression.c
   src/core/lib/compression/message_compress.c
   src/core/lib/compression/stream_compression.c
+  src/core/lib/debug/stats.c
+  src/core/lib/debug/stats_data.c
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
@@ -1895,6 +1901,8 @@ add_library(grpc_test_util_unsecure
   src/core/lib/compression/compression.c
   src/core/lib/compression/message_compress.c
   src/core/lib/compression/stream_compression.c
+  src/core/lib/debug/stats.c
+  src/core/lib/debug/stats_data.c
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
@@ -2144,6 +2152,8 @@ add_library(grpc_unsecure
   src/core/lib/compression/compression.c
   src/core/lib/compression/message_compress.c
   src/core/lib/compression/stream_compression.c
+  src/core/lib/debug/stats.c
+  src/core/lib/debug/stats_data.c
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c
@@ -2845,6 +2855,8 @@ add_library(grpc++_cronet
   src/core/lib/compression/compression.c
   src/core/lib/compression/message_compress.c
   src/core/lib/compression/stream_compression.c
+  src/core/lib/debug/stats.c
+  src/core/lib/debug/stats_data.c
   src/core/lib/http/format_request.c
   src/core/lib/http/httpcli.c
   src/core/lib/http/parser.c

Makefile

@@ -2907,6 +2907,8 @@ LIBGRPC_SRC = \
     src/core/lib/compression/compression.c \
     src/core/lib/compression/message_compress.c \
     src/core/lib/compression/stream_compression.c \
+    src/core/lib/debug/stats.c \
+    src/core/lib/debug/stats_data.c \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
@@ -3256,6 +3258,8 @@ LIBGRPC_CRONET_SRC = \
     src/core/lib/compression/compression.c \
     src/core/lib/compression/message_compress.c \
     src/core/lib/compression/stream_compression.c \
+    src/core/lib/debug/stats.c \
+    src/core/lib/debug/stats_data.c \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
@@ -3572,6 +3576,8 @@ LIBGRPC_TEST_UTIL_SRC = \
     src/core/lib/compression/compression.c \
     src/core/lib/compression/message_compress.c \
     src/core/lib/compression/stream_compression.c \
+    src/core/lib/debug/stats.c \
+    src/core/lib/debug/stats_data.c \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
@@ -3824,6 +3830,8 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
     src/core/lib/compression/compression.c \
     src/core/lib/compression/message_compress.c \
     src/core/lib/compression/stream_compression.c \
+    src/core/lib/debug/stats.c \
+    src/core/lib/debug/stats_data.c \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
@@ -4049,6 +4057,8 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/lib/compression/compression.c \
     src/core/lib/compression/message_compress.c \
     src/core/lib/compression/stream_compression.c \
+    src/core/lib/debug/stats.c \
+    src/core/lib/debug/stats_data.c \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \
@@ -4733,6 +4743,8 @@ LIBGRPC++_CRONET_SRC = \
     src/core/lib/compression/compression.c \
     src/core/lib/compression/message_compress.c \
     src/core/lib/compression/stream_compression.c \
+    src/core/lib/debug/stats.c \
+    src/core/lib/debug/stats_data.c \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \

bazel/grpc_build_system.bzl

@@ -105,3 +105,19 @@ def grpc_sh_test(name, srcs, args = [], data = []):
     srcs = srcs,
     args = args,
     data = data)
+
+def grpc_package(name, visibility = "private", features = []):
+  if visibility == "tests":
+    visibility = ["//test:__subpackages__"]
+  elif visibility == "public":
+    visibility = ["//visibility:public"]
+  elif visibility == "private":
+    visibility = []
+  else:
+    fail("Unknown visibility " + visibility)
+
+  if len(visibility) != 0:
+    native.package(
+      default_visibility = visibility,
+      features = features
+    )

binding.gyp

@@ -667,6 +667,8 @@
         'src/core/lib/compression/compression.c',
         'src/core/lib/compression/message_compress.c',
         'src/core/lib/compression/stream_compression.c',
+        'src/core/lib/debug/stats.c',
+        'src/core/lib/debug/stats_data.c',
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',

build.yaml

@@ -194,6 +194,8 @@ filegroups:
   - src/core/lib/compression/compression.c
   - src/core/lib/compression/message_compress.c
   - src/core/lib/compression/stream_compression.c
+  - src/core/lib/debug/stats.c
+  - src/core/lib/debug/stats_data.c
   - src/core/lib/http/format_request.c
   - src/core/lib/http/httpcli.c
   - src/core/lib/http/parser.c
@@ -346,6 +348,8 @@ filegroups:
   - src/core/lib/compression/algorithm_metadata.h
   - src/core/lib/compression/message_compress.h
   - src/core/lib/compression/stream_compression.h
+  - src/core/lib/debug/stats.h
+  - src/core/lib/debug/stats_data.h
   - src/core/lib/http/format_request.h
   - src/core/lib/http/httpcli.h
   - src/core/lib/http/parser.h

config.m4

@@ -96,6 +96,8 @@ if test "$PHP_GRPC" != "no"; then
     src/core/lib/compression/compression.c \
     src/core/lib/compression/message_compress.c \
     src/core/lib/compression/stream_compression.c \
+    src/core/lib/debug/stats.c \
+    src/core/lib/debug/stats_data.c \
     src/core/lib/http/format_request.c \
     src/core/lib/http/httpcli.c \
     src/core/lib/http/parser.c \

config.w32

@@ -73,6 +73,8 @@ if (PHP_GRPC != "no") {
     "src\\core\\lib\\compression\\compression.c " +
     "src\\core\\lib\\compression\\message_compress.c " +
     "src\\core\\lib\\compression\\stream_compression.c " +
+    "src\\core\\lib\\debug\\stats.c " +
+    "src\\core\\lib\\debug\\stats_data.c " +
     "src\\core\\lib\\http\\format_request.c " +
     "src\\core\\lib\\http\\httpcli.c " +
     "src\\core\\lib\\http\\parser.c " +

doc/compression.md

@@ -52,8 +52,8 @@ by the client WILL result in an `INTERNAL` error status on the client side.
 
 Note that a peer MAY choose to not disclose all the encodings it supports.
 However, if it receives a message compressed in an undisclosed but supported
-encoding, it MUST include said encoding in the response's `grpc-accept-encoding
-h`eader.
+encoding, it MUST include said encoding in the response's `grpc-accept-encoding`
+header.
 
 For every message a server is requested to compress using an algorithm it knows
 the client doesn't support (as indicated by the last `grpc-accept-encoding`

gRPC-Core.podspec

@@ -328,6 +328,8 @@ Pod::Spec.new do |s|
                       'src/core/lib/compression/algorithm_metadata.h',
                       'src/core/lib/compression/message_compress.h',
                       'src/core/lib/compression/stream_compression.h',
+                      'src/core/lib/debug/stats.h',
+                      'src/core/lib/debug/stats_data.h',
                       'src/core/lib/http/format_request.h',
                       'src/core/lib/http/httpcli.h',
                       'src/core/lib/http/parser.h',
@@ -478,6 +480,8 @@ Pod::Spec.new do |s|
                       'src/core/lib/compression/compression.c',
                       'src/core/lib/compression/message_compress.c',
                       'src/core/lib/compression/stream_compression.c',
+                      'src/core/lib/debug/stats.c',
+                      'src/core/lib/debug/stats_data.c',
                       'src/core/lib/http/format_request.c',
                       'src/core/lib/http/httpcli.c',
                       'src/core/lib/http/parser.c',
@@ -824,6 +828,8 @@ Pod::Spec.new do |s|
                               'src/core/lib/compression/algorithm_metadata.h',
                               'src/core/lib/compression/message_compress.h',
                               'src/core/lib/compression/stream_compression.h',
+                              'src/core/lib/debug/stats.h',
+                              'src/core/lib/debug/stats_data.h',
                               'src/core/lib/http/format_request.h',
                               'src/core/lib/http/httpcli.h',
                               'src/core/lib/http/parser.h',

grpc.gemspec

@@ -260,6 +260,8 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/compression/algorithm_metadata.h )
   s.files += %w( src/core/lib/compression/message_compress.h )
   s.files += %w( src/core/lib/compression/stream_compression.h )
+  s.files += %w( src/core/lib/debug/stats.h )
+  s.files += %w( src/core/lib/debug/stats_data.h )
   s.files += %w( src/core/lib/http/format_request.h )
   s.files += %w( src/core/lib/http/httpcli.h )
   s.files += %w( src/core/lib/http/parser.h )
@@ -414,6 +416,8 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/lib/compression/compression.c )
   s.files += %w( src/core/lib/compression/message_compress.c )
   s.files += %w( src/core/lib/compression/stream_compression.c )
+  s.files += %w( src/core/lib/debug/stats.c )
+  s.files += %w( src/core/lib/debug/stats_data.c )
   s.files += %w( src/core/lib/http/format_request.c )
   s.files += %w( src/core/lib/http/httpcli.c )
   s.files += %w( src/core/lib/http/parser.c )

grpc.gyp

@@ -233,6 +233,8 @@
         'src/core/lib/compression/compression.c',
         'src/core/lib/compression/message_compress.c',
         'src/core/lib/compression/stream_compression.c',
+        'src/core/lib/debug/stats.c',
+        'src/core/lib/debug/stats_data.c',
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
@@ -533,6 +535,8 @@
         'src/core/lib/compression/compression.c',
         'src/core/lib/compression/message_compress.c',
         'src/core/lib/compression/stream_compression.c',
+        'src/core/lib/debug/stats.c',
+        'src/core/lib/debug/stats_data.c',
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
@@ -738,6 +742,8 @@
         'src/core/lib/compression/compression.c',
         'src/core/lib/compression/message_compress.c',
         'src/core/lib/compression/stream_compression.c',
+        'src/core/lib/debug/stats.c',
+        'src/core/lib/debug/stats_data.c',
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',
@@ -928,6 +934,8 @@
         'src/core/lib/compression/compression.c',
         'src/core/lib/compression/message_compress.c',
         'src/core/lib/compression/stream_compression.c',
+        'src/core/lib/debug/stats.c',
+        'src/core/lib/debug/stats_data.c',
         'src/core/lib/http/format_request.c',
         'src/core/lib/http/httpcli.c',
         'src/core/lib/http/parser.c',

include/grpc++/impl/codegen/call.h

@@ -272,7 +272,7 @@ class CallOpSendInitialMetadata {
 
 class CallOpSendMessage {
  public:
-  CallOpSendMessage() : send_buf_(nullptr), own_buf_(false) {}
+  CallOpSendMessage() : send_buf_(nullptr) {}
 
   /// Send \a message using \a options for the write. The \a options are cleared
   /// after use.
@@ -295,20 +295,25 @@ class CallOpSendMessage {
     write_options_.Clear();
   }
   void FinishOp(bool* status) {
-    if (own_buf_) g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_);
+    g_core_codegen_interface->grpc_byte_buffer_destroy(send_buf_);
     send_buf_ = nullptr;
   }
 
  private:
   grpc_byte_buffer* send_buf_;
   WriteOptions write_options_;
-  bool own_buf_;
 };
 
 template <class M>
 Status CallOpSendMessage::SendMessage(const M& message, WriteOptions options) {
   write_options_ = options;
-  return SerializationTraits<M>::Serialize(message, &send_buf_, &own_buf_);
+  bool own_buf;
+  Status result =
+      SerializationTraits<M>::Serialize(message, &send_buf_, &own_buf);
+  if (!own_buf) {
+    send_buf_ = g_core_codegen_interface->grpc_byte_buffer_copy(send_buf_);
+  }
+  return result;
 }
 
 template <class M>

include/grpc++/impl/codegen/core_codegen.h

@@ -68,6 +68,7 @@ class CoreCodegen final : public CoreCodegenInterface {
   void grpc_call_unref(grpc_call* call) override;
   virtual void* grpc_call_arena_alloc(grpc_call* call, size_t length) override;
 
+  grpc_byte_buffer* grpc_byte_buffer_copy(grpc_byte_buffer* bb) override;
   void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) override;
 
   int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader,

include/grpc++/impl/codegen/core_codegen_interface.h

@@ -74,6 +74,7 @@ class CoreCodegenInterface {
   virtual void gpr_cv_signal(gpr_cv* cv) = 0;
   virtual void gpr_cv_broadcast(gpr_cv* cv) = 0;
 
+  virtual grpc_byte_buffer* grpc_byte_buffer_copy(grpc_byte_buffer* bb) = 0;
   virtual void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) = 0;
 
   virtual int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader,

include/grpc/impl/codegen/grpc_types.h

@@ -511,6 +511,11 @@ typedef struct grpc_op {
       } maybe_stream_compression_level;
     } send_initial_metadata;
     struct grpc_op_send_message {
+      /** This op takes ownership of the slices in send_message.  After
+       * a call completes, the contents of send_message are not guaranteed
+       * and likely empty.  The original owner should still call
+       * grpc_byte_buffer_destroy() on this object however.
+       */
       struct grpc_byte_buffer *send_message;
     } send_message;
     struct grpc_op_send_status_from_server {

package.xml

@@ -270,6 +270,8 @@
     <file baseinstalldir="/" name="src/core/lib/compression/algorithm_metadata.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/message_compress.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/stream_compression.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/debug/stats.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/debug/stats_data.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/format_request.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.h" role="src" />
@@ -424,6 +426,8 @@
     <file baseinstalldir="/" name="src/core/lib/compression/compression.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/message_compress.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/compression/stream_compression.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/debug/stats.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/debug/stats_data.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/format_request.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/httpcli.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/parser.c" role="src" />

src/core/ext/filters/client_channel/client_channel.c

@@ -810,15 +810,16 @@ typedef struct client_channel_call_data {
   // State for handling deadlines.
   // The code in deadline_filter.c requires this to be the first field.
   // TODO(roth): This is slightly sub-optimal in that grpc_deadline_state
-  // and this struct both independently store a pointer to the call
-  // combiner.  If/when we have time, find a way to avoid this without
-  // breaking the grpc_deadline_state abstraction.
+  // and this struct both independently store pointers to the call stack
+  // and call combiner.  If/when we have time, find a way to avoid this
+  // without breaking the grpc_deadline_state abstraction.
   grpc_deadline_state deadline_state;
 
   grpc_slice path;  // Request path.
   gpr_timespec call_start_time;
   grpc_millis deadline;
   gpr_arena *arena;
+  grpc_call_stack *owning_call;
   grpc_call_combiner *call_combiner;
 
   grpc_server_retry_throttle_data *retry_throttle_data;
@@ -829,7 +830,7 @@ typedef struct client_channel_call_data {
 
   grpc_lb_policy *lb_policy;  // Holds ref while LB pick is pending.
   grpc_closure lb_pick_closure;
-  grpc_closure cancel_closure;
+  grpc_closure lb_pick_cancel_closure;
 
   grpc_connected_subchannel *connected_subchannel;
   grpc_call_context_element subchannel_call_context[GRPC_CONTEXT_COUNT];
@@ -1048,8 +1049,9 @@ static bool pick_subchannel_locked(grpc_exec_ctx *exec_ctx,
 
 typedef struct {
   grpc_call_element *elem;
-  bool cancelled;
+  bool finished;
   grpc_closure closure;
+  grpc_closure cancel_closure;
 } pick_after_resolver_result_args;
 
 // Note: This runs under the client_channel combiner, but will NOT be
@@ -1057,71 +1059,71 @@ typedef struct {
 static void pick_after_resolver_result_cancel_locked(grpc_exec_ctx *exec_ctx,
                                                      void *arg,
                                                      grpc_error *error) {
-  grpc_call_element *elem = arg;
+  pick_after_resolver_result_args *args = arg;
+  if (args->finished) {
+    gpr_free(args);
+    return;
+  }
+  args->finished = true;
+  grpc_call_element *elem = args->elem;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
   // If we don't yet have a resolver result, then a closure for
   // pick_after_resolver_result_done_locked() will have been added to
   // chand->waiting_for_resolver_result_closures, and it may not be invoked
   // until after this call has been destroyed.  We mark the operation as
-  // cancelled, so that when pick_after_resolver_result_done_locked()
+  // finished, so that when pick_after_resolver_result_done_locked()
   // is called, it will be a no-op.  We also immediately invoke
   // subchannel_ready_locked() to propagate the error back to the caller.
-  for (grpc_closure *closure = chand->waiting_for_resolver_result_closures.head;
-       closure != NULL; closure = closure->next_data.next) {
-    pick_after_resolver_result_args *args = closure->cb_arg;
-    if (!args->cancelled && args->elem == elem) {
-      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-        gpr_log(GPR_DEBUG,
-                "chand=%p calld=%p: "
-                "cancelling pick waiting for resolver result",
-                chand, calld);
-      }
-      args->cancelled = true;
-      // Note: Although we are not in the call combiner here, we are
-      // basically stealing the call combiner from the pending pick, so
-      // it's safe to call subchannel_ready_locked() here -- we are
-      // essentially calling it here instead of calling it in
-      // pick_after_resolver_result_done_locked().
-      subchannel_ready_locked(exec_ctx, elem,
-                              GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING(
-                                  "Pick cancelled", &error, 1));
-    }
+  if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+    gpr_log(GPR_DEBUG,
+            "chand=%p calld=%p: cancelling pick waiting for resolver result",
+            chand, calld);
   }
+  // Note: Although we are not in the call combiner here, we are
+  // basically stealing the call combiner from the pending pick, so
+  // it's safe to call subchannel_ready_locked() here -- we are
+  // essentially calling it here instead of calling it in
+  // pick_after_resolver_result_done_locked().
+  subchannel_ready_locked(
+      exec_ctx, elem,
+      GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Pick cancelled",
+                                                       &error, 1));
 }
 
 static void pick_after_resolver_result_done_locked(grpc_exec_ctx *exec_ctx,
                                                    void *arg,
                                                    grpc_error *error) {
   pick_after_resolver_result_args *args = arg;
-  if (args->cancelled) {
+  if (args->finished) {
     /* cancelled, do nothing */
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG, "call cancelled before resolver result");
     }
+    gpr_free(args);
+    return;
+  }
+  args->finished = true;
+  grpc_call_element *elem = args->elem;
+  channel_data *chand = elem->channel_data;
+  call_data *calld = elem->call_data;
+  grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
+                                          NULL);
+  if (error != GRPC_ERROR_NONE) {
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data",
+              chand, calld);
+    }
+    subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
   } else {
-    grpc_call_element *elem = args->elem;
-    channel_data *chand = elem->channel_data;
-    call_data *calld = elem->call_data;
-    grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
-                                            NULL);
-    if (error != GRPC_ERROR_NONE) {
-      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-        gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver failed to return data",
-                chand, calld);
-      }
-      subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_REF(error));
-    } else {
-      if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
-        gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick",
-                chand, calld);
-      }
-      if (pick_subchannel_locked(exec_ctx, elem)) {
-        subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_NONE);
-      }
+    if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
+      gpr_log(GPR_DEBUG, "chand=%p calld=%p: resolver returned, doing pick",
+              chand, calld);
+    }
+    if (pick_subchannel_locked(exec_ctx, elem)) {
+      subchannel_ready_locked(exec_ctx, elem, GRPC_ERROR_NONE);
     }
   }
-  gpr_free(args);
 }
 
 static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx,
@@ -1142,8 +1144,8 @@ static void pick_after_resolver_result_start_locked(grpc_exec_ctx *exec_ctx,
                            &args->closure, GRPC_ERROR_NONE);
   grpc_call_combiner_set_notify_on_cancel(
       exec_ctx, calld->call_combiner,
-      GRPC_CLOSURE_INIT(&calld->cancel_closure,
-                        pick_after_resolver_result_cancel_locked, elem,
+      GRPC_CLOSURE_INIT(&args->cancel_closure,
+                        pick_after_resolver_result_cancel_locked, args,
                         grpc_combiner_scheduler(chand->combiner)));
 }
 
@@ -1154,7 +1156,7 @@ static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_call_element *elem = arg;
   channel_data *chand = elem->channel_data;
   call_data *calld = elem->call_data;
-  if (calld->lb_policy != NULL) {
+  if (error != GRPC_ERROR_NONE && calld->lb_policy != NULL) {
     if (GRPC_TRACER_ON(grpc_client_channel_trace)) {
       gpr_log(GPR_DEBUG, "chand=%p calld=%p: cancelling pick from LB policy %p",
               chand, calld, calld->lb_policy);
@@ -1163,6 +1165,7 @@ static void pick_callback_cancel_locked(grpc_exec_ctx *exec_ctx, void *arg,
                                       &calld->connected_subchannel,
                                       GRPC_ERROR_REF(error));
   }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "pick_callback_cancel");
 }
 
 // Callback invoked by grpc_lb_policy_pick_locked() for async picks.
@@ -1212,10 +1215,12 @@ static bool pick_callback_start_locked(grpc_exec_ctx *exec_ctx,
     GRPC_LB_POLICY_UNREF(exec_ctx, calld->lb_policy, "pick_subchannel");
     calld->lb_policy = NULL;
   } else {
+    GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback_cancel");
     grpc_call_combiner_set_notify_on_cancel(
         exec_ctx, calld->call_combiner,
-        GRPC_CLOSURE_INIT(&calld->cancel_closure, pick_callback_cancel_locked,
-                          elem, grpc_combiner_scheduler(chand->combiner)));
+        GRPC_CLOSURE_INIT(&calld->lb_pick_cancel_closure,
+                          pick_callback_cancel_locked, elem,
+                          grpc_combiner_scheduler(chand->combiner)));
   }
   return pick_done;
 }
@@ -1419,6 +1424,7 @@ static grpc_error *cc_init_call_elem(grpc_exec_ctx *exec_ctx,
   calld->call_start_time = args->start_time;
   calld->deadline = args->deadline;
   calld->arena = args->arena;
+  calld->owning_call = args->call_stack;
   calld->call_combiner = args->call_combiner;
   if (chand->deadline_checking_enabled) {
     grpc_deadline_state_init(exec_ctx, elem, args->call_stack,

src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.c

@@ -296,8 +296,6 @@ static void stop_connectivity_watchers(grpc_exec_ctx *exec_ctx,
 static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                              const grpc_lb_policy_args *args) {
   pick_first_lb_policy *p = (pick_first_lb_policy *)policy;
-  /* Find the number of backend addresses. We ignore balancer
-   * addresses, since we don't know how to handle them. */
   const grpc_arg *arg =
       grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
   if (arg == NULL || arg->type != GRPC_ARG_POINTER) {
@@ -317,11 +315,7 @@ static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
     return;
   }
   const grpc_lb_addresses *addresses = arg->value.pointer.p;
-  size_t num_addrs = 0;
-  for (size_t i = 0; i < addresses->num_addresses; i++) {
-    if (!addresses->addresses[i].is_balancer) ++num_addrs;
-  }
-  if (num_addrs == 0) {
+  if (addresses->num_addresses == 0) {
     // Empty update. Unsubscribe from all current subchannels and put the
     // channel in TRANSIENT_FAILURE.
     grpc_connectivity_state_set(
@@ -333,9 +327,10 @@ static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
   }
   if (GRPC_TRACER_ON(grpc_lb_pick_first_trace)) {
     gpr_log(GPR_INFO, "Pick First %p received update with %lu addresses",
-            (void *)p, (unsigned long)num_addrs);
+            (void *)p, (unsigned long)addresses->num_addresses);
   }
-  grpc_subchannel_args *sc_args = gpr_zalloc(sizeof(*sc_args) * num_addrs);
+  grpc_subchannel_args *sc_args =
+      gpr_zalloc(sizeof(*sc_args) * addresses->num_addresses);
   /* We remove the following keys in order for subchannel keys belonging to
    * subchannels point to the same address to match. */
   static const char *keys_to_remove[] = {GRPC_ARG_SUBCHANNEL_ADDRESS,
@@ -344,7 +339,8 @@ static void pf_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
 
   /* Create list of subchannel args for new addresses in \a args. */
   for (size_t i = 0; i < addresses->num_addresses; i++) {
-    if (addresses->addresses[i].is_balancer) continue;
+    // If there were any balancer, we would have chosen grpclb policy instead.
+    GPR_ASSERT(!addresses->addresses[i].is_balancer);
     if (addresses->addresses[i].user_data != NULL) {
       gpr_log(GPR_ERROR,
               "This LB policy doesn't support user data. It will be ignored");

src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.c

@@ -737,8 +737,6 @@ static void rr_ping_one_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol,
 static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                              const grpc_lb_policy_args *args) {
   round_robin_lb_policy *p = (round_robin_lb_policy *)policy;
-  /* Find the number of backend addresses. We ignore balancer addresses, since
-   * we don't know how to handle them. */
   const grpc_arg *arg =
       grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
   if (arg == NULL || arg->type != GRPC_ARG_POINTER) {
@@ -757,12 +755,9 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
     return;
   }
   grpc_lb_addresses *addresses = arg->value.pointer.p;
-  size_t num_addrs = 0;
-  for (size_t i = 0; i < addresses->num_addresses; i++) {
-    if (!addresses->addresses[i].is_balancer) ++num_addrs;
-  }
-  rr_subchannel_list *subchannel_list = rr_subchannel_list_create(p, num_addrs);
-  if (num_addrs == 0) {
+  rr_subchannel_list *subchannel_list =
+      rr_subchannel_list_create(p, addresses->num_addresses);
+  if (addresses->num_addresses == 0) {
     grpc_connectivity_state_set(
         exec_ctx, &p->state_tracker, GRPC_CHANNEL_TRANSIENT_FAILURE,
         GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty update"),
@@ -794,9 +789,8 @@ static void rr_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
                                          GRPC_ARG_LB_ADDRESSES};
   /* Create subchannels for addresses in the update. */
   for (size_t i = 0; i < addresses->num_addresses; i++) {
-    /* Skip balancer addresses, since we only know how to handle backends. */
-    if (addresses->addresses[i].is_balancer) continue;
-    GPR_ASSERT(i < num_addrs);
+    // If there were any balancer, we would have chosen grpclb policy instead.
+    GPR_ASSERT(!addresses->addresses[i].is_balancer);
     memset(&sc_args, 0, sizeof(grpc_subchannel_args));
     grpc_arg addr_arg =
         grpc_create_subchannel_address_arg(&addresses->addresses[i].address);

src/core/ext/filters/client_channel/subchannel.c

@@ -723,11 +723,12 @@ void grpc_subchannel_call_unref(grpc_exec_ctx *exec_ctx,
 
 void grpc_subchannel_call_process_op(grpc_exec_ctx *exec_ctx,
                                      grpc_subchannel_call *call,
-                                     grpc_transport_stream_op_batch *op) {
+                                     grpc_transport_stream_op_batch *batch) {
   GPR_TIMER_BEGIN("grpc_subchannel_call_process_op", 0);
   grpc_call_stack *call_stack = SUBCHANNEL_CALL_TO_CALL_STACK(call);
   grpc_call_element *top_elem = grpc_call_stack_element(call_stack, 0);
-  top_elem->filter->start_transport_stream_op_batch(exec_ctx, top_elem, op);
+  GRPC_CALL_LOG_OP(GPR_INFO, top_elem, batch);
+  top_elem->filter->start_transport_stream_op_batch(exec_ctx, top_elem, batch);
   GPR_TIMER_END("grpc_subchannel_call_process_op", 0);
 }
 

src/core/ext/filters/http/client/http_client_filter.c

@@ -303,7 +303,6 @@ static void hc_start_transport_stream_op_batch(
   call_data *calld = elem->call_data;
   channel_data *channeld = elem->channel_data;
   GPR_TIMER_BEGIN("hc_start_transport_stream_op_batch", 0);
-  GRPC_CALL_LOG_OP(GPR_INFO, elem, batch);
 
   if (batch->recv_initial_metadata) {
     /* substitute our callback for the higher callback */

src/core/lib/debug/stats.c

@@ -0,0 +1,67 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "src/core/lib/debug/stats.h"
+
+#include <inttypes.h>
+#include <string.h>
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/useful.h>
+
+#include "src/core/lib/support/string.h"
+
+grpc_stats_data *grpc_stats_per_cpu_storage = NULL;
+static size_t g_num_cores;
+
+void grpc_stats_init(void) {
+  g_num_cores = GPR_MAX(1, gpr_cpu_num_cores());
+  grpc_stats_per_cpu_storage =
+      gpr_zalloc(sizeof(grpc_stats_data) * g_num_cores);
+}
+
+void grpc_stats_shutdown(void) { gpr_free(grpc_stats_per_cpu_storage); }
+
+void grpc_stats_collect(grpc_stats_data *output) {
+  memset(output, 0, sizeof(*output));
+  for (size_t core = 0; core < g_num_cores; core++) {
+    for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+      output->counters[i] += gpr_atm_no_barrier_load(
+          &grpc_stats_per_cpu_storage[core].counters[i]);
+    }
+  }
+}
+
+char *grpc_stats_data_as_json(const grpc_stats_data *data) {
+  gpr_strvec v;
+  char *tmp;
+  bool is_first = true;
+  gpr_strvec_init(&v);
+  gpr_strvec_add(&v, gpr_strdup("{"));
+  for (size_t i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+    gpr_asprintf(&tmp, "%s\"%s\": %" PRIdPTR, is_first ? "" : ", ",
+                 grpc_stats_counter_name[i], data->counters[i]);
+    gpr_strvec_add(&v, tmp);
+    is_first = false;
+  }
+  gpr_strvec_add(&v, gpr_strdup("}"));
+  tmp = gpr_strvec_flatten(&v, NULL);
+  gpr_strvec_destroy(&v);
+  return tmp;
+}

src/core/lib/debug/stats.h

@@ -0,0 +1,44 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_DEBUG_STATS_H
+#define GRPC_CORE_LIB_DEBUG_STATS_H
+
+#include <grpc/support/atm.h>
+#include "src/core/lib/debug/stats_data.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
+
+typedef struct grpc_stats_data {
+  gpr_atm counters[GRPC_STATS_COUNTER_COUNT];
+} grpc_stats_data;
+
+extern grpc_stats_data *grpc_stats_per_cpu_storage;
+
+#define GRPC_THREAD_STATS_DATA(exec_ctx) \
+  (&grpc_stats_per_cpu_storage[(exec_ctx)->starting_cpu])
+
+#define GRPC_STATS_INC_COUNTER(exec_ctx, ctr) \
+  (gpr_atm_no_barrier_fetch_add(              \
+      &GRPC_THREAD_STATS_DATA((exec_ctx))->counters[(ctr)], 1))
+
+void grpc_stats_init(void);
+void grpc_stats_shutdown(void);
+void grpc_stats_collect(grpc_stats_data *output);
+char *grpc_stats_data_as_json(const grpc_stats_data *data);
+
+#endif

src/core/lib/debug/stats_data.c

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Automatically generated by tools/codegen/core/gen_stats_data.py
+ */
+
+#include "src/core/lib/debug/stats_data.h"
+const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
+    "client_calls_created", "server_calls_created", "syscall_write",
+    "syscall_read",         "syscall_poll",         "syscall_wait",
+};

src/core/lib/debug/stats_data.h

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Automatically generated by tools/codegen/core/gen_stats_data.py
+ */
+
+#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H
+#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H
+
+typedef enum {
+  GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED,
+  GRPC_STATS_COUNTER_SERVER_CALLS_CREATED,
+  GRPC_STATS_COUNTER_SYSCALL_WRITE,
+  GRPC_STATS_COUNTER_SYSCALL_READ,
+  GRPC_STATS_COUNTER_SYSCALL_POLL,
+  GRPC_STATS_COUNTER_SYSCALL_WAIT,
+  GRPC_STATS_COUNTER_COUNT
+} grpc_stats_counters;
+#define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED)
+#define GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_CALLS_CREATED)
+#define GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WRITE)
+#define GRPC_STATS_INC_SYSCALL_READ(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_READ)
+#define GRPC_STATS_INC_SYSCALL_POLL(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_POLL)
+#define GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx) \
+  GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SYSCALL_WAIT)
+extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];
+
+#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */

src/core/lib/debug/stats_data.yaml

@@ -0,0 +1,9 @@
+# Stats data declaration
+# use tools/codegen/core/gen_stats_data.py to turn this into stats_data.h
+
+- counter: client_calls_created
+- counter: server_calls_created
+- counter: syscall_write
+- counter: syscall_read
+- counter: syscall_poll
+- counter: syscall_wait

src/core/lib/iomgr/call_combiner.c

@@ -140,11 +140,33 @@ void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
     // If error is set, invoke the cancellation closure immediately.
     // Otherwise, store the new closure.
     if (original_error != GRPC_ERROR_NONE) {
+      if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+        gpr_log(GPR_DEBUG,
+                "call_combiner=%p: scheduling notify_on_cancel callback=%p "
+                "for pre-existing cancellation",
+                call_combiner, closure);
+      }
       GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_REF(original_error));
       break;
     } else {
       if (gpr_atm_full_cas(&call_combiner->cancel_state, original_state,
                            (gpr_atm)closure)) {
+        if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+          gpr_log(GPR_DEBUG, "call_combiner=%p: setting notify_on_cancel=%p",
+                  call_combiner, closure);
+        }
+        // If we replaced an earlier closure, invoke the original
+        // closure with GRPC_ERROR_NONE.  This allows callers to clean
+        // up any resources they may be holding for the callback.
+        if (original_state != 0) {
+          closure = (grpc_closure*)original_state;
+          if (GRPC_TRACER_ON(grpc_call_combiner_trace)) {
+            gpr_log(GPR_DEBUG,
+                    "call_combiner=%p: scheduling old cancel callback=%p",
+                    call_combiner, closure);
+          }
+          GRPC_CLOSURE_SCHED(exec_ctx, closure, GRPC_ERROR_NONE);
+        }
         break;
       }
     }

src/core/lib/iomgr/call_combiner.h

@@ -87,11 +87,20 @@ void grpc_call_combiner_stop(grpc_exec_ctx* exec_ctx,
                              const char* reason);
 #endif
 
-/// Tells \a call_combiner to invoke \a closure when
-/// grpc_call_combiner_cancel() is called.  If grpc_call_combiner_cancel()
-/// was previously called, \a closure will be invoked immediately.
+/// Tells \a call_combiner to schedule \a closure when
+/// grpc_call_combiner_cancel() is called.
+///
+/// If grpc_call_combiner_cancel() was previously called, \a closure will be
+/// scheduled immediately.
+///
 /// If \a closure is NULL, then no closure will be invoked on
 /// cancellation; this effectively unregisters the previously set closure.
+///
+/// If a closure was set via a previous call to
+/// grpc_call_combiner_set_notify_on_cancel(), the previous closure will be
+/// scheduled immediately with GRPC_ERROR_NONE.  This ensures that
+/// \a closure will be scheduled exactly once, which allows callers to clean
+/// up resources they may be holding for the callback.
 void grpc_call_combiner_set_notify_on_cancel(grpc_exec_ctx* exec_ctx,
                                              grpc_call_combiner* call_combiner,
                                              grpc_closure* closure);

src/core/lib/iomgr/ev_epoll1_linux.c

@@ -40,6 +40,7 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
@@ -631,6 +632,7 @@ static grpc_error *do_epoll_wait(grpc_exec_ctx *exec_ctx, grpc_pollset *ps,
     GRPC_SCHEDULING_START_BLOCKING_REGION;
   }
   do {
+    GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
     r = epoll_wait(g_epoll_set.epfd, g_epoll_set.events, MAX_EPOLL_EVENTS,
                    timeout);
   } while (r < 0 && errno == EINTR);
@@ -1189,10 +1191,6 @@ static const grpc_event_engine_vtable vtable = {
  * Create epoll_fd (epoll_set_init() takes care of that) to make sure epoll
  * support is available */
 const grpc_event_engine_vtable *grpc_init_epoll1_linux(bool explicit_request) {
-  if (!explicit_request) {
-    return NULL;
-  }
-
   if (!grpc_has_wakeup_fd()) {
     return NULL;
   }

src/core/lib/iomgr/ev_epoll_limited_pollers_linux.c

@@ -40,6 +40,7 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/ev_posix.h"
@@ -1291,6 +1292,7 @@ static bool acquire_polling_lease(grpc_exec_ctx *exec_ctx,
     } else {
       struct timespec sigwait_timeout = millis_to_timespec(timeout_ms);
       GRPC_SCHEDULING_START_BLOCKING_REGION;
+      GRPC_STATS_INC_SYSCALL_WAIT(exec_ctx);
       ret = sigtimedwait(&g_wakeup_sig_set, NULL, &sigwait_timeout);
       GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(exec_ctx);
     }
@@ -1371,6 +1373,7 @@ static void pollset_do_epoll_pwait(grpc_exec_ctx *exec_ctx, int epoll_fd,
   int timeout_ms = poll_deadline_to_millis_timeout(exec_ctx, deadline);
 
   GRPC_SCHEDULING_START_BLOCKING_REGION;
+  GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
   ep_rv =
       epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask);
   GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(exec_ctx);

src/core/lib/iomgr/ev_epoll_thread_pool_linux.c

@@ -41,8 +41,9 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/block_annotate.h"
-#include "src/core/lib/iomgr/ev_posix.h"
+#include "src/core/lib/iomgr/exec_ctx.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/lockfree_event.h"
 #include "src/core/lib/iomgr/timer.h"
@@ -50,17 +51,14 @@
 #include "src/core/lib/profiling/timers.h"
 
 /* TODO: sreek - Move this to init.c and initialize this like other tracers. */
-#define GRPC_POLLING_TRACE(fmt, ...)        \
-  if (GRPC_TRACER_ON(grpc_polling_trace)) { \
-    gpr_log(GPR_INFO, (fmt), __VA_ARGS__);  \
-  }
-
-/* The alarm system needs to be able to wakeup 'some poller' sometimes
- * (specifically when a new alarm needs to be triggered earlier than the next
- * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a
- * case occurs. */
-
-struct epoll_set;
+#define GRPC_POLLING_TRACE(fmt, ...)                                          \
+  if (GRPC_TRACER_ON(grpc_polling_trace)) {                                   \
+    gpr_log(GPR_INFO, (fmt), __VA_ARGS__);                                    \
+  } /* The alarm system needs to be able to wakeup 'some poller' sometimes    \
+ * (specifically when a new alarm needs to be triggered earlier than the next \
+ * alarm 'epoch'). This wakeup_fd gives us something to alert on when such a  \
+ * case occurs. */                                                            \
+  struct epoll_set;
 
 #define GRPC_POLLSET_KICK_BROADCAST ((grpc_pollset_worker *)1)
 
@@ -776,6 +774,7 @@ static void do_epoll_wait(grpc_exec_ctx *exec_ctx, int epoll_fd, epoll_set *eps,
 
   GRPC_SCHEDULING_START_BLOCKING_REGION;
   acquire_epoll_lease(eps);
+  GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
   ep_rv = epoll_wait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms);
   release_epoll_lease(eps);
   GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(exec_ctx);

src/core/lib/iomgr/ev_epollex_linux.c

@@ -38,8 +38,8 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/block_annotate.h"
-#include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/is_epollexclusive_available.h"
 #include "src/core/lib/iomgr/lockfree_event.h"
@@ -52,14 +52,13 @@
 /*******************************************************************************
  * Polling object
  */
-
 typedef enum {
   PO_POLLING_GROUP,
   PO_POLLSET_SET,
   PO_POLLSET,
-  PO_FD, /* ordering is important: we always want to lock pollsets before fds:
-            this guarantees that using an fd as a pollable is safe */
-  PO_EMPTY_POLLABLE,
+  PO_FD,
+  /* ordering is important: we always want to lock pollsets before fds:
+            this guarantees that using an fd as a pollable is safe */ PO_EMPTY_POLLABLE,
   PO_COUNT
 } polling_obj_type;
 
@@ -801,6 +800,7 @@ static grpc_error *pollset_epoll(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   }
   int r;
   do {
+    GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
     r = epoll_wait(p->epfd, pollset->events, MAX_EPOLL_EVENTS, timeout);
   } while (r < 0 && errno == EINTR);
   if (timeout != 0) {

src/core/lib/iomgr/ev_epollsig_linux.c

@@ -40,6 +40,7 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
@@ -1223,6 +1224,7 @@ static void pollset_work_and_unlock(grpc_exec_ctx *exec_ctx,
   g_current_thread_polling_island = pi;
 
   GRPC_SCHEDULING_START_BLOCKING_REGION;
+  GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
   ep_rv =
       epoll_pwait(epoll_fd, ep_ev, GRPC_EPOLL_MAX_EVENTS, timeout_ms, sig_mask);
   GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(exec_ctx);
@@ -1715,9 +1717,7 @@ const grpc_event_engine_vtable *grpc_init_epollsig_linux(
   }
 
   if (!is_grpc_wakeup_signal_initialized) {
-    /* TODO(ctiller): when other epoll engines are ready, remove the true || to
-     * force this to be explitly chosen if needed */
-    if (true || explicit_request) {
+    if (explicit_request) {
       grpc_use_signal(SIGRTMIN + 6);
     } else {
       return NULL;

src/core/lib/iomgr/ev_poll_posix.c

@@ -37,9 +37,9 @@
 #include <grpc/support/tls.h>
 #include <grpc/support/useful.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/block_annotate.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
-#include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/iomgr/wakeup_fd_cv.h"
 #include "src/core/lib/iomgr/wakeup_fd_posix.h"
 #include "src/core/lib/profiling/timers.h"
@@ -50,7 +50,6 @@
 /*******************************************************************************
  * FD declarations
  */
-
 typedef struct grpc_fd_watcher {
   struct grpc_fd_watcher *next;
   struct grpc_fd_watcher *prev;
@@ -984,6 +983,7 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid
          even going into the blocking annotation if possible */
       GRPC_SCHEDULING_START_BLOCKING_REGION;
+      GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
       r = grpc_poll_function(pfds, pfd_count, timeout);
       GRPC_SCHEDULING_END_BLOCKING_REGION_WITH_EXEC_CTX(exec_ctx);
 

src/core/lib/iomgr/ev_posix.c

@@ -64,8 +64,8 @@ typedef struct {
 } event_engine_factory;
 
 static const event_engine_factory g_factories[] = {
-    {"epollsig", grpc_init_epollsig_linux},
     {"epoll1", grpc_init_epoll1_linux},
+    {"epollsig", grpc_init_epollsig_linux},
     {"epoll-threadpool", grpc_init_epoll_thread_pool_linux},
     {"epoll-limited", grpc_init_epoll_limited_pollers_linux},
     {"poll", grpc_init_poll_posix},

src/core/lib/iomgr/exec_ctx.h

@@ -20,6 +20,7 @@
 #define GRPC_CORE_LIB_IOMGR_EXEC_CTX_H
 
 #include <grpc/support/atm.h>
+#include <grpc/support/cpu.h>
 
 #include "src/core/lib/iomgr/closure.h"
 
@@ -66,6 +67,7 @@ struct grpc_exec_ctx {
   /** last active combiner in the active combiner list */
   grpc_combiner *last_combiner;
   uintptr_t flags;
+  unsigned starting_cpu;
   void *check_ready_to_finish_arg;
   bool (*check_ready_to_finish)(grpc_exec_ctx *exec_ctx, void *arg);
 
@@ -75,10 +77,10 @@ struct grpc_exec_ctx {
 
 /* initializer for grpc_exec_ctx:
    prefer to use GRPC_EXEC_CTX_INIT whenever possible */
-#define GRPC_EXEC_CTX_INITIALIZER(flags, finish_check, finish_check_arg)       \
-  {                                                                            \
-    GRPC_CLOSURE_LIST_INIT, NULL, NULL, flags, finish_check_arg, finish_check, \
-        false, 0                                                               \
+#define GRPC_EXEC_CTX_INITIALIZER(flags, finish_check, finish_check_arg) \
+  {                                                                      \
+    GRPC_CLOSURE_LIST_INIT, NULL, NULL, flags, gpr_cpu_current_cpu(),    \
+        finish_check_arg, finish_check                                   \
   }
 
 /* initialize an execution context at the top level of an API call into grpc

src/core/lib/iomgr/iocp_windows.c

@@ -27,6 +27,7 @@
 #include <grpc/support/log_windows.h>
 #include <grpc/support/thd.h>
 
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/iocp_windows.h"
 #include "src/core/lib/iomgr/iomgr_internal.h"
 #include "src/core/lib/iomgr/socket_windows.h"
@@ -65,6 +66,7 @@ grpc_iocp_work_status grpc_iocp_work(grpc_exec_ctx *exec_ctx,
   LPOVERLAPPED overlapped;
   grpc_winsocket *socket;
   grpc_winsocket_callback_info *info;
+  GRPC_STATS_INC_SYSCALL_POLL(exec_ctx);
   success = GetQueuedCompletionStatus(
       g_iocp, &bytes, &completion_key, &overlapped,
       deadline_to_millis_timeout(deadline, gpr_now(deadline.clock_type)));

src/core/lib/iomgr/tcp_posix.c

@@ -40,6 +40,7 @@
 #include <grpc/support/useful.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/iomgr/ev_posix.h"
 #include "src/core/lib/profiling/timers.h"
@@ -258,6 +259,7 @@ static void tcp_do_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) {
 
   GPR_TIMER_BEGIN("recvmsg", 0);
   do {
+    GRPC_STATS_INC_SYSCALL_READ(exec_ctx);
     read_bytes = recvmsg(tcp->fd, &msg, 0);
   } while (read_bytes < 0 && errno == EINTR);
   GPR_TIMER_END("recvmsg", read_bytes >= 0);
@@ -361,7 +363,8 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
 
 /* returns true if done, false if pending; if returning true, *error is set */
 #define MAX_WRITE_IOVEC 1000
-static bool tcp_flush(grpc_tcp *tcp, grpc_error **error) {
+static bool tcp_flush(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp,
+                      grpc_error **error) {
   struct msghdr msg;
   struct iovec iov[MAX_WRITE_IOVEC];
   msg_iovlen_type iov_size;
@@ -403,6 +406,7 @@ static bool tcp_flush(grpc_tcp *tcp, grpc_error **error) {
     GPR_TIMER_BEGIN("sendmsg", 1);
     do {
       /* TODO(klempner): Cork if this is a partial write */
+      GRPC_STATS_INC_SYSCALL_WRITE(exec_ctx);
       sent_length = sendmsg(tcp->fd, &msg, SENDMSG_FLAGS);
     } while (sent_length < 0 && errno == EINTR);
     GPR_TIMER_END("sendmsg", 0);
@@ -459,7 +463,7 @@ static void tcp_handle_write(grpc_exec_ctx *exec_ctx, void *arg /* grpc_tcp */,
     return;
   }
 
-  if (!tcp_flush(tcp, &error)) {
+  if (!tcp_flush(exec_ctx, tcp, &error)) {
     if (GRPC_TRACER_ON(grpc_tcp_trace)) {
       gpr_log(GPR_DEBUG, "write: delayed");
     }
@@ -510,7 +514,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
   tcp->outgoing_slice_idx = 0;
   tcp->outgoing_byte_idx = 0;
 
-  if (!tcp_flush(tcp, &error)) {
+  if (!tcp_flush(exec_ctx, tcp, &error)) {
     TCP_REF(tcp, "write");
     tcp->write_cb = cb;
     if (GRPC_TRACER_ON(grpc_tcp_trace)) {

src/core/lib/security/transport/client_auth_filter.c

@@ -39,6 +39,7 @@
 
 /* We can have a per-call credentials. */
 typedef struct {
+  grpc_call_stack *owning_call;
   grpc_call_combiner *call_combiner;
   grpc_call_credentials *creds;
   bool have_host;
@@ -53,8 +54,9 @@ typedef struct {
   grpc_credentials_mdelem_array md_array;
   grpc_linked_mdelem md_links[MAX_CREDENTIALS_METADATA_COUNT];
   grpc_auth_metadata_context auth_md_context;
-  grpc_closure async_cancel_closure;
   grpc_closure async_result_closure;
+  grpc_closure check_call_host_cancel_closure;
+  grpc_closure get_request_metadata_cancel_closure;
 } call_data;
 
 /* We can have a per-channel credentials. */
@@ -151,8 +153,12 @@ static void cancel_get_request_metadata(grpc_exec_ctx *exec_ctx, void *arg,
                                         grpc_error *error) {
   grpc_call_element *elem = (grpc_call_element *)arg;
   call_data *calld = (call_data *)elem->call_data;
-  grpc_call_credentials_cancel_get_request_metadata(
-      exec_ctx, calld->creds, &calld->md_array, GRPC_ERROR_REF(error));
+  if (error != GRPC_ERROR_NONE) {
+    grpc_call_credentials_cancel_get_request_metadata(
+        exec_ctx, calld->creds, &calld->md_array, GRPC_ERROR_REF(error));
+  }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call,
+                        "cancel_get_request_metadata");
 }
 
 static void send_security_metadata(grpc_exec_ctx *exec_ctx,
@@ -208,10 +214,12 @@ static void send_security_metadata(grpc_exec_ctx *exec_ctx,
     GRPC_ERROR_UNREF(error);
   } else {
     // Async return; register cancellation closure with call combiner.
-    GRPC_CLOSURE_INIT(&calld->async_cancel_closure, cancel_get_request_metadata,
-                      elem, grpc_schedule_on_exec_ctx);
-    grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
-                                            &calld->async_cancel_closure);
+    GRPC_CALL_STACK_REF(calld->owning_call, "cancel_get_request_metadata");
+    grpc_call_combiner_set_notify_on_cancel(
+        exec_ctx, calld->call_combiner,
+        GRPC_CLOSURE_INIT(&calld->get_request_metadata_cancel_closure,
+                          cancel_get_request_metadata, elem,
+                          grpc_schedule_on_exec_ctx));
   }
 }
 
@@ -244,9 +252,12 @@ static void cancel_check_call_host(grpc_exec_ctx *exec_ctx, void *arg,
   grpc_call_element *elem = (grpc_call_element *)arg;
   call_data *calld = (call_data *)elem->call_data;
   channel_data *chand = (channel_data *)elem->channel_data;
-  grpc_channel_security_connector_cancel_check_call_host(
-      exec_ctx, chand->security_connector, &calld->async_result_closure,
-      GRPC_ERROR_REF(error));
+  if (error != GRPC_ERROR_NONE) {
+    grpc_channel_security_connector_cancel_check_call_host(
+        exec_ctx, chand->security_connector, &calld->async_result_closure,
+        GRPC_ERROR_REF(error));
+  }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "cancel_check_call_host");
 }
 
 static void auth_start_transport_stream_op_batch(
@@ -307,11 +318,21 @@ static void auth_start_transport_stream_op_batch(
         on_host_checked(exec_ctx, batch, error);
         GRPC_ERROR_UNREF(error);
       } else {
+// FIXME: if grpc_channel_security_connector_check_call_host() invokes
+// the callback in this thread before returning, then we'll call
+// grpc_call_combiner_set_notify_on_cancel() to set it "back" to NULL
+// *before* we call this to set it to the cancel function.
+// Can't just do this before calling
+// grpc_channel_security_connector_check_call_host(), because then the
+// cancellation might be invoked before we actually send the request.
+// May need to fix the credentials plugin API to deal with this.
         // Async return; register cancellation closure with call combiner.
-        GRPC_CLOSURE_INIT(&calld->async_cancel_closure, cancel_check_call_host,
-                          elem, grpc_schedule_on_exec_ctx);
-        grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,
-                                                &calld->async_cancel_closure);
+        GRPC_CALL_STACK_REF(calld->owning_call, "cancel_check_call_host");
+        grpc_call_combiner_set_notify_on_cancel(
+            exec_ctx, calld->call_combiner,
+            GRPC_CLOSURE_INIT(&calld->check_call_host_cancel_closure,
+                              cancel_check_call_host, elem,
+                              grpc_schedule_on_exec_ctx));
       }
       gpr_free(call_host);
       GPR_TIMER_END("auth_start_transport_stream_op_batch", 0);
@@ -329,6 +350,7 @@ static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx,
                                   grpc_call_element *elem,
                                   const grpc_call_element_args *args) {
   call_data *calld = elem->call_data;
+  calld->owning_call = args->call_stack;
   calld->call_combiner = args->call_combiner;
   return GRPC_ERROR_NONE;
 }

src/core/lib/security/transport/server_auth_filter.c

@@ -152,11 +152,13 @@ static void cancel_call(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) {
   grpc_call_element *elem = (grpc_call_element *)arg;
   call_data *calld = elem->call_data;
   // If the result was not already processed, invoke the callback now.
-  if (gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT,
+  if (error != GRPC_ERROR_NONE &&
+      gpr_atm_full_cas(&calld->state, (gpr_atm)STATE_INIT,
                        (gpr_atm)STATE_CANCELLED)) {
     on_md_processing_done_inner(exec_ctx, elem, NULL, 0, NULL, 0,
                                 GRPC_ERROR_REF(error));
   }
+  GRPC_CALL_STACK_UNREF(exec_ctx, calld->owning_call, "cancel_call");
 }
 
 static void recv_initial_metadata_ready(grpc_exec_ctx *exec_ctx, void *arg,
@@ -169,6 +171,7 @@ static void recv_initial_metadata_ready(grpc_exec_ctx *exec_ctx, void *arg,
     if (chand->creds != NULL && chand->creds->processor.process != NULL) {
       // We're calling out to the application, so we need to make sure
       // to drop the call combiner early if we get cancelled.
+      GRPC_CALL_STACK_REF(calld->owning_call, "cancel_call");
       GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem,
                         grpc_schedule_on_exec_ctx);
       grpc_call_combiner_set_notify_on_cancel(exec_ctx, calld->call_combiner,

src/core/lib/surface/call.c

@@ -32,6 +32,7 @@
 
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/compression/algorithm_metadata.h"
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/slice/slice_internal.h"
@@ -343,6 +344,11 @@ grpc_error *grpc_call_create(grpc_exec_ctx *exec_ctx,
   /* Always support no compression */
   GPR_BITSET(&call->encodings_accepted_by_peer, GRPC_COMPRESS_NONE);
   call->is_client = args->server_transport_data == NULL;
+  if (call->is_client) {
+    GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx);
+  } else {
+    GRPC_STATS_INC_SERVER_CALLS_CREATED(exec_ctx);
+  }
   call->stream_op_payload.context = call->context;
   grpc_slice path = grpc_empty_slice();
   if (call->is_client) {

src/core/lib/surface/init.c

@@ -28,6 +28,7 @@
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/channel/handshaker_registry.h"
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/debug/trace.h"
 #include "src/core/lib/http/parser.h"
 #include "src/core/lib/iomgr/call_combiner.h"
@@ -119,6 +120,7 @@ void grpc_init(void) {
   gpr_mu_lock(&g_init_mu);
   if (++g_initializations == 1) {
     gpr_time_init();
+    grpc_stats_init();
     grpc_slice_intern_init();
     grpc_mdctx_global_init();
     grpc_channel_init_init();
@@ -188,6 +190,7 @@ void grpc_shutdown(void) {
     grpc_mdctx_global_shutdown(&exec_ctx);
     grpc_handshaker_factory_registry_shutdown(&exec_ctx);
     grpc_slice_intern_shutdown();
+    grpc_stats_shutdown();
   }
   gpr_mu_unlock(&g_init_mu);
   grpc_exec_ctx_finish(&exec_ctx);

src/core/lib/transport/byte_stream.c

@@ -85,6 +85,7 @@ static void slice_buffer_stream_shutdown(grpc_exec_ctx *exec_ctx,
 static void slice_buffer_stream_destroy(grpc_exec_ctx *exec_ctx,
                                         grpc_byte_stream *byte_stream) {
   grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream;
+  grpc_slice_buffer_reset_and_unref_internal(exec_ctx, stream->backing_buffer);
   GRPC_ERROR_UNREF(stream->shutdown_error);
 }
 

src/core/lib/transport/byte_stream.h

@@ -81,7 +81,9 @@ void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx,
 
 // grpc_slice_buffer_stream
 //
-// A grpc_byte_stream that wraps a slice buffer.
+// A grpc_byte_stream that wraps a slice buffer.  The stream takes
+// ownership of the slices in the buffer, and on destruction will
+// reset the contents of the buffer.
 
 typedef struct grpc_slice_buffer_stream {
   grpc_byte_stream base;

src/cpp/common/channel_filter.cc

@@ -18,7 +18,9 @@
 
 #include <string.h>
 
+extern "C" {
 #include "src/core/lib/channel/channel_stack.h"
+}
 #include "src/cpp/common/channel_filter.h"
 
 #include <grpc++/impl/codegen/slice.h>

src/cpp/common/channel_filter.h

@@ -26,9 +26,11 @@
 #include <functional>
 #include <vector>
 
+extern "C" {
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/transport/metadata_batch.h"
+}
 
 /// An interface to define filters.
 ///
@@ -193,6 +195,15 @@ class TransportStreamOpBatch {
     op_->payload->send_message.send_message = send_message;
   }
 
+  grpc_byte_stream **recv_message() const {
+    return op_->recv_message ? op_->payload->recv_message.recv_message
+                             : nullptr;
+  }
+  void set_recv_message(grpc_byte_stream **recv_message) {
+    op_->recv_message = true;
+    op_->payload->recv_message.recv_message = recv_message;
+  }
+
   census_context *get_census_context() const {
     return (census_context *)op_->payload->context[GRPC_CONTEXT_TRACING].value;
   }

src/cpp/common/core_codegen.cc

@@ -89,6 +89,10 @@ int CoreCodegen::gpr_cv_wait(gpr_cv* cv, gpr_mu* mu,
 void CoreCodegen::gpr_cv_signal(gpr_cv* cv) { ::gpr_cv_signal(cv); }
 void CoreCodegen::gpr_cv_broadcast(gpr_cv* cv) { ::gpr_cv_broadcast(cv); }
 
+grpc_byte_buffer* CoreCodegen::grpc_byte_buffer_copy(grpc_byte_buffer* bb) {
+  return ::grpc_byte_buffer_copy(bb);
+}
+
 void CoreCodegen::grpc_byte_buffer_destroy(grpc_byte_buffer* bb) {
   ::grpc_byte_buffer_destroy(bb);
 }

src/php/ext/grpc/php_grpc.c

@@ -49,9 +49,7 @@ const zend_function_entry grpc_functions[] = {
 /* {{{ grpc_module_entry
  */
 zend_module_entry grpc_module_entry = {
-#if ZEND_MODULE_API_NO >= 20010901
   STANDARD_MODULE_HEADER,
-#endif
   "grpc",
   grpc_functions,
   PHP_MINIT(grpc),
@@ -59,9 +57,7 @@ zend_module_entry grpc_module_entry = {
   PHP_RINIT(grpc),
   NULL,
   PHP_MINFO(grpc),
-#if ZEND_MODULE_API_NO >= 20010901
   PHP_GRPC_VERSION,
-#endif
   PHP_MODULE_GLOBALS(grpc),
   PHP_GINIT(grpc),
   NULL,

src/proto/grpc/health/v1/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
 
-load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
+grpc_package(name = "health", visibility = "public")
 
 grpc_proto_library(
     name = "health_proto",

src/proto/grpc/lb/v1/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
 
-load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
+grpc_package(name = "lb", visibility = "public")
 
 grpc_proto_library(
     name = "load_balancer_proto",

src/proto/grpc/reflection/v1alpha/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
 
-load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
+grpc_package(name = "reflection", visibility = "public")
 
 grpc_proto_library(
     name = "reflection_proto",

src/proto/grpc/status/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
 
-load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
+grpc_package(name = "status", visibility = "public")
 
 grpc_proto_library(
     name = "status_proto",

src/proto/grpc/testing/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
 
-load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
+grpc_package(name = "testing", visibility = "public")
 
 grpc_proto_library(
     name = "compiler_test_proto",

src/proto/grpc/testing/duplicate/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+load("//bazel:grpc_build_system.bzl", "grpc_proto_library", "grpc_package")
 
-load("//bazel:grpc_build_system.bzl", "grpc_proto_library")
+grpc_package(name = "duplicate", visibility = "public")
 
 grpc_proto_library(
     name = "echo_duplicate_proto",

src/python/grpcio/grpc_core_dependencies.py

@@ -72,6 +72,8 @@ CORE_SOURCE_FILES = [
   'src/core/lib/compression/compression.c',
   'src/core/lib/compression/message_compress.c',
   'src/core/lib/compression/stream_compression.c',
+  'src/core/lib/debug/stats.c',
+  'src/core/lib/debug/stats_data.c',
   'src/core/lib/http/format_request.c',
   'src/core/lib/http/httpcli.c',
   'src/core/lib/http/parser.c',

test/core/bad_client/BUILD

@@ -12,14 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/bad_client")
 
 licenses(["notice"])  # Apache v2
 

test/core/bad_ssl/BUILD

@@ -12,14 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/bad_ssl")
 
 licenses(["notice"])  # Apache v2
 

test/core/census/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/census")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 grpc_cc_test(
     name = "context_test",

test/core/channel/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/channel")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 grpc_cc_test(
     name = "channel_args_test",

test/core/client_channel/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/client_channel")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/client_channel/resolvers/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/client_channel_resolvers")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 grpc_cc_test(
     name = "dns_resolver_connectivity_test",

test/core/compression/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/compression")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 grpc_cc_test(
     name = "algorithm_test",

test/core/end2end/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/end2end")
 
 load(":generate_tests.bzl", "grpc_end2end_tests")
 

test/core/end2end/fuzzers/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/end2end/fuzzers")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/end2end/tests/cancel_after_round_trip.c

@@ -114,7 +114,9 @@ static void test_cancel_after_round_trip(grpc_end2end_test_config config,
       grpc_slice_from_copied_string("hello you");
   grpc_byte_buffer *request_payload =
       grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-  grpc_byte_buffer *response_payload =
+  grpc_byte_buffer *response_payload1 =
+      grpc_raw_byte_buffer_create(&response_payload_slice, 1);
+  grpc_byte_buffer *response_payload2 =
       grpc_raw_byte_buffer_create(&response_payload_slice, 1);
   int was_cancelled = 2;
 
@@ -199,7 +201,7 @@ static void test_cancel_after_round_trip(grpc_end2end_test_config config,
   op->reserved = NULL;
   op++;
   op->op = GRPC_OP_SEND_MESSAGE;
-  op->data.send_message.send_message = response_payload;
+  op->data.send_message.send_message = response_payload1;
   op->flags = 0;
   op->reserved = NULL;
   op++;
@@ -242,7 +244,7 @@ static void test_cancel_after_round_trip(grpc_end2end_test_config config,
   op->reserved = NULL;
   op++;
   op->op = GRPC_OP_SEND_MESSAGE;
-  op->data.send_message.send_message = response_payload;
+  op->data.send_message.send_message = response_payload2;
   op->flags = 0;
   op->reserved = NULL;
   op++;
@@ -262,7 +264,8 @@ static void test_cancel_after_round_trip(grpc_end2end_test_config config,
   grpc_call_details_destroy(&call_details);
 
   grpc_byte_buffer_destroy(request_payload);
-  grpc_byte_buffer_destroy(response_payload);
+  grpc_byte_buffer_destroy(response_payload1);
+  grpc_byte_buffer_destroy(response_payload2);
   grpc_byte_buffer_destroy(request_payload_recv);
   grpc_byte_buffer_destroy(response_payload_recv);
   grpc_slice_unref(details);

test/core/end2end/tests/resource_quota_server.c

@@ -143,6 +143,8 @@ void resource_quota_server(grpc_end2end_test_config config) {
       malloc(sizeof(grpc_call_details) * NUM_CALLS);
   grpc_status_code *status = malloc(sizeof(grpc_status_code) * NUM_CALLS);
   grpc_slice *details = malloc(sizeof(grpc_slice) * NUM_CALLS);
+  grpc_byte_buffer **request_payload =
+      malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS);
   grpc_byte_buffer **request_payload_recv =
       malloc(sizeof(grpc_byte_buffer *) * NUM_CALLS);
   int *was_cancelled = malloc(sizeof(int) * NUM_CALLS);
@@ -156,9 +158,6 @@ void resource_quota_server(grpc_end2end_test_config config) {
   int deadline_exceeded = 0;
   int unavailable = 0;
 
-  grpc_byte_buffer *request_payload =
-      grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-
   grpc_op ops[6];
   grpc_op *op;
 
@@ -167,6 +166,7 @@ void resource_quota_server(grpc_end2end_test_config config) {
     grpc_metadata_array_init(&trailing_metadata_recv[i]);
     grpc_metadata_array_init(&request_metadata_recv[i]);
     grpc_call_details_init(&call_details[i]);
+    request_payload[i] = grpc_raw_byte_buffer_create(&request_payload_slice, 1);
     request_payload_recv[i] = NULL;
     was_cancelled[i] = 0;
   }
@@ -195,7 +195,7 @@ void resource_quota_server(grpc_end2end_test_config config) {
     op->reserved = NULL;
     op++;
     op->op = GRPC_OP_SEND_MESSAGE;
-    op->data.send_message.send_message = request_payload;
+    op->data.send_message.send_message = request_payload[i];
     op->flags = 0;
     op->reserved = NULL;
     op++;
@@ -261,6 +261,7 @@ void resource_quota_server(grpc_end2end_test_config config) {
       grpc_metadata_array_destroy(&trailing_metadata_recv[call_id]);
       grpc_call_unref(client_calls[call_id]);
       grpc_slice_unref(details[call_id]);
+      grpc_byte_buffer_destroy(request_payload[call_id]);
 
       pending_client_calls--;
     } else if (ev_tag < SERVER_RECV_BASE_TAG) {
@@ -351,7 +352,6 @@ void resource_quota_server(grpc_end2end_test_config config) {
           NUM_CALLS, cancelled_calls_on_server, cancelled_calls_on_client,
           deadline_exceeded, unavailable);
 
-  grpc_byte_buffer_destroy(request_payload);
   grpc_slice_unref(request_payload_slice);
   grpc_resource_quota_unref(resource_quota);
 
@@ -366,6 +366,7 @@ void resource_quota_server(grpc_end2end_test_config config) {
   free(call_details);
   free(status);
   free(details);
+  free(request_payload);
   free(request_payload_recv);
   free(was_cancelled);
 }

test/core/end2end/tests/simple_request.c

@@ -27,6 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+#include "src/core/lib/debug/stats.h"
 #include "src/core/lib/support/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
@@ -102,6 +103,10 @@ static void simple_request_body(grpc_end2end_test_config config,
   grpc_slice details;
   int was_cancelled = 2;
   char *peer;
+  grpc_stats_data before;
+  grpc_stats_data after;
+
+  grpc_stats_collect(&before);
 
   gpr_timespec deadline = five_seconds_from_now();
   c = grpc_channel_create_call(
@@ -208,6 +213,23 @@ static void simple_request_body(grpc_end2end_test_config config,
   grpc_call_unref(s);
 
   cq_verifier_destroy(cqv);
+
+  grpc_stats_collect(&after);
+
+  char *stats = grpc_stats_data_as_json(&after);
+  gpr_log(GPR_DEBUG, "%s", stats);
+  gpr_free(stats);
+
+  int expected_calls = 1;
+  if (config.feature_mask & FEATURE_MASK_SUPPORTS_REQUEST_PROXYING) {
+    expected_calls *= 2;
+  }
+  GPR_ASSERT(after.counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] -
+                 before.counters[GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED] ==
+             expected_calls);
+  GPR_ASSERT(after.counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] -
+                 before.counters[GRPC_STATS_COUNTER_SERVER_CALLS_CREATED] ==
+             expected_calls);
 }
 
 static void test_invoke_simple_request(grpc_end2end_test_config config) {

test/core/fling/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/fling")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/handshake/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/handshake")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 grpc_cc_test(
     name = "client_ssl",

test/core/http/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/http")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/iomgr/BUILD

@@ -12,19 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 
-package(
-    default_visibility = ["//visibility:public"], # Useful for third party devs to test their io manager implementation.
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/iomgr", visibility = "public") # Useful for third party devs to test their io manager implementation.
 
 grpc_cc_library(
     name = "endpoint_tests",

test/core/iomgr/ev_epollsig_linux_test.c

@@ -26,6 +26,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/thd.h>
@@ -306,9 +307,8 @@ static void test_threading(void) {
 int main(int argc, char **argv) {
   const char *poll_strategy = NULL;
   grpc_test_init(argc, argv);
+  grpc_init();
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_iomgr_init(&exec_ctx);
-  grpc_iomgr_start(&exec_ctx);
 
   poll_strategy = grpc_get_poll_strategy_name();
   if (poll_strategy != NULL && strcmp(poll_strategy, "epollsig") == 0) {
@@ -321,8 +321,8 @@ int main(int argc, char **argv) {
             poll_strategy);
   }
 
-  grpc_iomgr_shutdown(&exec_ctx);
   grpc_exec_ctx_finish(&exec_ctx);
+  grpc_shutdown();
   return 0;
 }
 #else /* defined(GRPC_LINUX_EPOLL) */

test/core/iomgr/fd_posix_test.c

@@ -35,6 +35,7 @@
 #include <sys/time.h>
 #include <unistd.h>
 
+#include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -522,8 +523,7 @@ int main(int argc, char **argv) {
   grpc_closure destroyed;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
   grpc_test_init(argc, argv);
-  grpc_iomgr_init(&exec_ctx);
-  grpc_iomgr_start(&exec_ctx);
+  grpc_init();
   g_pollset = gpr_zalloc(grpc_pollset_size());
   grpc_pollset_init(g_pollset, &g_mu);
   test_grpc_fd();
@@ -533,8 +533,8 @@ int main(int argc, char **argv) {
   grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
   grpc_exec_ctx_flush(&exec_ctx);
   gpr_free(g_pollset);
-  grpc_iomgr_shutdown(&exec_ctx);
   grpc_exec_ctx_finish(&exec_ctx);
+  grpc_shutdown();
   return 0;
 }
 

test/core/iomgr/resolve_address_posix_test.c

@@ -21,12 +21,14 @@
 #include <string.h>
 #include <sys/un.h>
 
+#include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
 #include <grpc/support/thd.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
+
 #include "src/core/lib/iomgr/executor.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "test/core/util/test_config.h"
@@ -157,13 +159,12 @@ static void test_unix_socket_path_name_too_long(void) {
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
+  grpc_init();
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_iomgr_init(&exec_ctx);
-  grpc_iomgr_start(&exec_ctx);
   test_unix_socket();
   test_unix_socket_path_name_too_long();
   grpc_executor_shutdown(&exec_ctx);
-  grpc_iomgr_shutdown(&exec_ctx);
   grpc_exec_ctx_finish(&exec_ctx);
+  grpc_shutdown();
   return 0;
 }

test/core/iomgr/resolve_address_test.c

@@ -17,6 +17,7 @@
  */
 
 #include "src/core/lib/iomgr/resolve_address.h"
+#include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/sync.h>
@@ -248,9 +249,8 @@ static void test_unparseable_hostports(void) {
 
 int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
+  grpc_init();
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_iomgr_init(&exec_ctx);
-  grpc_iomgr_start(&exec_ctx);
   test_localhost();
   test_default_port();
   test_non_numeric_default_port();
@@ -260,7 +260,7 @@ int main(int argc, char **argv) {
   test_invalid_ip_addresses();
   test_unparseable_hostports();
   grpc_executor_shutdown(&exec_ctx);
-  grpc_iomgr_shutdown(&exec_ctx);
   grpc_exec_ctx_finish(&exec_ctx);
+  grpc_shutdown();
   return 0;
 }

test/core/json/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/json")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/nanopb/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/nanopb")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/network_benchmarks/BUILD

@@ -12,17 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
-
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
+grpc_package(name = "test/core/network_benchmarks",
+             features = ["-layering_check", "-parse_headers" ]
 )
 
+licenses(["notice"])  # Apache v2
+
 grpc_cc_binary(
     name = "low_level_ping_pong",
     srcs = ["low_level_ping_pong.c"],

test/core/security/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/security")
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/slice/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-licenses(["notice"])  # Apache v2
+grpc_package(name = "test/core/slice")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+licenses(["notice"])  # Apache v2
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/support/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/support")
 
 grpc_cc_test(
     name = "alloc_test",

test/core/surface/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/surface")
 
 grpc_cc_test(
     name = "alarm_test",

test/core/transport/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/transport")
 
 grpc_cc_test(
     name = "bdp_estimator_test",

test/core/transport/chttp2/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/transport/chttp2")
 
 load("//test/core/util:grpc_fuzzer.bzl", "grpc_fuzzer")
 

test/core/tsi/BUILD

@@ -12,16 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/tsi")
 
 grpc_cc_test(
     name = "transport_security_test",

test/core/util/BUILD

@@ -12,17 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
 licenses(["notice"])  # Apache v2
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/core/util", visibility = "public")
 
 grpc_cc_library(
     name = "gpr_test_util",

test/cpp/codegen/BUILD

@@ -14,14 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_test")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_package")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/cpp/codegen")
 
 grpc_cc_test(
     name = "codegen_test_full",

test/cpp/common/BUILD

@@ -14,14 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_test")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_package")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/cpp/common")
 
 grpc_cc_test(
     name = "alarm_cpp_test",

test/cpp/end2end/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_package")
 
-package(
-    default_visibility=["//visibility:public"], # Allows external users to implement end2end tests.
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/cpp/end2end", visibility = "public") # Allows external users to implement end2end tests.
 
 grpc_cc_library(
     name = "test_service_impl",

test/cpp/interop/BUILD

@@ -14,14 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_test", "grpc_cc_binary", "grpc_package")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/cpp/interop")
 
 grpc_cc_library(
     name = "server_helper_lib",

test/cpp/microbenchmarks/BUILD

@@ -14,14 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_cc_library", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_cc_library", "grpc_cc_binary", "grpc_package")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/cpp/microbenchmarks")
 
 grpc_cc_test(
     name = "noop-benchmark",

test/cpp/microbenchmarks/bm_cq_multiple_threads.cc

@@ -38,6 +38,8 @@ struct grpc_pollset {
 namespace grpc {
 namespace testing {
 
+auto& force_library_initialization = Library::get();
+
 static void* g_tag = (void*)(intptr_t)10;  // Some random number
 static grpc_completion_queue* g_cq;
 static grpc_event_engine_vtable g_vtable;

test/cpp/microbenchmarks/helpers.cc

@@ -29,6 +29,13 @@ void TrackCounters::Finish(benchmark::State &state) {
 }
 
 void TrackCounters::AddToLabel(std::ostream &out, benchmark::State &state) {
+  grpc_stats_data stats_end;
+  grpc_stats_collect(&stats_end);
+  for (int i = 0; i < GRPC_STATS_COUNTER_COUNT; i++) {
+    out << " " << grpc_stats_counter_name[i] << "/iter:"
+        << ((double)(stats_end.counters[i] - stats_begin_.counters[i]) /
+            (double)state.iterations());
+  }
 #ifdef GPR_LOW_LEVEL_COUNTERS
   grpc_memory_counters counters_at_end = grpc_memory_counters_snapshot();
   out << " locks/iter:" << ((double)(gpr_atm_no_barrier_load(&gpr_mu_locks) -

test/cpp/microbenchmarks/helpers.h

@@ -23,6 +23,7 @@
 
 extern "C" {
 #include <grpc/support/port_platform.h>
+#include "src/core/lib/debug/stats.h"
 #include "test/core/util/memory_counters.h"
 }
 
@@ -62,10 +63,12 @@ extern "C" gpr_atm gpr_now_call_count;
 
 class TrackCounters {
  public:
+  TrackCounters() { grpc_stats_collect(&stats_begin_); }
   virtual void Finish(benchmark::State& state);
   virtual void AddToLabel(std::ostream& out, benchmark::State& state);
 
  private:
+  grpc_stats_data stats_begin_;
 #ifdef GPR_LOW_LEVEL_COUNTERS
   const size_t mu_locks_at_start_ = gpr_atm_no_barrier_load(&gpr_mu_locks);
   const size_t atm_cas_at_start_ =

test/cpp/qps/BUILD

@@ -14,14 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_cc_library", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_cc_library", "grpc_cc_binary", "grpc_package")
 
-package(
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/cpp/qps")
 
 grpc_cc_library(
     name = "parse_json",

test/cpp/server/BUILD

@@ -14,7 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_cc_library", "grpc_cc_binary")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_test", "grpc_cc_library", "grpc_cc_binary", "grpc_package")
+
+grpc_package(name = "test/cpp/server")
 
 grpc_cc_test(
     name = "server_builder_test",

test/cpp/util/BUILD

@@ -14,15 +14,9 @@
 
 licenses(["notice"])  # Apache v2
 
-load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_binary", "grpc_cc_test")
+load("//bazel:grpc_build_system.bzl", "grpc_cc_library", "grpc_cc_binary", "grpc_cc_test", "grpc_package")
 
-package(
-    default_visibility = ["//visibility:public"],
-    features = [
-        "-layering_check",
-        "-parse_headers",
-    ],
-)
+grpc_package(name = "test/cpp/util", visibility = "public")
 
 grpc_cc_library(
     name = "test_config",

tools/codegen/core/gen_stats_data.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python2.7
+
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import sys
+import yaml
+
+with open('src/core/lib/debug/stats_data.yaml') as f:
+  attrs = yaml.load(f.read())
+
+Counter = collections.namedtuple('Counter', 'name')
+
+counters = []
+
+for attr in attrs:
+  if 'counter' in attr:
+    counters.append(Counter(name=attr['counter']))
+  else:
+    print 'Error: bad attr %r' % attr
+
+# utility: print a big comment block into a set of files
+def put_banner(files, banner):
+  for f in files:
+    print >>f, '/*'
+    for line in banner:
+      print >>f, ' * %s' % line
+    print >>f, ' */'
+    print >>f
+
+with open('src/core/lib/debug/stats_data.h', 'w') as H:
+  # copy-paste copyright notice from this file
+  with open(sys.argv[0]) as my_source:
+    copyright = []
+    for line in my_source:
+      if line[0] != '#': break
+    for line in my_source:
+      if line[0] == '#':
+        copyright.append(line)
+        break
+    for line in my_source:
+      if line[0] != '#':
+        break
+      copyright.append(line)
+    put_banner([H], [line[2:].rstrip() for line in copyright])
+
+  put_banner([H], ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
+
+  print >>H, "#ifndef GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
+  print >>H, "#define GRPC_CORE_LIB_DEBUG_STATS_DATA_H"
+  print >>H
+
+  print >>H, "typedef enum {"
+  for ctr in counters:
+    print >>H, "  GRPC_STATS_COUNTER_%s," % ctr.name.upper()
+  print >>H, "  GRPC_STATS_COUNTER_COUNT"
+  print >>H, "} grpc_stats_counters;"
+
+  for ctr in counters:
+    print >>H, "#define GRPC_STATS_INC_%s(exec_ctx) GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_%s)" % (ctr.name.upper(), ctr.name.upper())
+
+  print >>H, "extern const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT];"
+
+  print >>H
+  print >>H, "#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */"
+
+with open('src/core/lib/debug/stats_data.c', 'w') as C:
+  # copy-paste copyright notice from this file
+  with open(sys.argv[0]) as my_source:
+    copyright = []
+    for line in my_source:
+      if line[0] != '#': break
+    for line in my_source:
+      if line[0] == '#':
+        copyright.append(line)
+        break
+    for line in my_source:
+      if line[0] != '#':
+        break
+      copyright.append(line)
+    put_banner([C], [line[2:].rstrip() for line in copyright])
+
+  put_banner([C], ["Automatically generated by tools/codegen/core/gen_stats_data.py"])
+
+  print >>C, "#include \"src/core/lib/debug/stats_data.h\""
+
+  print >>C, "const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {";
+  for ctr in counters:
+    print >>C, "  \"%s\"," % ctr.name
+  print >>C, "};"

tools/distrib/docker_for_windows.rb

@@ -23,11 +23,21 @@ def docker_for_windows_image()
   dockerfile = File.join(grpc_root, 'third_party', 'rake-compiler-dock', 'Dockerfile') 
   dockerpath = File.dirname(dockerfile)
   version = Digest::SHA1.file(dockerfile).hexdigest
-  image_name = 'grpc/rake-compiler-dock:' + version
-  cmd = "docker build -t #{image_name} --file #{dockerfile} #{dockerpath}"
-  puts cmd
-  system cmd
-  raise "Failed to build the docker image." unless $? == 0
+  image_name = 'rake-compiler-dock_' + version
+  # if "DOCKERHUB_ORGANIZATION" env is set, we try to pull the pre-built
+  # rake-compiler-dock image from dockerhub rather then building from scratch.
+  if ENV.has_key?('DOCKERHUB_ORGANIZATION')
+    image_name = ENV['DOCKERHUB_ORGANIZATION'] + '/' + image_name
+    cmd = "docker pull #{image_name}"
+    puts cmd
+    system cmd
+    raise "Failed to pull the docker image." unless $? == 0
+  else
+    cmd = "docker build -t #{image_name} --file #{dockerfile} #{dockerpath}"
+    puts cmd
+    system cmd
+    raise "Failed to build the docker image." unless $? == 0
+  end
   image_name
 end